diff options
282 files changed, 3930 insertions, 2743 deletions
diff --git a/client/go/cmd/api_key.go b/client/go/cmd/api_key.go index 438bf017a54..c6b6ae1d1e3 100644 --- a/client/go/cmd/api_key.go +++ b/client/go/cmd/api_key.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "log" + "github.com/fatih/color" "github.com/spf13/cobra" "github.com/vespa-engine/vespa/client/go/util" "github.com/vespa-engine/vespa/client/go/vespa" @@ -124,10 +125,10 @@ func printPublicKey(system vespa.System, apiKeyFile, tenant string) error { if err != nil { return fmt.Errorf("failed to extract fingerprint: %w", err) } - log.Printf("\nThis is your public key:\n%s", color.Green(pemPublicKey)) - log.Printf("Its fingerprint is:\n%s\n", color.Cyan(fingerprint)) + log.Printf("\nThis is your public key:\n%s", color.GreenString(string(pemPublicKey))) + log.Printf("Its fingerprint is:\n%s\n", color.CyanString(fingerprint)) log.Print("\nTo use this key in Vespa Cloud click 'Add custom key' at") - log.Printf(color.Cyan("%s/tenant/%s/keys").String(), system.ConsoleURL, tenant) + log.Printf(color.CyanString("%s/tenant/%s/keys"), system.ConsoleURL, tenant) log.Print("and paste the entire public key including the BEGIN and END lines.") return nil } diff --git a/client/go/cmd/cert.go b/client/go/cmd/cert.go index 5120459f7ac..672442b9ad9 100644 --- a/client/go/cmd/cert.go +++ b/client/go/cmd/cert.go @@ -10,6 +10,7 @@ import ( "os" "path/filepath" + "github.com/fatih/color" "github.com/spf13/cobra" "github.com/vespa-engine/vespa/client/go/util" "github.com/vespa-engine/vespa/client/go/vespa" @@ -97,7 +98,7 @@ command (default '.').`, $ vespa auth cert add -a my-tenant.my-app.my-instance path/to/application/package`, DisableAutoGenTag: true, SilenceUsage: true, - Args: cobra.MinimumNArgs(1), + Args: cobra.MaximumNArgs(1), RunE: doCertAdd, } @@ -134,10 +135,10 @@ func doCert(_ *cobra.Command, args []string) error { } } if util.PathExists(privateKeyFile) { - return errHint(fmt.Errorf("private key %s already exists", color.Cyan(privateKeyFile)), hint) + return errHint(fmt.Errorf("private key %s already exists", color.CyanString(privateKeyFile)), hint) } if util.PathExists(certificateFile) { - return errHint(fmt.Errorf("certificate %s already exists", color.Cyan(certificateFile)), hint) + return errHint(fmt.Errorf("certificate %s already exists", color.CyanString(certificateFile)), hint) } } if !noApplicationPackage { @@ -168,10 +169,10 @@ func doCert(_ *cobra.Command, args []string) error { return fmt.Errorf("could not write private key: %w", err) } if !noApplicationPackage { - printSuccess("Certificate written to ", color.Cyan(pkgCertificateFile)) + printSuccess("Certificate written to ", color.CyanString(pkgCertificateFile)) } - printSuccess("Certificate written to ", color.Cyan(certificateFile)) - printSuccess("Private key written to ", color.Cyan(privateKeyFile)) + printSuccess("Certificate written to ", color.CyanString(certificateFile)) + printSuccess("Private key written to ", color.CyanString(privateKeyFile)) return nil } @@ -225,6 +226,6 @@ func doCertAdd(_ *cobra.Command, args []string) error { return fmt.Errorf("could not copy certificate file to application: %w", err) } - printSuccess("Certificate written to ", color.Cyan(pkgCertificateFile)) + printSuccess("Certificate written to ", color.CyanString(pkgCertificateFile)) return nil } diff --git a/client/go/cmd/clone.go b/client/go/cmd/clone.go index 1329e868a9c..a1add299712 100644 --- a/client/go/cmd/clone.go +++ b/client/go/cmd/clone.go @@ -17,6 +17,7 @@ import ( "strings" "time" + "github.com/fatih/color" "github.com/spf13/cobra" "github.com/vespa-engine/vespa/client/go/util" ) @@ -77,7 +78,7 @@ func cloneApplication(applicationName string, applicationDir string) error { r, err := zip.OpenReader(zipFile.Name()) if err != nil { - return fmt.Errorf("could not open sample apps zip '%s': %w", color.Cyan(zipFile.Name()), err) + return fmt.Errorf("could not open sample apps zip '%s': %w", color.CyanString(zipFile.Name()), err) } defer r.Close() @@ -88,20 +89,20 @@ func cloneApplication(applicationName string, applicationDir string) error { if !found { // Create destination directory lazily when source is found createErr := os.Mkdir(applicationDir, 0755) if createErr != nil { - return fmt.Errorf("could not create directory '%s': %w", color.Cyan(applicationDir), createErr) + return fmt.Errorf("could not create directory '%s': %w", color.CyanString(applicationDir), createErr) } } found = true if err := copy(f, applicationDir, dirPrefix); err != nil { - return fmt.Errorf("could not copy zip entry '%s': %w", color.Cyan(f.Name), err) + return fmt.Errorf("could not copy zip entry '%s': %w", color.CyanString(f.Name), err) } } } if !found { - return errHint(fmt.Errorf("could not find source application '%s'", color.Cyan(applicationName)), "Use -f to ignore the cache") + return errHint(fmt.Errorf("could not find source application '%s'", color.CyanString(applicationName)), "Use -f to ignore the cache") } else { - log.Print("Created ", color.Cyan(applicationDir)) + log.Print("Created ", color.CyanString(applicationDir)) } return nil } @@ -120,7 +121,7 @@ func fetchSampleAppsZip(destination string) error { return fmt.Errorf("could not create temporary file: %w", err) } defer f.Close() - return util.Spinner(stderr, color.Yellow("Downloading sample apps ...").String(), func() error { + return util.Spinner(stderr, color.YellowString("Downloading sample apps ..."), func() error { request, err := http.NewRequest("GET", "https://github.com/vespa-engine/sample-apps/archive/refs/heads/master.zip", nil) if err != nil { return fmt.Errorf("invalid url: %w", err) @@ -163,7 +164,7 @@ func openSampleAppsZip() (*os.File, error) { return nil, errHint(fmt.Errorf("could not determine cache status: %w", err), "Try ignoring the cache with the -f flag") } if useCache { - log.Print(color.Yellow("Using cached sample apps ...")) + log.Print(color.YellowString("Using cached sample apps ...")) return os.Open(path) } } diff --git a/client/go/cmd/command_tester_test.go b/client/go/cmd/command_tester_test.go index d71cdde0e8f..efaca6a7258 100644 --- a/client/go/cmd/command_tester_test.go +++ b/client/go/cmd/command_tester_test.go @@ -53,7 +53,7 @@ func setEnv(env map[string]string) map[string]string { } func resetEnv(env map[string]string, original map[string]string) { - for k, _ := range env { + for k := range env { if v, ok := original[k]; ok { os.Setenv(k, v) } else { diff --git a/client/go/cmd/config.go b/client/go/cmd/config.go index 0997be2c899..59739d1c342 100644 --- a/client/go/cmd/config.go +++ b/client/go/cmd/config.go @@ -15,6 +15,7 @@ import ( "strconv" "strings" + "github.com/fatih/color" "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/vespa-engine/vespa/client/go/auth0" @@ -183,14 +184,14 @@ func (c *Config) X509KeyPair(app vespa.ApplicationID) (KeyPair, error) { } func (c *Config) APIKeyPath(tenantName string) string { - if override, err := c.Get(apiKeyFileFlag); err == nil && override != "" { + if override, ok := c.Get(apiKeyFileFlag); ok { return override } return filepath.Join(c.Home, tenantName+".api-key.pem") } func (c *Config) ReadAPIKey(tenantName string) ([]byte, error) { - if override, err := c.Get(apiKeyFlag); err == nil && override != "" { + if override, ok := c.Get(apiKeyFlag); ok { return []byte(override), nil } return ioutil.ReadFile(c.APIKeyPath(tenantName)) @@ -198,19 +199,20 @@ func (c *Config) ReadAPIKey(tenantName string) ([]byte, error) { // UseAPIKey checks if api key should be used be checking if api-key or api-key-file has been set. func (c *Config) UseAPIKey(system vespa.System, tenantName string) bool { - if _, err := c.Get(apiKeyFlag); err == nil { + if _, ok := c.Get(apiKeyFlag); ok { return true } - if _, err := c.Get(apiKeyFileFlag); err == nil { + if _, ok := c.Get(apiKeyFileFlag); ok { return true } - // If no Auth0 token is created, fall back to tenant api key, but warn that this functionality is deprecated // TODO: Remove this when users have had time to migrate over to Auth0 device flow authentication - a, err := auth0.GetAuth0(c.AuthConfigPath(), system.Name, system.URL) - if err != nil || !a.HasSystem() { - fmt.Fprintln(stderr, "Defaulting to tenant API key is deprecated. Use Auth0 device flow: 'vespa auth login' instead") - return util.PathExists(c.APIKeyPath(tenantName)) + if !isCI() { + a, err := auth0.GetAuth0(c.AuthConfigPath(), system.Name, system.URL) + if err != nil || !a.HasSystem() { + printWarning("Use of API key is deprecated", "Authenticate with Auth0 instead: 'vespa auth login'") + return util.PathExists(c.APIKeyPath(tenantName)) + } } return false } @@ -267,12 +269,12 @@ func (c *Config) load() error { return err } -func (c *Config) Get(option string) (string, error) { +func (c *Config) Get(option string) (string, bool) { value := viper.GetString(option) if value == "" { - return "", fmt.Errorf("no such option: %q", option) + return "", false } - return value, nil + return value, true } func (c *Config) Set(option, value string) error { @@ -305,6 +307,12 @@ func (c *Config) Set(option, value string) error { viper.Set(option, value) return nil } + case quietFlag: + switch value { + case "true", "false": + viper.Set(option, value) + return nil + } case apiKeyFileFlag: viper.Set(option, value) return nil @@ -313,11 +321,12 @@ func (c *Config) Set(option, value string) error { } func printOption(cfg *Config, option string) { - value, err := cfg.Get(option) - if err != nil { - value = color.Faint("<unset>").String() + value, ok := cfg.Get(option) + if !ok { + faintColor := color.New(color.FgWhite, color.Faint) + value = faintColor.Sprint("<unset>") } else { - value = color.Cyan(value).String() + value = color.CyanString(value) } log.Printf("%s = %s", option, value) } diff --git a/client/go/cmd/config_test.go b/client/go/cmd/config_test.go index 2f0ccbb29e1..1ca51652340 100644 --- a/client/go/cmd/config_test.go +++ b/client/go/cmd/config_test.go @@ -38,6 +38,9 @@ func TestConfig(t *testing.T) { assertConfigCommand(t, "", homeDir, "config", "set", "wait", "60") assertConfigCommandErr(t, "Error: wait option must be an integer >= 0, got \"foo\"\n", homeDir, "config", "set", "wait", "foo") assertConfigCommand(t, "wait = 60\n", homeDir, "config", "get", "wait") + + assertConfigCommand(t, "", homeDir, "config", "set", "quiet", "true") + assertConfigCommand(t, "", homeDir, "config", "set", "quiet", "false") } func assertConfigCommand(t *testing.T, expected, homeDir string, args ...string) { @@ -98,10 +101,17 @@ func TestUseAPIKey(t *testing.T) { } }` withEnv("VESPA_CLI_CLOUD_SYSTEM", "public", func() { + ci, ok := os.LookupEnv("CI") + if ok { + os.Unsetenv("CI") // Test depends on unset variable + } _, err := os.Create(filepath.Join(homeDir, "t2.api-key.pem")) require.Nil(t, err) assert.True(t, c.UseAPIKey(vespa.PublicSystem, "t2")) require.Nil(t, ioutil.WriteFile(filepath.Join(homeDir, "auth.json"), []byte(authContent), 0600)) assert.False(t, c.UseAPIKey(vespa.PublicSystem, "t2")) + if ok { + os.Setenv("CI", ci) + } }) } diff --git a/client/go/cmd/curl.go b/client/go/cmd/curl.go index 1ede2cccae3..289a65465bd 100644 --- a/client/go/cmd/curl.go +++ b/client/go/cmd/curl.go @@ -42,11 +42,11 @@ $ vespa curl -- -v --data-urlencode "yql=select * from music where album contain if err != nil { return err } - app, err := getApplication() + target, err := getTarget() if err != nil { return err } - service, err := getService(curlService, 0, "") + service, err := target.Service(curlService, 0, 0, "") if err != nil { return err } @@ -57,27 +57,15 @@ $ vespa curl -- -v --data-urlencode "yql=select * from music where album contain return err } switch curlService { - case "deploy": - t, err := getTarget() - if err != nil { - return err - } - if t.Type() == vespa.TargetCloud { - if err := addCloudAuth0Authentication(t.Deployment().System, cfg, c); err != nil { + case vespa.DeployService: + if target.Type() == vespa.TargetCloud { + if err := addCloudAuth0Authentication(target.Deployment().System, cfg, c); err != nil { return err } } - case "document", "query": - privateKeyFile, err := cfg.PrivateKeyPath(app) - if err != nil { - return err - } - certificateFile, err := cfg.CertificatePath(app) - if err != nil { - return err - } - c.PrivateKey = privateKeyFile - c.Certificate = certificateFile + case vespa.DocumentService, vespa.QueryService: + c.PrivateKey = service.TLSOptions.PrivateKeyFile + c.Certificate = service.TLSOptions.CertificateFile default: return fmt.Errorf("service not found: %s", curlService) } diff --git a/client/go/cmd/curl_test.go b/client/go/cmd/curl_test.go index 253943f2b04..50b837e0d85 100644 --- a/client/go/cmd/curl_test.go +++ b/client/go/cmd/curl_test.go @@ -3,6 +3,7 @@ package cmd import ( "fmt" + "os" "path/filepath" "testing" @@ -13,14 +14,27 @@ import ( func TestCurl(t *testing.T) { homeDir := filepath.Join(t.TempDir(), ".vespa") httpClient := &mock.HTTPClient{} - out, _ := execute(command{homeDir: homeDir, args: []string{"curl", "-n", "-a", "t1.a1.i1", "--", "-v", "--data-urlencode", "arg=with space", "/search"}}, t, httpClient) + _, outErr := execute(command{args: []string{"config", "set", "application", "t1.a1.i1"}, homeDir: homeDir}, t, nil) + assert.Equal(t, "", outErr) + _, outErr = execute(command{args: []string{"config", "set", "target", "cloud"}, homeDir: homeDir}, t, nil) + assert.Equal(t, "", outErr) + _, outErr = execute(command{args: []string{"auth", "api-key"}, homeDir: homeDir}, t, nil) + assert.Equal(t, "", outErr) + _, outErr = execute(command{args: []string{"auth", "cert", "--no-add"}, homeDir: homeDir}, t, nil) + assert.Equal(t, "", outErr) + + os.Setenv("VESPA_CLI_ENDPOINTS", "{\"endpoints\":[{\"cluster\":\"container\",\"url\":\"http://127.0.0.1:8080\"}]}") + out, _ := execute(command{homeDir: homeDir, args: []string{"curl", "-n", "--", "-v", "--data-urlencode", "arg=with space", "/search"}}, t, httpClient) expected := fmt.Sprintf("curl --key %s --cert %s -v --data-urlencode 'arg=with space' http://127.0.0.1:8080/search\n", filepath.Join(homeDir, "t1.a1.i1", "data-plane-private-key.pem"), filepath.Join(homeDir, "t1.a1.i1", "data-plane-public-cert.pem")) assert.Equal(t, expected, out) - out, _ = execute(command{homeDir: homeDir, args: []string{"curl", "-a", "t1.a1.i1", "-s", "deploy", "-n", "/application/v4/tenant/foo"}}, t, httpClient) + _, outErr = execute(command{args: []string{"config", "set", "target", "local"}, homeDir: homeDir}, t, nil) + assert.Equal(t, "", outErr) + out, outErr = execute(command{homeDir: homeDir, args: []string{"curl", "-a", "t1.a1.i1", "-s", "deploy", "-n", "/application/v4/tenant/foo"}}, t, httpClient) + assert.Equal(t, "", outErr) expected = "curl http://127.0.0.1:19071/application/v4/tenant/foo\n" assert.Equal(t, expected, out) } diff --git a/client/go/cmd/deploy.go b/client/go/cmd/deploy.go index 4a7fae243a0..396f42fae67 100644 --- a/client/go/cmd/deploy.go +++ b/client/go/cmd/deploy.go @@ -7,7 +7,9 @@ package cmd import ( "fmt" "log" + "strconv" + "github.com/fatih/color" "github.com/spf13/cobra" "github.com/vespa-engine/vespa/client/go/util" "github.com/vespa-engine/vespa/client/go/vespa" @@ -70,9 +72,9 @@ $ vespa deploy -t cloud -z perf.aws-us-east-1c`, return err } - var sessionOrRunID int64 + var result vespa.PrepareResult err = util.Spinner(stderr, "Uploading application package ...", func() error { - sessionOrRunID, err = vespa.Deploy(opts) + result, err = vespa.Deploy(opts) return err }) if err != nil { @@ -81,19 +83,20 @@ $ vespa deploy -t cloud -z perf.aws-us-east-1c`, log.Println() if opts.IsCloud() { - printSuccess("Triggered deployment of ", color.Cyan(pkg.Path), " with run ID ", color.Cyan(sessionOrRunID)) + printSuccess("Triggered deployment of ", color.CyanString(pkg.Path), " with run ID ", color.CyanString(strconv.FormatInt(result.ID, 10))) } else { - printSuccess("Deployed ", color.Cyan(pkg.Path)) + printSuccess("Deployed ", color.CyanString(pkg.Path)) + printPrepareLog(result) } if opts.IsCloud() { - log.Printf("\nUse %s for deployment status, or follow this deployment at", color.Cyan("vespa status")) - log.Print(color.Cyan(fmt.Sprintf("%s/tenant/%s/application/%s/dev/instance/%s/job/%s-%s/run/%d", + log.Printf("\nUse %s for deployment status, or follow this deployment at", color.CyanString("vespa status")) + log.Print(color.CyanString(fmt.Sprintf("%s/tenant/%s/application/%s/dev/instance/%s/job/%s-%s/run/%d", opts.Target.Deployment().System.ConsoleURL, opts.Target.Deployment().Application.Tenant, opts.Target.Deployment().Application.Application, opts.Target.Deployment().Application.Instance, opts.Target.Deployment().Zone.Environment, opts.Target.Deployment().Zone.Region, - sessionOrRunID))) + result.ID))) } - return waitForQueryService(sessionOrRunID) + return waitForQueryService(result.ID) }, } @@ -116,9 +119,9 @@ var prepareCmd = &cobra.Command{ if err != nil { return err } - var sessionID int64 + var result vespa.PrepareResult err = util.Spinner(stderr, "Uploading application package ...", func() error { - sessionID, err = vespa.Prepare(vespa.DeploymentOptions{ + result, err = vespa.Prepare(vespa.DeploymentOptions{ ApplicationPackage: pkg, Target: target, }) @@ -127,10 +130,10 @@ var prepareCmd = &cobra.Command{ if err != nil { return err } - if err := cfg.WriteSessionID(vespa.DefaultApplication, sessionID); err != nil { + if err := cfg.WriteSessionID(vespa.DefaultApplication, result.ID); err != nil { return fmt.Errorf("could not write session id: %w", err) } - printSuccess("Prepared ", color.Cyan(pkg.Path), " with session ", sessionID) + printSuccess("Prepared ", color.CyanString(pkg.Path), " with session ", result.ID) return nil }, } @@ -165,7 +168,7 @@ var activateCmd = &cobra.Command{ if err != nil { return err } - printSuccess("Activated ", color.Cyan(pkg.Path), " with session ", sessionID) + printSuccess("Activated ", color.CyanString(pkg.Path), " with session ", sessionID) return waitForQueryService(sessionID) }, } @@ -173,7 +176,20 @@ var activateCmd = &cobra.Command{ func waitForQueryService(sessionOrRunID int64) error { if waitSecsArg > 0 { log.Println() - return waitForService("query", sessionOrRunID) + return waitForService(vespa.QueryService, sessionOrRunID) } return nil } + +func printPrepareLog(result vespa.PrepareResult) { + for _, entry := range result.LogLines { + level := entry.Level + switch level { + case "ERROR": + level = color.RedString(level) + case "WARNING": + level = color.YellowString(level) + } + fmt.Fprintf(stderr, "%s %s", level, entry.Message) + } +} diff --git a/client/go/cmd/document.go b/client/go/cmd/document.go index 7c22a67a560..5e5108d117d 100644 --- a/client/go/cmd/document.go +++ b/client/go/cmd/document.go @@ -11,6 +11,7 @@ import ( "strings" "time" + "github.com/fatih/color" "github.com/spf13/cobra" "github.com/vespa-engine/vespa/client/go/util" "github.com/vespa-engine/vespa/client/go/vespa" @@ -143,7 +144,7 @@ var documentGetCmd = &cobra.Command{ }, } -func documentService() (*vespa.Service, error) { return getService("document", 0, "") } +func documentService() (*vespa.Service, error) { return getService(vespa.DocumentService, 0, "") } func operationOptions() vespa.OperationOptions { return vespa.OperationOptions{ @@ -166,13 +167,13 @@ func printResult(result util.OperationResult, payloadOnlyOnSuccess bool) error { } if !result.Success { - fmt.Fprintln(out, color.Red("Error:"), result.Message) + fmt.Fprintln(out, color.RedString("Error:"), result.Message) } else if !(payloadOnlyOnSuccess && result.Payload != "") { - fmt.Fprintln(out, color.Green("Success:"), result.Message) + fmt.Fprintln(out, color.GreenString("Success:"), result.Message) } if result.Detail != "" { - fmt.Fprintln(out, color.Yellow(result.Detail)) + fmt.Fprintln(out, color.YellowString(result.Detail)) } if result.Payload != "" { diff --git a/client/go/cmd/helpers.go b/client/go/cmd/helpers.go index ab47a0e6d88..d45fda58a2f 100644 --- a/client/go/cmd/helpers.go +++ b/client/go/cmd/helpers.go @@ -12,23 +12,32 @@ import ( "log" "os" "path/filepath" + "strconv" "strings" "time" + "github.com/fatih/color" "github.com/vespa-engine/vespa/client/go/build" "github.com/vespa-engine/vespa/client/go/version" "github.com/vespa-engine/vespa/client/go/vespa" ) func printErrHint(err error, hints ...string) { - fmt.Fprintln(stderr, color.Red("Error:"), err) + fmt.Fprintln(stderr, color.RedString("Error:"), err) for _, hint := range hints { - fmt.Fprintln(stderr, color.Cyan("Hint:"), hint) + fmt.Fprintln(stderr, color.CyanString("Hint:"), hint) } } func printSuccess(msg ...interface{}) { - log.Print(color.Green("Success: "), fmt.Sprint(msg...)) + log.Print(color.GreenString("Success: "), fmt.Sprint(msg...)) +} + +func printWarning(msg string, hints ...string) { + fmt.Fprintln(stderr, color.YellowString("Warning:"), msg) + for _, hint := range hints { + fmt.Fprintln(stderr, color.CyanString("Hint:"), hint) + } } func athenzPath(filename string) (string, error) { @@ -39,30 +48,30 @@ func athenzPath(filename string) (string, error) { return filepath.Join(userHome, ".athenz", filename), nil } -func athenzKeyPair() (tls.Certificate, error) { +func athenzKeyPair() (KeyPair, error) { certFile, err := athenzPath("cert") if err != nil { - return tls.Certificate{}, err + return KeyPair{}, err } keyFile, err := athenzPath("key") if err != nil { - return tls.Certificate{}, err + return KeyPair{}, err } kp, err := tls.LoadX509KeyPair(certFile, keyFile) if err != nil { - return tls.Certificate{}, err + return KeyPair{}, err } cert, err := x509.ParseCertificate(kp.Certificate[0]) if err != nil { - return tls.Certificate{}, err + return KeyPair{}, err } now := time.Now() expiredAt := cert.NotAfter if expiredAt.Before(now) { delta := now.Sub(expiredAt).Truncate(time.Second) - return tls.Certificate{}, errHint(fmt.Errorf("certificate %s expired at %s (%s ago)", certFile, cert.NotAfter, delta), "Try renewing certificate with 'athenz-user-cert'") + return KeyPair{}, errHint(fmt.Errorf("certificate %s expired at %s (%s ago)", certFile, cert.NotAfter, delta), "Try renewing certificate with 'athenz-user-cert'") } - return kp, nil + return KeyPair{KeyPair: kp, CertificateFile: certFile, PrivateKeyFile: keyFile}, nil } func vespaCliHome() (string, error) { @@ -123,8 +132,8 @@ func getApplication() (vespa.ApplicationID, error) { if err != nil { return vespa.ApplicationID{}, err } - app, err := cfg.Get(applicationFlag) - if err != nil { + app, ok := cfg.Get(applicationFlag) + if !ok { return vespa.ApplicationID{}, errHint(fmt.Errorf("no application specified: %w", err), "Try the --"+applicationFlag+" flag") } application, err := vespa.ApplicationFromString(app) @@ -139,9 +148,9 @@ func getTargetType() (string, error) { if err != nil { return "", err } - target, err := cfg.Get(targetFlag) - if err != nil { - return "", fmt.Errorf("invalid target: %w", err) + target, ok := cfg.Get(targetFlag) + if !ok { + return "", fmt.Errorf("target is unset") } return target, nil } @@ -153,7 +162,7 @@ func getService(service string, sessionOrRunID int64, cluster string) (*vespa.Se } timeout := time.Duration(waitSecsArg) * time.Second if timeout > 0 { - log.Printf("Waiting up to %d %s for %s service to become available ...", color.Cyan(waitSecsArg), color.Cyan("seconds"), color.Cyan(service)) + log.Printf("Waiting up to %s %s for %s service to become available ...", color.CyanString(strconv.Itoa(waitSecsArg)), color.CyanString("seconds"), color.CyanString(service)) } s, err := t.Service(service, timeout, sessionOrRunID, cluster) if err != nil { @@ -187,8 +196,10 @@ func getTarget() (vespa.Target, error) { if err != nil { return nil, err } - if err := target.CheckVersion(clientVersion); err != nil { - printErrHint(err, "This is not a fatal error, but this version may not work as expected", "Try 'vespa version' to check for a new version") + if !isCloudCI() { // Vespa Cloud always runs an up-to-date version + if err := target.CheckVersion(clientVersion); err != nil { + printErrHint(err, "This is not a fatal error, but this version may not work as expected", "Try 'vespa version' to check for a new version") + } } return target, nil } @@ -255,7 +266,11 @@ func createCloudTarget(targetType string) (vespa.Target, error) { if err != nil { return nil, err } - apiTLSOptions = vespa.TLSOptions{KeyPair: kp} + apiTLSOptions = vespa.TLSOptions{ + KeyPair: kp.KeyPair, + CertificateFile: kp.CertificateFile, + PrivateKeyFile: kp.PrivateKeyFile, + } deploymentTLSOptions = apiTLSOptions } else { return nil, fmt.Errorf("invalid cloud target: %s", targetType) @@ -285,16 +300,16 @@ func waitForService(service string, sessionOrRunID int64) error { } timeout := time.Duration(waitSecsArg) * time.Second if timeout > 0 { - log.Printf("Waiting up to %d %s for service to become ready ...", color.Cyan(waitSecsArg), color.Cyan("seconds")) + log.Printf("Waiting up to %s %s for service to become ready ...", color.CyanString(strconv.Itoa(waitSecsArg)), color.CyanString("seconds")) } status, err := s.Wait(timeout) if status/100 == 2 { - log.Print(s.Description(), " at ", color.Cyan(s.BaseURL), " is ", color.Green("ready")) + log.Print(s.Description(), " at ", color.CyanString(s.BaseURL), " is ", color.GreenString("ready")) } else { if err == nil { err = fmt.Errorf("status %d", status) } - return fmt.Errorf("%s at %s is %s: %w", s.Description(), color.Cyan(s.BaseURL), color.Red("not ready"), err) + return fmt.Errorf("%s at %s is %s: %w", s.Description(), color.CyanString(s.BaseURL), color.RedString("not ready"), err) } return nil } @@ -331,6 +346,18 @@ func getEndpointsFromEnv() (map[string]string, error) { return urlsByCluster, nil } +// isCI returns true if running inside a continuous integration environment. +func isCI() bool { + _, ok := os.LookupEnv("CI") + return ok +} + +// isCloudCI returns true if running inside a Vespa Cloud deployment job. +func isCloudCI() bool { + _, ok := os.LookupEnv("VESPA_CLI_CLOUD_CI") + return ok +} + type endpoints struct { Endpoints []endpoint `json:"endpoints"` } diff --git a/client/go/cmd/prod.go b/client/go/cmd/prod.go index 10fc9f92368..4ce126bebb4 100644 --- a/client/go/cmd/prod.go +++ b/client/go/cmd/prod.go @@ -12,6 +12,7 @@ import ( "path/filepath" "strings" + "github.com/fatih/color" "github.com/spf13/cobra" "github.com/vespa-engine/vespa/client/go/util" "github.com/vespa-engine/vespa/client/go/vespa" @@ -78,11 +79,11 @@ https://cloud.vespa.ai/en/reference/deployment`, return err } - fmt.Fprint(stdout, "This will modify any existing ", color.Yellow("deployment.xml"), " and ", color.Yellow("services.xml"), + fmt.Fprint(stdout, "This will modify any existing ", color.YellowString("deployment.xml"), " and ", color.YellowString("services.xml"), "!\nBefore modification a backup of the original file will be created.\n\n") fmt.Fprint(stdout, "A default value is suggested (shown inside brackets) based on\nthe files' existing contents. Press enter to use it.\n\n") fmt.Fprint(stdout, "Abort the configuration at any time by pressing Ctrl-C. The\nfiles will remain untouched.\n\n") - fmt.Fprint(stdout, "See this guide for sizing a Vespa deployment:\n", color.Green("https://docs.vespa.ai/en/performance/sizing-search.html\n\n")) + fmt.Fprint(stdout, "See this guide for sizing a Vespa deployment:\n", color.GreenString("https://docs.vespa.ai/en/performance/sizing-search.html\n\n")) r := bufio.NewReader(stdin) deploymentXML, err = updateRegions(r, deploymentXML, target.Deployment().System) if err != nil { @@ -152,14 +153,11 @@ $ vespa prod submit`, "The application must be a Java maven project, or include basic HTTP tests under src/test/application/", "See https://cloud.vespa.ai/en/getting-to-production") } - // TODO: Always verify tests. Do it before packaging, when running Maven from this CLI. - if !pkg.IsZip() { - verifyTests(pkg.TestPath, target) + if err := verifyTests(pkg, target); err != nil { + return err } - isCI := os.Getenv("CI") != "" - if !isCI { - fmt.Fprintln(stderr, color.Yellow("Warning:"), "We recommend doing this only from a CD job") - printErrHint(nil, "See https://cloud.vespa.ai/en/getting-to-production") + if !isCI() { + printWarning("We recommend doing this only from a CD job", "See https://cloud.vespa.ai/en/getting-to-production") } opts, err := getDeploymentOptions(cfg, pkg, target) if err != nil { @@ -168,8 +166,8 @@ $ vespa prod submit`, if err := vespa.Submit(opts); err != nil { return fmt.Errorf("could not submit application for deployment: %w", err) } else { - printSuccess("Submitted ", color.Cyan(pkg.Path), " for deployment") - log.Printf("See %s for deployment progress\n", color.Cyan(fmt.Sprintf("%s/tenant/%s/application/%s/prod/deployment", + printSuccess("Submitted ", color.CyanString(pkg.Path), " for deployment") + log.Printf("See %s for deployment progress\n", color.CyanString(fmt.Sprintf("%s/tenant/%s/application/%s/prod/deployment", opts.Target.Deployment().System.ConsoleURL, opts.Target.Deployment().Application.Tenant, opts.Target.Deployment().Application.Application))) } return nil @@ -184,14 +182,14 @@ func writeWithBackup(pkg vespa.ApplicationPackage, filename, contents string) er return err } if bytes.Equal(data, []byte(contents)) { - fmt.Fprintf(stdout, "Not writing %s: File is unchanged\n", color.Yellow(filename)) + fmt.Fprintf(stdout, "Not writing %s: File is unchanged\n", color.YellowString(filename)) return nil } renamed := false for i := 1; i <= 1000; i++ { bak := fmt.Sprintf("%s.%d.bak", dst, i) if !util.PathExists(bak) { - fmt.Fprintf(stdout, "Backing up existing %s to %s\n", color.Yellow(filename), color.Yellow(bak)) + fmt.Fprintf(stdout, "Backing up existing %s to %s\n", color.YellowString(filename), color.YellowString(bak)) if err := os.Rename(dst, bak); err != nil { return err } @@ -203,7 +201,7 @@ func writeWithBackup(pkg vespa.ApplicationPackage, filename, contents string) er return fmt.Errorf("could not find an unused backup name for %s", dst) } } - fmt.Fprintf(stdout, "Writing %s\n", color.Green(dst)) + fmt.Fprintf(stdout, "Writing %s\n", color.GreenString(dst)) return ioutil.WriteFile(dst, []byte(contents), 0644) } @@ -228,9 +226,9 @@ func updateRegions(r *bufio.Reader, deploymentXML xml.Deployment, system vespa.S } func promptRegions(r *bufio.Reader, deploymentXML xml.Deployment, system vespa.System) (string, error) { - fmt.Fprintln(stdout, color.Cyan("> Deployment regions")) - fmt.Fprintf(stdout, "Documentation: %s\n", color.Green("https://cloud.vespa.ai/en/reference/zones")) - fmt.Fprintf(stdout, "Example: %s\n\n", color.Yellow("aws-us-east-1c,aws-us-west-2a")) + fmt.Fprintln(stdout, color.CyanString("> Deployment regions")) + fmt.Fprintf(stdout, "Documentation: %s\n", color.GreenString("https://cloud.vespa.ai/en/reference/zones")) + fmt.Fprintf(stdout, "Example: %s\n\n", color.YellowString("aws-us-east-1c,aws-us-west-2a")) var currentRegions []string for _, r := range deploymentXML.Prod.Regions { currentRegions = append(currentRegions, r.Name) @@ -302,20 +300,20 @@ func promptNodes(r *bufio.Reader, clusterID string, defaultValue xml.Nodes) (xml } func promptNodeCount(r *bufio.Reader, clusterID string, nodeCount string) (string, error) { - fmt.Fprintln(stdout, color.Cyan("\n> Node count: "+clusterID+" cluster")) - fmt.Fprintf(stdout, "Documentation: %s\n", color.Green("https://cloud.vespa.ai/en/reference/services")) - fmt.Fprintf(stdout, "Example: %s\nExample: %s\n\n", color.Yellow("4"), color.Yellow("[2,8]")) + fmt.Fprintln(stdout, color.CyanString("\n> Node count: "+clusterID+" cluster")) + fmt.Fprintf(stdout, "Documentation: %s\n", color.GreenString("https://cloud.vespa.ai/en/reference/services")) + fmt.Fprintf(stdout, "Example: %s\nExample: %s\n\n", color.YellowString("4"), color.YellowString("[2,8]")) validator := func(input string) error { _, _, err := xml.ParseNodeCount(input) return err } - return prompt(r, fmt.Sprintf("How many nodes should the %s cluster have?", color.Cyan(clusterID)), nodeCount, validator) + return prompt(r, fmt.Sprintf("How many nodes should the %s cluster have?", color.CyanString(clusterID)), nodeCount, validator) } func promptResources(r *bufio.Reader, clusterID string, resources string) (string, error) { - fmt.Fprintln(stdout, color.Cyan("\n> Node resources: "+clusterID+" cluster")) - fmt.Fprintf(stdout, "Documentation: %s\n", color.Green("https://cloud.vespa.ai/en/reference/services")) - fmt.Fprintf(stdout, "Example: %s\nExample: %s\n\n", color.Yellow("auto"), color.Yellow("vcpu=4,memory=8Gb,disk=100Gb")) + fmt.Fprintln(stdout, color.CyanString("\n> Node resources: "+clusterID+" cluster")) + fmt.Fprintf(stdout, "Documentation: %s\n", color.GreenString("https://cloud.vespa.ai/en/reference/services")) + fmt.Fprintf(stdout, "Example: %s\nExample: %s\n\n", color.YellowString("auto"), color.YellowString("vcpu=4,memory=8Gb,disk=100Gb")) validator := func(input string) error { if input == "auto" { return nil @@ -323,7 +321,7 @@ func promptResources(r *bufio.Reader, clusterID string, resources string) (strin _, err := xml.ParseResources(input) return err } - return prompt(r, fmt.Sprintf("Which resources should each node in the %s cluster have?", color.Cyan(clusterID)), resources, validator) + return prompt(r, fmt.Sprintf("Which resources should each node in the %s cluster have?", color.CyanString(clusterID)), resources, validator) } func readDeploymentXML(pkg vespa.ApplicationPackage) (xml.Deployment, error) { @@ -352,7 +350,7 @@ func prompt(r *bufio.Reader, question, defaultAnswer string, validator func(inpu for input == "" { fmt.Fprint(stdout, question) if defaultAnswer != "" { - fmt.Fprint(stdout, " [", color.Yellow(defaultAnswer), "]") + fmt.Fprint(stdout, " [", color.YellowString(defaultAnswer), "]") } fmt.Fprint(stdout, " ") @@ -375,11 +373,30 @@ func prompt(r *bufio.Reader, question, defaultAnswer string, validator func(inpu return input, nil } -func verifyTests(testsParent string, target vespa.Target) { - verifyTest(testsParent, "system-test", target, true) - verifyTest(testsParent, "staging-setup", target, true) - verifyTest(testsParent, "staging-test", target, true) - verifyTest(testsParent, "production-test", target, false) +func verifyTests(app vespa.ApplicationPackage, target vespa.Target) error { + // TODO: system-test, staging-setup and staging-test should be required if the application + // does not have any Java tests. + suites := map[string]bool{ + "system-test": false, + "staging-setup": false, + "staging-test": false, + "production-test": false, + } + testPath := app.TestPath + if app.IsZip() { + path, err := app.Unzip(true) + if err != nil { + return err + } + defer os.RemoveAll(path) + testPath = path + } + for suite, required := range suites { + if err := verifyTest(testPath, suite, target, required); err != nil { + return err + } + } + return nil } func verifyTest(testsParent string, suite string, target vespa.Target, required bool) error { diff --git a/client/go/cmd/query.go b/client/go/cmd/query.go index ed7541288b6..cb0fd923c4e 100644 --- a/client/go/cmd/query.go +++ b/client/go/cmd/query.go @@ -14,6 +14,7 @@ import ( "strings" "time" + "github.com/fatih/color" "github.com/spf13/cobra" "github.com/vespa-engine/vespa/client/go/curl" "github.com/vespa-engine/vespa/client/go/util" @@ -62,7 +63,7 @@ func printCurl(url string, service *vespa.Service) error { } func query(cmd *cobra.Command, arguments []string) error { - service, err := getService("query", 0, "") + service, err := getService(vespa.QueryService, 0, "") if err != nil { return err } @@ -97,7 +98,7 @@ func query(cmd *cobra.Command, arguments []string) error { } else if response.StatusCode/100 == 4 { return fmt.Errorf("invalid query: %s\n%s", response.Status, util.ReaderToJSON(response.Body)) } else { - return fmt.Errorf("%s from container at %s\n%s", response.Status, color.Cyan(url.Host), util.ReaderToJSON(response.Body)) + return fmt.Errorf("%s from container at %s\n%s", response.Status, color.CyanString(url.Host), util.ReaderToJSON(response.Body)) } return nil } diff --git a/client/go/cmd/root.go b/client/go/cmd/root.go index 96521737ba9..6f29a09e177 100644 --- a/client/go/cmd/root.go +++ b/client/go/cmd/root.go @@ -11,7 +11,7 @@ import ( "log" "os" - "github.com/logrusorgru/aurora/v3" + "github.com/fatih/color" "github.com/mattn/go-colorable" "github.com/mattn/go-isatty" "github.com/spf13/cobra" @@ -56,7 +56,6 @@ Vespa documentation: https://docs.vespa.ai`, apiKeyFileArg string stdin io.ReadWriter = os.Stdin - color = aurora.NewAurora(false) stdout = colorable.NewColorableStdout() stderr = colorable.NewColorableStderr() ) @@ -82,21 +81,16 @@ func isTerminal() bool { } func configureOutput() error { - if quietArg { - stdout = ioutil.Discard - } - log.SetFlags(0) // No timestamps - log.SetOutput(stdout) - config, err := LoadConfig() if err != nil { return err } - colorValue, err := config.Get(colorFlag) - if err != nil { - return err + if quiet, _ := config.Get(quietFlag); quiet == "true" { + stdout = ioutil.Discard } - + log.SetFlags(0) // No timestamps + log.SetOutput(stdout) + colorValue, _ := config.Get(colorFlag) colorize := false switch colorValue { case "auto": @@ -108,7 +102,7 @@ func configureOutput() error { default: return errHint(fmt.Errorf("invalid value for %s option", colorFlag), "Must be \"auto\", \"never\" or \"always\"") } - color = aurora.NewAurora(colorize) + color.NoColor = !colorize return nil } diff --git a/client/go/cmd/status.go b/client/go/cmd/status.go index 711dba4aa9d..93316b7b6de 100644 --- a/client/go/cmd/status.go +++ b/client/go/cmd/status.go @@ -6,6 +6,7 @@ package cmd import ( "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/vespa" ) func init() { @@ -23,7 +24,7 @@ var statusCmd = &cobra.Command{ SilenceUsage: true, Args: cobra.MaximumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - return waitForService("query", 0) + return waitForService(vespa.QueryService, 0) }, } @@ -35,7 +36,7 @@ var statusQueryCmd = &cobra.Command{ SilenceUsage: true, Args: cobra.ExactArgs(0), RunE: func(cmd *cobra.Command, args []string) error { - return waitForService("query", 0) + return waitForService(vespa.QueryService, 0) }, } @@ -47,7 +48,7 @@ var statusDocumentCmd = &cobra.Command{ SilenceUsage: true, Args: cobra.ExactArgs(0), RunE: func(cmd *cobra.Command, args []string) error { - return waitForService("document", 0) + return waitForService(vespa.DocumentService, 0) }, } @@ -59,6 +60,6 @@ var statusDeployCmd = &cobra.Command{ SilenceUsage: true, Args: cobra.ExactArgs(0), RunE: func(cmd *cobra.Command, args []string) error { - return waitForService("deploy", 0) + return waitForService(vespa.DeployService, 0) }, } diff --git a/client/go/cmd/test.go b/client/go/cmd/test.go index d12059a8d12..56ea9277842 100644 --- a/client/go/cmd/test.go +++ b/client/go/cmd/test.go @@ -15,9 +15,11 @@ import ( "net/url" "os" "path/filepath" + "strconv" "strings" "time" + "github.com/fatih/color" "github.com/spf13/cobra" "github.com/vespa-engine/vespa/client/go/util" "github.com/vespa-engine/vespa/client/go/vespa" @@ -29,7 +31,7 @@ func init() { } var testCmd = &cobra.Command{ - Use: "test <tests directory or test file>", + Use: "test test-directory-or-file", Short: "Run a test suite, or a single test", Long: `Run a test suite, or a single test @@ -51,7 +53,7 @@ $ vespa test src/test/application/tests/system-test/feed-and-query.json`, if count == 1 { plural = "" } - fmt.Fprintf(stdout, "\n%s %d of %d test%s failed:\n", color.Red("Failure:"), len(failed), count, plural) + fmt.Fprintf(stdout, "\n%s %d of %d test%s failed:\n", color.RedString("Failure:"), len(failed), count, plural) for _, test := range failed { fmt.Fprintln(stdout, test) } @@ -61,7 +63,7 @@ $ vespa test src/test/application/tests/system-test/feed-and-query.json`, if count == 1 { plural = "" } - fmt.Fprintf(stdout, "\n%s %d test%s OK\n", color.Green("Success:"), count, plural) + fmt.Fprintf(stdout, "\n%s %d test%s OK\n", color.GreenString("Success:"), count, plural) return nil } }, @@ -154,7 +156,7 @@ func runTest(testPath string, context testContext) (string, error) { } if !context.dryRun { if failure != "" { - fmt.Fprintf(stdout, " %s\n%s:\n%s\n", color.Red("failed"), stepName, longFailure) + fmt.Fprintf(stdout, " %s\n%s:\n%s\n", color.RedString("failed"), stepName, longFailure) return fmt.Sprintf("%s: %s: %s", testName, stepName, failure), nil } if i == 0 { @@ -164,7 +166,7 @@ func runTest(testPath string, context testContext) (string, error) { } } if !context.dryRun { - fmt.Fprintln(stdout, color.Green(" OK")) + fmt.Fprintln(stdout, color.GreenString(" OK")) } return "", nil } @@ -214,7 +216,7 @@ func verify(step step, defaultCluster string, defaultParameters map[string]strin if err != nil { return "", "", err } - service, err = target.Service("query", 0, 0, cluster) + service, err = target.Service(vespa.QueryService, 0, 0, cluster) if err != nil { return "", "", err } @@ -274,12 +276,12 @@ func verify(step step, defaultCluster string, defaultParameters map[string]strin defer response.Body.Close() if statusCode != response.StatusCode { - return fmt.Sprintf("Unexpected status code: %d", color.Red(response.StatusCode)), - fmt.Sprintf("Unexpected status code\nExpected: %d\nActual: %d\nRequested: %s at %s\nResponse:\n%s", - color.Cyan(statusCode), - color.Red(response.StatusCode), - color.Cyan(method), - color.Cyan(requestUrl), + return fmt.Sprintf("Unexpected status code: %s", color.RedString(strconv.Itoa(response.StatusCode))), + fmt.Sprintf("Unexpected status code\nExpected: %s\nActual: %s\nRequested: %s at %s\nResponse:\n%s", + color.CyanString(strconv.Itoa(statusCode)), + color.RedString(strconv.Itoa(response.StatusCode)), + color.CyanString(method), + color.CyanString(requestUrl.String()), util.ReaderToJSON(response.Body)), nil } @@ -308,7 +310,7 @@ func verify(step step, defaultCluster string, defaultParameters map[string]strin failure += ": " + actual longFailure += "\nActual: " + actual } - longFailure += fmt.Sprintf("\nRequested: %s at %s\nResponse:\n%s", color.Cyan(method), color.Cyan(requestUrl), string(responsePretty)) + longFailure += fmt.Sprintf("\nRequested: %s at %s\nResponse:\n%s", color.CyanString(method), color.CyanString(requestUrl.String()), string(responsePretty)) return failure, longFailure, err } return "", "", err @@ -345,9 +347,9 @@ func compare(expected interface{}, actual interface{}, path string) (string, str } valueMatch = true } else { - return fmt.Sprintf("Unexpected number of elements at %s", color.Cyan(path)), - fmt.Sprintf("%d", color.Cyan(len(u))), - fmt.Sprintf("%d", color.Red(len(v))), + return fmt.Sprintf("Unexpected number of elements at %s", color.CyanString(path)), + color.CyanString(strconv.Itoa(len(u))), + color.RedString(strconv.Itoa(len(v))), nil } } @@ -359,7 +361,7 @@ func compare(expected interface{}, actual interface{}, path string) (string, str childPath := fmt.Sprintf("%s/%s", path, strings.ReplaceAll(strings.ReplaceAll(n, "~", "~0"), "/", "~1")) f, ok := v[n] if !ok { - return fmt.Sprintf("Missing expected field at %s", color.Red(childPath)), "", "", nil + return fmt.Sprintf("Missing expected field at %s", color.RedString(childPath)), "", "", nil } if failure, expected, actual, err := compare(e, f, childPath); failure != "" || err != nil { return failure, expected, actual, err @@ -381,9 +383,9 @@ func compare(expected interface{}, actual interface{}, path string) (string, str } expectedJson, _ := json.Marshal(expected) actualJson, _ := json.Marshal(actual) - return fmt.Sprintf("Unexpected %s at %s", mismatched, color.Cyan(path)), - fmt.Sprintf("%s", color.Cyan(expectedJson)), - fmt.Sprintf("%s", color.Red(actualJson)), + return fmt.Sprintf("Unexpected %s at %s", mismatched, color.CyanString(path)), + fmt.Sprintf("%s", color.CyanString(string(expectedJson))), + fmt.Sprintf("%s", color.RedString(string(actualJson))), nil } return "", "", "", nil diff --git a/client/go/cmd/version.go b/client/go/cmd/version.go index 2660dfe7d61..875fc0cc3c1 100644 --- a/client/go/cmd/version.go +++ b/client/go/cmd/version.go @@ -13,6 +13,7 @@ import ( "strings" "time" + "github.com/fatih/color" "github.com/spf13/cobra" "github.com/vespa-engine/vespa/client/go/build" "github.com/vespa-engine/vespa/client/go/util" @@ -73,10 +74,10 @@ func checkVersion() error { if usingHomebrew && latest.isRecent() { return nil // Allow some time for new release to appear in Homebrew repo } - log.Printf("\nNew release available: %s", color.Green(latest.Version)) + log.Printf("\nNew release available: %s", color.GreenString(latest.Version.String())) log.Printf("https://github.com/vespa-engine/vespa/releases/tag/v%s", latest.Version) if usingHomebrew { - log.Printf("\nUpgrade by running:\n%s", color.Cyan("brew update && brew upgrade vespa-cli")) + log.Printf("\nUpgrade by running:\n%s", color.CyanString("brew update && brew upgrade vespa-cli")) } return nil } diff --git a/client/go/go.mod b/client/go/go.mod index 263114fd517..36060ec2014 100644 --- a/client/go/go.mod +++ b/client/go/go.mod @@ -4,10 +4,9 @@ go 1.15 require ( github.com/briandowns/spinner v1.16.0 - github.com/fatih/color v1.10.0 // indirect + github.com/fatih/color v1.10.0 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/lestrrat-go/jwx v1.2.9 - github.com/logrusorgru/aurora/v3 v3.0.0 github.com/mattn/go-colorable v0.1.8 github.com/mattn/go-isatty v0.0.13 github.com/pkg/browser v0.0.0-20210706143420-7d21f8c997e2 diff --git a/client/go/go.sum b/client/go/go.sum index d27d422e5cc..35c8fb6bea7 100644 --- a/client/go/go.sum +++ b/client/go/go.sum @@ -205,8 +205,6 @@ github.com/lestrrat-go/jwx v1.2.9 h1:kS8kLI4oaBYJJ6u6rpbPI0tDYVCqo0P5u8vv1zoQ49U github.com/lestrrat-go/jwx v1.2.9/go.mod h1:25DcLbNWArPA/Ew5CcBmewl32cJKxOk5cbepBsIJFzw= github.com/lestrrat-go/option v1.0.0 h1:WqAWL8kh8VcSoD6xjSH34/1m8yxluXQbDeKNfvFeEO4= github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= -github.com/logrusorgru/aurora/v3 v3.0.0 h1:R6zcoZZbvVcGMvDCKo45A9U/lzYyzl5NfYIvznmDfE4= -github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxAlxggSjiwMnCUc= github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= diff --git a/client/go/util/http.go b/client/go/util/http.go index f95d52dabad..bb70c3ec6db 100644 --- a/client/go/util/http.go +++ b/client/go/util/http.go @@ -8,7 +8,6 @@ import ( "crypto/tls" "fmt" "net/http" - "net/url" "time" "github.com/vespa-engine/vespa/client/go/build" @@ -45,15 +44,6 @@ func CreateClient(timeout time.Duration) HttpClient { } } -// Convenience function for doing a HTTP GET -func HttpGet(host string, path string, description string) (*http.Response, error) { - url, err := url.Parse(host + path) - if err != nil { - return nil, fmt.Errorf("invalid target url: %s: %w", host+path, err) - } - return HttpDo(&http.Request{URL: url}, time.Second*10, description) -} - func HttpDo(request *http.Request, timeout time.Duration, description string) (*http.Response, error) { if request.Header == nil { request.Header = make(http.Header) diff --git a/client/go/util/http_test.go b/client/go/util/http_test.go index e87a1e5ada4..ccb809d198b 100644 --- a/client/go/util/http_test.go +++ b/client/go/util/http_test.go @@ -41,11 +41,15 @@ func (c mockHttpClient) UseCertificate(certificates []tls.Certificate) {} func TestHttpRequest(t *testing.T) { ActiveHttpClient = mockHttpClient{} - response, err := HttpGet("http://host", "/okpath", "description") + req, err := http.NewRequest("GET", "http://host/okpath", nil) + assert.Nil(t, err) + response, err := HttpDo(req, time.Second*10, "description") assert.Nil(t, err) assert.Equal(t, 200, response.StatusCode) - response, err = HttpGet("http://host", "/otherpath", "description") + req, err = http.NewRequest("GET", "http://host/otherpath", nil) + assert.Nil(t, err) + response, err = HttpDo(req, time.Second*10, "description") assert.Nil(t, err) assert.Equal(t, 500, response.StatusCode) } diff --git a/client/go/vespa/application.go b/client/go/vespa/application.go new file mode 100644 index 00000000000..80965987b66 --- /dev/null +++ b/client/go/vespa/application.go @@ -0,0 +1,231 @@ +package vespa + +import ( + "archive/zip" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + + "github.com/vespa-engine/vespa/client/go/util" +) + +type ApplicationPackage struct { + Path string + TestPath string +} + +func (ap *ApplicationPackage) HasCertificate() bool { + return ap.hasFile(filepath.Join("security", "clients.pem"), "security/clients.pem") +} + +func (ap *ApplicationPackage) HasDeployment() bool { return ap.hasFile("deployment.xml", "") } + +func (ap *ApplicationPackage) hasFile(filename, zipName string) bool { + if zipName == "" { + zipName = filename + } + if ap.IsZip() { + r, err := zip.OpenReader(ap.Path) + if err != nil { + return false + } + defer r.Close() + for _, f := range r.File { + if f.Name == zipName { + return true + } + } + return false + } + return util.PathExists(filepath.Join(ap.Path, filename)) +} + +func (ap *ApplicationPackage) IsZip() bool { return isZip(ap.Path) } + +func (ap *ApplicationPackage) IsJava() bool { + if ap.IsZip() { + r, err := zip.OpenReader(ap.Path) + if err != nil { + return false + } + defer r.Close() + for _, f := range r.File { + if filepath.Ext(f.Name) == ".jar" { + return true + } + } + return false + } + return util.PathExists(filepath.Join(ap.Path, "pom.xml")) +} + +func isZip(filename string) bool { return filepath.Ext(filename) == ".zip" } + +func zipDir(dir string, destination string) error { + if filepath.IsAbs(dir) { + message := "Path must be relative, but '" + dir + "'" + return errors.New(message) + } + if !util.PathExists(dir) { + message := "'" + dir + "' should be an application package zip or dir, but does not exist" + return errors.New(message) + } + if !util.IsDirectory(dir) { + message := "'" + dir + "' should be an application package dir, but is a (non-zip) file" + return errors.New(message) + } + + file, err := os.Create(destination) + if err != nil { + message := "Could not create a temporary zip file for the application package: " + err.Error() + return errors.New(message) + } + defer file.Close() + + w := zip.NewWriter(file) + defer w.Close() + + walker := func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + zippath, err := filepath.Rel(dir, path) + if err != nil { + return err + } + zipfile, err := w.Create(zippath) + if err != nil { + return err + } + + _, err = io.Copy(zipfile, file) + if err != nil { + return err + } + return nil + } + return filepath.Walk(dir, walker) +} + +func (ap *ApplicationPackage) zipReader(test bool) (io.ReadCloser, error) { + zipFile := ap.Path + if test { + zipFile = ap.TestPath + } + if !ap.IsZip() { + tempZip, err := ioutil.TempFile("", "vespa") + if err != nil { + return nil, fmt.Errorf("could not create a temporary zip file for the application package: %w", err) + } + defer func() { + tempZip.Close() + os.Remove(tempZip.Name()) + // TODO: Caller must remove temporary file + }() + if err := zipDir(zipFile, tempZip.Name()); err != nil { + return nil, err + } + zipFile = tempZip.Name() + } + f, err := os.Open(zipFile) + if err != nil { + return nil, fmt.Errorf("could not open application package at %s: %w", ap.Path, err) + } + return f, nil +} + +func (ap *ApplicationPackage) Unzip(test bool) (string, error) { + if !ap.IsZip() { + return "", fmt.Errorf("can't unzip a package that is a directory structure") + } + cleanTemp := true + tmp, err := os.MkdirTemp(os.TempDir(), "vespa-test-pkg") + if err != nil { + return "", err + } + defer func() { + if cleanTemp { + os.RemoveAll(tmp) + } + }() + path := ap.Path + if test { + path = ap.TestPath + } + f, err := zip.OpenReader(path) + if err != nil { + return "", err + } + defer f.Close() + for _, f := range f.File { + dst := filepath.Join(tmp, f.Name) + if f.FileInfo().IsDir() { + if err := os.Mkdir(dst, f.FileInfo().Mode()); err != nil { + return "", err + } + continue + } + if err := copyFile(f, dst); err != nil { + return "", fmt.Errorf("copyFile: %w", err) + } + + } + cleanTemp = false + return tmp, nil +} + +func copyFile(src *zip.File, dst string) error { + from, err := src.Open() + if err != nil { + return err + } + defer from.Close() + to, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY, src.FileInfo().Mode()) + if err != nil { + return err + } + defer to.Close() + _, err = io.Copy(to, from) + return err +} + +// FindApplicationPackage finds the path to an application package from the zip file or directory zipOrDir. +func FindApplicationPackage(zipOrDir string, requirePackaging bool) (ApplicationPackage, error) { + if isZip(zipOrDir) { + return ApplicationPackage{Path: zipOrDir}, nil + } + if util.PathExists(filepath.Join(zipOrDir, "pom.xml")) { + zip := filepath.Join(zipOrDir, "target", "application.zip") + if util.PathExists(zip) { + if testZip := filepath.Join(zipOrDir, "target", "application-test.zip"); util.PathExists(testZip) { + return ApplicationPackage{Path: zip, TestPath: testZip}, nil + } + return ApplicationPackage{Path: zip}, nil + } + if requirePackaging { + return ApplicationPackage{}, errors.New("pom.xml exists but no target/application.zip. Run mvn package first") + } + } + if path := filepath.Join(zipOrDir, "src", "main", "application"); util.PathExists(path) { + if testPath := filepath.Join(zipOrDir, "src", "test", "application"); util.PathExists(testPath) { + return ApplicationPackage{Path: path, TestPath: testPath}, nil + } + return ApplicationPackage{Path: path}, nil + } + if util.PathExists(filepath.Join(zipOrDir, "services.xml")) { + return ApplicationPackage{Path: zipOrDir}, nil + } + return ApplicationPackage{}, fmt.Errorf("could not find an application package source in '%s'", zipOrDir) +} diff --git a/client/go/vespa/deploy.go b/client/go/vespa/deploy.go index fdc083631f7..2a6d51c7b9f 100644 --- a/client/go/vespa/deploy.go +++ b/client/go/vespa/deploy.go @@ -5,18 +5,14 @@ package vespa import ( - "archive/zip" "bytes" "encoding/json" - "errors" "fmt" "io" "io/ioutil" "mime/multipart" "net/http" "net/url" - "os" - "path/filepath" "strconv" "strings" "time" @@ -49,9 +45,16 @@ type DeploymentOptions struct { Timeout time.Duration } -type ApplicationPackage struct { - Path string - TestPath string +type LogLinePrepareResponse struct { + Time int64 + Level string + Message string +} + +type PrepareResult struct { + // Session or Run ID + ID int64 + LogLines []LogLinePrepareResponse } func (a ApplicationID) String() string { @@ -76,112 +79,13 @@ func (d *DeploymentOptions) IsCloud() bool { } func (d *DeploymentOptions) url(path string) (*url.URL, error) { - service, err := d.Target.Service(deployService, 0, 0, "") + service, err := d.Target.Service(DeployService, 0, 0, "") if err != nil { return nil, err } return url.Parse(service.BaseURL + path) } -func (ap *ApplicationPackage) HasCertificate() bool { - return ap.hasFile(filepath.Join("security", "clients.pem"), "security/clients.pem") -} - -func (ap *ApplicationPackage) HasDeployment() bool { return ap.hasFile("deployment.xml", "") } - -func (ap *ApplicationPackage) hasFile(filename, zipName string) bool { - if zipName == "" { - zipName = filename - } - if ap.IsZip() { - r, err := zip.OpenReader(ap.Path) - if err != nil { - return false - } - defer r.Close() - for _, f := range r.File { - if f.Name == zipName { - return true - } - } - return false - } - return util.PathExists(filepath.Join(ap.Path, filename)) -} - -func (ap *ApplicationPackage) IsZip() bool { return isZip(ap.Path) } - -func (ap *ApplicationPackage) IsJava() bool { - if ap.IsZip() { - r, err := zip.OpenReader(ap.Path) - if err != nil { - return false - } - defer r.Close() - for _, f := range r.File { - if filepath.Ext(f.Name) == ".jar" { - return true - } - } - return false - } - return util.PathExists(filepath.Join(ap.Path, "pom.xml")) -} - -func (ap *ApplicationPackage) zipReader(test bool) (io.ReadCloser, error) { - zipFile := ap.Path - if test { - zipFile = ap.TestPath - } - if !ap.IsZip() { - tempZip, err := ioutil.TempFile("", "vespa") - if err != nil { - return nil, fmt.Errorf("could not create a temporary zip file for the application package: %w", err) - } - defer func() { - tempZip.Close() - os.Remove(tempZip.Name()) - }() - if err := zipDir(zipFile, tempZip.Name()); err != nil { - return nil, err - } - zipFile = tempZip.Name() - } - f, err := os.Open(zipFile) - if err != nil { - return nil, fmt.Errorf("could not open application package at %s: %w", ap.Path, err) - } - return f, nil -} - -// FindApplicationPackage finds the path to an application package from the zip file or directory zipOrDir. -func FindApplicationPackage(zipOrDir string, requirePackaging bool) (ApplicationPackage, error) { - if isZip(zipOrDir) { - return ApplicationPackage{Path: zipOrDir}, nil - } - if util.PathExists(filepath.Join(zipOrDir, "pom.xml")) { - zip := filepath.Join(zipOrDir, "target", "application.zip") - if util.PathExists(zip) { - testZip := filepath.Join(zipOrDir, "target", "application-test.zip") - return ApplicationPackage{Path: zip, TestPath: testZip}, nil - } - if requirePackaging { - return ApplicationPackage{}, errors.New("pom.xml exists but no target/application.zip. Run mvn package first") - } - } - if util.PathExists(filepath.Join(zipOrDir, "src", "main", "application")) { - if util.PathExists(filepath.Join(zipOrDir, "src", "test", "application")) { - return ApplicationPackage{Path: filepath.Join(zipOrDir, "src", "main", "application"), - TestPath: filepath.Join(zipOrDir, "src", "test", "application")}, nil - } - return ApplicationPackage{Path: filepath.Join(zipOrDir, "src", "main", "application")}, nil - } - if util.PathExists(filepath.Join(zipOrDir, "services.xml")) { - return ApplicationPackage{Path: zipOrDir}, nil - } - return ApplicationPackage{}, errors.New("Could not find an application package source in '" + zipOrDir + "'") -} - func ApplicationFromString(s string) (ApplicationID, error) { parts := strings.Split(s, ".") if len(parts) != 3 { @@ -199,36 +103,36 @@ func ZoneFromString(s string) (ZoneID, error) { } // Prepare deployment and return the session ID -func Prepare(deployment DeploymentOptions) (int64, error) { +func Prepare(deployment DeploymentOptions) (PrepareResult, error) { if deployment.IsCloud() { - return 0, fmt.Errorf("prepare is not supported with %s target", deployment.Target.Type()) + return PrepareResult{}, fmt.Errorf("prepare is not supported with %s target", deployment.Target.Type()) } sessionURL, err := deployment.url("/application/v2/tenant/default/session") if err != nil { - return 0, err + return PrepareResult{}, err } - sessionID, err := uploadApplicationPackage(sessionURL, deployment) + result, err := uploadApplicationPackage(sessionURL, deployment) if err != nil { - return 0, err + return PrepareResult{}, err } - prepareURL, err := deployment.url(fmt.Sprintf("/application/v2/tenant/default/session/%d/prepared", sessionID)) + prepareURL, err := deployment.url(fmt.Sprintf("/application/v2/tenant/default/session/%d/prepared", result.ID)) if err != nil { - return 0, err + return PrepareResult{}, err } req, err := http.NewRequest("PUT", prepareURL.String(), nil) if err != nil { - return 0, err + return PrepareResult{}, err } serviceDescription := "Deploy service" response, err := util.HttpDo(req, time.Second*30, serviceDescription) if err != nil { - return 0, err + return PrepareResult{}, err } defer response.Body.Close() if err := checkResponse(req, response, serviceDescription); err != nil { - return 0, err + return PrepareResult{}, err } - return sessionID, nil + return result, nil } // Activate deployment with sessionID from a past prepare @@ -253,14 +157,14 @@ func Activate(sessionID int64, deployment DeploymentOptions) error { return checkResponse(req, response, serviceDescription) } -func Deploy(opts DeploymentOptions) (int64, error) { +func Deploy(opts DeploymentOptions) (PrepareResult, error) { path := "/application/v2/tenant/default/prepareandactivate" if opts.IsCloud() { if err := checkDeploymentOpts(opts); err != nil { - return 0, err + return PrepareResult{}, err } if opts.Target.Deployment().Zone.Environment == "" || opts.Target.Deployment().Zone.Region == "" { - return 0, fmt.Errorf("%s: missing zone", opts) + return PrepareResult{}, fmt.Errorf("%s: missing zone", opts) } path = fmt.Sprintf("/application/v4/tenant/%s/application/%s/instance/%s/deploy/%s-%s", opts.Target.Deployment().Application.Tenant, @@ -271,7 +175,7 @@ func Deploy(opts DeploymentOptions) (int64, error) { } u, err := opts.url(path) if err != nil { - return 0, err + return PrepareResult{}, err } return uploadApplicationPackage(u, opts) } @@ -354,10 +258,10 @@ func checkDeploymentOpts(opts DeploymentOptions) error { return nil } -func uploadApplicationPackage(url *url.URL, opts DeploymentOptions) (int64, error) { +func uploadApplicationPackage(url *url.URL, opts DeploymentOptions) (PrepareResult, error) { zipReader, err := opts.ApplicationPackage.zipReader(false) if err != nil { - return 0, err + return PrepareResult{}, err } header := http.Header{} header.Add("Content-Type", "application/zip") @@ -367,35 +271,44 @@ func uploadApplicationPackage(url *url.URL, opts DeploymentOptions) (int64, erro Header: header, Body: ioutil.NopCloser(zipReader), } - service, err := opts.Target.Service(deployService, opts.Timeout, 0, "") + service, err := opts.Target.Service(DeployService, opts.Timeout, 0, "") if err != nil { - return 0, err + return PrepareResult{}, err } keyID := opts.Target.Deployment().Application.SerializedForm() if err := opts.Target.SignRequest(request, keyID); err != nil { - return 0, err + return PrepareResult{}, err } response, err := service.Do(request, time.Minute*10) if err != nil { - return 0, err + return PrepareResult{}, err } defer response.Body.Close() var jsonResponse struct { SessionID string `json:"session-id"` // Config server RunID int64 `json:"run"` // Controller + + Log []LogLinePrepareResponse `json:"log"` } jsonResponse.SessionID = "0" // Set a default session ID for responses that don't contain int (e.g. cloud deployment) if err := checkResponse(request, response, service.Description()); err != nil { - return 0, err + return PrepareResult{}, err } jsonDec := json.NewDecoder(response.Body) jsonDec.Decode(&jsonResponse) // Ignore error in case this is a non-JSON response - if jsonResponse.RunID > 0 { - return jsonResponse.RunID, nil + id := jsonResponse.RunID + if id == 0 { + id, err = strconv.ParseInt(jsonResponse.SessionID, 10, 64) + if err != nil { + return PrepareResult{}, err + } } - return strconv.ParseInt(jsonResponse.SessionID, 10, 64) + return PrepareResult{ + ID: id, + LogLines: jsonResponse.Log, + }, err } func checkResponse(req *http.Request, response *http.Response, serviceDescription string) error { @@ -407,63 +320,6 @@ func checkResponse(req *http.Request, response *http.Response, serviceDescriptio return nil } -func isZip(filename string) bool { return filepath.Ext(filename) == ".zip" } - -func zipDir(dir string, destination string) error { - if filepath.IsAbs(dir) { - message := "Path must be relative, but '" + dir + "'" - return errors.New(message) - } - if !util.PathExists(dir) { - message := "'" + dir + "' should be an application package zip or dir, but does not exist" - return errors.New(message) - } - if !util.IsDirectory(dir) { - message := "'" + dir + "' should be an application package dir, but is a (non-zip) file" - return errors.New(message) - } - - file, err := os.Create(destination) - if err != nil { - message := "Could not create a temporary zip file for the application package: " + err.Error() - return errors.New(message) - } - defer file.Close() - - w := zip.NewWriter(file) - defer w.Close() - - walker := func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - if info.IsDir() { - return nil - } - file, err := os.Open(path) - if err != nil { - return err - } - defer file.Close() - - zippath, err := filepath.Rel(dir, path) - if err != nil { - return err - } - zipfile, err := w.Create(zippath) - if err != nil { - return err - } - - _, err = io.Copy(zipfile, file) - if err != nil { - return err - } - return nil - } - return filepath.Walk(dir, walker) -} - // Returns the error message in the given JSON, or the entire content if it could not be extracted func extractError(reader io.Reader) string { responseData, _ := ioutil.ReadAll(reader) diff --git a/client/go/vespa/document.go b/client/go/vespa/document.go index 6424113bd52..3bddfc87e82 100644 --- a/client/go/vespa/document.go +++ b/client/go/vespa/document.go @@ -104,7 +104,7 @@ func sendOperation(documentId string, jsonFile string, service *Service, operati Body: ioutil.NopCloser(bytes.NewReader(documentData)), } response, err := serviceDo(service, request, jsonFile, options) - if response == nil { + if err != nil { return util.Failure("Request failed: " + err.Error()) } @@ -179,7 +179,7 @@ func Get(documentId string, service *Service, options OperationOptions) util.Ope Method: "GET", } response, err := serviceDo(service, request, "", options) - if response == nil { + if err != nil { return util.Failure("Request failed: " + err.Error()) } diff --git a/client/go/vespa/target.go b/client/go/vespa/target.go index f620f3b865c..8d48f1ffa3a 100644 --- a/client/go/vespa/target.go +++ b/client/go/vespa/target.go @@ -3,23 +3,15 @@ package vespa import ( - "bytes" "crypto/tls" - "encoding/json" "fmt" "io" "io/ioutil" - "math" "net/http" - "net/url" - "sort" - "strconv" "time" - "github.com/vespa-engine/vespa/client/go/auth0" "github.com/vespa-engine/vespa/client/go/util" "github.com/vespa-engine/vespa/client/go/version" - "github.com/vespa-engine/vespa/client/go/zts" ) const ( @@ -35,9 +27,14 @@ const ( // A hosted Vespa target TargetHosted = "hosted" - deployService = "deploy" - queryService = "query" - documentService = "document" + // A Vespa service that handles deployments, either a config server or a controller + DeployService = "deploy" + + // A Vespa service that handles queries. + QueryService = "query" + + // A Vespa service that handles feeding of document. This may point to the same service as QueryService. + DocumentService = "document" retryInterval = 2 * time.Second ) @@ -89,26 +86,6 @@ type LogOptions struct { Level int } -// CloudOptions configures URL and authentication for a cloud target. -type APIOptions struct { - System System - TLSOptions TLSOptions - APIKey []byte - AuthConfigPath string -} - -// CloudDeploymentOptions configures the deployment to manage through a cloud target. -type CloudDeploymentOptions struct { - Deployment Deployment - TLSOptions TLSOptions - ClusterURLs map[string]string // Endpoints keyed on cluster name -} - -type customTarget struct { - targetType string - baseURL string -} - // Do sends request to this service. Any required authentication happens automatically. func (s *Service) Do(request *http.Request, timeout time.Duration) (*http.Response, error) { if s.TLSOptions.KeyPair.Certificate != nil { @@ -131,469 +108,45 @@ func (s *Service) Do(request *http.Request, timeout time.Duration) (*http.Respon func (s *Service) Wait(timeout time.Duration) (int, error) { url := s.BaseURL switch s.Name { - case deployService: + case DeployService: url += "/status.html" // because /ApplicationStatus is not publicly reachable in Vespa Cloud - case queryService, documentService: + case QueryService, DocumentService: url += "/ApplicationStatus" default: return 0, fmt.Errorf("invalid service: %s", s.Name) } - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return 0, err - } - okFunc := func(status int, response []byte) (bool, error) { return status/100 == 2, nil } - return wait(okFunc, func() *http.Request { return req }, &s.TLSOptions.KeyPair, timeout) + return waitForOK(url, &s.TLSOptions.KeyPair, timeout) } func (s *Service) Description() string { switch s.Name { - case queryService: + case QueryService: return "Container (query API)" - case documentService: + case DocumentService: return "Container (document API)" - case deployService: + case DeployService: return "Deploy API" } return fmt.Sprintf("No description of service %s", s.Name) } -func (t *customTarget) Type() string { return t.targetType } - -func (t *customTarget) Deployment() Deployment { return Deployment{} } - -func (t *customTarget) Service(name string, timeout time.Duration, sessionOrRunID int64, cluster string) (*Service, error) { - if timeout > 0 && name != deployService { - if err := t.waitForConvergence(timeout); err != nil { - return nil, err - } - } - switch name { - case deployService, queryService, documentService: - url, err := t.urlWithPort(name) - if err != nil { - return nil, err - } - return &Service{BaseURL: url, Name: name}, nil - } - return nil, fmt.Errorf("unknown service: %s", name) -} - -func (t *customTarget) PrintLog(options LogOptions) error { - return fmt.Errorf("reading logs from non-cloud deployment is unsupported") -} - -func (t *customTarget) SignRequest(req *http.Request, sigKeyId string) error { return nil } +func isOK(status int) bool { return status/100 == 2 } -func (t *customTarget) CheckVersion(version version.Version) error { return nil } +type responseFunc func(status int, response []byte) (bool, error) -func (t *customTarget) urlWithPort(serviceName string) (string, error) { - u, err := url.Parse(t.baseURL) - if err != nil { - return "", err - } - port := u.Port() - if port == "" { - switch serviceName { - case deployService: - port = "19071" - case queryService, documentService: - port = "8080" - default: - return "", fmt.Errorf("unknown service: %s", serviceName) - } - u.Host = u.Host + ":" + port - } - return u.String(), nil -} +type requestFunc func() *http.Request -func (t *customTarget) waitForConvergence(timeout time.Duration) error { - deployer, err := t.Service(deployService, 0, 0, "") - if err != nil { - return err - } - url := fmt.Sprintf("%s/application/v2/tenant/default/application/default/environment/prod/region/default/instance/default/serviceconverge", deployer.BaseURL) +// waitForOK queries url and returns its status code. If the url returns a non-200 status code, it is repeatedly queried +// until timeout elapses. +func waitForOK(url string, certificate *tls.Certificate, timeout time.Duration) (int, error) { req, err := http.NewRequest("GET", url, nil) if err != nil { - return err - } - converged := false - convergedFunc := func(status int, response []byte) (bool, error) { - if status/100 != 2 { - return false, nil - } - var resp serviceConvergeResponse - if err := json.Unmarshal(response, &resp); err != nil { - return false, nil - } - converged = resp.Converged - return converged, nil - } - if _, err := wait(convergedFunc, func() *http.Request { return req }, nil, timeout); err != nil { - return err - } - if !converged { - return fmt.Errorf("services have not converged") - } - return nil -} - -type cloudTarget struct { - apiOptions APIOptions - deploymentOptions CloudDeploymentOptions - logOptions LogOptions - ztsClient ztsClient -} - -type ztsClient interface { - AccessToken(domain string, certficiate tls.Certificate) (string, error) -} - -func (t *cloudTarget) resolveEndpoint(cluster string) (string, error) { - if cluster == "" { - for _, u := range t.deploymentOptions.ClusterURLs { - if len(t.deploymentOptions.ClusterURLs) == 1 { - return u, nil - } else { - return "", fmt.Errorf("multiple clusters, none chosen: %v", t.deploymentOptions.ClusterURLs) - } - } - } else { - u := t.deploymentOptions.ClusterURLs[cluster] - if u == "" { - clusters := make([]string, len(t.deploymentOptions.ClusterURLs)) - for c := range t.deploymentOptions.ClusterURLs { - clusters = append(clusters, c) - } - return "", fmt.Errorf("unknown cluster '%s': must be one of %v", cluster, clusters) - } - return u, nil - } - - return "", fmt.Errorf("no endpoints") -} - -func (t *cloudTarget) Type() string { - switch t.apiOptions.System.Name { - case MainSystem.Name, CDSystem.Name: - return TargetHosted - } - return TargetCloud -} - -func (t *cloudTarget) Deployment() Deployment { return t.deploymentOptions.Deployment } - -func (t *cloudTarget) Service(name string, timeout time.Duration, runID int64, cluster string) (*Service, error) { - if name != deployService && t.deploymentOptions.ClusterURLs == nil { - if err := t.waitForEndpoints(timeout, runID); err != nil { - return nil, err - } - } - switch name { - case deployService: - return &Service{Name: name, BaseURL: t.apiOptions.System.URL, TLSOptions: t.apiOptions.TLSOptions, ztsClient: t.ztsClient}, nil - case queryService, documentService: - url, err := t.resolveEndpoint(cluster) - if err != nil { - return nil, err - } - t.deploymentOptions.TLSOptions.AthenzDomain = t.apiOptions.System.AthenzDomain - return &Service{Name: name, BaseURL: url, TLSOptions: t.deploymentOptions.TLSOptions, ztsClient: t.ztsClient}, nil - } - return nil, fmt.Errorf("unknown service: %s", name) -} - -func (t *cloudTarget) SignRequest(req *http.Request, keyID string) error { - if t.apiOptions.System.IsPublic() { - if t.apiOptions.APIKey != nil { - signer := NewRequestSigner(keyID, t.apiOptions.APIKey) - return signer.SignRequest(req) - } else { - return t.addAuth0AccessToken(req) - } - } else { - if t.apiOptions.TLSOptions.KeyPair.Certificate == nil { - return fmt.Errorf("system %s requires a certificate for authentication", t.apiOptions.System.Name) - } - return nil - } -} - -func (t *cloudTarget) CheckVersion(clientVersion version.Version) error { - if clientVersion.IsZero() { // development version is always fine - return nil - } - req, err := http.NewRequest("GET", fmt.Sprintf("%s/cli/v1/", t.apiOptions.System.URL), nil) - if err != nil { - return err - } - response, err := util.HttpDo(req, 10*time.Second, "") - if err != nil { - return err - } - defer response.Body.Close() - var cliResponse struct { - MinVersion string `json:"minVersion"` - } - dec := json.NewDecoder(response.Body) - if err := dec.Decode(&cliResponse); err != nil { - return err - } - minVersion, err := version.Parse(cliResponse.MinVersion) - if err != nil { - return err - } - if clientVersion.Less(minVersion) { - return fmt.Errorf("client version %s is less than the minimum supported version: %s", clientVersion, minVersion) - } - return nil -} - -func (t *cloudTarget) addAuth0AccessToken(request *http.Request) error { - a, err := auth0.GetAuth0(t.apiOptions.AuthConfigPath, t.apiOptions.System.Name, t.apiOptions.System.URL) - if err != nil { - return err - } - system, err := a.PrepareSystem(auth0.ContextWithCancel()) - if err != nil { - return err - } - request.Header.Set("Authorization", "Bearer "+system.AccessToken) - return nil -} - -func (t *cloudTarget) logsURL() string { - return fmt.Sprintf("%s/application/v4/tenant/%s/application/%s/instance/%s/environment/%s/region/%s/logs", - t.apiOptions.System.URL, - t.deploymentOptions.Deployment.Application.Tenant, t.deploymentOptions.Deployment.Application.Application, t.deploymentOptions.Deployment.Application.Instance, - t.deploymentOptions.Deployment.Zone.Environment, t.deploymentOptions.Deployment.Zone.Region) -} - -func (t *cloudTarget) PrintLog(options LogOptions) error { - req, err := http.NewRequest("GET", t.logsURL(), nil) - if err != nil { - return err - } - lastFrom := options.From - requestFunc := func() *http.Request { - fromMillis := lastFrom.Unix() * 1000 - q := req.URL.Query() - q.Set("from", strconv.FormatInt(fromMillis, 10)) - if !options.To.IsZero() { - toMillis := options.To.Unix() * 1000 - q.Set("to", strconv.FormatInt(toMillis, 10)) - } - req.URL.RawQuery = q.Encode() - t.SignRequest(req, t.deploymentOptions.Deployment.Application.SerializedForm()) - return req - } - logFunc := func(status int, response []byte) (bool, error) { - if ok, err := isOK(status); !ok { - return ok, err - } - logEntries, err := ReadLogEntries(bytes.NewReader(response)) - if err != nil { - return true, err - } - for _, le := range logEntries { - if !le.Time.After(lastFrom) { - continue - } - if LogLevel(le.Level) > options.Level { - continue - } - fmt.Fprintln(options.Writer, le.Format(options.Dequote)) - } - if len(logEntries) > 0 { - lastFrom = logEntries[len(logEntries)-1].Time - } - return false, nil - } - var timeout time.Duration - if options.Follow { - timeout = math.MaxInt64 // No timeout - } - _, err = wait(logFunc, requestFunc, &t.apiOptions.TLSOptions.KeyPair, timeout) - return err -} - -func (t *cloudTarget) waitForEndpoints(timeout time.Duration, runID int64) error { - if runID > 0 { - if err := t.waitForRun(runID, timeout); err != nil { - return err - } - } - return t.discoverEndpoints(timeout) -} - -func (t *cloudTarget) waitForRun(runID int64, timeout time.Duration) error { - runURL := fmt.Sprintf("%s/application/v4/tenant/%s/application/%s/instance/%s/job/%s-%s/run/%d", - t.apiOptions.System.URL, - t.deploymentOptions.Deployment.Application.Tenant, t.deploymentOptions.Deployment.Application.Application, t.deploymentOptions.Deployment.Application.Instance, - t.deploymentOptions.Deployment.Zone.Environment, t.deploymentOptions.Deployment.Zone.Region, runID) - req, err := http.NewRequest("GET", runURL, nil) - if err != nil { - return err - } - lastID := int64(-1) - requestFunc := func() *http.Request { - q := req.URL.Query() - q.Set("after", strconv.FormatInt(lastID, 10)) - req.URL.RawQuery = q.Encode() - if err := t.SignRequest(req, t.deploymentOptions.Deployment.Application.SerializedForm()); err != nil { - panic(err) - } - return req - } - jobSuccessFunc := func(status int, response []byte) (bool, error) { - if ok, err := isOK(status); !ok { - return ok, err - } - var resp jobResponse - if err := json.Unmarshal(response, &resp); err != nil { - return false, nil - } - if t.logOptions.Writer != nil { - lastID = t.printLog(resp, lastID) - } - if resp.Active { - return false, nil - } - if resp.Status != "success" { - return false, fmt.Errorf("run %d ended with unsuccessful status: %s", runID, resp.Status) - } - return true, nil - } - _, err = wait(jobSuccessFunc, requestFunc, &t.apiOptions.TLSOptions.KeyPair, timeout) - return err -} - -func (t *cloudTarget) printLog(response jobResponse, last int64) int64 { - if response.LastID == 0 { - return last - } - var msgs []logMessage - for step, stepMsgs := range response.Log { - for _, msg := range stepMsgs { - if step == "copyVespaLogs" && LogLevel(msg.Type) > t.logOptions.Level || LogLevel(msg.Type) == 3 { - continue - } - msgs = append(msgs, msg) - } - } - sort.Slice(msgs, func(i, j int) bool { return msgs[i].At < msgs[j].At }) - for _, msg := range msgs { - tm := time.Unix(msg.At/1000, (msg.At%1000)*1000) - fmtTime := tm.Format("15:04:05") - fmt.Fprintf(t.logOptions.Writer, "[%s] %-7s %s\n", fmtTime, msg.Type, msg.Message) - } - return response.LastID -} - -func (t *cloudTarget) discoverEndpoints(timeout time.Duration) error { - deploymentURL := fmt.Sprintf("%s/application/v4/tenant/%s/application/%s/instance/%s/environment/%s/region/%s", - t.apiOptions.System.URL, - t.deploymentOptions.Deployment.Application.Tenant, t.deploymentOptions.Deployment.Application.Application, t.deploymentOptions.Deployment.Application.Instance, - t.deploymentOptions.Deployment.Zone.Environment, t.deploymentOptions.Deployment.Zone.Region) - req, err := http.NewRequest("GET", deploymentURL, nil) - if err != nil { - return err - } - if err := t.SignRequest(req, t.deploymentOptions.Deployment.Application.SerializedForm()); err != nil { - return err - } - urlsByCluster := make(map[string]string) - endpointFunc := func(status int, response []byte) (bool, error) { - if ok, err := isOK(status); !ok { - return ok, err - } - var resp deploymentResponse - if err := json.Unmarshal(response, &resp); err != nil { - return false, nil - } - if len(resp.Endpoints) == 0 { - return false, nil - } - for _, endpoint := range resp.Endpoints { - if endpoint.Scope != "zone" { - continue - } - urlsByCluster[endpoint.Cluster] = endpoint.URL - } - return true, nil - } - if _, err = wait(endpointFunc, func() *http.Request { return req }, &t.apiOptions.TLSOptions.KeyPair, timeout); err != nil { - return err - } - if len(urlsByCluster) == 0 { - return fmt.Errorf("no endpoints discovered") - } - t.deploymentOptions.ClusterURLs = urlsByCluster - return nil -} - -func isOK(status int) (bool, error) { - if status == 401 { - return false, fmt.Errorf("status %d: invalid api key", status) - } - return status/100 == 2, nil -} - -// LocalTarget creates a target for a Vespa platform running locally. -func LocalTarget() Target { - return &customTarget{targetType: TargetLocal, baseURL: "http://127.0.0.1"} -} - -// CustomTarget creates a Target for a Vespa platform running at baseURL. -func CustomTarget(baseURL string) Target { - return &customTarget{targetType: TargetCustom, baseURL: baseURL} -} - -// CloudTarget creates a Target for the Vespa Cloud or hosted Vespa platform. -func CloudTarget(apiOptions APIOptions, deploymentOptions CloudDeploymentOptions, logOptions LogOptions) (Target, error) { - ztsClient, err := zts.NewClient(zts.DefaultURL, util.ActiveHttpClient) - if err != nil { - return nil, err + return 0, err } - return &cloudTarget{ - apiOptions: apiOptions, - deploymentOptions: deploymentOptions, - logOptions: logOptions, - ztsClient: ztsClient, - }, nil -} - -type deploymentEndpoint struct { - Cluster string `json:"cluster"` - URL string `json:"url"` - Scope string `json:"scope"` -} - -type deploymentResponse struct { - Endpoints []deploymentEndpoint `json:"endpoints"` + okFunc := func(status int, response []byte) (bool, error) { return isOK(status), nil } + return wait(okFunc, func() *http.Request { return req }, certificate, timeout) } -type serviceConvergeResponse struct { - Converged bool `json:"converged"` -} - -type jobResponse struct { - Active bool `json:"active"` - Status string `json:"status"` - Log map[string][]logMessage `json:"log"` - LastID int64 `json:"lastId"` -} - -type logMessage struct { - At int64 `json:"at"` - Type string `json:"type"` - Message string `json:"message"` -} - -type responseFunc func(status int, response []byte) (bool, error) - -type requestFunc func() *http.Request - func wait(fn responseFunc, reqFn requestFunc, certificate *tls.Certificate, timeout time.Duration) (int, error) { if certificate != nil { util.ActiveHttpClient.UseCertificate([]tls.Certificate{*certificate}) diff --git a/client/go/vespa/target_cloud.go b/client/go/vespa/target_cloud.go new file mode 100644 index 00000000000..f4eccaacab6 --- /dev/null +++ b/client/go/vespa/target_cloud.go @@ -0,0 +1,382 @@ +package vespa + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "fmt" + "math" + "net/http" + "sort" + "strconv" + "time" + + "github.com/vespa-engine/vespa/client/go/auth0" + "github.com/vespa-engine/vespa/client/go/util" + "github.com/vespa-engine/vespa/client/go/version" + "github.com/vespa-engine/vespa/client/go/zts" +) + +// CloudOptions configures URL and authentication for a cloud target. +type APIOptions struct { + System System + TLSOptions TLSOptions + APIKey []byte + AuthConfigPath string +} + +// CloudDeploymentOptions configures the deployment to manage through a cloud target. +type CloudDeploymentOptions struct { + Deployment Deployment + TLSOptions TLSOptions + ClusterURLs map[string]string // Endpoints keyed on cluster name +} + +type cloudTarget struct { + apiOptions APIOptions + deploymentOptions CloudDeploymentOptions + logOptions LogOptions + ztsClient ztsClient +} + +type deploymentEndpoint struct { + Cluster string `json:"cluster"` + URL string `json:"url"` + Scope string `json:"scope"` +} + +type deploymentResponse struct { + Endpoints []deploymentEndpoint `json:"endpoints"` +} + +type jobResponse struct { + Active bool `json:"active"` + Status string `json:"status"` + Log map[string][]logMessage `json:"log"` + LastID int64 `json:"lastId"` +} + +type logMessage struct { + At int64 `json:"at"` + Type string `json:"type"` + Message string `json:"message"` +} + +type ztsClient interface { + AccessToken(domain string, certficiate tls.Certificate) (string, error) +} + +// CloudTarget creates a Target for the Vespa Cloud or hosted Vespa platform. +func CloudTarget(apiOptions APIOptions, deploymentOptions CloudDeploymentOptions, logOptions LogOptions) (Target, error) { + ztsClient, err := zts.NewClient(zts.DefaultURL, util.ActiveHttpClient) + if err != nil { + return nil, err + } + return &cloudTarget{ + apiOptions: apiOptions, + deploymentOptions: deploymentOptions, + logOptions: logOptions, + ztsClient: ztsClient, + }, nil +} + +func (t *cloudTarget) resolveEndpoint(cluster string) (string, error) { + if cluster == "" { + for _, u := range t.deploymentOptions.ClusterURLs { + if len(t.deploymentOptions.ClusterURLs) == 1 { + return u, nil + } else { + return "", fmt.Errorf("multiple clusters, none chosen: %v", t.deploymentOptions.ClusterURLs) + } + } + } else { + u := t.deploymentOptions.ClusterURLs[cluster] + if u == "" { + clusters := make([]string, len(t.deploymentOptions.ClusterURLs)) + for c := range t.deploymentOptions.ClusterURLs { + clusters = append(clusters, c) + } + return "", fmt.Errorf("unknown cluster '%s': must be one of %v", cluster, clusters) + } + return u, nil + } + + return "", fmt.Errorf("no endpoints") +} + +func (t *cloudTarget) Type() string { + switch t.apiOptions.System.Name { + case MainSystem.Name, CDSystem.Name: + return TargetHosted + } + return TargetCloud +} + +func (t *cloudTarget) Deployment() Deployment { return t.deploymentOptions.Deployment } + +func (t *cloudTarget) Service(name string, timeout time.Duration, runID int64, cluster string) (*Service, error) { + switch name { + case DeployService: + service := &Service{Name: name, BaseURL: t.apiOptions.System.URL, TLSOptions: t.apiOptions.TLSOptions, ztsClient: t.ztsClient} + if timeout > 0 { + status, err := service.Wait(timeout) + if err != nil { + return nil, err + } + if !isOK(status) { + return nil, fmt.Errorf("got status %d from deploy service at %s", status, service.BaseURL) + } + } + return service, nil + case QueryService, DocumentService: + if t.deploymentOptions.ClusterURLs == nil { + if err := t.waitForEndpoints(timeout, runID); err != nil { + return nil, err + } + } + url, err := t.resolveEndpoint(cluster) + if err != nil { + return nil, err + } + t.deploymentOptions.TLSOptions.AthenzDomain = t.apiOptions.System.AthenzDomain + return &Service{Name: name, BaseURL: url, TLSOptions: t.deploymentOptions.TLSOptions, ztsClient: t.ztsClient}, nil + } + return nil, fmt.Errorf("unknown service: %s", name) +} + +func (t *cloudTarget) SignRequest(req *http.Request, keyID string) error { + if t.apiOptions.System.IsPublic() { + if t.apiOptions.APIKey != nil { + signer := NewRequestSigner(keyID, t.apiOptions.APIKey) + return signer.SignRequest(req) + } else { + return t.addAuth0AccessToken(req) + } + } else { + if t.apiOptions.TLSOptions.KeyPair.Certificate == nil { + return fmt.Errorf("system %s requires a certificate for authentication", t.apiOptions.System.Name) + } + return nil + } +} + +func (t *cloudTarget) CheckVersion(clientVersion version.Version) error { + if clientVersion.IsZero() { // development version is always fine + return nil + } + req, err := http.NewRequest("GET", fmt.Sprintf("%s/cli/v1/", t.apiOptions.System.URL), nil) + if err != nil { + return err + } + response, err := util.HttpDo(req, 10*time.Second, "") + if err != nil { + return err + } + defer response.Body.Close() + var cliResponse struct { + MinVersion string `json:"minVersion"` + } + dec := json.NewDecoder(response.Body) + if err := dec.Decode(&cliResponse); err != nil { + return err + } + minVersion, err := version.Parse(cliResponse.MinVersion) + if err != nil { + return err + } + if clientVersion.Less(minVersion) { + return fmt.Errorf("client version %s is less than the minimum supported version: %s", clientVersion, minVersion) + } + return nil +} + +func (t *cloudTarget) addAuth0AccessToken(request *http.Request) error { + a, err := auth0.GetAuth0(t.apiOptions.AuthConfigPath, t.apiOptions.System.Name, t.apiOptions.System.URL) + if err != nil { + return err + } + system, err := a.PrepareSystem(auth0.ContextWithCancel()) + if err != nil { + return err + } + request.Header.Set("Authorization", "Bearer "+system.AccessToken) + return nil +} + +func (t *cloudTarget) logsURL() string { + return fmt.Sprintf("%s/application/v4/tenant/%s/application/%s/instance/%s/environment/%s/region/%s/logs", + t.apiOptions.System.URL, + t.deploymentOptions.Deployment.Application.Tenant, t.deploymentOptions.Deployment.Application.Application, t.deploymentOptions.Deployment.Application.Instance, + t.deploymentOptions.Deployment.Zone.Environment, t.deploymentOptions.Deployment.Zone.Region) +} + +func (t *cloudTarget) PrintLog(options LogOptions) error { + req, err := http.NewRequest("GET", t.logsURL(), nil) + if err != nil { + return err + } + lastFrom := options.From + requestFunc := func() *http.Request { + fromMillis := lastFrom.Unix() * 1000 + q := req.URL.Query() + q.Set("from", strconv.FormatInt(fromMillis, 10)) + if !options.To.IsZero() { + toMillis := options.To.Unix() * 1000 + q.Set("to", strconv.FormatInt(toMillis, 10)) + } + req.URL.RawQuery = q.Encode() + t.SignRequest(req, t.deploymentOptions.Deployment.Application.SerializedForm()) + return req + } + logFunc := func(status int, response []byte) (bool, error) { + if ok, err := isCloudOK(status); !ok { + return ok, err + } + logEntries, err := ReadLogEntries(bytes.NewReader(response)) + if err != nil { + return true, err + } + for _, le := range logEntries { + if !le.Time.After(lastFrom) { + continue + } + if LogLevel(le.Level) > options.Level { + continue + } + fmt.Fprintln(options.Writer, le.Format(options.Dequote)) + } + if len(logEntries) > 0 { + lastFrom = logEntries[len(logEntries)-1].Time + } + return false, nil + } + var timeout time.Duration + if options.Follow { + timeout = math.MaxInt64 // No timeout + } + _, err = wait(logFunc, requestFunc, &t.apiOptions.TLSOptions.KeyPair, timeout) + return err +} + +func (t *cloudTarget) waitForEndpoints(timeout time.Duration, runID int64) error { + if runID > 0 { + if err := t.waitForRun(runID, timeout); err != nil { + return err + } + } + return t.discoverEndpoints(timeout) +} + +func (t *cloudTarget) waitForRun(runID int64, timeout time.Duration) error { + runURL := fmt.Sprintf("%s/application/v4/tenant/%s/application/%s/instance/%s/job/%s-%s/run/%d", + t.apiOptions.System.URL, + t.deploymentOptions.Deployment.Application.Tenant, t.deploymentOptions.Deployment.Application.Application, t.deploymentOptions.Deployment.Application.Instance, + t.deploymentOptions.Deployment.Zone.Environment, t.deploymentOptions.Deployment.Zone.Region, runID) + req, err := http.NewRequest("GET", runURL, nil) + if err != nil { + return err + } + lastID := int64(-1) + requestFunc := func() *http.Request { + q := req.URL.Query() + q.Set("after", strconv.FormatInt(lastID, 10)) + req.URL.RawQuery = q.Encode() + if err := t.SignRequest(req, t.deploymentOptions.Deployment.Application.SerializedForm()); err != nil { + panic(err) + } + return req + } + jobSuccessFunc := func(status int, response []byte) (bool, error) { + if ok, err := isCloudOK(status); !ok { + return ok, err + } + var resp jobResponse + if err := json.Unmarshal(response, &resp); err != nil { + return false, nil + } + if t.logOptions.Writer != nil { + lastID = t.printLog(resp, lastID) + } + if resp.Active { + return false, nil + } + if resp.Status != "success" { + return false, fmt.Errorf("run %d ended with unsuccessful status: %s", runID, resp.Status) + } + return true, nil + } + _, err = wait(jobSuccessFunc, requestFunc, &t.apiOptions.TLSOptions.KeyPair, timeout) + return err +} + +func (t *cloudTarget) printLog(response jobResponse, last int64) int64 { + if response.LastID == 0 { + return last + } + var msgs []logMessage + for step, stepMsgs := range response.Log { + for _, msg := range stepMsgs { + if step == "copyVespaLogs" && LogLevel(msg.Type) > t.logOptions.Level || LogLevel(msg.Type) == 3 { + continue + } + msgs = append(msgs, msg) + } + } + sort.Slice(msgs, func(i, j int) bool { return msgs[i].At < msgs[j].At }) + for _, msg := range msgs { + tm := time.Unix(msg.At/1000, (msg.At%1000)*1000) + fmtTime := tm.Format("15:04:05") + fmt.Fprintf(t.logOptions.Writer, "[%s] %-7s %s\n", fmtTime, msg.Type, msg.Message) + } + return response.LastID +} + +func (t *cloudTarget) discoverEndpoints(timeout time.Duration) error { + deploymentURL := fmt.Sprintf("%s/application/v4/tenant/%s/application/%s/instance/%s/environment/%s/region/%s", + t.apiOptions.System.URL, + t.deploymentOptions.Deployment.Application.Tenant, t.deploymentOptions.Deployment.Application.Application, t.deploymentOptions.Deployment.Application.Instance, + t.deploymentOptions.Deployment.Zone.Environment, t.deploymentOptions.Deployment.Zone.Region) + req, err := http.NewRequest("GET", deploymentURL, nil) + if err != nil { + return err + } + if err := t.SignRequest(req, t.deploymentOptions.Deployment.Application.SerializedForm()); err != nil { + return err + } + urlsByCluster := make(map[string]string) + endpointFunc := func(status int, response []byte) (bool, error) { + if ok, err := isCloudOK(status); !ok { + return ok, err + } + var resp deploymentResponse + if err := json.Unmarshal(response, &resp); err != nil { + return false, nil + } + if len(resp.Endpoints) == 0 { + return false, nil + } + for _, endpoint := range resp.Endpoints { + if endpoint.Scope != "zone" { + continue + } + urlsByCluster[endpoint.Cluster] = endpoint.URL + } + return true, nil + } + if _, err = wait(endpointFunc, func() *http.Request { return req }, &t.apiOptions.TLSOptions.KeyPair, timeout); err != nil { + return err + } + if len(urlsByCluster) == 0 { + return fmt.Errorf("no endpoints discovered") + } + t.deploymentOptions.ClusterURLs = urlsByCluster + return nil +} + +func isCloudOK(status int) (bool, error) { + if status == 401 { + // when retrying we should give up immediately if we're not authorized + return false, fmt.Errorf("status %d: invalid credentials", status) + } + return isOK(status), nil +} diff --git a/client/go/vespa/target_custom.go b/client/go/vespa/target_custom.go new file mode 100644 index 00000000000..072ec8649e4 --- /dev/null +++ b/client/go/vespa/target_custom.go @@ -0,0 +1,128 @@ +package vespa + +import ( + "encoding/json" + "fmt" + "net/http" + "net/url" + "time" + + "github.com/vespa-engine/vespa/client/go/version" +) + +type customTarget struct { + targetType string + baseURL string +} + +type serviceConvergeResponse struct { + Converged bool `json:"converged"` +} + +// LocalTarget creates a target for a Vespa platform running locally. +func LocalTarget() Target { + return &customTarget{targetType: TargetLocal, baseURL: "http://127.0.0.1"} +} + +// CustomTarget creates a Target for a Vespa platform running at baseURL. +func CustomTarget(baseURL string) Target { + return &customTarget{targetType: TargetCustom, baseURL: baseURL} +} + +func (t *customTarget) Type() string { return t.targetType } + +func (t *customTarget) Deployment() Deployment { return Deployment{} } + +func (t *customTarget) createService(name string) (*Service, error) { + switch name { + case DeployService, QueryService, DocumentService: + url, err := t.urlWithPort(name) + if err != nil { + return nil, err + } + return &Service{BaseURL: url, Name: name}, nil + } + return nil, fmt.Errorf("unknown service: %s", name) +} + +func (t *customTarget) Service(name string, timeout time.Duration, sessionOrRunID int64, cluster string) (*Service, error) { + service, err := t.createService(name) + if err != nil { + return nil, err + } + if timeout > 0 { + if name == DeployService { + status, err := service.Wait(timeout) + if err != nil { + return nil, err + } + if !isOK(status) { + return nil, fmt.Errorf("got status %d from deploy service at %s", status, service.BaseURL) + } + } else { + if err := t.waitForConvergence(timeout); err != nil { + return nil, err + } + } + } + return service, nil +} + +func (t *customTarget) PrintLog(options LogOptions) error { + return fmt.Errorf("reading logs from non-cloud deployment is unsupported") +} + +func (t *customTarget) SignRequest(req *http.Request, sigKeyId string) error { return nil } + +func (t *customTarget) CheckVersion(version version.Version) error { return nil } + +func (t *customTarget) urlWithPort(serviceName string) (string, error) { + u, err := url.Parse(t.baseURL) + if err != nil { + return "", err + } + port := u.Port() + if port == "" { + switch serviceName { + case DeployService: + port = "19071" + case QueryService, DocumentService: + port = "8080" + default: + return "", fmt.Errorf("unknown service: %s", serviceName) + } + u.Host = u.Host + ":" + port + } + return u.String(), nil +} + +func (t *customTarget) waitForConvergence(timeout time.Duration) error { + deployURL, err := t.urlWithPort(DeployService) + if err != nil { + return err + } + url := fmt.Sprintf("%s/application/v2/tenant/default/application/default/environment/prod/region/default/instance/default/serviceconverge", deployURL) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return err + } + converged := false + convergedFunc := func(status int, response []byte) (bool, error) { + if !isOK(status) { + return false, nil + } + var resp serviceConvergeResponse + if err := json.Unmarshal(response, &resp); err != nil { + return false, nil + } + converged = resp.Converged + return converged, nil + } + if _, err := wait(convergedFunc, func() *http.Request { return req }, nil, timeout); err != nil { + return err + } + if !converged { + return fmt.Errorf("services have not converged") + } + return nil +} diff --git a/client/go/zts/zts.go b/client/go/zts/zts.go index b1a47db8e48..538971ebdd8 100644 --- a/client/go/zts/zts.go +++ b/client/go/zts/zts.go @@ -44,6 +44,9 @@ func (c *Client) AccessToken(domain string, certificate tls.Certificate) (string } defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return "", fmt.Errorf("got status %d from %s", response.StatusCode, c.tokenURL.String()) + } var ztsResponse struct { AccessToken string `json:"access_token"` } diff --git a/client/go/zts/zts_test.go b/client/go/zts/zts_test.go index f1bd9c1ba75..0eec085aadb 100644 --- a/client/go/zts/zts_test.go +++ b/client/go/zts/zts_test.go @@ -13,6 +13,11 @@ func TestAccessToken(t *testing.T) { if err != nil { t.Fatal(err) } + httpClient.NextResponse(400, `{"message": "bad request"}`) + _, err = client.AccessToken("vespa.vespa", tls.Certificate{}) + if err == nil { + t.Fatal("want error for non-ok response status") + } httpClient.NextResponse(200, `{"access_token": "foo bar"}`) token, err := client.AccessToken("vespa.vespa", tls.Certificate{}) if err != nil { diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index e0e4318ccc8..7cb36374568 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -116,6 +116,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"arnej"}) default boolean useQrserverServiceName() { return true; } @ModelFeatureFlag(owners = {"bjorncs", "baldersheim"}) default boolean enableJdiscPreshutdownCommand() { return true; } @ModelFeatureFlag(owners = {"arnej"}) default boolean avoidRenamingSummaryFeatures() { return false; } + @ModelFeatureFlag(owners = {"bjorncs", "baldersheim"}) default boolean mergeGroupingResultInSearchInvoker() { return false; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 300a77d4a6b..5a1e5ec401b 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -23,7 +23,7 @@ import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.provision.HostsXmlProvisioner; -import com.yahoo.config.model.provision .SingleNodeProvisioner; +import com.yahoo.config.model.provision.SingleNodeProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Zone; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java index 2f4ddb1bda7..16bf37902f5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java @@ -42,7 +42,7 @@ public class FieldOperationApplierForStructs extends FieldOperationApplier { if (structUsedByField.getName().equals(structType.getName())) { //this field is using this type!! field.populateWithStructFields(sdoc, field.getName(), field.getDataType(), 0); - field.populateWithStructMatching(sdoc, field.getName(), field.getDataType(), field.getMatching()); + field.populateWithStructMatching(sdoc, field.getDataType(), field.getMatching()); } } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java index 26b4b78fcaa..e419b8c93a7 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java @@ -783,11 +783,14 @@ public class RankProfile implements Cloneable { } private final List<MutateOperation> mutateOperations = new ArrayList<>(); + public void addMutateOperation(MutateOperation op) { + mutateOperations.add(op); + String prefix = "vespa.mutate." + op.phase.toString(); + addRankProperty(prefix + ".attribute", op.attribute); + addRankProperty(prefix + ".operation", op.operation); + } public void addMutateOperation(MutateOperation.Phase phase, String attribute, String operation) { - mutateOperations.add(new MutateOperation(phase, attribute, operation)); - String prefix = "vespa.mutate." + phase.toString(); - addRankProperty(prefix + ".attribute", attribute); - addRankProperty(prefix + ".operation", operation); + addMutateOperation(new MutateOperation(phase, attribute, operation)); } public List<MutateOperation> getMutateOperations() { return mutateOperations; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java index 495c3da5d3a..b65af3a8f02 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java @@ -17,6 +17,7 @@ import com.yahoo.searchdefinition.document.FieldSet; import com.yahoo.searchdefinition.document.GeoPos; import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.searchdefinition.document.MatchType; import com.yahoo.searchdefinition.document.Stemming; import com.yahoo.searchdefinition.processing.ExactMatch; import com.yahoo.searchdefinition.processing.NGramMatch; @@ -153,7 +154,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { if (normalizeAccents(field)) { addIndexCommand(field, CMD_NORMALIZE); } - if (field.getMatching() == null || field.getMatching().getType().equals(Matching.Type.TEXT)) { + if (field.getMatching() == null || field.getMatching().getType().equals(MatchType.TEXT)) { addIndexCommand(field, CMD_PLAIN_TOKENS); } } @@ -383,7 +384,7 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { .indexname(fieldSet.getName()) .command(CMD_INDEX)); if ( ! isExactMatch(fieldSetMatching)) { - if (fieldSetMatching == null || fieldSetMatching.getType().equals(Matching.Type.TEXT)) { + if (fieldSetMatching == null || fieldSetMatching.getType().equals(MatchType.TEXT)) { iiB.command( new IndexInfoConfig.Indexinfo.Command.Builder() .indexname(fieldSet.getName()) @@ -420,24 +421,24 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { } if (fieldSetMatching != null) { // Explicit matching set on fieldset - if (fieldSetMatching.getType().equals(Matching.Type.EXACT)) { + if (fieldSetMatching.getType().equals(MatchType.EXACT)) { String term = fieldSetMatching.getExactMatchTerminator(); if (term==null) term=ExactMatch.DEFAULT_EXACT_TERMINATOR; iiB.command( new IndexInfoConfig.Indexinfo.Command.Builder() .indexname(fieldSet.getName()) .command("exact "+term)); - } else if (fieldSetMatching.getType().equals(Matching.Type.WORD)) { + } else if (fieldSetMatching.getType().equals(MatchType.WORD)) { iiB.command( new IndexInfoConfig.Indexinfo.Command.Builder() .indexname(fieldSet.getName()) .command(CMD_WORD)); - } else if (fieldSetMatching.getType().equals(Matching.Type.GRAM)) { + } else if (fieldSetMatching.getType().equals(MatchType.GRAM)) { iiB.command( new IndexInfoConfig.Indexinfo.Command.Builder() .indexname(fieldSet.getName()) .command("ngram "+(fieldSetMatching.getGramSize()>0 ? fieldSetMatching.getGramSize() : NGramMatch.DEFAULT_GRAM_SIZE))); - } else if (fieldSetMatching.getType().equals(Matching.Type.TEXT)) { + } else if (fieldSetMatching.getType().equals(MatchType.TEXT)) { } } @@ -477,8 +478,8 @@ public class IndexInfo extends Derived implements IndexInfoConfig.Producer { private boolean isExactMatch(Matching m) { if (m == null) return false; - if (m.getType().equals(Matching.Type.EXACT)) return true; - if (m.getType().equals(Matching.Type.WORD)) return true; + if (m.getType().equals(MatchType.EXACT)) return true; + if (m.getType().equals(MatchType.WORD)) return true; return false; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java index 7310a3fbd88..3edc93fc7c7 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java @@ -16,6 +16,7 @@ import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.document.FieldSet; import com.yahoo.searchdefinition.document.GeoPos; import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.searchdefinition.document.MatchType; import com.yahoo.searchdefinition.document.SDDocumentType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig; @@ -226,9 +227,9 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { public VsmfieldsConfig.Fieldspec.Builder getFieldSpecConfig() { VsmfieldsConfig.Fieldspec.Builder fB = new VsmfieldsConfig.Fieldspec.Builder(); String matchingName = matching.getType().getName(); - if (matching.getType().equals(Matching.Type.TEXT)) + if (matching.getType().equals(MatchType.TEXT)) matchingName = ""; - if (matching.getType() != Matching.Type.EXACT) { + if (matching.getType() != MatchType.EXACT) { if (matching.isPrefix()) { matchingName = "prefix"; } else if (matching.isSubstring()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/BooleanIndexDefinition.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/BooleanIndexDefinition.java index 5719a74aaa6..3b6228029c7 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/BooleanIndexDefinition.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/BooleanIndexDefinition.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.document; +import java.util.Optional; import java.util.OptionalDouble; import java.util.OptionalInt; import java.util.OptionalLong; @@ -22,6 +23,17 @@ public final class BooleanIndexDefinition { private final OptionalLong upperBound; private final OptionalDouble densePostingListThreshold; + public BooleanIndexDefinition(Optional<Integer> arity, + Optional<Long> lowerBound, + Optional<Long> upperBound, + Optional<Double> densePLT) + { + this.arity = arity.isPresent() ? OptionalInt.of(arity.get()) : OptionalInt.empty(); + this.lowerBound = lowerBound.isPresent() ? OptionalLong.of(lowerBound.get()) : OptionalLong.empty(); + this.upperBound = upperBound.isPresent() ? OptionalLong.of(upperBound.get()) : OptionalLong.empty(); + this.densePostingListThreshold = densePLT.isPresent() ? OptionalDouble.of(densePLT.get()) : OptionalDouble.empty(); + } + public BooleanIndexDefinition(OptionalInt arity, OptionalLong lowerBound, OptionalLong upperBound, OptionalDouble densePostingListThreshold) { this.arity = arity; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/MatchAlgorithm.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/MatchAlgorithm.java index 92dc75a9eda..545f2e293dc 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/MatchAlgorithm.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/MatchAlgorithm.java @@ -1,11 +1,16 @@ -package com.yahoo.searchdefinition.parser; +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.document; + +/** Which match algorithm is used by this matching setup */ public enum MatchAlgorithm { NORMAL("normal"), PREFIX("prefix"), SUBSTRING("substring"), SUFFIX("suffix"); + private String name; MatchAlgorithm(String name) { this.name = name; } + public String getName() { return name; } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/MatchType.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/MatchType.java index cd24db6d72e..dd4af0965d8 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/MatchType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/MatchType.java @@ -1,11 +1,14 @@ -package com.yahoo.searchdefinition.parser; +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.document; public enum MatchType { TEXT("text"), WORD("word"), EXACT("exact"), GRAM("gram"); + private String name; MatchType(String name) { this.name = name; } + public String getName() { return name; } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java index d506b22297d..ffac35ba61e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java @@ -11,34 +11,13 @@ import java.io.Serializable; */ public class Matching implements Cloneable, Serializable { - public static final Type defaultType = Type.TEXT; - - public enum Type { - TEXT("text"), - WORD("word"), - EXACT("exact"), - GRAM("gram"); - private String name; - Type(String name) { this.name = name; } - public String getName() { return name; } - } - - /** Which match algorithm is used by this matching setup */ - public enum Algorithm { - NORMAL("normal"), - PREFIX("prefix"), - SUBSTRING("substring"), - SUFFIX("suffix"); - private String name; - Algorithm(String name) { this.name = name; } - public String getName() { return name; } - } + public static final MatchType defaultType = MatchType.TEXT; - private Type type = Type.TEXT; + private MatchType type = MatchType.TEXT; private Case casing = Case.UNCASED; /** The basic match algorithm */ - private Algorithm algorithm = Algorithm.NORMAL; + private MatchAlgorithm algorithm = MatchAlgorithm.NORMAL; private boolean typeUserSet = false; @@ -55,14 +34,14 @@ public class Matching implements Cloneable, Serializable { /** Creates a matching of type "text" */ public Matching() {} - public Matching(Type type) { + public Matching(MatchType type) { this.type = type; } - public Type getType() { return type; } + public MatchType getType() { return type; } public Case getCase() { return casing; } - public void setType(Type type) { + public void setType(MatchType type) { this.type = type; typeUserSet = true; } @@ -73,20 +52,20 @@ public class Matching implements Cloneable, Serializable { public Matching maxLength(int maxLength) { this.maxLength = maxLength; return this; } public boolean isTypeUserSet() { return typeUserSet; } - public Algorithm getAlgorithm() { return algorithm; } + public MatchAlgorithm getAlgorithm() { return algorithm; } - public void setAlgorithm(Algorithm algorithm) { + public void setAlgorithm(MatchAlgorithm algorithm) { this.algorithm = algorithm; algorithmUserSet = true; } public boolean isAlgorithmUserSet() { return algorithmUserSet; } - public boolean isPrefix() { return algorithm == Algorithm.PREFIX; } + public boolean isPrefix() { return algorithm == MatchAlgorithm.PREFIX; } - public boolean isSubstring() { return algorithm == Algorithm.SUBSTRING; } + public boolean isSubstring() { return algorithm == MatchAlgorithm.SUBSTRING; } - public boolean isSuffix() { return algorithm == Algorithm.SUFFIX; } + public boolean isSuffix() { return algorithm == MatchAlgorithm.SUFFIX; } /** Returns the gram size, or -1 if not set. Should only be set with gram matching. */ public int getGramSize() { return gramSize; } @@ -97,12 +76,13 @@ public class Matching implements Cloneable, Serializable { * Merge data from another matching object */ public void merge(Matching m) { + if (m == null) return; if (m.isAlgorithmUserSet()) { this.setAlgorithm(m.getAlgorithm()); } if (m.isTypeUserSet()) { this.setType(m.getType()); - if (m.getType() == Type.GRAM) + if (m.getType() == MatchType.GRAM) gramSize = m.gramSize; } if (m.getExactMatchTerminator() != null) { @@ -127,7 +107,7 @@ public class Matching implements Cloneable, Serializable { @Override public String toString() { - return type + " matching [" + (type==Type.GRAM ? "gram size " + gramSize : "supports " + algorithm) + + return type + " matching [" + (type==MatchType.GRAM ? "gram size " + gramSize : "supports " + algorithm) + "], [exact-terminator "+exactMatchTerminator+"]"; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java index edb317f4a99..d9bdf5dc917 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java @@ -7,6 +7,7 @@ import com.yahoo.document.DocumentType; import com.yahoo.document.Field; import com.yahoo.document.MapDataType; import com.yahoo.document.StructDataType; +import com.yahoo.document.TemporaryStructuredDataType; import com.yahoo.document.TensorDataType; import com.yahoo.language.Linguistics; import com.yahoo.language.process.Embedder; @@ -146,7 +147,6 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, /** Creates a new field */ protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner, boolean populate) { super(name, dataType, owner == null ? null : owner.getDocumentType()); - this.ownerDocType = owner; populate(populate, repo, name, dataType); } @@ -161,17 +161,17 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, protected SDField(SDDocumentType repo, String name, DataType dataType, SDDocumentType owner, Matching fieldMatching, boolean populate, int recursion) { super(name, dataType, owner == null ? null : owner.getDocumentType()); - this.ownerDocType = owner; if (fieldMatching != null) this.setMatching(fieldMatching); populate(populate, repo, name, dataType, fieldMatching, recursion); } - public SDField(SDDocumentType repo, String name, DataType dataType) { - this(repo, name,dataType, true); + public SDField(SDDocumentType repo, String name, DataType dataType) { + this(repo, name, dataType, true); } + public SDField(String name, DataType dataType) { - this(null, name,dataType); + this(null, name, dataType); } private void populate(boolean populate, SDDocumentType repo, String name, DataType dataType) { @@ -191,7 +191,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, } if (populate || (dataType instanceof MapDataType)) { populateWithStructFields(repo, name, dataType, recursion); - populateWithStructMatching(repo, name, dataType, fieldMatching); + populateWithStructMatching(repo, dataType, fieldMatching); } } @@ -268,93 +268,78 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, } } + @SuppressWarnings("deprecation") public void populateWithStructFields(SDDocumentType sdoc, String name, DataType dataType, int recursion) { DataType dt = getFirstStructOrMapRecursive(); if (dt == null) return; + java.util.function.BiConsumer<String, DataType> supplyStructField = (fieldName, fieldType) -> { + if (structFields.containsKey(fieldName)) return; + String subName = name.concat(".").concat(fieldName); + var subField = new SDField(sdoc, subName, fieldType, + ownerDocType, new Matching(), + true, recursion + 1); + structFields.put(fieldName, subField); + }; + if (dataType instanceof MapDataType) { MapDataType mdt = (MapDataType) dataType; - SDField keyField = new SDField(sdoc, name.concat(".key"), mdt.getKeyType(), - getOwnerDocType(), new Matching(), true, recursion + 1); - structFields.put("key", keyField); - SDField valueField = new SDField(sdoc, name.concat(".value"), mdt.getValueType(), - getOwnerDocType(), new Matching(), true, recursion + 1); - structFields.put("value", valueField); + supplyStructField.accept("key", mdt.getKeyType()); + supplyStructField.accept("value", mdt.getValueType()); } else { if (recursion >= 10) return; if (dataType instanceof CollectionDataType) { dataType = ((CollectionDataType)dataType).getNestedType(); } - if (dataType instanceof StructDataType) { + if (dataType instanceof TemporaryStructuredDataType) { SDDocumentType subType = sdoc != null ? sdoc.getType(dataType.getName()) : null; if (subType == null) { throw new IllegalArgumentException("Could not find struct '" + dataType.getName() + "'."); } for (Field field : subType.fieldSet()) { - SDField subField = new SDField(sdoc, name.concat(".").concat(field.getName()), field.getDataType(), - subType, new Matching(), true, recursion + 1); - structFields.putIfAbsent(field.getName(), subField); + supplyStructField.accept(field.getName(), field.getDataType()); + } + } else if (dataType instanceof StructDataType) { + var sdt = (StructDataType) dataType; + for (Field field : sdt.getFields()) { + supplyStructField.accept(field.getName(), field.getDataType()); } } } } - public void populateWithStructMatching(SDDocumentType sdoc, String name, DataType dataType, - Matching superFieldMatching) { + public void populateWithStructMatching(SDDocumentType sdoc, DataType dataType, Matching superFieldMatching) { + if (sdoc == null) return; + if (superFieldMatching == null) return; DataType dt = getFirstStructOrMapRecursive(); if (dt == null) return; if (dataType instanceof MapDataType) { - MapDataType mdt = (MapDataType) dataType; - - Matching keyFieldMatching = new Matching(); - if (superFieldMatching != null) { - keyFieldMatching.merge(superFieldMatching); - } - SDField keyField = structFields.get(name.concat(".key")); - if (keyField != null) { - keyField.populateWithStructMatching(sdoc, name.concat(".key"), mdt.getKeyType(), keyFieldMatching); - keyField.setMatching(keyFieldMatching); - } - - Matching valueFieldMatching = new Matching(); - if (superFieldMatching != null) { - valueFieldMatching.merge(superFieldMatching); - } - SDField valueField = structFields.get(name.concat(".value")); - if (valueField != null) { - valueField.populateWithStructMatching(sdoc, name.concat(".value"), mdt.getValueType(), - valueFieldMatching); - valueField.setMatching(valueFieldMatching); - } - + // old code here would never do anything useful, should we do something here? + return; } else { if (dataType instanceof CollectionDataType) { dataType = ((CollectionDataType)dataType).getNestedType(); } if (dataType instanceof StructDataType) { - SDDocumentType subType = sdoc != null ? sdoc.getType(dataType.getName()) : null; - if (subType != null) { - for (Field f : subType.fieldSet()) { - if (f instanceof SDField) { - SDField field = (SDField) f; - Matching subFieldMatching = new Matching(); - if (superFieldMatching != null) { - subFieldMatching.merge(superFieldMatching); - } - subFieldMatching.merge(field.getMatching()); - SDField subField = structFields.get(field.getName()); - if (subField != null) { - subField.populateWithStructMatching(sdoc, name.concat(".").concat(field.getName()), field.getDataType(), - subFieldMatching); - subField.setMatching(subFieldMatching); - } - } else { - throw new IllegalArgumentException("Field in struct is not SDField " + f.getName()); + SDDocumentType subType = sdoc.getType(dataType.getName()); + if (subType == null) { + throw new IllegalArgumentException("Could not find struct " + dataType.getName()); + } + for (Field f : subType.fieldSet()) { + if (f instanceof SDField) { + SDField field = (SDField) f; + Matching subFieldMatching = new Matching(); + subFieldMatching.merge(superFieldMatching); + subFieldMatching.merge(field.getMatching()); + SDField subField = structFields.get(field.getName()); + if (subField != null) { + subField.populateWithStructMatching(sdoc, field.getDataType(), subFieldMatching); + subField.setMatching(subFieldMatching); } + } else { + throw new IllegalArgumentException("Field in struct is not SDField " + f.getName()); } - } else { - throw new IllegalArgumentException("Could not find struct " + dataType.getName()); } } } @@ -551,7 +536,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, */ // TODO: When this is not the same as getMatching().setthis we have a potential for inconsistency. Find the right // Matching object for struct fields at lookup time instead. - public void setMatchingType(Matching.Type type) { + public void setMatchingType(MatchType type) { this.getMatching().setType(type); for (SDField structField : getStructFields()) { structField.setMatchingType(type); @@ -574,7 +559,7 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, */ // TODO: When this is not the same as getMatching().setthis we have a potential for inconsistency. Find the right // Matching object for struct fields at lookup time instead. - public void setMatchingAlgorithm(Matching.Algorithm algorithm) { + public void setMatchingAlgorithm(MatchAlgorithm algorithm) { this.getMatching().setAlgorithm(algorithm); for (SDField structField : getStructFields()) { structField.getMatching().setAlgorithm(algorithm); @@ -776,11 +761,6 @@ public class SDField extends Field implements TypedKey, FieldOperationContainer, @Override public List<String> getQueryCommands() { return queryCommands; } - /** Returns the document that this field was declared in, or null */ - private SDDocumentType getOwnerDocType() { - return ownerDocType; - } - @Override public boolean equals(Object other) { if ( ! (other instanceof SDField)) return false; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/MatchOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/MatchOperation.java index d6e3a898420..322f26bc0e3 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/MatchOperation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/MatchOperation.java @@ -2,7 +2,9 @@ package com.yahoo.searchdefinition.fieldoperation; import com.yahoo.searchdefinition.document.Case; +import com.yahoo.searchdefinition.document.MatchAlgorithm; import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.searchdefinition.document.MatchType; import com.yahoo.searchdefinition.document.SDField; /** @@ -10,14 +12,14 @@ import com.yahoo.searchdefinition.document.SDField; */ public class MatchOperation implements FieldOperation { - private Matching.Type matchingType; + private MatchType matchingType; private Case casing; private Integer gramSize; private Integer maxLength; - private Matching.Algorithm matchingAlgorithm; + private MatchAlgorithm matchingAlgorithm; private String exactMatchTerminator; - public void setMatchingType(Matching.Type matchingType) { + public void setMatchingType(MatchType matchingType) { this.matchingType = matchingType; } @@ -28,7 +30,7 @@ public class MatchOperation implements FieldOperation { this.maxLength = maxLength; } - public void setMatchingAlgorithm(Matching.Algorithm matchingAlgorithm) { + public void setMatchingAlgorithm(MatchAlgorithm matchingAlgorithm) { this.matchingAlgorithm = matchingAlgorithm; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java new file mode 100644 index 00000000000..ea6c0a338a6 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java @@ -0,0 +1,183 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.parser; + +import com.yahoo.searchdefinition.Index; +import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.searchdefinition.document.Case; +import com.yahoo.searchdefinition.document.Dictionary; +import com.yahoo.searchdefinition.document.NormalizeLevel; +import com.yahoo.searchdefinition.document.RankType; +import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.searchdefinition.document.Sorting; +import com.yahoo.vespa.documentmodel.SummaryField; + +import java.util.Locale; + +/** + * Helper for converting ParsedField etc to SDField with settings + * + * @author arnej27959 + **/ +public class ConvertParsedFields { + + void convertMatchSettings(SDField field, ParsedMatchSettings parsed) { + parsed.getMatchType().ifPresent(matchingType -> field.setMatchingType(matchingType)); + parsed.getMatchCase().ifPresent(casing -> field.setMatchingCase(casing)); + parsed.getGramSize().ifPresent(gramSize -> field.getMatching().setGramSize(gramSize)); + parsed.getMaxLength().ifPresent(maxLength -> field.getMatching().maxLength(maxLength)); + parsed.getMatchAlgorithm().ifPresent + (matchingAlgorithm -> field.setMatchingAlgorithm(matchingAlgorithm)); + parsed.getExactTerminator().ifPresent + (exactMatchTerminator -> field.getMatching().setExactMatchTerminator(exactMatchTerminator)); + } + + void convertSorting(SDField field, ParsedSorting parsed, String name) { + Attribute attribute = field.getAttributes().get(name); + if (attribute == null) { + attribute = new Attribute(name, field.getDataType()); + field.addAttribute(attribute); + } + Sorting sorting = attribute.getSorting(); + if (parsed.getAscending()) { + sorting.setAscending(); + } else { + sorting.setDescending(); + } + parsed.getFunction().ifPresent(function -> sorting.setFunction(function)); + parsed.getStrength().ifPresent(strength -> sorting.setStrength(strength)); + parsed.getLocale().ifPresent(locale -> sorting.setLocale(locale)); + } + + void convertAttribute(SDField field, ParsedAttribute parsed) { + String name = parsed.name(); + String fieldName = field.getName(); + Attribute attribute = null; + if (fieldName.endsWith("." + name)) { + attribute = field.getAttributes().get(field.getName()); + } + if (attribute == null) { + attribute = field.getAttributes().get(name); + if (attribute == null) { + attribute = new Attribute(name, field.getDataType()); + field.addAttribute(attribute); + } + } + attribute.setHuge(parsed.getHuge()); + attribute.setPaged(parsed.getPaged()); + attribute.setFastSearch(parsed.getFastSearch()); + attribute.setFastAccess(parsed.getFastAccess()); + attribute.setMutable(parsed.getMutable()); + attribute.setEnableBitVectors(parsed.getEnableBitVectors()); + attribute.setEnableOnlyBitVector(parsed.getEnableOnlyBitVector()); + + // attribute.setTensorType(?) + + for (String alias : parsed.getAliases()) { + field.getAliasToName().put(alias, parsed.lookupAliasedFrom(alias)); + } + var distanceMetric = parsed.getDistanceMetric(); + if (distanceMetric.isPresent()) { + String upper = distanceMetric.get().toUpperCase(Locale.ENGLISH); + attribute.setDistanceMetric(Attribute.DistanceMetric.valueOf(upper)); + } + var sorting = parsed.getSorting(); + if (sorting.isPresent()) { + convertSorting(field, sorting.get(), name); + } + } + + private void convertRankType(SDField field, String indexName, String rankType) { + RankType type = RankType.fromString(rankType); + if (indexName == null || indexName.equals("")) { + field.setRankType(type); // Set default if the index is not specified. + } else { + Index index = field.getIndex(indexName); + if (index == null) { + index = new Index(indexName); + field.addIndex(index); + } + index.setRankType(type); + } + } + + private void convertNormalizing(SDField field, String setting) { + NormalizeLevel.Level level; + if ("none".equals(setting)) { + level = NormalizeLevel.Level.NONE; + } else if ("codepoint".equals(setting)) { + level = NormalizeLevel.Level.CODEPOINT; + } else if ("lowercase".equals(setting)) { + level = NormalizeLevel.Level.LOWERCASE; + } else if ("accent".equals(setting)) { + level = NormalizeLevel.Level.ACCENT; + } else if ("all".equals(setting)) { + level = NormalizeLevel.Level.ACCENT; + } else { + throw new IllegalArgumentException("invalid normalizing setting: " + setting); + } + field.setNormalizing(new NormalizeLevel(level, true)); + } + + // from grammar, things that can be inside struct-field block + private void convertCommonFieldSettings(SDField field, ParsedField parsed) { + convertMatchSettings(field, parsed.matchSettings()); + var indexing = parsed.getIndexing(); + if (indexing.isPresent()) { + field.setIndexingScript(indexing.get().script()); + } + parsed.getStemming().ifPresent(value -> field.setStemming(value)); + parsed.getNormalizing().ifPresent(value -> convertNormalizing(field, value)); + for (var attribute : parsed.getAttributes()) { + convertAttribute(field, attribute); + } + // MISSING: parsed.getSummaryFields() + for (String command : parsed.getQueryCommands()) { + field.addQueryCommand(command); + } + for (var structField : parsed.getStructFields()) { + convertStructField(field, structField); + } + } + + private void convertStructField(SDField field, ParsedField parsed) { + SDField structField = field.getStructField(parsed.name()); + if (structField == null ) { + throw new IllegalArgumentException("Struct field '" + parsed.name() + "' has not been defined in struct " + + "for field '" + field.getName() + "'."); + } + convertCommonFieldSettings(structField, parsed); + } + + private void convertExtraFieldSettings(SDField field, ParsedField parsed) { + String name = parsed.name(); + for (var dictOp : parsed.getDictionaryOptions()) { + var dictionary = field.getOrSetDictionary(); + switch (dictOp) { + case HASH: dictionary.updateType(Dictionary.Type.HASH); break; + case BTREE: dictionary.updateType(Dictionary.Type.BTREE); break; + case CASED: dictionary.updateMatch(Case.CASED); break; + case UNCASED: dictionary.updateMatch(Case.UNCASED); break; + } + } + // MISSING: parsed.getIndexes() + for (var alias : parsed.getAliases()) { + field.getAliasToName().put(alias, parsed.lookupAliasedFrom(alias)); + } + parsed.getRankTypes().forEach((indexName, rankType) -> convertRankType(field, indexName, rankType)); + parsed.getSorting().ifPresent(sortInfo -> convertSorting(field, sortInfo, name)); + if (parsed.getBolding()) { + // TODO must it be so ugly: + SummaryField summaryField = field.getSummaryField(name, true); + summaryField.addSource(name); + summaryField.addDestination("default"); + summaryField.setTransform(summaryField.getTransform().bold()); + } + if (parsed.getLiteral()) { + field.getRanking().setLiteral(true); + } + if (parsed.getFilter()) { + field.getRanking().setFilter(true); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/InheritanceResolver.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/InheritanceResolver.java index edcbf85b5dc..d9132d3aa24 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/InheritanceResolver.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/InheritanceResolver.java @@ -39,7 +39,7 @@ public class InheritanceResolver { for (String inherit : schema.getInherited()) { var parent = parsedSchemas.get(inherit); if (parent == null) { - throw new IllegalArgumentException("schema " + schema.name() + " inherits from unavailable schema " + inherit); + throw new IllegalArgumentException("schema '" + schema.name() + "' inherits '" + inherit + "', but this schema does not exist"); } schema.resolveInherit(inherit, parent); } @@ -56,15 +56,8 @@ public class InheritanceResolver { private void resolveDocumentInheritance() { for (ParsedSchema schema : parsedSchemas.values()) { if (! schema.hasDocument()) { - // TODO: is schema without a document even valid? - // could make sense for schemas with just rank-profile functions - // it makes life easier to behave as if there was en empty - // document block here. - var doc = new ParsedDocument(schema.name()); - for (String inherit : schema.getInherited()) { - doc.inherit(inherit); - } - schema.addDocument(doc); + throw new IllegalArgumentException("For schema '" + schema.name() + + "': A search specification must have an equally named document inside of it."); } ParsedDocument doc = schema.getDocument(); var old = parsedDocs.put(doc.name(), doc); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java index a93fb441563..536caf55111 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java @@ -8,6 +8,7 @@ import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; +import com.yahoo.yolean.Exceptions; import java.io.File; import java.util.ArrayList; @@ -42,18 +43,24 @@ public class IntermediateCollection { public ParsedSchema getParsedSchema(String name) { return parsedSchemas.get(name); } - ParsedSchema addSchemaFromString(String input) throws ParseException { - var stream = new SimpleCharStream(input); - var parser = new IntermediateParser(stream, deployLogger, modelProperties); + public ParsedSchema addSchemaFromString(String input) throws ParseException { + var stream = new SimpleCharStream(input); + var parser = new IntermediateParser(stream, deployLogger, modelProperties); + try { var schema = parser.schema(); if (parsedSchemas.containsKey(schema.name())) { throw new IllegalArgumentException("Duplicate schemas named: " + schema.name()); } parsedSchemas.put(schema.name(), schema); return schema; + } catch (TokenMgrException e) { + throw new ParseException("Unknown symbol: " + e.getMessage()); + } catch (ParseException pe) { + throw new ParseException(stream.formatException(Exceptions.toMessageString(pe))); + } } - private void addSchemaFromStringWithFileName(String input, String fileName) throws ParseException { + private String addSchemaFromStringWithFileName(String input, String fileName) throws ParseException { var parsed = addSchemaFromString(input); String nameFromFile = baseName(fileName); if (! parsed.name().equals(nameFromFile)) { @@ -62,6 +69,7 @@ public class IntermediateCollection { + parsed.name() + ApplicationPackage.SD_NAME_SUFFIX + "', was '" + stripDirs(fileName) + "'"); } + return parsed.name(); } private String baseName(String filename) { @@ -87,25 +95,28 @@ public class IntermediateCollection { /** * parse a schema from the given reader and add result to collection **/ - public void addSchemaFromReader(NamedReader reader) { + public String addSchemaFromReader(NamedReader reader) throws ParseException { try { - addSchemaFromStringWithFileName(IOUtils.readAll(reader.getReader()), reader.getName()); + var nameParsed = addSchemaFromStringWithFileName(IOUtils.readAll(reader.getReader()), reader.getName()); + reader.close(); + return nameParsed; + } catch (ParseException ex) { + throw new ParseException("Failed parsing schema from " + reader.getName() + ": " + ex.getMessage()); } catch (java.io.IOException ex) { throw new IllegalArgumentException("Failed reading from " + reader.getName() + ": " + ex.getMessage()); - } catch (ParseException ex) { - throw new IllegalArgumentException("Failed parsing schema from " + reader.getName() + ": " + ex.getMessage()); } - } /** for unit tests */ - public void addSchemaFromFile(String fileName) { + public String addSchemaFromFile(String fileName) throws ParseException { try { - addSchemaFromStringWithFileName(IOUtils.readFile(new File(fileName)), fileName); + // return addSchemaFromStringWithFileName(IOUtils.readFile(new File(fileName)), fileName); + var parsed = addSchemaFromString(IOUtils.readFile(new File(fileName))); + return parsed.name(); + } catch (ParseException ex) { + throw new ParseException("Failed parsing schema from " + fileName + ": " + ex.getMessage()); } catch (java.io.IOException ex) { throw new IllegalArgumentException("Could not read file " + fileName + ": " + ex.getMessage()); - } catch (ParseException ex) { - throw new IllegalArgumentException("Failed parsing schema file " + fileName + ": " + ex.getMessage()); } } @@ -113,7 +124,7 @@ public class IntermediateCollection { * parse a rank profile from the given reader and add to the schema identified by name. * note: the named schema must have been parsed already. **/ - public void addRankProfileFile(String schemaName, NamedReader reader) { + public void addRankProfileFile(String schemaName, NamedReader reader) throws ParseException { try { ParsedSchema schema = parsedSchemas.get(schemaName); if (schema == null) { @@ -121,16 +132,19 @@ public class IntermediateCollection { } var stream = new SimpleCharStream(IOUtils.readAll(reader.getReader())); var parser = new IntermediateParser(stream, deployLogger, modelProperties); - parser.rankProfile(schema); + try { + parser.rankProfile(schema); + } catch (ParseException pe) { + throw new ParseException("Failed parsing rank-profile from " + reader.getName() + ": " + + stream.formatException(Exceptions.toMessageString(pe))); + } } catch (java.io.IOException ex) { throw new IllegalArgumentException("Failed reading from " + reader.getName() + ": " + ex.getMessage()); - } catch (ParseException ex) { - throw new IllegalArgumentException("Failed parsing rank-profile from " + reader.getName() + ": " + ex.getMessage()); } } // for unit test - void addRankProfileFile(String schemaName, String fileName) { + void addRankProfileFile(String schemaName, String fileName) throws ParseException { try { var reader = IOUtils.createReader(fileName, "UTF-8"); addRankProfileFile(schemaName, new NamedReader(fileName, reader)); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/MatchCase.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/MatchCase.java deleted file mode 100644 index 30e968873ec..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/MatchCase.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.yahoo.searchdefinition.parser; - -public enum MatchCase { - CASED("cased"), - UNCASED("uncased"); - private String name; - MatchCase(String name) { this.name = name; } - public String getName() { return name;} -} - diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAttribute.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAttribute.java index 484d56b6e3e..b5dabcdb608 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAttribute.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAttribute.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.searchdefinition.parser; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -21,8 +21,8 @@ class ParsedAttribute extends ParsedBlock { private boolean enableHuge = false; private boolean enableMutable = false; private boolean enablePaged = false; - private final Map<String, String> aliases = new HashMap<>(); - private ParsedSorting sortInfo = null; + private final Map<String, String> aliases = new LinkedHashMap<>(); + private ParsedSorting sortSettings = null; private String distanceMetric = null; ParsedAttribute(String name) { @@ -39,16 +39,23 @@ class ParsedAttribute extends ParsedBlock { boolean getHuge() { return this.enableHuge; } boolean getMutable() { return this.enableMutable; } boolean getPaged() { return this.enablePaged; } - Optional<ParsedSorting> getSorting() { return Optional.ofNullable(sortInfo); } + Optional<ParsedSorting> getSorting() { return Optional.ofNullable(sortSettings); } void addAlias(String from, String to) { verifyThat(! aliases.containsKey(to), "already has alias", to); aliases.put(to, from); } + void setDistanceMetric(String value) { verifyThat(distanceMetric == null, "already has distance-metric", distanceMetric); this.distanceMetric = value; } + + ParsedSorting sortInfo() { + if (sortSettings == null) sortSettings = new ParsedSorting(name(), "attribute.sorting"); + return this.sortSettings; + } + void setEnableBitVectors(boolean value) { this.enableBitVectors = value; } void setEnableOnlyBitVector(boolean value) { this.enableOnlyBitVector = value; } void setFastAccess(boolean value) { this.enableFastAccess = true; } @@ -56,8 +63,4 @@ class ParsedAttribute extends ParsedBlock { void setHuge(boolean value) { this.enableHuge = true; } void setMutable(boolean value) { this.enableMutable = true; } void setPaged(boolean value) { this.enablePaged = true; } - void setSorting(ParsedSorting sorting) { - verifyThat(this.sortInfo == null, "already has sorting"); - this.sortInfo = sorting; - } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java index dd61124c3a7..065b66e22b1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java @@ -4,7 +4,7 @@ package com.yahoo.searchdefinition.parser; import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -17,10 +17,10 @@ import java.util.Optional; **/ public class ParsedDocument extends ParsedBlock { private final List<String> inherited = new ArrayList<>(); - private final Map<String, ParsedDocument> resolvedInherits = new HashMap(); - private final Map<String, ParsedField> docFields = new HashMap<>(); - private final Map<String, ParsedStruct> docStructs = new HashMap<>(); - private final Map<String, ParsedAnnotation> docAnnotations = new HashMap<>(); + private final Map<String, ParsedDocument> resolvedInherits = new LinkedHashMap(); + private final Map<String, ParsedField> docFields = new LinkedHashMap<>(); + private final Map<String, ParsedStruct> docStructs = new LinkedHashMap<>(); + private final Map<String, ParsedAnnotation> docAnnotations = new LinkedHashMap<>(); public ParsedDocument(String name) { super(name, "document"); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocumentSummary.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocumentSummary.java index 65cfa433dc0..08f4946a218 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocumentSummary.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocumentSummary.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.parser; import java.util.ArrayList; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -17,7 +17,7 @@ class ParsedDocumentSummary extends ParsedBlock { private boolean omitSummaryFeatures; private boolean fromDisk; private final List<String> inherited = new ArrayList<>(); - private final Map<String, ParsedSummaryField> fields = new HashMap<>(); + private final Map<String, ParsedSummaryField> fields = new LinkedHashMap<>(); ParsedDocumentSummary(String name) { super(name, "document-summary"); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedField.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedField.java index 9c7400136ab..5ee73abc28d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedField.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedField.java @@ -4,7 +4,7 @@ package com.yahoo.searchdefinition.parser; import com.yahoo.searchdefinition.document.Stemming; import java.util.ArrayList; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -24,17 +24,17 @@ class ParsedField extends ParsedBlock { private boolean isLiteral = false; private boolean isNormal = false; private Integer weight; - private String normalizing; + private String normalizing = null; private final ParsedMatchSettings matchInfo = new ParsedMatchSettings(); private Stemming stemming = null; private ParsedIndexingOp indexingOp = null; private ParsedSorting sortSettings = null; - private final Map<String, ParsedAttribute> attributes = new HashMap<>(); - private final Map<String, ParsedIndex> fieldIndexes = new HashMap<>(); - private final Map<String, String> aliases = new HashMap<>(); - private final Map<String, String> rankTypes = new HashMap<>(); - private final Map<String, ParsedField> structFields = new HashMap<>(); - private final Map<String, ParsedSummaryField> summaryFields = new HashMap<>(); + private final Map<String, ParsedAttribute> attributes = new LinkedHashMap<>(); + private final Map<String, ParsedIndex> fieldIndexes = new LinkedHashMap<>(); + private final Map<String, String> aliases = new LinkedHashMap<>(); + private final Map<String, String> rankTypes = new LinkedHashMap<>(); + private final Map<String, ParsedField> structFields = new LinkedHashMap<>(); + private final Map<String, ParsedSummaryField> summaryFields = new LinkedHashMap<>(); private final List<DictionaryOption> dictionaryOptions = new ArrayList<>(); private final List<String> queryCommands = new ArrayList<>(); @@ -46,6 +46,7 @@ class ParsedField extends ParsedBlock { ParsedType getType() { return this.type; } boolean getBolding() { return this.hasBolding; } boolean getFilter() { return this.isFilter; } + boolean getLiteral() { return this.isLiteral; } boolean hasIdOverride() { return overrideId != 0; } int idOverride() { return overrideId; } List<DictionaryOption> getDictionaryOptions() { return List.copyOf(dictionaryOptions); } @@ -58,6 +59,7 @@ class ParsedField extends ParsedBlock { String lookupAliasedFrom(String alias) { return aliases.get(alias); } ParsedMatchSettings matchSettings() { return this.matchInfo; } Optional<Stemming> getStemming() { return Optional.ofNullable(stemming); } + Optional<String> getNormalizing() { return Optional.ofNullable(normalizing); } Optional<ParsedIndexingOp> getIndexing() { return Optional.ofNullable(indexingOp); } Optional<ParsedSorting> getSorting() { return Optional.ofNullable(sortSettings); } Map<String, String> getRankTypes() { return Map.copyOf(rankTypes); } @@ -130,9 +132,9 @@ class ParsedField extends ParsedBlock { indexingOp = idxOp; } - void setSorting(ParsedSorting sorting) { - verifyThat(sortSettings == null, "already has sorting"); - this.sortSettings = sorting; + ParsedSorting sortInfo() { + if (sortSettings == null) sortSettings = new ParsedSorting(name(), "field.sorting"); + return this.sortSettings; } void addQueryCommand(String command) { @@ -148,6 +150,9 @@ class ParsedField extends ParsedBlock { void addSummaryField(ParsedSummaryField summaryField) { String fieldName = summaryField.name(); verifyThat(! summaryFields.containsKey(fieldName), "already has summary field", fieldName); + if (summaryField.getType() == null) { + summaryField.setType(getType()); + } summaryFields.put(fieldName, summaryField); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedFieldSet.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedFieldSet.java index 30597a6dc42..9103ed46631 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedFieldSet.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedFieldSet.java @@ -2,6 +2,7 @@ package com.yahoo.searchdefinition.parser; import java.util.ArrayList; import java.util.List; +import java.util.Optional; /** * This class holds the extracted information after parsing a "fieldset" @@ -13,15 +14,22 @@ class ParsedFieldSet extends ParsedBlock { private final List<String> fields = new ArrayList<>(); private final List<String> queryCommands = new ArrayList<>(); - private final ParsedMatchSettings matchInfo = new ParsedMatchSettings(); + private ParsedMatchSettings matchInfo = null; ParsedFieldSet(String name) { super(name, "fieldset"); } - ParsedMatchSettings matchSettings() { return this.matchInfo; } + ParsedMatchSettings matchSettings() { + if (matchInfo == null) matchInfo = new ParsedMatchSettings(); + return this.matchInfo; + } + List<String> getQueryCommands() { return List.copyOf(queryCommands); } List<String> getFieldNames() { return List.copyOf(fields); } + Optional<ParsedMatchSettings> getMatchSettings() { + return Optional.ofNullable(this.matchInfo); + } void addField(String field) { fields.add(field); } void addQueryCommand(String command) { queryCommands.add(command); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedIndex.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedIndex.java index dfc9d911f57..a3504cd2bf7 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedIndex.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedIndex.java @@ -6,6 +6,7 @@ import com.yahoo.searchdefinition.document.Stemming; import java.util.ArrayList; import java.util.List; +import java.util.Optional; /** * This class holds the extracted information after parsing an "index" @@ -15,30 +16,30 @@ import java.util.List; **/ class ParsedIndex extends ParsedBlock { - private boolean enableBm25 = false; - private boolean isPrefix = false; - private HnswIndexParams hnswParams; + private Boolean enableBm25 = null; + private Boolean isPrefix = null; + private HnswIndexParams hnswParams = null; private final List<String> aliases = new ArrayList<>(); private Stemming stemming = null; - private Integer arity; - private Long lowerBound; - private Long upperBound; - private Double densePLT; + private Integer arity = null; + private Long lowerBound = null; + private Long upperBound = null; + private Double densePLT = null; ParsedIndex(String name) { super(name, "index"); } - boolean getEnableBm25() { return this.enableBm25; } - boolean getPrefix() { return this.isPrefix; } - HnswIndexParams getHnswIndexParams() { return this.hnswParams; } + Optional<Boolean> getEnableBm25() { return Optional.ofNullable(this.enableBm25); } + Optional<Boolean> getPrefix() { return Optional.ofNullable(this.isPrefix); } + Optional<HnswIndexParams> getHnswIndexParams() { return Optional.ofNullable(this.hnswParams); } List<String> getAliases() { return List.copyOf(aliases); } boolean hasStemming() { return stemming != null; } - Stemming getStemming() { return stemming; } - Integer getArity() { return this.arity; } - Long getLowerBound() { return this.lowerBound; } - Long getUpperBound() { return this.upperBound; } - Double getDensePostingListThreshold() { return this.densePLT; } + Optional<Stemming> getStemming() { return Optional.ofNullable(stemming); } + Optional<Integer> getArity() { return Optional.ofNullable(this.arity); } + Optional<Long> getLowerBound() { return Optional.ofNullable(this.lowerBound); } + Optional<Long> getUpperBound() { return Optional.ofNullable(this.upperBound); } + Optional<Double> getDensePostingListThreshold() { return Optional.ofNullable(this.densePLT); } void addAlias(String alias) { aliases.add(alias); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedMatchSettings.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedMatchSettings.java index 03b27d8c9ef..9b51521ad2b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedMatchSettings.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedMatchSettings.java @@ -1,5 +1,9 @@ package com.yahoo.searchdefinition.parser; +import com.yahoo.searchdefinition.document.Case; +import com.yahoo.searchdefinition.document.MatchType; +import com.yahoo.searchdefinition.document.MatchAlgorithm; + import java.util.Optional; /** @@ -11,14 +15,14 @@ import java.util.Optional; public class ParsedMatchSettings { private MatchType matchType = null; - private MatchCase matchCase = null; + private Case matchCase = null; private MatchAlgorithm matchAlgorithm = null; private String exactTerminator = null; private Integer gramSize = null; private Integer maxLength = null; Optional<MatchType> getMatchType() { return Optional.ofNullable(matchType); } - Optional<MatchCase> getMatchCase() { return Optional.ofNullable(matchCase); } + Optional<Case> getMatchCase() { return Optional.ofNullable(matchCase); } Optional<MatchAlgorithm> getMatchAlgorithm() { return Optional.ofNullable(matchAlgorithm); } Optional<String> getExactTerminator() { return Optional.ofNullable(exactTerminator); } Optional<Integer> getGramSize() { return Optional.ofNullable(gramSize); } @@ -26,7 +30,7 @@ public class ParsedMatchSettings { // TODO - consider allowing each set only once: void setType(MatchType value) { this.matchType = value; } - void setCase(MatchCase value) { this.matchCase = value; } + void setCase(Case value) { this.matchCase = value; } void setAlgorithm(MatchAlgorithm value) { this.matchAlgorithm = value; } void setExactTerminator(String value) { this.exactTerminator = value; } void setGramSize(int value) { this.gramSize = value; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java index 2011ac24148..2c9ead4a446 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java @@ -6,9 +6,10 @@ import com.yahoo.searchdefinition.RankProfile.MutateOperation; import com.yahoo.searchlib.rankingexpression.FeatureList; import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; import com.yahoo.searchlib.rankingexpression.evaluation.Value; +import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import java.util.ArrayList; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -25,9 +26,9 @@ class ParsedRankProfile extends ParsedBlock { private boolean ignoreDefaultRankFeatures = false; private Double rankScoreDropLimit = null; private Double termwiseLimit = null; - private FeatureList matchFeatures = null; - private FeatureList rankFeatures = null; - private FeatureList summaryFeatures = null; + private List<ReferenceNode> matchFeatures = new ArrayList<>(); + private List<ReferenceNode> rankFeatures = new ArrayList<>(); + private List<ReferenceNode> summaryFeatures = new ArrayList<>(); private Integer keepRankCount = null; private Integer minHitsPerThread = null; private Integer numSearchPartitions = null; @@ -41,12 +42,12 @@ class ParsedRankProfile extends ParsedBlock { private Boolean strict = null; private final List<MutateOperation> mutateOperations = new ArrayList<>(); private final List<String> inherited = new ArrayList<>(); - private final Map<String, Boolean> fieldsRankFilter = new HashMap<>(); - private final Map<String, Integer> fieldsRankWeight = new HashMap<>(); - private final Map<String, ParsedRankFunction> functions = new HashMap<>(); - private final Map<String, String> fieldsRankType = new HashMap<>(); - private final Map<String, String> rankProperties = new HashMap<>(); - private final Map<String, Value> constants = new HashMap<>(); + private final Map<String, Boolean> fieldsRankFilter = new LinkedHashMap<>(); + private final Map<String, Integer> fieldsRankWeight = new LinkedHashMap<>(); + private final Map<String, ParsedRankFunction> functions = new LinkedHashMap<>(); + private final Map<String, String> fieldsRankType = new LinkedHashMap<>(); + private final Map<String, List<String>> rankProperties = new LinkedHashMap<>(); + private final Map<String, Value> constants = new LinkedHashMap<>(); ParsedRankProfile(String name) { super(name, "rank-profile"); @@ -55,9 +56,9 @@ class ParsedRankProfile extends ParsedBlock { boolean getIgnoreDefaultRankFeatures() { return this.ignoreDefaultRankFeatures; } Optional<Double> getRankScoreDropLimit() { return Optional.ofNullable(this.rankScoreDropLimit); } Optional<Double> getTermwiseLimit() { return Optional.ofNullable(this.termwiseLimit); } - Optional<FeatureList> getMatchFeatures() { return Optional.ofNullable(this.matchFeatures); } - Optional<FeatureList> getRankFeatures() { return Optional.ofNullable(this.rankFeatures); } - Optional<FeatureList> getSummaryFeatures() { return Optional.ofNullable(this.summaryFeatures); } + List<ReferenceNode> getMatchFeatures() { return List.copyOf(this.matchFeatures); } + List<ReferenceNode> getRankFeatures() { return List.copyOf(this.rankFeatures); } + List<ReferenceNode> getSummaryFeatures() { return List.copyOf(this.summaryFeatures); } Optional<Integer> getKeepRankCount() { return Optional.ofNullable(this.keepRankCount); } Optional<Integer> getMinHitsPerThread() { return Optional.ofNullable(this.minHitsPerThread); } Optional<Integer> getNumSearchPartitions() { return Optional.ofNullable(this.numSearchPartitions); } @@ -72,25 +73,28 @@ class ParsedRankProfile extends ParsedBlock { Map<String, Boolean> getFieldsWithRankFilter() { return Map.copyOf(fieldsRankFilter); } Map<String, Integer> getFieldsWithRankWeight() { return Map.copyOf(fieldsRankWeight); } Map<String, String> getFieldsWithRankType() { return Map.copyOf(fieldsRankType); } - Map<String, String> getRankProperties() { return Map.copyOf(rankProperties); } + Map<String, List<String>> getRankProperties() { return Map.copyOf(rankProperties); } Map<String, Value> getConstants() { return Map.copyOf(constants); } Optional<String> getInheritedSummaryFeatures() { return Optional.ofNullable(this.inheritedSummaryFeatures); } Optional<String> getSecondPhaseExpression() { return Optional.ofNullable(this.secondPhaseExpression); } Optional<Boolean> isStrict() { return Optional.ofNullable(this.strict); } void addSummaryFeatures(FeatureList features) { - verifyThat(summaryFeatures == null, "already has summary-features"); - this.summaryFeatures = features; + for (var feature : features) { + this.summaryFeatures.add(feature); + } } void addMatchFeatures(FeatureList features) { - verifyThat(matchFeatures == null, "already has match-features"); - this.matchFeatures = features; + for (var feature : features) { + this.matchFeatures.add(feature); + } } void addRankFeatures(FeatureList features) { - verifyThat(rankFeatures == null, "already has rank-features"); - this.rankFeatures = features; + for (var feature : features) { + this.rankFeatures.add(feature); + } } void inherit(String other) { inherited.add(other); } @@ -134,8 +138,8 @@ class ParsedRankProfile extends ParsedBlock { } void addRankProperty(String key, String value) { - verifyThat(! rankProperties.containsKey(key), "already has value for rank property", key); - rankProperties.put(key, value); + List<String> values = rankProperties.computeIfAbsent(key, k -> new ArrayList<String>()); + values.add(value); } void setFirstPhaseRanking(String expression) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java index a0b238f1f43..bcbf14d9398 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java @@ -6,7 +6,7 @@ import com.yahoo.searchdefinition.RankingConstant; import com.yahoo.searchdefinition.document.Stemming; import java.util.ArrayList; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -29,6 +29,7 @@ public class ParsedSchema extends ParsedBlock { } } + private boolean documentWithoutSchema = false; private boolean rawAsBase64 = false; // TODO Vespa 8 flip default private ParsedDocument myDocument = null; private Stemming defaultStemming = null; @@ -37,20 +38,21 @@ public class ParsedSchema extends ParsedBlock { private final List<RankingConstant> rankingConstants = new ArrayList<>(); private final List<String> inherited = new ArrayList<>(); private final List<String> inheritedByDocument = new ArrayList<>(); - private final Map<String, ParsedSchema> resolvedInherits = new HashMap(); - private final Map<String, ParsedSchema> allResolvedInherits = new HashMap(); - private final Map<String, ParsedAnnotation> extraAnnotations = new HashMap<>(); - private final Map<String, ParsedDocumentSummary> docSums = new HashMap<>(); - private final Map<String, ParsedField> extraFields = new HashMap<>(); - private final Map<String, ParsedFieldSet> fieldSets = new HashMap<>(); - private final Map<String, ParsedIndex> extraIndexes = new HashMap<>(); - private final Map<String, ParsedRankProfile> rankProfiles = new HashMap<>(); - private final Map<String, ParsedStruct> extraStructs = new HashMap<>(); + private final Map<String, ParsedSchema> resolvedInherits = new LinkedHashMap(); + private final Map<String, ParsedSchema> allResolvedInherits = new LinkedHashMap(); + private final Map<String, ParsedAnnotation> extraAnnotations = new LinkedHashMap<>(); + private final Map<String, ParsedDocumentSummary> docSums = new LinkedHashMap<>(); + private final Map<String, ParsedField> extraFields = new LinkedHashMap<>(); + private final Map<String, ParsedFieldSet> fieldSets = new LinkedHashMap<>(); + private final Map<String, ParsedIndex> extraIndexes = new LinkedHashMap<>(); + private final Map<String, ParsedRankProfile> rankProfiles = new LinkedHashMap<>(); + private final Map<String, ParsedStruct> extraStructs = new LinkedHashMap<>(); public ParsedSchema(String name) { super(name, "schema"); } + boolean getDocumentWithoutSchema() { return documentWithoutSchema; } boolean getRawAsBase64() { return rawAsBase64; } boolean hasDocument() { return myDocument != null; } ParsedDocument getDocument() { return myDocument; } @@ -67,7 +69,7 @@ public class ParsedSchema extends ParsedBlock { List<RankingConstant> getRankingConstants() { return List.copyOf(rankingConstants); } List<String> getInherited() { return List.copyOf(inherited); } List<String> getInheritedByDocument() { return List.copyOf(inheritedByDocument); } - Map<String, ParsedRankProfile> getRankProfiles() { return Map.copyOf(rankProfiles); } + List<ParsedRankProfile> getRankProfiles() { return List.copyOf(rankProfiles.values()); } List<ParsedSchema> getResolvedInherits() { return List.copyOf(resolvedInherits.values()); } List<ParsedSchema> getAllResolvedInherits() { return List.copyOf(allResolvedInherits.values()); } @@ -85,6 +87,8 @@ public class ParsedSchema extends ParsedBlock { this.myDocument = document; } + void setDocumentWithoutSchema() { this.documentWithoutSchema = true; } + void addDocumentSummary(ParsedDocumentSummary docsum) { String dsName = docsum.name(); verifyThat(! docSums.containsKey(dsName), "already has document-summary", dsName); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSorting.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSorting.java index d4686c594da..32c822bf25b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSorting.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSorting.java @@ -1,6 +1,9 @@ package com.yahoo.searchdefinition.parser; +import com.yahoo.searchdefinition.document.Sorting.Function; +import com.yahoo.searchdefinition.document.Sorting.Strength; + import java.util.Optional; /** @@ -11,10 +14,6 @@ import java.util.Optional; **/ class ParsedSorting extends ParsedBlock { - enum Function { RAW, LOWERCASE, UCA } - - enum Strength { PRIMARY, SECONDARY, TERTIARY, QUATERNARY, IDENTICAL } - private boolean ascending = true; private Function sortFunction = null; private Strength sortStrength = null; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedStruct.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedStruct.java index cc3b2425726..17b20459c9c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedStruct.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedStruct.java @@ -2,7 +2,7 @@ package com.yahoo.searchdefinition.parser; import java.util.ArrayList; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -14,7 +14,7 @@ import java.util.Map; **/ public class ParsedStruct extends ParsedBlock { private final List<String> inherited = new ArrayList<>(); - private final Map<String, ParsedField> fields = new HashMap<>(); + private final Map<String, ParsedField> fields = new LinkedHashMap<>(); private String ownedBy = null; public ParsedStruct(String name) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java index d3e85bc1b11..3aed90a58e1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java @@ -43,6 +43,8 @@ class ParsedType { case "double": return Variant.BUILTIN; case "uri": return Variant.BUILTIN; case "predicate": return Variant.BUILTIN; + case "raw": return Variant.BUILTIN; + case "tag": return Variant.BUILTIN; case "position": return Variant.POSITION; } return Variant.UNKNOWN; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java index d3fa3282f4f..3c10ccb92dc 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java @@ -7,6 +7,7 @@ import com.yahoo.document.DataType; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.searchdefinition.document.MatchType; import com.yahoo.document.NumericDataType; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -38,7 +39,7 @@ public class AttributesImplicitWord extends Processor { private void processField(ImmutableSDField field) { if (fieldImplicitlyWordMatch(field)) { - field.getMatching().setType(Matching.Type.WORD); + field.getMatching().setType(MatchType.WORD); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java index a516667c524..ca473a2029f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java @@ -7,6 +7,7 @@ import com.yahoo.document.DataType; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.searchdefinition.document.MatchType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.document.Stemming; import com.yahoo.vespa.indexinglanguage.ExpressionSearcher; @@ -39,8 +40,8 @@ public class ExactMatch extends Processor { } private void processField(SDField field, Schema schema) { - Matching.Type matching = field.getMatching().getType(); - if (matching.equals(Matching.Type.EXACT) || matching.equals(Matching.Type.WORD)) { + MatchType matching = field.getMatching().getType(); + if (matching.equals(MatchType.EXACT) || matching.equals(MatchType.WORD)) { implementExactMatch(field, schema); } else if (field.getMatching().getExactMatchTerminator() != null) { warn(schema, field, "exact-terminator requires 'exact' matching to have any effect."); @@ -54,7 +55,7 @@ public class ExactMatch extends Processor { field.setStemming(Stemming.NONE); field.getNormalizing().inferLowercase(); - if (field.getMatching().getType().equals(Matching.Type.WORD)) { + if (field.getMatching().getType().equals(MatchType.WORD)) { field.addQueryCommand("word"); } else { // exact String exactTerminator = DEFAULT_EXACT_TERMINATOR; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java index c302ef63cfd..6cb0a86b481 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java @@ -5,7 +5,7 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.Matching.Type; +import com.yahoo.searchdefinition.document.MatchType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; import com.yahoo.vespa.indexinglanguage.expressions.Expression; @@ -33,14 +33,14 @@ public class MatchConsistency extends Processor { public void process(boolean validate, boolean documentsOnly) { if ( ! validate) return; - Map<String, Matching.Type> types = new HashMap<>(); + Map<String, MatchType> types = new HashMap<>(); for (SDField field : schema.allConcreteFields()) { new MyVisitor(schema, field, types).visit(field.getIndexingScript()); } } - private void checkMatching(Schema schema, SDField field, Map<String, Type> types, String indexTo) { - Type prevType = types.get(indexTo); + private void checkMatching(Schema schema, SDField field, Map<String, MatchType> types, String indexTo) { + MatchType prevType = types.get(indexTo); if (prevType == null) { types.put(indexTo, field.getMatching().getType()); } else if ( ! field.getMatching().getType().equals(prevType)) { @@ -54,9 +54,9 @@ public class MatchConsistency extends Processor { final Schema schema; final SDField field; - final Map<String, Type> types; + final Map<String, MatchType> types; - MyVisitor(Schema schema, SDField field, Map<String, Type> types) { + MyVisitor(Schema schema, SDField field, Map<String, MatchType> types) { this.schema = schema; this.field = field; this.types = types; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java index b5f573ecf71..2ba149925e9 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java @@ -7,6 +7,7 @@ import com.yahoo.document.DataType; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.searchdefinition.document.MatchType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.document.Stemming; import com.yahoo.vespa.indexinglanguage.expressions.*; @@ -29,7 +30,7 @@ public class NGramMatch extends Processor { @Override public void process(boolean validate, boolean documentsOnly) { for (SDField field : schema.allConcreteFields()) { - if (field.getMatching().getType().equals(Matching.Type.GRAM)) + if (field.getMatching().getType().equals(MatchType.GRAM)) implementGramMatch(schema, field, validate); else if (validate && field.getMatching().getGramSize() >= 0) throw new IllegalArgumentException("gram-size can only be set when the matching mode is 'gram'"); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java index 8cebfc89458..357575660ea 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java @@ -6,6 +6,7 @@ import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.document.*; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.searchdefinition.document.MatchType; import com.yahoo.searchdefinition.document.RankType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -38,7 +39,7 @@ public class TagType extends Processor { if (!field.doesIndexing() && !field.doesAttributing()) return; Matching m = field.getMatching(); if ( ! m.isTypeUserSet()) - m.setType(Matching.Type.WORD); + m.setType(MatchType.WORD); if (field.getRankType() == null || field.getRankType() == RankType.DEFAULT) field.setRankType((RankType.TAGS)); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java index a8065ec4bb5..c2b41edf454 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java @@ -7,6 +7,7 @@ import com.yahoo.document.CollectionDataType; import com.yahoo.document.DataType; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.searchdefinition.document.MatchType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.document.Stemming; import com.yahoo.vespa.indexinglanguage.ExpressionConverter; @@ -36,7 +37,7 @@ public class TextMatch extends Processor { @Override public void process(boolean validate, boolean documentsOnly) { for (SDField field : schema.allConcreteFields()) { - if (field.getMatching().getType() != Matching.Type.TEXT) continue; + if (field.getMatching().getType() != MatchType.TEXT) continue; ScriptExpression script = field.getIndexingScript(); if (script == null) continue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java b/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java index c4989604457..2a1afd616bc 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java @@ -4,6 +4,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.searchdefinition.RankProfileRegistry; import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.searchdefinition.document.MatchType; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.document.Stemming; import com.yahoo.searchdefinition.Schema; @@ -39,7 +40,7 @@ public class WordMatch extends Processor { } private void processField(SDField field) { - if (!field.getMatching().getType().equals(Matching.Type.WORD)) { + if (!field.getMatching().getType().equals(MatchType.WORD)) { return; } field.setStemming(Stemming.NONE); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java index 67667674ea9..2b474cb3b1b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java @@ -5,6 +5,7 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.searchdefinition.document.MatchAlgorithm; import com.yahoo.searchdefinition.Index; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.vespa.model.VespaModel; @@ -32,7 +33,7 @@ public class NoPrefixForIndexes extends Validator { for (ImmutableSDField field : schema.allConcreteFields()) { if (field.doesIndexing()) { //if (!field.getIndexTo().isEmpty() && !field.getIndexTo().contains(field.getName())) continue; - if (field.getMatching().getAlgorithm().equals(Matching.Algorithm.PREFIX)) { + if (field.getMatching().getAlgorithm().equals(MatchAlgorithm.PREFIX)) { failField(schema, field); } for (Map.Entry<String, Index> e : field.getIndices().entrySet()) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java index f27efe07941..dc5ffa7ecc7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java @@ -9,6 +9,7 @@ import com.yahoo.document.ReferenceDataType; import com.yahoo.searchdefinition.document.Attribute; import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.searchdefinition.document.MatchType; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.search.AbstractSearchCluster; import com.yahoo.vespa.model.search.SearchCluster; @@ -39,7 +40,7 @@ public class StreamingValidator extends Validator { private static void warnStreamingGramMatching(SearchCluster sc, DeployLogger logger) { if (sc.getSdConfig() != null) { for (ImmutableSDField sd : sc.getSdConfig().getSearch().allConcreteFields()) { - if (sd.getMatching().getType().equals(Matching.Type.GRAM)) { + if (sd.getMatching().getType().equals(MatchType.GRAM)) { logger.logApplicationPackage(Level.WARNING, "For streaming search cluster '" + sc.getClusterName() + "', SD field '" + sd.getName() + "': n-gram matching is not supported for streaming search."); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidator.java index 41710181706..59b2af0835f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidator.java @@ -16,7 +16,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; /** - * Compares the startup command for the services in the next model with the ones in the old model. + * Compares the startup command for the services in the next model with the ones in the current model. * If the startup command has changes, a change entry is created and reported back. * * @author bjorncs @@ -26,11 +26,11 @@ public class StartupCommandChangeValidator implements ChangeValidator { @Override public List<ConfigChangeAction> validate(VespaModel currentModel, VespaModel nextModel, ValidationOverrides overrides, Instant now) { - return findServicesWithChangedStartupCommmand(currentModel, nextModel).collect(Collectors.toList()); + return findServicesWithChangedStartupCommand(currentModel, nextModel).collect(Collectors.toList()); } - public Stream<ConfigChangeAction> findServicesWithChangedStartupCommmand(AbstractConfigProducerRoot currentModel, - AbstractConfigProducerRoot nextModel) { + public Stream<ConfigChangeAction> findServicesWithChangedStartupCommand(AbstractConfigProducerRoot currentModel, + AbstractConfigProducerRoot nextModel) { return nextModel.getDescendantServices().stream() .map(nextService -> currentModel.getService(nextService.getConfigId()) .flatMap(currentService -> compareStartupCommand(currentService, nextService))) @@ -44,7 +44,7 @@ public class StartupCommandChangeValidator implements ChangeValidator { if (Objects.equals(currentCommand, nextCommand)) return Optional.empty(); - String message = String.format("Startup command for '%s' has changed.\nNew command: %s.\nOld command: %s.", + String message = String.format("Startup command for '%s' has changed.\nNew command: %s\nCurrent command: %s", currentService.getServiceName(), nextCommand, currentCommand); return Optional.of(new VespaRestartAction(ClusterSpec.Id.from(currentService.getConfigId()), message, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java index 5e258fde821..b461b38c75f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.application.validation.change.search; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.document.ImmutableSDField; import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.searchdefinition.document.MatchType; import com.yahoo.searchdefinition.document.NormalizeLevel; import com.yahoo.searchdefinition.document.Stemming; import com.yahoo.vespa.documentmodel.SummaryField; @@ -85,9 +86,9 @@ public class IndexingScriptChangeMessageBuilder { } private static String toString(Matching matching) { - Matching.Type type = matching.getType(); + MatchType type = matching.getType(); String retval = type.getName(); - if (type == Matching.Type.GRAM) { + if (type == MatchType.GRAM) { retval += " (size " + matching.getGramSize() + ")"; } return retval; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 22b752777e9..e483351a25a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -158,7 +158,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> String clusterName, ContentSearchCluster search) { List<ModelElement> indexedDefs = getIndexedSchemas(clusterElem); if (!indexedDefs.isEmpty()) { - IndexedSearchCluster isc = new IndexedSearchCluster(search, clusterName, 0); + IndexedSearchCluster isc = new IndexedSearchCluster(deployState, search, clusterName, 0); isc.setRoutingSelector(clusterElem.childAsString("documents.selection")); Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index fb7c6696b54..53aac23135a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -6,7 +6,6 @@ import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.searchdefinition.DocumentOnlySchema; -import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.derived.DerivedConfiguration; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.config.search.DispatchConfig; @@ -34,6 +33,7 @@ public class IndexedSearchCluster extends SearchCluster IlscriptsConfig.Producer, DispatchConfig.Producer { + private final boolean mergeGroupingResultInSearchInvoker; private String indexingClusterName = null; // The name of the docproc cluster to run indexing, by config. private String indexingChainName = null; @@ -63,10 +63,11 @@ public class IndexedSearchCluster extends SearchCluster return routingSelector; } - public IndexedSearchCluster(AbstractConfigProducer<SearchCluster> parent, String clusterName, int index) { + public IndexedSearchCluster(DeployState deployState, AbstractConfigProducer<SearchCluster> parent, String clusterName, int index) { super(parent, clusterName, index); unionCfg = new UnionConfiguration(this, documentDbs); rootDispatch = new DispatchGroup(this); + mergeGroupingResultInSearchInvoker = deployState.featureFlags().mergeGroupingResultInSearchInvoker(); } @Override @@ -320,6 +321,7 @@ public class IndexedSearchCluster extends SearchCluster builder.maxWaitAfterCoverageFactor(searchCoverage.getMaxWaitAfterCoverageFactor()); } builder.warmuptime(5.0); + builder.mergeGroupingResultInSearchInvokerEnabled(mergeGroupingResultInSearchInvoker); } @Override diff --git a/config-model/src/main/javacc/IntermediateParser.jj b/config-model/src/main/javacc/IntermediateParser.jj index a8e77dead6e..90260cb9895 100644 --- a/config-model/src/main/javacc/IntermediateParser.jj +++ b/config-model/src/main/javacc/IntermediateParser.jj @@ -30,7 +30,11 @@ import com.yahoo.searchdefinition.RankProfile.MatchPhaseSettings; import com.yahoo.searchdefinition.RankProfile; import com.yahoo.searchdefinition.RankingConstant; import com.yahoo.searchdefinition.Schema; +import com.yahoo.searchdefinition.document.Case; +import com.yahoo.searchdefinition.document.MatchType; +import com.yahoo.searchdefinition.document.MatchAlgorithm; import com.yahoo.searchdefinition.document.HnswIndexParams; +import com.yahoo.searchdefinition.document.Sorting; import com.yahoo.searchdefinition.document.Stemming; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.fieldoperation.IndexingOperation; @@ -449,6 +453,7 @@ ParsedSchema rootDocumentItem(ParsedSchema schema) : ( doc = namedDocument() { if (schema == null) schema = new ParsedSchema(doc.name()); schema.addDocument(doc); + schema.setDocumentWithoutSchema(); return schema; } ) } @@ -1024,7 +1029,7 @@ void weightedsetBody(ParsedType type) : { } void rankType(ParsedField field) : { String typeName; - String indexName = field.name(); + String indexName = ""; } { <RANKTYPE> [indexName = identifier()] <COLON> typeName = identifier() @@ -1055,34 +1060,22 @@ void attribute(ParsedField field) : } /* pick up sorting in field block */ -void fieldSorting(ParsedField field) : -{ - ParsedSorting sortInfo; -} +void fieldSorting(ParsedField field) : { } { - sortInfo = sorting(field.name(), "field") { field.setSorting(sortInfo); } + sorting(field.sortInfo()) } /* pick up sorting in field block */ -void attributeSorting(ParsedAttribute attribute) : +void attributeSorting(ParsedAttribute attribute) : { } { - ParsedSorting sortInfo; -} -{ - sortInfo = sorting(attribute.name(), "attribute") { attribute.setSorting(sortInfo); } + sorting(attribute.sortInfo()) } -ParsedSorting sorting(String blockName, String blockType) : -{ - ParsedSorting sort = new ParsedSorting(blockName, blockType); -} +void sorting(ParsedSorting sort) : { } { <SORTING> ( (<COLON> sortingSetting(sort)) | (lbrace() (sortingSetting(sort) (<NL>)*)* <RBRACE>) ) - { - return sort; - } } void sortingSetting(ParsedSorting sorting) : @@ -1094,16 +1087,16 @@ void sortingSetting(ParsedSorting sorting) : <ASCENDING> { sorting.setAscending(); } | <DESCENDING> { sorting.setDescending(); } | <FUNCTION> <COLON> ( - <UCA> { sorting.setFunction(ParsedSorting.Function.UCA); } - | <RAW> { sorting.setFunction(ParsedSorting.Function.RAW); } - | <LOWERCASE> { sorting.setFunction(ParsedSorting.Function.LOWERCASE); } + <UCA> { sorting.setFunction(Sorting.Function.UCA); } + | <RAW> { sorting.setFunction(Sorting.Function.RAW); } + | <LOWERCASE> { sorting.setFunction(Sorting.Function.LOWERCASE); } ) | <STRENGTH> <COLON> ( - <PRIMARY> { sorting.setStrength(ParsedSorting.Strength.PRIMARY); } - | <SECONDARY> { sorting.setStrength(ParsedSorting.Strength.SECONDARY); } - | <TERTIARY> { sorting.setStrength(ParsedSorting.Strength.TERTIARY); } - | <QUATERNARY> { sorting.setStrength(ParsedSorting.Strength.QUATERNARY); } - | <IDENTICAL> { sorting.setStrength(ParsedSorting.Strength.IDENTICAL); } + <PRIMARY> { sorting.setStrength(Sorting.Strength.PRIMARY); } + | <SECONDARY> { sorting.setStrength(Sorting.Strength.SECONDARY); } + | <TERTIARY> { sorting.setStrength(Sorting.Strength.TERTIARY); } + | <QUATERNARY> { sorting.setStrength(Sorting.Strength.QUATERNARY); } + | <IDENTICAL> { sorting.setStrength(Sorting.Strength.IDENTICAL); } ) | <LOCALE> <COLON> locale = identifierWithDash() { sorting.setLocale(locale); } ) @@ -1456,8 +1449,8 @@ void matchType(ParsedMatchSettings matchInfo) : { } | <WORD> { matchInfo.setType(MatchType.WORD); } | <EXACT> { matchInfo.setType(MatchType.EXACT); } | <GRAM> { matchInfo.setType(MatchType.GRAM); } - | <CASED> { matchInfo.setCase(MatchCase.CASED); } - | <UNCASED> { matchInfo.setCase(MatchCase.UNCASED); } + | <CASED> { matchInfo.setCase(Case.CASED); } + | <UNCASED> { matchInfo.setCase(Case.UNCASED); } | <PREFIX> { matchInfo.setAlgorithm(MatchAlgorithm.PREFIX); } | <SUBSTRING> { matchInfo.setAlgorithm(MatchAlgorithm.SUBSTRING); } | <SUFFIX> { matchInfo.setAlgorithm(MatchAlgorithm.SUFFIX); } ) @@ -1807,12 +1800,10 @@ void rankProfile(ParsedSchema schema) : ParsedRankProfile profile; } { - ( ( <MODEL> | <RANKPROFILE> ) name = identifierWithDash() - { - profile = new ParsedRankProfile(name); - } - [inheritsRankProfile(profile)] - lbrace() (rankProfileItem(profile) (<NL>)*)* <RBRACE> ) + ( <MODEL> | <RANKPROFILE> ) name = identifierWithDash() + { profile = new ParsedRankProfile(name); } + [inheritsRankProfile(profile)] + lbrace() (rankProfileItem(profile) (<NL>)*)* <RBRACE> { schema.addRankProfile(profile); } diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index f18107f574a..1a0b518bbb2 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -1584,16 +1584,16 @@ void matchType(FieldOperationContainer container) : MatchOperation matchOp = new MatchOperation(); } { - ( <MTOKEN> { matchOp.setMatchingType(Matching.Type.TEXT); } // Deprecated synonym to TEXT - | <TEXT> { matchOp.setMatchingType(Matching.Type.TEXT); } - | <WORD> { matchOp.setMatchingType(Matching.Type.WORD); } - | <EXACT> { matchOp.setMatchingType(Matching.Type.EXACT); } - | <GRAM> { matchOp.setMatchingType(Matching.Type.GRAM); } + ( <MTOKEN> { matchOp.setMatchingType(MatchType.TEXT); } // Deprecated synonym to TEXT + | <TEXT> { matchOp.setMatchingType(MatchType.TEXT); } + | <WORD> { matchOp.setMatchingType(MatchType.WORD); } + | <EXACT> { matchOp.setMatchingType(MatchType.EXACT); } + | <GRAM> { matchOp.setMatchingType(MatchType.GRAM); } | <CASED> { matchOp.setCase(Case.CASED); } | <UNCASED> { matchOp.setCase(Case.UNCASED); } - | <PREFIX> { matchOp.setMatchingAlgorithm(Matching.Algorithm.PREFIX); } - | <SUBSTRING> { matchOp.setMatchingAlgorithm(Matching.Algorithm.SUBSTRING); } - | <SUFFIX> { matchOp.setMatchingAlgorithm(Matching.Algorithm.SUFFIX); } ) + | <PREFIX> { matchOp.setMatchingAlgorithm(MatchAlgorithm.PREFIX); } + | <SUBSTRING> { matchOp.setMatchingAlgorithm(MatchAlgorithm.SUBSTRING); } + | <SUFFIX> { matchOp.setMatchingAlgorithm(MatchAlgorithm.SUFFIX); } ) { container.addOperation(matchOp); } diff --git a/config-model/src/test/derived/exactmatch/exactmatch.sd b/config-model/src/test/derived/exactmatch/exactmatch.sd index cad44111978..e6e002d5449 100644 --- a/config-model/src/test/derived/exactmatch/exactmatch.sd +++ b/config-model/src/test/derived/exactmatch/exactmatch.sd @@ -17,7 +17,7 @@ search exactmatch { indexing: index | summary | attribute match { exact - exact-terminator: "*!!!*" + exact-terminator: "*!A!!*" } } @@ -27,7 +27,7 @@ search exactmatch { indexing: attribute match { exact - exact-terminator: "*!!!*" + exact-terminator: "*!B!!*" } } } @@ -47,6 +47,16 @@ search exactmatch { match: exact } } - } + field another_map type map<string, elem> { + struct-field value.name { + indexing: attribute + match { + exact + exact-terminator: "*!C!!*" + } + } + } + + } } diff --git a/config-model/src/test/derived/exactmatch/ilscripts.cfg b/config-model/src/test/derived/exactmatch/ilscripts.cfg index 5595f954c4a..c24b656c4e7 100644 --- a/config-model/src/test/derived/exactmatch/ilscripts.cfg +++ b/config-model/src/test/derived/exactmatch/ilscripts.cfg @@ -6,8 +6,10 @@ ilscript[].docfield[] "screweduserids" ilscript[].docfield[] "string_map" ilscript[].docfield[] "elem_map" ilscript[].docfield[] "elem_array" +ilscript[].docfield[] "another_map" ilscript[].content[] "clear_state | guard { input tag | exact | summary tag | index tag; }" ilscript[].content[] "clear_state | guard { input screweduserids | exact | index screweduserids | summary screweduserids | attribute screweduserids; }" +ilscript[].content[] "input another_map | passthrough another_map" ilscript[].content[] "input elem_array | passthrough elem_array" ilscript[].content[] "input elem_map | passthrough elem_map" ilscript[].content[] "input string_map | passthrough string_map" diff --git a/config-model/src/test/derived/exactmatch/index-info.cfg b/config-model/src/test/derived/exactmatch/index-info.cfg index f5d928a8fd0..bbc2f2c761d 100644 --- a/config-model/src/test/derived/exactmatch/index-info.cfg +++ b/config-model/src/test/derived/exactmatch/index-info.cfg @@ -18,7 +18,7 @@ indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "screweduserids" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "screweduserids" -indexinfo[].command[].command "exact *!!!*" +indexinfo[].command[].command "exact *!A!!*" indexinfo[].command[].indexname "string_map.key" indexinfo[].command[].command "index" indexinfo[].command[].indexname "string_map.key" @@ -28,7 +28,7 @@ indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "string_map.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "string_map.key" -indexinfo[].command[].command "exact *!!!*" +indexinfo[].command[].command "exact *!B!!*" indexinfo[].command[].indexname "string_map.value" indexinfo[].command[].command "index" indexinfo[].command[].indexname "string_map.value" @@ -91,3 +91,33 @@ indexinfo[].command[].indexname "elem_array" indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "elem_array" indexinfo[].command[].command "type Array<elem>" +indexinfo[].command[].indexname "another_map.key" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "another_map.key" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "another_map.value.name" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "another_map.value.name" +indexinfo[].command[].command "lowercase" +indexinfo[].command[].indexname "another_map.value.name" +indexinfo[].command[].command "attribute" +indexinfo[].command[].indexname "another_map.value.name" +indexinfo[].command[].command "type string" +indexinfo[].command[].indexname "another_map.value.name" +indexinfo[].command[].command "exact *!C!!*" +indexinfo[].command[].indexname "another_map.value.weight" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "another_map.value.weight" +indexinfo[].command[].command "numerical" +indexinfo[].command[].indexname "another_map.value.weight" +indexinfo[].command[].command "type int" +indexinfo[].command[].indexname "another_map.value" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "another_map.value" +indexinfo[].command[].command "type elem" +indexinfo[].command[].indexname "another_map" +indexinfo[].command[].command "index" +indexinfo[].command[].indexname "another_map" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "another_map" +indexinfo[].command[].command "type Map<string,elem>" diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NameCollisionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NameCollisionTestCase.java index 35593be4843..5fce5c06943 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NameCollisionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/derived/NameCollisionTestCase.java @@ -5,7 +5,7 @@ package com.yahoo.searchdefinition.derived; import org.junit.Test; /** - * Verifies that a struct in a document type is preferred over another dopcument type + * Verifies that a struct in a document type is preferred over another document type * of the same name. * * @author bratseth diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateCollectionTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateCollectionTestCase.java index da5d0da146f..cd78f1faeb2 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateCollectionTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateCollectionTestCase.java @@ -9,6 +9,7 @@ import java.nio.charset.StandardCharsets; import java.io.BufferedReader; import java.io.File; import java.io.FileReader; +import java.util.List; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -79,6 +80,13 @@ public class IntermediateCollectionTestCase { assertEquals(schema.name(), "grandparent"); } + ParsedRankProfile get(List<ParsedRankProfile> all, String name) { + for (var rp : all) { + if (rp.name().equals(name)) return rp; + } + return null; + } + @Test public void can_add_extra_rank_profiles() throws Exception { var collection = new IntermediateCollection(); @@ -92,13 +100,13 @@ public class IntermediateCollectionTestCase { assertEquals(schema.name(), "test"); var rankProfiles = schema.getRankProfiles(); assertEquals(rankProfiles.size(), 7); - var outside = rankProfiles.get("outside_schema1"); + var outside = get(rankProfiles, "outside_schema1"); assertTrue(outside != null); assertEquals(outside.name(), "outside_schema1"); var functions = outside.getFunctions(); assertEquals(functions.size(), 1); assertEquals(functions.get(0).name(), "fo1"); - outside = rankProfiles.get("outside_schema2"); + outside = get(rankProfiles, "outside_schema2"); assertTrue(outside != null); assertEquals(outside.name(), "outside_schema2"); functions = outside.getFunctions(); @@ -118,17 +126,17 @@ public class IntermediateCollectionTestCase { @Test public void bad_parse_throws() throws Exception { var collection = new IntermediateCollection(); - var ex = assertThrows(IllegalArgumentException.class, () -> + var ex = assertThrows(ParseException.class, () -> collection.addSchemaFromFile("src/test/examples/structoutsideofdocument.sd")); - assertTrue(ex.getMessage().startsWith("Failed parsing schema file src/test/examples/structoutsideofdocument.sd: ")); - ex = assertThrows(IllegalArgumentException.class, () -> + assertTrue(ex.getMessage().startsWith("Failed parsing schema from src/test/examples/structoutsideofdocument.sd: Encountered")); + ex = assertThrows(ParseException.class, () -> collection.addSchemaFromReader(readerOf("src/test/examples/structoutsideofdocument.sd"))); - assertTrue(ex.getMessage().startsWith("Failed parsing schema from src/test/examples/structoutsideofdocument.sd: ")); + assertTrue(ex.getMessage().startsWith("Failed parsing schema from src/test/examples/structoutsideofdocument.sd: Encountered")); collection.addSchemaFromFile("src/test/derived/rankprofilemodularity/test.sd"); collection.addRankProfileFile("test", "src/test/derived/rankprofilemodularity/test/outside_schema1.profile"); - ex = assertThrows(IllegalArgumentException.class, () -> + ex = assertThrows(ParseException.class, () -> collection.addRankProfileFile("test", "src/test/examples/structoutsideofdocument.sd")); - assertTrue(ex.getMessage().startsWith("Failed parsing rank-profile from src/test/examples/structoutsideofdocument.sd: ")); + assertTrue(ex.getMessage().startsWith("Failed parsing rank-profile from src/test/examples/structoutsideofdocument.sd: Encountered")); } @Test diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateParserTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateParserTestCase.java index 6f04b37bb5b..4e212ccd574 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateParserTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateParserTestCase.java @@ -84,12 +84,128 @@ public class IntermediateParserTestCase { checkFileParses("src/test/configmodel/types/other_doc.sd"); checkFileParses("src/test/configmodel/types/types.sd"); checkFileParses("src/test/configmodel/types/type_with_doc_field.sd"); + checkFileParses("src/test/derived/advanced/advanced.sd"); + checkFileParses("src/test/derived/annotationsimplicitstruct/annotationsimplicitstruct.sd"); + checkFileParses("src/test/derived/annotationsinheritance2/annotationsinheritance2.sd"); + checkFileParses("src/test/derived/annotationsinheritance/annotationsinheritance.sd"); + checkFileParses("src/test/derived/annotationsoutsideofdocument/annotationsoutsideofdocument.sd"); + checkFileParses("src/test/derived/annotationspolymorphy/annotationspolymorphy.sd"); + checkFileParses("src/test/derived/annotationsreference2/annotationsreference2.sd"); + checkFileParses("src/test/derived/annotationsreference/annotationsreference.sd"); + checkFileParses("src/test/derived/annotationssimple/annotationssimple.sd"); + checkFileParses("src/test/derived/annotationsstruct/annotationsstruct.sd"); + checkFileParses("src/test/derived/annotationsstructarray/annotationsstructarray.sd"); checkFileParses("src/test/derived/array_of_struct_attribute/test.sd"); + checkFileParses("src/test/derived/arrays/arrays.sd"); + checkFileParses("src/test/derived/attributeprefetch/attributeprefetch.sd"); + checkFileParses("src/test/derived/attributerank/attributerank.sd"); + checkFileParses("src/test/derived/attributes/attributes.sd"); + checkFileParses("src/test/derived/combinedattributeandindexsearch/combinedattributeandindexsearch.sd"); + checkFileParses("src/test/derived/complex/complex.sd"); checkFileParses("src/test/derived/deriver/child.sd"); checkFileParses("src/test/derived/deriver/grandparent.sd"); checkFileParses("src/test/derived/deriver/parent.sd"); + checkFileParses("src/test/derived/emptychild/child.sd"); + checkFileParses("src/test/derived/emptychild/parent.sd"); + checkFileParses("src/test/derived/emptydefault/emptydefault.sd"); + checkFileParses("src/test/derived/exactmatch/exactmatch.sd"); + checkFileParses("src/test/derived/fieldset/test.sd"); + checkFileParses("src/test/derived/flickr/flickrphotos.sd"); + checkFileParses("src/test/derived/function_arguments/test.sd"); + checkFileParses("src/test/derived/function_arguments_with_expressions/test.sd"); + checkFileParses("src/test/derived/gemini2/gemini.sd"); + checkFileParses("src/test/derived/hnsw_index/test.sd"); + checkFileParses("src/test/derived/id/id.sd"); + checkFileParses("src/test/derived/importedfields/child.sd"); + checkFileParses("src/test/derived/importedfields/grandparent.sd"); + checkFileParses("src/test/derived/imported_fields_inherited_reference/child_a.sd"); + checkFileParses("src/test/derived/imported_fields_inherited_reference/child_b.sd"); + checkFileParses("src/test/derived/imported_fields_inherited_reference/child_c.sd"); + checkFileParses("src/test/derived/imported_fields_inherited_reference/parent.sd"); + checkFileParses("src/test/derived/importedfields/parent_a.sd"); + checkFileParses("src/test/derived/importedfields/parent_b.sd"); + checkFileParses("src/test/derived/imported_position_field/child.sd"); + checkFileParses("src/test/derived/imported_position_field/parent.sd"); + checkFileParses("src/test/derived/imported_position_field_summary/child.sd"); + checkFileParses("src/test/derived/imported_position_field_summary/parent.sd"); + checkFileParses("src/test/derived/imported_struct_fields/child.sd"); + checkFileParses("src/test/derived/imported_struct_fields/parent.sd"); + checkFileParses("src/test/derived/indexinfo_fieldsets/indexinfo_fieldsets.sd"); + checkFileParses("src/test/derived/indexinfo_lowercase/indexinfo_lowercase.sd"); + checkFileParses("src/test/derived/indexschema/indexschema.sd"); + checkFileParses("src/test/derived/indexswitches/indexswitches.sd"); + checkFileParses("src/test/derived/inheritance/child.sd"); + checkFileParses("src/test/derived/inheritance/father.sd"); + checkFileParses("src/test/derived/inheritance/grandparent.sd"); + checkFileParses("src/test/derived/inheritance/mother.sd"); + checkFileParses("src/test/derived/inheritdiamond/child.sd"); + checkFileParses("src/test/derived/inheritdiamond/father.sd"); + checkFileParses("src/test/derived/inheritdiamond/grandparent.sd"); + checkFileParses("src/test/derived/inheritdiamond/mother.sd"); + checkFileParses("src/test/derived/inheritfromgrandparent/child.sd"); + checkFileParses("src/test/derived/inheritfromgrandparent/grandparent.sd"); + checkFileParses("src/test/derived/inheritfromgrandparent/parent.sd"); + checkFileParses("src/test/derived/inheritfromnull/inheritfromnull.sd"); + checkFileParses("src/test/derived/inheritfromparent/child.sd"); + checkFileParses("src/test/derived/inheritfromparent/parent.sd"); + checkFileParses("src/test/derived/inheritstruct/child.sd"); + checkFileParses("src/test/derived/inheritstruct/parent.sd"); + checkFileParses("src/test/derived/integerattributetostringindex/integerattributetostringindex.sd"); + checkFileParses("src/test/derived/language/language.sd"); + checkFileParses("src/test/derived/lowercase/lowercase.sd"); + checkFileParses("src/test/derived/mail/mail.sd"); checkFileParses("src/test/derived/map_attribute/test.sd"); checkFileParses("src/test/derived/map_of_struct_attribute/test.sd"); + checkFileParses("src/test/derived/mlr/mlr.sd"); + checkFileParses("src/test/derived/multiplesummaries/multiplesummaries.sd"); + checkFileParses("src/test/derived/music3/music3.sd"); + checkFileParses("src/test/derived/music/music.sd"); + checkFileParses("src/test/derived/namecollision/collision.sd"); + checkFileParses("src/test/derived/namecollision/collisionstruct.sd"); + checkFileParses("src/test/derived/nearestneighbor/test.sd"); + checkFileParses("src/test/derived/neuralnet/neuralnet.sd"); + checkFileParses("src/test/derived/newrank/newrank.sd"); + checkFileParses("src/test/derived/nuwa/newsindex.sd"); + checkFileParses("src/test/derived/orderilscripts/orderilscripts.sd"); + checkFileParses("src/test/derived/position_array/position_array.sd"); + checkFileParses("src/test/derived/position_attribute/position_attribute.sd"); + checkFileParses("src/test/derived/position_extra/position_extra.sd"); + checkFileParses("src/test/derived/position_nosummary/position_nosummary.sd"); + checkFileParses("src/test/derived/position_summary/position_summary.sd"); + checkFileParses("src/test/derived/predicate_attribute/predicate_attribute.sd"); + checkFileParses("src/test/derived/prefixexactattribute/prefixexactattribute.sd"); + checkFileParses("src/test/derived/rankexpression/rankexpression.sd"); + checkFileParses("src/test/derived/rankprofileinheritance/child.sd"); + checkFileParses("src/test/derived/rankprofileinheritance/parent1.sd"); + checkFileParses("src/test/derived/rankprofileinheritance/parent2.sd"); + checkFileParses("src/test/derived/rankprofilemodularity/test.sd"); + checkFileParses("src/test/derived/rankprofiles/rankprofiles.sd"); + checkFileParses("src/test/derived/rankproperties/rankproperties.sd"); + checkFileParses("src/test/derived/ranktypes/ranktypes.sd"); + checkFileParses("src/test/derived/reference_fields/ad.sd"); + checkFileParses("src/test/derived/reference_fields/campaign.sd"); + checkFileParses("src/test/derived/renamedfeatures/foo.sd"); + checkFileParses("src/test/derived/reserved_position/reserved_position.sd"); + checkFileParses("src/test/derived/schemainheritance/child.sd"); + checkFileParses("src/test/derived/schemainheritance/importedschema.sd"); + checkFileParses("src/test/derived/schemainheritance/parent.sd"); + checkFileParses("src/test/derived/slice/test.sd"); + checkFileParses("src/test/derived/streamingjuniper/streamingjuniper.sd"); + checkFileParses("src/test/derived/streamingstructdefault/streamingstructdefault.sd"); + checkFileParses("src/test/derived/streamingstruct/streamingstruct.sd"); + checkFileParses("src/test/derived/structandfieldset/test.sd"); + checkFileParses("src/test/derived/structanyorder/structanyorder.sd"); + checkFileParses("src/test/derived/structinheritance/bad.sd"); + checkFileParses("src/test/derived/structinheritance/simple.sd"); + checkFileParses("src/test/derived/tensor2/first.sd"); + checkFileParses("src/test/derived/tensor2/second.sd"); + checkFileParses("src/test/derived/tensor/tensor.sd"); + checkFileParses("src/test/derived/tokenization/tokenization.sd"); + checkFileParses("src/test/derived/twostreamingstructs/streamingstruct.sd"); + checkFileParses("src/test/derived/twostreamingstructs/whatever.sd"); + checkFileParses("src/test/derived/types/types.sd"); + checkFileParses("src/test/derived/uri_array/uri_array.sd"); + checkFileParses("src/test/derived/uri_wset/uri_wset.sd"); checkFileParses("src/test/examples/arrays.sd"); checkFileParses("src/test/examples/arraysweightedsets.sd"); checkFileParses("src/test/examples/attributesettings.sd"); @@ -141,5 +257,4 @@ public class IntermediateParserTestCase { checkFileParses("src/test/examples/summaryfieldcollision.sd"); checkFileParses("src/test/examples/weightedset-summaryto.sd"); } - } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java index 67b369fd951..e30ac8f8e00 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java @@ -5,6 +5,7 @@ import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; import com.yahoo.searchdefinition.document.Matching; +import com.yahoo.searchdefinition.document.MatchType; import com.yahoo.searchdefinition.parser.ParseException; import org.junit.Test; @@ -21,19 +22,19 @@ public class AttributesExactMatchTestCase extends AbstractSchemaTestCase { @Test public void testAttributesExactMatch() throws IOException, ParseException { Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/attributesexactmatch.sd"); - assertEquals(schema.getConcreteField("color").getMatching().getType(), Matching.Type.EXACT); - assertEquals(schema.getConcreteField("artist").getMatching().getType(), Matching.Type.WORD); - assertEquals(schema.getConcreteField("drummer").getMatching().getType(), Matching.Type.WORD); - assertEquals(schema.getConcreteField("guitarist").getMatching().getType(), Matching.Type.TEXT); - assertEquals(schema.getConcreteField("saxophonist_arr").getMatching().getType(), Matching.Type.WORD); - assertEquals(schema.getConcreteField("flutist").getMatching().getType(), Matching.Type.TEXT); + assertEquals(schema.getConcreteField("color").getMatching().getType(), MatchType.EXACT); + assertEquals(schema.getConcreteField("artist").getMatching().getType(), MatchType.WORD); + assertEquals(schema.getConcreteField("drummer").getMatching().getType(), MatchType.WORD); + assertEquals(schema.getConcreteField("guitarist").getMatching().getType(), MatchType.TEXT); + assertEquals(schema.getConcreteField("saxophonist_arr").getMatching().getType(), MatchType.WORD); + assertEquals(schema.getConcreteField("flutist").getMatching().getType(), MatchType.TEXT); - assertFalse(schema.getConcreteField("genre").getMatching().getType().equals(Matching.Type.EXACT)); - assertFalse(schema.getConcreteField("title").getMatching().getType().equals(Matching.Type.EXACT)); - assertFalse(schema.getConcreteField("trumpetist").getMatching().getType().equals(Matching.Type.EXACT)); - assertFalse(schema.getConcreteField("genre").getMatching().getType().equals(Matching.Type.WORD)); - assertFalse(schema.getConcreteField("title").getMatching().getType().equals(Matching.Type.WORD)); - assertFalse(schema.getConcreteField("trumpetist").getMatching().getType().equals(Matching.Type.WORD)); + assertFalse(schema.getConcreteField("genre").getMatching().getType().equals(MatchType.EXACT)); + assertFalse(schema.getConcreteField("title").getMatching().getType().equals(MatchType.EXACT)); + assertFalse(schema.getConcreteField("trumpetist").getMatching().getType().equals(MatchType.EXACT)); + assertFalse(schema.getConcreteField("genre").getMatching().getType().equals(MatchType.WORD)); + assertFalse(schema.getConcreteField("title").getMatching().getType().equals(MatchType.WORD)); + assertFalse(schema.getConcreteField("trumpetist").getMatching().getType().equals(MatchType.WORD)); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java b/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java index 7169a97f3fc..ab78297bcf9 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java +++ b/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java @@ -4,6 +4,7 @@ package com.yahoo.searchdefinition.processing; import com.yahoo.searchdefinition.Schema; import com.yahoo.searchdefinition.ApplicationBuilder; import com.yahoo.searchdefinition.AbstractSchemaTestCase; +import com.yahoo.searchdefinition.document.MatchType; import com.yahoo.searchdefinition.document.Matching; import com.yahoo.searchdefinition.document.SDField; import com.yahoo.searchdefinition.document.Stemming; @@ -29,15 +30,15 @@ public class NGramTestCase extends AbstractSchemaTestCase { assertNotNull(schema); SDField gram1 = schema.getConcreteField("gram_1"); - assertEquals(Matching.Type.GRAM, gram1.getMatching().getType()); + assertEquals(MatchType.GRAM, gram1.getMatching().getType()); assertEquals(1, gram1.getMatching().getGramSize()); SDField gram2 = schema.getConcreteField("gram_2"); - assertEquals(Matching.Type.GRAM, gram2.getMatching().getType()); + assertEquals(MatchType.GRAM, gram2.getMatching().getType()); assertEquals(-1, gram2.getMatching().getGramSize()); // Not set explicitly SDField gram3= schema.getConcreteField("gram_3"); - assertEquals(Matching.Type.GRAM,gram3.getMatching().getType()); + assertEquals(MatchType.GRAM,gram3.getMatching().getType()); assertEquals(3, gram3.getMatching().getGramSize()); assertEquals("input gram_1 | ngram 1 | index gram_1 | summary gram_1", gram1.getIndexingScript().iterator().next().toString()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidatorTest.java index dddab5c2a2a..952b89a6ab1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/StartupCommandChangeValidatorTest.java @@ -9,7 +9,6 @@ import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.Host; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.PortAllocBridge; -import com.yahoo.vespa.model.application.validation.change.StartupCommandChangeValidator; import org.junit.Test; import java.util.Arrays; @@ -48,7 +47,7 @@ public class StartupCommandChangeValidatorTest { private static List<ConfigChangeAction> getStartupCommandChanges( AbstractConfigProducerRoot currentModel, AbstractConfigProducerRoot nextModel) { StartupCommandChangeValidator validator = new StartupCommandChangeValidator(); - return validator.findServicesWithChangedStartupCommmand(currentModel, nextModel).collect(Collectors.toList()); + return validator.findServicesWithChangedStartupCommand(currentModel, nextModel).collect(Collectors.toList()); } private static MockRoot createRootWithChildren(AbstractConfigProducer<?>... children) { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java index 7be4f7809d5..8fcdebc5d97 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java @@ -199,6 +199,12 @@ public class NodeResources { return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType, architecture); } + public NodeResources with(Architecture architecture) { + ensureSpecified(); + if (architecture == this.architecture) return this; + return new NodeResources(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType, architecture); + } + /** Returns this with disk speed and storage type set to any */ public NodeResources justNumbers() { if (isUnspecified()) return unspecified(); @@ -221,8 +227,7 @@ public class NodeResources { diskGb - other.diskGb, bandwidthGbps - other.bandwidthGbps, this.diskSpeed.combineWith(other.diskSpeed), - this.storageType.combineWith(other.storageType), - this.architecture.combineWith(other.architecture)); + this.storageType.combineWith(other.storageType)); } public NodeResources add(NodeResources other) { @@ -234,8 +239,7 @@ public class NodeResources { diskGb + other.diskGb, bandwidthGbps + other.bandwidthGbps, this.diskSpeed.combineWith(other.diskSpeed), - this.storageType.combineWith(other.storageType), - this.architecture.combineWith(other.architecture)); + this.storageType.combineWith(other.storageType)); } private boolean isInterchangeableWith(NodeResources other) { @@ -245,8 +249,6 @@ public class NodeResources { return false; if (this.storageType != StorageType.any && other.storageType != StorageType.any && this.storageType != other.storageType) return false; - if (this.architecture != other.architecture) - return false; return true; } diff --git a/config/src/tests/configagent/configagent.cpp b/config/src/tests/configagent/configagent.cpp index 9684d401a37..81422d198e0 100644 --- a/config/src/tests/configagent/configagent.cpp +++ b/config/src/tests/configagent/configagent.cpp @@ -89,9 +89,8 @@ public: return std::move(_update); } - bool wait(vespalib::duration timeout) override + bool wait_until(vespalib::steady_time) override { - (void) timeout; return true; } @@ -104,7 +103,7 @@ public: } bool poll() override { return true; } - void interrupt() override { } + void close() override { } private: std::unique_ptr<ConfigUpdate> _update; }; diff --git a/config/src/tests/configholder/configholder.cpp b/config/src/tests/configholder/configholder.cpp index b2f6cd83693..e22ef55d747 100644 --- a/config/src/tests/configholder/configholder.cpp +++ b/config/src/tests/configholder/configholder.cpp @@ -34,12 +34,12 @@ TEST("Require that waiting is done") ConfigHolder holder; vespalib::Timer timer; - holder.wait(1000ms); + holder.wait_for(1000ms); EXPECT_GREATER_EQUAL(timer.elapsed(), ONE_SEC); EXPECT_LESS(timer.elapsed(), ONE_MINUTE); holder.handle(std::make_unique<ConfigUpdate>(value, true, 0)); - ASSERT_TRUE(holder.wait(100ms)); + ASSERT_TRUE(holder.wait_for(100ms)); } TEST("Require that polling for elements work") @@ -58,26 +58,26 @@ TEST("Require that negative time does not mean forever.") { ConfigHolder holder; vespalib::Timer timer; ASSERT_FALSE(holder.poll()); - ASSERT_FALSE(holder.wait(10ms)); - ASSERT_FALSE(holder.wait(0ms)); - ASSERT_FALSE(holder.wait(-1ms)); - ASSERT_FALSE(holder.wait(-7ms)); + ASSERT_FALSE(holder.wait_for(10ms)); + ASSERT_FALSE(holder.wait_for(0ms)); + ASSERT_FALSE(holder.wait_for(-1ms)); + ASSERT_FALSE(holder.wait_for(-7ms)); EXPECT_LESS(timer.elapsed(), ONE_MINUTE); } -TEST_MT_F("Require that wait is interrupted", 2, ConfigHolder) +TEST_MT_F("Require that wait is interrupted on close", 2, ConfigHolder) { if (thread_id == 0) { vespalib::Timer timer; TEST_BARRIER(); - f.wait(1000ms); + f.wait_for(1000ms); EXPECT_LESS(timer.elapsed(), ONE_MINUTE); EXPECT_GREATER(timer.elapsed(), 400ms); TEST_BARRIER(); } else { TEST_BARRIER(); std::this_thread::sleep_for(500ms); - f.interrupt(); + f.close(); TEST_BARRIER(); } } diff --git a/config/src/tests/configretriever/configretriever.cpp b/config/src/tests/configretriever/configretriever.cpp index 1515632cf4b..cc36614d4c5 100644 --- a/config/src/tests/configretriever/configretriever.cpp +++ b/config/src/tests/configretriever/configretriever.cpp @@ -123,7 +123,7 @@ struct SubscriptionFixture sub(std::make_shared<ConfigSubscription>(0, key, holder, std::make_unique<MySource>())) { holder->handle(std::make_unique<ConfigUpdate>(value, 3, 3)); - ASSERT_TRUE(sub->nextUpdate(0, 0ms)); + ASSERT_TRUE(sub->nextUpdate(0, steady_clock::now())); sub->flip(); } }; diff --git a/config/src/tests/subscription/subscription.cpp b/config/src/tests/subscription/subscription.cpp index 13eafbf876e..fa184b90100 100644 --- a/config/src/tests/subscription/subscription.cpp +++ b/config/src/tests/subscription/subscription.cpp @@ -10,42 +10,47 @@ using namespace config; namespace { - struct SourceFixture - { - int numClose; - int numGetConfig; - int numReload; - SourceFixture() - : numClose(0), - numGetConfig(0), - numReload(0) - { } - }; - - struct MySource : public Source - { - MySource(SourceFixture * src) - : source(src) - {} +struct SourceFixture +{ + int numClose; + int numGetConfig; + int numReload; + SourceFixture() + : numClose(0), + numGetConfig(0), + numReload(0) + { } +}; + +struct MySource : public Source +{ + MySource(SourceFixture * src) + : source(src) + {} - void getConfig() override { source->numGetConfig++; } - void close() override { source->numClose++; } - void reload(int64_t gen) override { (void) gen; source->numReload++; } + void getConfig() override { source->numGetConfig++; } + void close() override { source->numClose++; } + void reload(int64_t gen) override { (void) gen; source->numReload++; } - SourceFixture * source; - }; + SourceFixture * source; +}; - struct SubscriptionFixture +struct SubscriptionFixture +{ + std::shared_ptr<IConfigHolder> holder; + ConfigSubscription sub; + SourceFixture src; + SubscriptionFixture(const ConfigKey & key) + : holder(new ConfigHolder()), + sub(0, key, holder, std::make_unique<MySource>(&src)) { - std::shared_ptr<IConfigHolder> holder; - ConfigSubscription sub; - SourceFixture src; - SubscriptionFixture(const ConfigKey & key) - : holder(new ConfigHolder()), - sub(0, key, holder, std::make_unique<MySource>(&src)) - { - } - }; + } +}; + +vespalib::steady_time +deadline(vespalib::duration timeout) { + return vespalib::steady_clock::now() + timeout; +} } TEST_FF("requireThatKeyIsReturned", ConfigKey("foo", "bar", "bim", "boo"), SubscriptionFixture(f1)) @@ -56,17 +61,17 @@ TEST_FF("requireThatKeyIsReturned", ConfigKey("foo", "bar", "bim", "boo"), Subsc TEST_F("requireThatUpdateReturns", SubscriptionFixture(ConfigKey::create<MyConfig>("myid"))) { f1.holder->handle(std::make_unique<ConfigUpdate>(ConfigValue(), 1, 1)); - ASSERT_TRUE(f1.sub.nextUpdate(0, 0ms)); + ASSERT_TRUE(f1.sub.nextUpdate(0, deadline(0ms))); ASSERT_TRUE(f1.sub.hasChanged()); ASSERT_EQUAL(1, f1.sub.getGeneration()); } TEST_F("requireThatNextUpdateBlocks", SubscriptionFixture(ConfigKey::create<MyConfig>("myid"))) { - ASSERT_FALSE(f1.sub.nextUpdate(0, 0ms)); + ASSERT_FALSE(f1.sub.nextUpdate(0, deadline(0ms))); f1.holder->handle(std::make_unique<ConfigUpdate>(ConfigValue(), 1, 1)); vespalib::Timer timer; - ASSERT_FALSE(f1.sub.nextUpdate(1, 500ms)); + ASSERT_FALSE(f1.sub.nextUpdate(1, deadline(500ms))); ASSERT_TRUE(timer.elapsed() > 400ms); } @@ -75,7 +80,7 @@ TEST_MT_F("requireThatNextUpdateReturnsWhenNotified", 2, SubscriptionFixture(Con if (thread_id == 0) { vespalib::Timer timer; f1.holder->handle(std::make_unique<ConfigUpdate>(ConfigValue(), 1, 1)); - ASSERT_TRUE(f1.sub.nextUpdate(2, 5000ms)); + ASSERT_TRUE(f1.sub.nextUpdate(2, deadline(5000ms))); ASSERT_TRUE(timer.elapsed() > 200ms); } else { std::this_thread::sleep_for(500ms); @@ -89,7 +94,7 @@ TEST_MT_F("requireThatNextUpdateReturnsInterrupted", 2, SubscriptionFixture(Conf if (thread_id == 0) { vespalib::Timer timer; f1.holder->handle(std::make_unique<ConfigUpdate>(ConfigValue(), 1, 1)); - ASSERT_TRUE(f1.sub.nextUpdate(1, 5000ms)); + ASSERT_TRUE(f1.sub.nextUpdate(1, deadline(5000ms))); ASSERT_TRUE(timer.elapsed() > 300ms); } else { std::this_thread::sleep_for(500ms); @@ -100,15 +105,15 @@ TEST_MT_F("requireThatNextUpdateReturnsInterrupted", 2, SubscriptionFixture(Conf TEST_F("Require that isChanged takes generation into account", SubscriptionFixture(ConfigKey::create<MyConfig>("myid"))) { f1.holder->handle(std::make_unique<ConfigUpdate>(ConfigValue(StringVector(), "a"), true, 1)); - ASSERT_TRUE(f1.sub.nextUpdate(0, 0ms)); + ASSERT_TRUE(f1.sub.nextUpdate(0, deadline(0ms))); f1.sub.flip(); ASSERT_EQUAL(1, f1.sub.getLastGenerationChanged()); f1.holder->handle(std::make_unique<ConfigUpdate>(ConfigValue(StringVector(), "b"), true, 2)); - ASSERT_TRUE(f1.sub.nextUpdate(1, 0ms)); + ASSERT_TRUE(f1.sub.nextUpdate(1, deadline(0ms))); f1.sub.flip(); ASSERT_EQUAL(2, f1.sub.getLastGenerationChanged()); f1.holder->handle(std::make_unique<ConfigUpdate>(ConfigValue(), false, 3)); - ASSERT_TRUE(f1.sub.nextUpdate(2, 0ms)); + ASSERT_TRUE(f1.sub.nextUpdate(2, deadline(0ms))); f1.sub.flip(); ASSERT_EQUAL(2, f1.sub.getLastGenerationChanged()); } diff --git a/config/src/vespa/config/common/configholder.cpp b/config/src/vespa/config/common/configholder.cpp index 95fa3fecba8..e0f1795a1d4 100644 --- a/config/src/vespa/config/common/configholder.cpp +++ b/config/src/vespa/config/common/configholder.cpp @@ -32,10 +32,10 @@ ConfigHolder::handle(std::unique_ptr<ConfigUpdate> update) } bool -ConfigHolder::wait(vespalib::duration timeout) +ConfigHolder::wait_until(vespalib::steady_time deadline) { std::unique_lock guard(_lock); - return static_cast<bool>(_current) || (_cond.wait_for(guard, timeout) == std::cv_status::no_timeout); + return static_cast<bool>(_current) || (_cond.wait_until(guard, deadline) == std::cv_status::no_timeout); } bool @@ -46,8 +46,10 @@ ConfigHolder::poll() } void -ConfigHolder::interrupt() +ConfigHolder::close() { + std::lock_guard guard(_lock); + _current.reset(); _cond.notify_all(); } diff --git a/config/src/vespa/config/common/configholder.h b/config/src/vespa/config/common/configholder.h index 1cbbb383f98..b3cb2d01abf 100644 --- a/config/src/vespa/config/common/configholder.h +++ b/config/src/vespa/config/common/configholder.h @@ -18,9 +18,9 @@ public: std::unique_ptr<ConfigUpdate> provide() override; void handle(std::unique_ptr<ConfigUpdate> update) override; - bool wait(vespalib::duration timeoutI) override; + bool wait_until(vespalib::steady_time deadline) override; bool poll() override; - void interrupt() override; + void close() override; public: std::mutex _lock; std::condition_variable _cond; diff --git a/config/src/vespa/config/common/configmanager.cpp b/config/src/vespa/config/common/configmanager.cpp index 1417aed79ae..d95fb12b06a 100644 --- a/config/src/vespa/config/common/configmanager.cpp +++ b/config/src/vespa/config/common/configmanager.cpp @@ -33,7 +33,7 @@ ConfigManager::subscribe(const ConfigKey & key, vespalib::duration timeout) source->reload(_generation); source->getConfig(); - ConfigSubscription::SP subscription(new ConfigSubscription(id, key, holder, std::move(source))); + auto subscription = std::make_shared<ConfigSubscription>(id, key, holder, std::move(source)); vespalib::steady_time endTime = vespalib::steady_clock::now() + timeout; while (vespalib::steady_clock::now() < endTime) { diff --git a/config/src/vespa/config/common/configupdate.cpp b/config/src/vespa/config/common/configupdate.cpp index 55f12775a46..b18186363e6 100644 --- a/config/src/vespa/config/common/configupdate.cpp +++ b/config/src/vespa/config/common/configupdate.cpp @@ -10,8 +10,5 @@ ConfigUpdate::ConfigUpdate(ConfigValue value, bool changed, int64_t generation) { } ConfigUpdate::~ConfigUpdate() = default; -const ConfigValue & ConfigUpdate::getValue() const { return _value; } -bool ConfigUpdate::hasChanged() const { return _hasChanged; } -int64_t ConfigUpdate::getGeneration() const { return _generation; } } // namespace config diff --git a/config/src/vespa/config/common/configupdate.h b/config/src/vespa/config/common/configupdate.h index 35d641615a0..5263f429718 100644 --- a/config/src/vespa/config/common/configupdate.h +++ b/config/src/vespa/config/common/configupdate.h @@ -16,10 +16,10 @@ public: ConfigUpdate(const ConfigUpdate &) = delete; ConfigUpdate & operator = (const ConfigUpdate &) = delete; ~ConfigUpdate(); - const ConfigValue & getValue() const; - bool hasChanged() const; - int64_t getGeneration() const; - void merge(const ConfigUpdate & b) { _hasChanged = _hasChanged || b.hasChanged(); } + const ConfigValue & getValue() const noexcept { return _value; } + bool hasChanged() const noexcept { return _hasChanged; } + int64_t getGeneration() const noexcept { return _generation; } + void merge(const ConfigUpdate & b) noexcept { _hasChanged = _hasChanged || b.hasChanged(); } private: ConfigValue _value; bool _hasChanged; diff --git a/config/src/vespa/config/common/handler.h b/config/src/vespa/config/common/handler.h deleted file mode 100644 index 5458f916357..00000000000 --- a/config/src/vespa/config/common/handler.h +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <memory> -#include "configupdate.h" - -namespace config { - -/** - * A Handler is a component to whom you can pass an object. - **/ -template <typename T> -struct Handler -{ - virtual void handle(std::unique_ptr<T> obj) = 0; - virtual ~Handler() {} -}; - -typedef Handler<ConfigUpdate> ConfigHandler; - -} // namespace config - diff --git a/config/src/vespa/config/common/iconfigholder.h b/config/src/vespa/config/common/iconfigholder.h index 9b474ccf70b..5b7e8d6bbca 100644 --- a/config/src/vespa/config/common/iconfigholder.h +++ b/config/src/vespa/config/common/iconfigholder.h @@ -1,23 +1,23 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/config/common/handler.h> -#include <vespa/config/common/provider.h> -#include <vespa/config/common/waitable.h> -#include <vespa/config/common/pollable.h> -#include <vespa/config/common/interruptable.h> #include <vespa/config/common/configupdate.h> +#include <vespa/vespalib/util/time.h> namespace config { -class IConfigHolder : public ConfigHandler, - public Provider<ConfigUpdate>, - public Waitable, - public Pollable, - public Interruptable +class IConfigHolder { public: virtual ~IConfigHolder() = default; + virtual std::unique_ptr<ConfigUpdate> provide() = 0; + virtual void handle(std::unique_ptr<ConfigUpdate> obj) = 0; + virtual void close() = 0; + virtual bool poll() = 0; + bool wait_for(vespalib::duration timeout) { + return wait_until(vespalib::steady_clock::now() + timeout); + } + virtual bool wait_until(vespalib::steady_time deadline) = 0; }; } // namespace config diff --git a/config/src/vespa/config/common/interruptable.h b/config/src/vespa/config/common/interruptable.h deleted file mode 100644 index 843ce8bc838..00000000000 --- a/config/src/vespa/config/common/interruptable.h +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <memory> - -namespace config { - -/** - * An Interruptable is a component that can be notified that it should abort its current activities. - */ -struct Interruptable -{ - virtual void interrupt() = 0; - virtual ~Interruptable() {} -}; - -} // namespace config - diff --git a/config/src/vespa/config/common/pollable.h b/config/src/vespa/config/common/pollable.h deleted file mode 100644 index 6799f06cef5..00000000000 --- a/config/src/vespa/config/common/pollable.h +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <memory> - -namespace config { - -/** - * A Pollable is a component that can be polled, and returns either true or - * false. - */ -struct Pollable -{ - virtual bool poll() = 0; - virtual ~Pollable() {} -}; - -} // namespace config - diff --git a/config/src/vespa/config/common/provider.h b/config/src/vespa/config/common/provider.h deleted file mode 100644 index 923d440558a..00000000000 --- a/config/src/vespa/config/common/provider.h +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <memory> - -namespace config { - -/** - * A Provider is a component from which you can request an object. - **/ -template <typename T> -struct Provider -{ - virtual std::unique_ptr<T> provide() = 0; - virtual ~Provider() {} -}; - -} // namespace config - diff --git a/config/src/vespa/config/common/waitable.h b/config/src/vespa/config/common/waitable.h deleted file mode 100644 index e15d1a0ace7..00000000000 --- a/config/src/vespa/config/common/waitable.h +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/vespalib/util/time.h> - -namespace config { - -/** - * A Waitable is a component that can be used to wait for some event to happen. - */ -struct Waitable -{ - virtual bool wait(vespalib::duration timeout) = 0; - virtual ~Waitable() = default; -}; - -} // namespace config - diff --git a/config/src/vespa/config/retriever/configretriever.cpp b/config/src/vespa/config/retriever/configretriever.cpp index b2de6337434..6c48c62deff 100644 --- a/config/src/vespa/config/retriever/configretriever.cpp +++ b/config/src/vespa/config/retriever/configretriever.cpp @@ -42,7 +42,7 @@ ConfigRetriever::getBootstrapConfigs(vespalib::duration timeout) ConfigSnapshot ConfigRetriever::getConfigs(const ConfigKeySet & keySet, vespalib::duration timeout) { - if (_closed) + if (isClosed()) return ConfigSnapshot(); if (_bootstrapRequired) { throw ConfigRuntimeException("Cannot change keySet until bootstrap getBootstrapConfigs() has been called"); @@ -52,7 +52,7 @@ ConfigRetriever::getConfigs(const ConfigKeySet & keySet, vespalib::duration time _lastKeySet = keySet; { std::lock_guard guard(_lock); - if (_closed) + if (isClosed()) return ConfigSnapshot(); _configSubscriber = std::make_unique<GenericConfigSubscriber>(_context); } @@ -83,7 +83,7 @@ void ConfigRetriever::close() { std::lock_guard guard(_lock); - _closed = true; + _closed.store(true, std::memory_order_relaxed); _bootstrapSubscriber.close(); if (_configSubscriber) _configSubscriber->close(); @@ -92,7 +92,7 @@ ConfigRetriever::close() bool ConfigRetriever::isClosed() const { - return (_closed); + return _closed.load(std::memory_order_relaxed); } } diff --git a/config/src/vespa/config/retriever/configretriever.h b/config/src/vespa/config/retriever/configretriever.h index 8fea6af0863..bf86a2e69ee 100644 --- a/config/src/vespa/config/retriever/configretriever.h +++ b/config/src/vespa/config/retriever/configretriever.h @@ -101,7 +101,7 @@ private: int64_t _generation; vespalib::duration _subscribeTimeout; bool _bootstrapRequired; - bool _closed; + std::atomic<bool> _closed; }; } // namespace config diff --git a/config/src/vespa/config/subscription/configsubscription.cpp b/config/src/vespa/config/subscription/configsubscription.cpp index 859e5b6f056..5eea0895eed 100644 --- a/config/src/vespa/config/subscription/configsubscription.cpp +++ b/config/src/vespa/config/subscription/configsubscription.cpp @@ -29,20 +29,20 @@ ConfigSubscription::~ConfigSubscription() bool -ConfigSubscription::nextUpdate(int64_t generation, vespalib::duration timeout) +ConfigSubscription::nextUpdate(int64_t generation, vespalib::steady_time deadline) { - if (_closed || !_holder->poll()) { - return false; - } + if (_closed || !_holder->wait_until(deadline)) { return false; } + if (_closed) { return false; } // The above wait_until can be interrupted auto old = std::move(_next); _next = _holder->provide(); + if ( ! _next) { return false; } if (old) { _next->merge(*old); } if (isGenerationNewer(_next->getGeneration(), generation)) { return true; } - return (!_closed && _holder->wait(timeout)); + return (!_closed && _holder->wait_until(deadline)); } bool @@ -63,41 +63,16 @@ ConfigSubscription::getGeneration() const return _next->getGeneration(); } -const ConfigKey & -ConfigSubscription::getKey() const -{ - return _key; -} - void ConfigSubscription::close() { - if (!_closed) { - _closed = true; - _holder->interrupt(); + if (!_closed.exchange(true)) { + _holder->close(); _source->close(); } } void -ConfigSubscription::reset() -{ - _isChanged = false; -} - -bool -ConfigSubscription::isChanged() const -{ - return _isChanged; -} - -int64_t -ConfigSubscription::getLastGenerationChanged() const -{ - return _lastGenerationChanged; -} - -void ConfigSubscription::flip() { bool change = hasChanged(); diff --git a/config/src/vespa/config/subscription/configsubscription.h b/config/src/vespa/config/subscription/configsubscription.h index 063af97dcec..c6ce7eb6188 100644 --- a/config/src/vespa/config/subscription/configsubscription.h +++ b/config/src/vespa/config/subscription/configsubscription.h @@ -39,22 +39,22 @@ public: * * @return true if changed, false if not. */ - bool isChanged() const; + bool isChanged() const noexcept { return _isChanged; } /** * Returns the last generation that actually changed the config. */ - int64_t getLastGenerationChanged() const; + int64_t getLastGenerationChanged() const noexcept { return _lastGenerationChanged; } /// Used by ConfigSubscriptionSet - SubscriptionId getSubscriptionId() const { return _id; } - const ConfigKey & getKey() const; - bool nextUpdate(int64_t generation, vespalib::duration timeout); + SubscriptionId getSubscriptionId() const noexcept { return _id; } + const ConfigKey & getKey() const noexcept { return _key; } + bool nextUpdate(int64_t generation, vespalib::steady_time deadline); int64_t getGeneration() const; bool hasChanged() const; bool hasGenerationChanged() const; void flip(); - void reset(); + void reset() noexcept { _isChanged = false; } void close(); // Used by ConfigManager @@ -65,11 +65,11 @@ private: const ConfigKey _key; std::unique_ptr<Source> _source; std::shared_ptr<IConfigHolder> _holder; - std::unique_ptr<ConfigUpdate> _next; - std::unique_ptr<ConfigUpdate> _current; - bool _isChanged; - int64_t _lastGenerationChanged; - std::atomic<bool> _closed; + std::unique_ptr<ConfigUpdate> _next; + std::unique_ptr<ConfigUpdate> _current; + bool _isChanged; + int64_t _lastGenerationChanged; + std::atomic<bool> _closed; }; typedef std::vector<ConfigSubscription::SP> SubscriptionList; diff --git a/config/src/vespa/config/subscription/configsubscriptionset.cpp b/config/src/vespa/config/subscription/configsubscriptionset.cpp index 40dc40d11ef..1d5db778dda 100644 --- a/config/src/vespa/config/subscription/configsubscriptionset.cpp +++ b/config/src/vespa/config/subscription/configsubscriptionset.cpp @@ -13,11 +13,12 @@ LOG_SETUP(".config.subscription.configsubscriptionset"); using vespalib::duration; using vespalib::steady_clock; +using vespalib::steady_time; namespace config { ConfigSubscriptionSet::ConfigSubscriptionSet(std::shared_ptr<IConfigContext> context) - : _maxNapTime(vespalib::from_s(10*1.0/vespalib::getVespaTimerHz())), //10x slower than default timer frequency. + : _maxNapTime(vespalib::adjustTimeoutByDetectedHz(20ms)), _context(std::move(context)), _mgr(_context->getManagerInstance()), _currentGeneration(-1), @@ -39,13 +40,13 @@ ConfigSubscriptionSet::acquireSnapshot(duration timeout, bool ignoreChange) _state = FROZEN; } - steady_clock::time_point startTime = steady_clock::now(); - duration timeLeft = timeout; + steady_time now = steady_clock::now(); + const steady_time deadline = now + timeout; int64_t lastGeneration = _currentGeneration; bool inSync = false; - LOG(spam, "Going into nextConfig loop, time left is %f", vespalib::to_s(timeLeft)); - while (!isClosed() && (timeLeft >= duration::zero()) && !inSync) { + LOG(spam, "Going into nextConfig loop, time left is %f", vespalib::to_s(deadline - now)); + while (!isClosed() && (now <= deadline)) { size_t numChanged = 0; size_t numGenerationChanged = 0; bool generationsInSync = true; @@ -54,7 +55,7 @@ ConfigSubscriptionSet::acquireSnapshot(duration timeout, bool ignoreChange) // Run nextUpdate on all subscribers to get them in sync. for (const auto & subscription : _subscriptionList) { - if (!subscription->nextUpdate(_currentGeneration, timeLeft) && !subscription->hasGenerationChanged()) { + if (!subscription->nextUpdate(_currentGeneration, deadline) && !subscription->hasGenerationChanged()) { subscription->reset(); continue; } @@ -76,14 +77,12 @@ ConfigSubscriptionSet::acquireSnapshot(duration timeout, bool ignoreChange) if (subscription->getGeneration() != generation) { generationsInSync = false; } - // Adjust timeout - timeLeft = timeout - (steady_clock::now() - startTime); } inSync = generationsInSync && (_subscriptionList.size() == numGenerationChanged) && (ignoreChange || numChanged > 0); lastGeneration = generation; - timeLeft = timeout - (steady_clock::now() - startTime); - if (!inSync && (timeLeft > duration::zero())) { - std::this_thread::sleep_for(std::min(_maxNapTime, timeLeft)); + now = steady_clock::now(); + if (!inSync && (now < deadline)) { + std::this_thread::sleep_for(std::min(_maxNapTime, deadline - now)); } else { break; } @@ -117,12 +116,6 @@ ConfigSubscriptionSet::close() } } -bool -ConfigSubscriptionSet::isClosed() const -{ - return (_state.load(std::memory_order_relaxed) == CLOSED); -} - std::shared_ptr<ConfigSubscription> ConfigSubscriptionSet::subscribe(const ConfigKey & key, duration timeout) { @@ -136,10 +129,4 @@ ConfigSubscriptionSet::subscribe(const ConfigKey & key, duration timeout) return s; } -int64_t -ConfigSubscriptionSet::getGeneration() const -{ - return _currentGeneration; -} - } // namespace config diff --git a/config/src/vespa/config/subscription/configsubscriptionset.h b/config/src/vespa/config/subscription/configsubscriptionset.h index 9cee3894aa9..4b6d970770d 100644 --- a/config/src/vespa/config/subscription/configsubscriptionset.h +++ b/config/src/vespa/config/subscription/configsubscriptionset.h @@ -38,7 +38,9 @@ public: * * @return generation number */ - int64_t getGeneration() const; + int64_t getGeneration() const noexcept { + return _currentGeneration; + } /** * Closes the set, which will interrupt acquireSnapshot and unsubscribe all @@ -49,7 +51,9 @@ public: /** * Checks if this subscription set is closed. */ - bool isClosed() const; + bool isClosed() const noexcept { + return (_state.load(std::memory_order_relaxed) == CLOSED); + } // Helpers for doing the subscription std::shared_ptr<ConfigSubscription> subscribe(const ConfigKey & key, vespalib::duration timeout); diff --git a/configdefinitions/src/vespa/dispatch.def b/configdefinitions/src/vespa/dispatch.def index 17f42a73bfd..fef9300a410 100644 --- a/configdefinitions/src/vespa/dispatch.def +++ b/configdefinitions/src/vespa/dispatch.def @@ -71,3 +71,6 @@ node[].host string # The rpc port of this search node node[].port int + +# Temporary feature flag +mergeGroupingResultInSearchInvokerEnabled bool default=false diff --git a/configdefinitions/src/vespa/stor-filestor.def b/configdefinitions/src/vespa/stor-filestor.def index e54b503ed93..531805d3039 100644 --- a/configdefinitions/src/vespa/stor-filestor.def +++ b/configdefinitions/src/vespa/stor-filestor.def @@ -80,7 +80,7 @@ resource_usage_reporter_noise_level double default=0.001 ## - DYNAMIC uses DynamicThrottlePolicy under the hood and will block if the window ## is full (if a blocking throttler API call is invoked). ## -async_operation_throttler.type enum { UNLIMITED, DYNAMIC } default=UNLIMITED restart +async_operation_throttler.type enum { UNLIMITED, DYNAMIC } default=UNLIMITED ## Internal throttler tuning parameters that only apply when type == DYNAMIC: async_operation_throttler.window_size_increment int default=20 async_operation_throttler.window_size_decrement_factor double default=1.2 @@ -104,7 +104,7 @@ async_operation_throttler.throttle_individual_merge_feed_ops bool default=true ## is full (if a blocking throttler API call is invoked). ## ## TODO deprecate in favor of the async_operation_throttler struct instead. -async_operation_throttler_type enum { UNLIMITED, DYNAMIC } default=UNLIMITED restart +async_operation_throttler_type enum { UNLIMITED, DYNAMIC } default=UNLIMITED ## Specifies the extent the throttling window is increased by when the async throttle ## policy has decided that more concurrent operations are desirable. Also affects the diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 5a813c7886a..6222e7b1788 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -209,6 +209,7 @@ public class ModelContextImpl implements ModelContext { private final boolean inhibitDefaultMergesWhenGlobalMergesPending; private final boolean useQrserverServiceName; private final boolean avoidRenamingSummaryFeatures; + private final boolean mergeGroupingResultInSearchInvoker; public FeatureFlags(FlagSource source, ApplicationId appId) { this.defaultTermwiseLimit = flagValue(source, appId, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -256,6 +257,7 @@ public class ModelContextImpl implements ModelContext { this.inhibitDefaultMergesWhenGlobalMergesPending = flagValue(source, appId, Flags.INHIBIT_DEFAULT_MERGES_WHEN_GLOBAL_MERGES_PENDING); this.useQrserverServiceName = flagValue(source, appId, Flags.USE_QRSERVER_SERVICE_NAME); this.avoidRenamingSummaryFeatures = flagValue(source, appId, Flags.AVOID_RENAMING_SUMMARY_FEATURES); + this.mergeGroupingResultInSearchInvoker = flagValue(source, appId, Flags.MERGE_GROUPING_RESULT_IN_SEARCH_INVOKER); } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @@ -305,6 +307,7 @@ public class ModelContextImpl implements ModelContext { @Override public boolean inhibitDefaultMergesWhenGlobalMergesPending() { return inhibitDefaultMergesWhenGlobalMergesPending; } @Override public boolean useQrserverServiceName() { return useQrserverServiceName; } @Override public boolean avoidRenamingSummaryFeatures() { return avoidRenamingSummaryFeatures; } + @Override public boolean mergeGroupingResultInSearchInvoker() { return mergeGroupingResultInSearchInvoker; } private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/GroupingResultAggregator.java b/container-search/src/main/java/com/yahoo/search/dispatch/GroupingResultAggregator.java new file mode 100644 index 00000000000..5ce7accfdd4 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/dispatch/GroupingResultAggregator.java @@ -0,0 +1,50 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.dispatch; + +import com.yahoo.prelude.fastsearch.DocsumDefinitionSet; +import com.yahoo.prelude.fastsearch.GroupingListHit; +import com.yahoo.searchlib.aggregation.Grouping; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Incrementally merges underlying {@link Grouping} instances from {@link GroupingListHit} hits. + * + * @author bjorncs + */ +class GroupingResultAggregator { + private static final Logger log = Logger.getLogger(GroupingResultAggregator.class.getName()); + + private final Map<Integer, Grouping> groupings = new LinkedHashMap<>(); + private DocsumDefinitionSet documentDefinitions = null; + private int groupingHitsMerged = 0; + + void mergeWith(GroupingListHit result) { + if (groupingHitsMerged == 0) documentDefinitions = result.getDocsumDefinitionSet(); + ++groupingHitsMerged; + log.log(Level.FINE, () -> + String.format("Merging hit #%d having %d groupings", + groupingHitsMerged, result.getGroupingList().size())); + for (Grouping grouping : result.getGroupingList()) { + groupings.merge(grouping.getId(), grouping, (existingGrouping, newGrouping) -> { + existingGrouping.merge(newGrouping); + return existingGrouping; + }); + } + } + + Optional<GroupingListHit> toAggregatedHit() { + if (groupingHitsMerged == 0) return Optional.empty(); + log.log(Level.FINE, () -> + String.format("Creating aggregated hit containing %d groupings from %d hits", + groupings.size(), groupingHitsMerged)); + groupings.values().forEach(Grouping::postMerge); + return Optional.of(new GroupingListHit(List.copyOf(groupings.values()), documentDefinitions)); + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java index 4e658122cdf..d7c9f1dce53 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/InterleavedSearchInvoker.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.dispatch; +import com.yahoo.prelude.fastsearch.GroupingListHit; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.dispatch.searchcluster.Group; @@ -44,6 +45,7 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM private final Group group; private final LinkedBlockingQueue<SearchInvoker> availableForProcessing; private final Set<Integer> alreadyFailedNodes; + private final boolean mergeGroupingResult; private Query query; private boolean adaptiveTimeoutCalculated = false; @@ -71,6 +73,7 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM this.group = group; this.availableForProcessing = newQueue(); this.alreadyFailedNodes = alreadyFailedNodes; + this.mergeGroupingResult = searchCluster.dispatchConfig().mergeGroupingResultInSearchInvokerEnabled(); } /** @@ -115,6 +118,7 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM long nextTimeout = query.getTimeLeft(); boolean extraDebug = (query.getOffset() == 0) && (query.getHits() == 7) && log.isLoggable(java.util.logging.Level.FINE); List<InvokerResult> processed = new ArrayList<>(); + var groupingResultAggregator = new GroupingResultAggregator(); try { while (!invokers.isEmpty() && nextTimeout >= 0) { SearchInvoker invoker = availableForProcessing.poll(nextTimeout, TimeUnit.MILLISECONDS); @@ -126,7 +130,7 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM if (extraDebug) { processed.add(toMerge); } - merged = mergeResult(result.getResult(), toMerge, merged); + merged = mergeResult(result.getResult(), toMerge, merged, groupingResultAggregator); ejectInvoker(invoker); } nextTimeout = nextTimeout(); @@ -134,6 +138,7 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM } catch (InterruptedException e) { throw new RuntimeException("Interrupted while waiting for search results", e); } + groupingResultAggregator.toAggregatedHit().ifPresent(h -> result.getResult().hits().add(h)); insertNetworkErrors(result.getResult()); result.getResult().setCoverage(createCoverage()); @@ -238,14 +243,20 @@ public class InterleavedSearchInvoker extends SearchInvoker implements ResponseM return nextAdaptive; } - private List<LeanHit> mergeResult(Result result, InvokerResult partialResult, List<LeanHit> current) { + private List<LeanHit> mergeResult(Result result, InvokerResult partialResult, List<LeanHit> current, + GroupingResultAggregator groupingResultAggregator) { collectCoverage(partialResult.getResult().getCoverage(true)); result.mergeWith(partialResult.getResult()); List<Hit> partialNonLean = partialResult.getResult().hits().asUnorderedHits(); for(Hit hit : partialNonLean) { if (hit.isAuxiliary()) { - result.hits().add(hit); + if (hit instanceof GroupingListHit && mergeGroupingResult) { + var groupingHit = (GroupingListHit) hit; + groupingResultAggregator.mergeWith(groupingHit); + } else { + result.hits().add(hit); + } } } if (current.isEmpty() ) { diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/Group.java b/container-search/src/main/java/com/yahoo/search/grouping/result/Group.java index be50b572f5b..8e3d0269bbe 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/result/Group.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/result/Group.java @@ -6,7 +6,7 @@ import com.yahoo.search.result.HitGroup; import com.yahoo.search.result.Relevance; /** - * This class represents a single group in the grouping result model. A group may contain any number of results (stored + * A single group in the grouping result model. A group may contain any number of results (stored * as fields, use {@link #getField(String)} to access), {@link GroupList} and {@link HitList}. Use the {@link * com.yahoo.search.grouping.GroupingRequest#getResultGroup(com.yahoo.search.Result)} to retrieve an instance of this. * diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index 2ba507a83b8..d5303d205a0 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -401,8 +401,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { private void renderGroupingGroupSyntheticFields(Hit hit) throws IOException { renderGroupMetadata(((Group) hit).getGroupId()); if (hit instanceof RootGroup) { - renderContinuations(Collections.singletonMap( - Continuation.THIS_PAGE, ((RootGroup) hit).continuation())); + renderContinuations(Map.of(Continuation.THIS_PAGE, ((RootGroup) hit).continuation())); } } @@ -732,23 +731,6 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { return object; } - private static Inspector wrapAsMap(Inspector data) { - if (data.type() != Type.ARRAY) return null; - if (data.entryCount() == 0) return null; - Value.ObjectValue map = new Value.ObjectValue(); - for (int i = 0; i < data.entryCount(); i++) { - Inspector obj = data.entry(i); - if (obj.type() != Type.OBJECT) return null; - if (obj.fieldCount() != 2) return null; - Inspector key = obj.field("key"); - Inspector value = obj.field("value"); - if (key.type() != Type.STRING) return null; - if (! value.valid()) return null; - map.put(key.asString(), value); - } - return map; - } - private Inspector deepMaybeConvert(Inspector data) { if (data.type() == Type.ARRAY) { if (settings.jsonDeepMaps) { @@ -786,7 +768,7 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { return data; } - private Inspector maybeConvertData(Inspector data) throws IOException { + private Inspector maybeConvertData(Inspector data) { if (data.type() == Type.ARRAY) { return convertTopLevelArray(data); } diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java index 6e08b1c6fa5..347276d680d 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/InterleavedSearchInvokerTest.java @@ -4,6 +4,7 @@ package com.yahoo.search.dispatch; import com.yahoo.document.GlobalId; import com.yahoo.document.idstring.IdString; import com.yahoo.prelude.fastsearch.FastHit; +import com.yahoo.prelude.fastsearch.GroupingListHit; import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.dispatch.searchcluster.Group; @@ -13,6 +14,11 @@ import com.yahoo.search.result.Coverage; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; import com.yahoo.search.result.Relevance; +import com.yahoo.searchlib.aggregation.Grouping; +import com.yahoo.searchlib.aggregation.MaxAggregationResult; +import com.yahoo.searchlib.aggregation.MinAggregationResult; +import com.yahoo.searchlib.expression.IntegerResultNode; +import com.yahoo.searchlib.expression.StringResultNode; import com.yahoo.test.ManualClock; import org.junit.Test; @@ -320,6 +326,39 @@ public class InterleavedSearchInvokerTest { assertEquals(3, result.getQuery().getHits()); } + @Test + public void requireThatGroupingsAreMerged() throws IOException { + SearchCluster cluster = new MockSearchCluster("!", 1, 2); + List<SearchInvoker> invokers = new ArrayList<>(); + + Grouping grouping1 = new Grouping(0); + grouping1.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group() + .setId(new StringResultNode("uniqueA")) + .addAggregationResult(new MaxAggregationResult().setMax(new IntegerResultNode(6)).setTag(4))) + .addChild(new com.yahoo.searchlib.aggregation.Group() + .setId(new StringResultNode("common")) + .addAggregationResult(new MaxAggregationResult().setMax(new IntegerResultNode(9)).setTag(4)))); + invokers.add(new MockInvoker(0).setHits(List.of(new GroupingListHit(List.of(grouping1))))); + + Grouping grouping2 = new Grouping(0); + grouping2.setRoot(new com.yahoo.searchlib.aggregation.Group() + .addChild(new com.yahoo.searchlib.aggregation.Group() + .setId(new StringResultNode("uniqueB")) + .addAggregationResult(new MaxAggregationResult().setMax(new IntegerResultNode(9)).setTag(4))) + .addChild(new com.yahoo.searchlib.aggregation.Group() + .setId(new StringResultNode("common")) + .addAggregationResult(new MinAggregationResult().setMin(new IntegerResultNode(6)).setTag(3)))); + invokers.add(new MockInvoker(0).setHits(List.of(new GroupingListHit(List.of(grouping2))))); + + InterleavedSearchInvoker invoker = new InterleavedSearchInvoker(invokers, cluster, new Group(0, List.of()), Collections.emptySet()); + invoker.responseAvailable(invokers.get(0)); + invoker.responseAvailable(invokers.get(1)); + Result result = invoker.search(query, null); + assertEquals(1, ((GroupingListHit) result.hits().get(0)).getGroupingList().size()); + + } + private static InterleavedSearchInvoker createInterLeavedTestInvoker(List<Double> a, List<Double> b, Group group) { SearchCluster cluster = new MockSearchCluster("!", 1, 2); List<SearchInvoker> invokers = new ArrayList<>(); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java index 2acce0f8d2d..54c8c1e0522 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/MockSearchCluster.java @@ -120,7 +120,8 @@ public class MockSearchCluster extends SearchCluster { builder.minActivedocsPercentage(88.0); builder.minGroupCoverage(99.0); builder.minSearchCoverage(minSearchCoverage); - builder.distributionPolicy(DispatchConfig.DistributionPolicy.Enum.ROUNDROBIN); + builder.distributionPolicy(DispatchConfig.DistributionPolicy.Enum.ROUNDROBIN) + .mergeGroupingResultInSearchInvokerEnabled(true); if (minSearchCoverage < 100.0) { builder.minWaitAfterCoverageFactor(0); builder.maxWaitAfterCoverageFactor(0.5); diff --git a/document/src/main/java/com/yahoo/document/BucketDistribution.java b/document/src/main/java/com/yahoo/document/BucketDistribution.java index abacd4fdc2f..e963f57e22b 100644 --- a/document/src/main/java/com/yahoo/document/BucketDistribution.java +++ b/document/src/main/java/com/yahoo/document/BucketDistribution.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.document; -import com.yahoo.document.BucketId; - import java.util.ArrayList; import java.util.List; import java.util.logging.Level; @@ -14,7 +12,7 @@ import java.util.logging.Logger; public class BucketDistribution { // A logger object to enable proper logging. - private static Logger log = Logger.getLogger(BucketDistribution.class.getName()); + private static final Logger log = Logger.getLogger(BucketDistribution.class.getName()); // A map from bucket id to column index. private int[] bucketToColumn; diff --git a/document/src/main/java/com/yahoo/document/DocumentId.java b/document/src/main/java/com/yahoo/document/DocumentId.java index 8c35eaa0329..3512c5cb7b7 100644 --- a/document/src/main/java/com/yahoo/document/DocumentId.java +++ b/document/src/main/java/com/yahoo/document/DocumentId.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.objects.Identifiable; import com.yahoo.vespa.objects.Serializer; import java.io.Serializable; +import java.util.Objects; /** * The id of a document @@ -18,11 +19,9 @@ import java.io.Serializable; public class DocumentId extends Identifiable implements Serializable { private IdString id; - private GlobalId globalId; + private GlobalId globalId = null; - /** - * Constructor used for deserialization. - */ + /** Constructor used for deserialization. */ public DocumentId(Deserializer buf) { deserialize(buf); } @@ -33,21 +32,14 @@ public class DocumentId extends Identifiable implements Serializable { * The document id string can only contain text characters. */ public DocumentId(String id) { - if (id == null) { - throw new IllegalArgumentException("Cannot create DocumentId from null id."); - } - if (id.length() > IdString.MAX_LENGTH) { - throw new IllegalArgumentException("The document id(" + id.length() + ") is too long(" + IdString.MAX_LENGTH + "). " + - "However if you have already fed a document earlier on and want to remove it, you can do so by " + - "calling new DocumentId(IdString.createIdStringLessStrict()) that will bypass this restriction."); - } - this.id = IdString.createIdString(id); - globalId = null; + this.id = IdString.createIdString(Objects.requireNonNull(id)); + if (id.length() > IdString.MAX_LENGTH) + throw new IllegalArgumentException("Document id of length " + id.length() + + " is longer than the max " + IdString.MAX_LENGTH); } public DocumentId(IdString id) { this.id = id; - globalId = null; } /** @@ -86,14 +78,17 @@ public class DocumentId extends Identifiable implements Serializable { return id.toString().compareTo(cmp.id.toString()); } + @Override public boolean equals(Object o) { return o instanceof DocumentId && id.equals(((DocumentId)o).id); } + @Override public int hashCode() { return id.hashCode(); } + @Override public String toString() { return id.toString(); } @@ -107,7 +102,6 @@ public class DocumentId extends Identifiable implements Serializable { } } - public void onDeserialize(Deserializer data) throws DeserializationException { if (data instanceof DocumentReader) { id = ((DocumentReader)data).readDocumentId().getScheme(); @@ -123,4 +117,5 @@ public class DocumentId extends Identifiable implements Serializable { public String getDocType() { return id.getDocType(); } + } diff --git a/document/src/main/java/com/yahoo/document/GlobalId.java b/document/src/main/java/com/yahoo/document/GlobalId.java index 9e90b59171c..b9d454dd007 100644 --- a/document/src/main/java/com/yahoo/document/GlobalId.java +++ b/document/src/main/java/com/yahoo/document/GlobalId.java @@ -40,10 +40,10 @@ public class GlobalId implements Comparable { /** * Constructs a new global id from a document id string. * - * @param id The document id to derive from. + * @param id the document id to derive from */ public GlobalId(IdString id) { - byte [] raw = MD5.md5.get().digest(id.toUtf8().wrap().array()); + byte[] raw = MD5.md5.get().digest(id.toUtf8().wrap().array()); long location = id.getLocation(); this.raw = new byte [LENGTH]; for (int i = 0; i < 4; ++i) { @@ -57,7 +57,7 @@ public class GlobalId implements Comparable { /** * Constructs a global id by deserializing content from the given byte buffer. * - * @param buf The buffer to deserialize from. + * @param buf the buffer to deserialize from */ public GlobalId(Deserializer buf) { raw = buf.getBytes(null, LENGTH); @@ -66,7 +66,7 @@ public class GlobalId implements Comparable { /** * Serializes the content of this global id into the given byte buffer. * - * @param buf The buffer to serialize to. + * @param buf the buffer to serialize to */ public void serialize(Serializer buf) { buf.put(null, raw); diff --git a/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java b/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java index 63a0f8d25ed..0d3b07fd6ee 100644 --- a/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentIdTestCase.java @@ -266,8 +266,7 @@ public class DocumentIdTestCase { new DocumentId(sb.toString()); fail("Expected an IllegalArgumentException to be thrown"); } catch (IllegalArgumentException ex) { - assertTrue(ex.getMessage().contains("However if you have already fed a document earlier on and want to remove it, " + - "you can do so by calling new DocumentId(IdString.createIdStringLessStrict()) that will bypass this restriction.")); + assertEquals("Document id length 65548 is longer than max length of 65536", ex.getMessage()); } assertEquals(65548, new DocumentId(IdString.createIdStringLessStrict(sb.toString())).toString().length()); } diff --git a/document/src/vespa/document/base/documentid.cpp b/document/src/vespa/document/base/documentid.cpp index f3045cd0758..cb5320db538 100644 --- a/document/src/vespa/document/base/documentid.cpp +++ b/document/src/vespa/document/base/documentid.cpp @@ -59,8 +59,9 @@ DocumentId::calculateGlobalId() const IdString::LocationType location(_id.getLocation()); memcpy(key, &location, 4); - _globalId.first = true; + // FIXME this lazy init mechanic does not satisfy const thread safety _globalId.second.set(key); + _globalId.first = true; } std::ostream & diff --git a/document/src/vespa/document/fieldvalue/referencefieldvalue.cpp b/document/src/vespa/document/fieldvalue/referencefieldvalue.cpp index 83890c1acfc..328e5b67151 100644 --- a/document/src/vespa/document/fieldvalue/referencefieldvalue.cpp +++ b/document/src/vespa/document/fieldvalue/referencefieldvalue.cpp @@ -75,6 +75,8 @@ void ReferenceFieldValue::setDeserializedDocumentId(const DocumentId& id) { assert(_dataType != nullptr); requireIdOfMatchingType(id, _dataType->getTargetType()); _documentId = id; + // Pre-cache GID to ensure it's not attempted lazily initialized later in a racing manner. + (void) _documentId.getGlobalId(); _altered = false; } diff --git a/fastos/src/tests/processtest.cpp b/fastos/src/tests/processtest.cpp index 6b947523364..204b87938de 100644 --- a/fastos/src/tests/processtest.cpp +++ b/fastos/src/tests/processtest.cpp @@ -88,7 +88,7 @@ public: { TestHeader("PollWait Test"); - FastOS_Process *xproc = new FastOS_Process("sort", true); + FastOS_Process *xproc = new FastOS_Process("sort"); if(xproc->Create()) { @@ -168,7 +168,7 @@ public: for(j=0; j<numEachTime; j++) { FastOS_ProcessInterface *xproc = - new FastOS_Process("sort", true, + new FastOS_Process("sort", new MyListener("STDOUT"), new MyListener("STDERR")); diff --git a/fastos/src/vespa/fastos/process.cpp b/fastos/src/vespa/fastos/process.cpp index 332d82c6aad..8e4f4afdc98 100644 --- a/fastos/src/vespa/fastos/process.cpp +++ b/fastos/src/vespa/fastos/process.cpp @@ -3,15 +3,11 @@ #include "process.h" FastOS_ProcessInterface::FastOS_ProcessInterface (const char *cmdLine, - bool pipeStdin, FastOS_ProcessRedirectListener *stdoutListener, - FastOS_ProcessRedirectListener *stderrListener, - int bufferSize) : + FastOS_ProcessRedirectListener *stderrListener) : _cmdLine(cmdLine), - _pipeStdin(pipeStdin), _stdoutListener(stdoutListener), _stderrListener(stderrListener), - _bufferSize(bufferSize), _next(nullptr), _prev(nullptr) { diff --git a/fastos/src/vespa/fastos/process.h b/fastos/src/vespa/fastos/process.h index 25d5224817a..f520fcd30f8 100644 --- a/fastos/src/vespa/fastos/process.h +++ b/fastos/src/vespa/fastos/process.h @@ -47,19 +47,12 @@ class FastOS_ApplicationInterface; */ class FastOS_ProcessInterface { -private: - FastOS_ProcessInterface(const FastOS_ProcessInterface&); - FastOS_ProcessInterface &operator=(const FastOS_ProcessInterface &); - protected: std::string _cmdLine; - bool _pipeStdin; - FastOS_ProcessRedirectListener *_stdoutListener; FastOS_ProcessRedirectListener *_stderrListener; - int _bufferSize; public: FastOS_ProcessInterface *_next, *_prev; static FastOS_ApplicationInterface *_app; @@ -74,17 +67,16 @@ public: * Constructor. Does not start the process, use @ref Create or * @ref CreateWithShell to actually start the process. * @param cmdLine Command line - * @param pipeStdin set to true in order to redirect stdin * @param stdoutListener non-nullptr to redirect stdout * @param stderrListener non-nullptr to redirect stderr * @param bufferSize Size of redirect buffers */ FastOS_ProcessInterface (const char *cmdLine, - bool pipeStdin = false, FastOS_ProcessRedirectListener *stdoutListener = nullptr, - FastOS_ProcessRedirectListener *stderrListener = nullptr, - int bufferSize = 65535); + FastOS_ProcessRedirectListener *stderrListener = nullptr); + FastOS_ProcessInterface(const FastOS_ProcessInterface&) = delete; + FastOS_ProcessInterface &operator=(const FastOS_ProcessInterface &) = delete; /** * Destructor. * If @ref Wait has not been called yet, it is called here. diff --git a/fastos/src/vespa/fastos/unix_app.cpp b/fastos/src/vespa/fastos/unix_app.cpp index baf7960dc1d..b0baf3990ad 100644 --- a/fastos/src/vespa/fastos/unix_app.cpp +++ b/fastos/src/vespa/fastos/unix_app.cpp @@ -17,7 +17,7 @@ FastOS_UNIX_Application::FastOS_UNIX_Application () - : _processStarter(nullptr), + : _processStarter(), _ipcHelper(nullptr) { } @@ -31,8 +31,8 @@ extern char **environ; int FastOS_UNIX_Application::GetOpt (const char *optionsString, - const char* &optionArgument, - int &optionIndex) + const char* &optionArgument, + int &optionIndex) { int rc = getopt(_argc, _argv, optionsString); optionArgument = optarg; @@ -74,7 +74,7 @@ bool FastOS_UNIX_Application::PreThreadInit () sigaction(SIGPIPE, &act, nullptr); if (useProcessStarter()) { - _processStarter = new FastOS_UNIX_ProcessStarter(this); + _processStarter = std::make_unique<FastOS_UNIX_ProcessStarter>(this); } } else { rc = false; @@ -107,15 +107,14 @@ void FastOS_UNIX_Application::Cleanup () if(_ipcHelper != nullptr) _ipcHelper->Exit(); - if (_processStarter != nullptr) { + if (_processStarter) { { std::unique_lock<std::mutex> guard; if (_processListMutex) { guard = getProcessGuard(); } } - delete _processStarter; - _processStarter = nullptr; + _processStarter.reset(); } FastOS_ApplicationInterface::Cleanup(); @@ -124,7 +123,7 @@ void FastOS_UNIX_Application::Cleanup () FastOS_UNIX_ProcessStarter * FastOS_UNIX_Application::GetProcessStarter () { - return _processStarter; + return _processStarter.get(); } void FastOS_UNIX_Application:: diff --git a/fastos/src/vespa/fastos/unix_app.h b/fastos/src/vespa/fastos/unix_app.h index 0c51b268299..5bb6f14ad26 100644 --- a/fastos/src/vespa/fastos/unix_app.h +++ b/fastos/src/vespa/fastos/unix_app.h @@ -11,6 +11,7 @@ #include "types.h" #include "app.h" +#include <memory> class FastOS_UNIX_ProcessStarter; class FastOS_UNIX_IPCHelper; @@ -25,7 +26,7 @@ private: FastOS_UNIX_Application(const FastOS_UNIX_Application&); FastOS_UNIX_Application& operator=(const FastOS_UNIX_Application&); - FastOS_UNIX_ProcessStarter *_processStarter; + std::unique_ptr<FastOS_UNIX_ProcessStarter> _processStarter; FastOS_UNIX_IPCHelper *_ipcHelper; protected: diff --git a/fastos/src/vespa/fastos/unix_process.cpp b/fastos/src/vespa/fastos/unix_process.cpp index 7536ac74477..cd29108c3d6 100644 --- a/fastos/src/vespa/fastos/unix_process.cpp +++ b/fastos/src/vespa/fastos/unix_process.cpp @@ -7,7 +7,6 @@ #include <cstdlib> #include <unistd.h> #include <fcntl.h> -#include <sys/socket.h> #include <sys/wait.h> #ifndef __linux__ #include <signal.h> @@ -63,7 +62,6 @@ public: private: pid_t _pid; - bool _died; bool _terse; // Set if using direct fork (bypassing proxy process) int _streamMask; @@ -84,10 +82,6 @@ private: public: void SetRunDir(const char * runDir) { _runDir = runDir; } - int GetStdinDescriptor() const { return _stdinDes[1]; } - int GetStdoutDescriptor() const { return _stdoutDes[0]; } - int GetStderrDescriptor() const { return _stderrDes[0]; } - int HandoverStdinDescriptor() { int ret = _stdinDes[1]; _stdinDes[1] = -1; @@ -106,10 +100,6 @@ public: return ret; } - void CloseStdinDescriptor(); - void CloseStdoutDescriptor(); - void CloseStderrDescriptor(); - FastOS_UNIX_RealProcess *_prev, *_next; FastOS_UNIX_RealProcess (int streamMask); @@ -164,11 +154,9 @@ public: bool ForkAndExec(const char *command, char **environmentVariables, - FastOS_UNIX_Process *process, - FastOS_UNIX_ProcessStarter *processStarter); + FastOS_UNIX_Process *process); bool Setup(); - pid_t GetProcessId() const { return _pid; } void SetTerse() { _terse = true; } ssize_t HandshakeRead(void *buf, size_t len); void HandshakeWrite(int val); @@ -206,30 +194,8 @@ FastOS_UNIX_RealProcess::CloseDescriptors() } -void -FastOS_UNIX_RealProcess::CloseStdinDescriptor() -{ - CloseAndResetDescriptor(&_stdinDes[1]); -} - - -void -FastOS_UNIX_RealProcess::CloseStdoutDescriptor() -{ - CloseAndResetDescriptor(&_stdoutDes[0]); -} - - -void -FastOS_UNIX_RealProcess::CloseStderrDescriptor() -{ - CloseAndResetDescriptor(&_stderrDes[0]); -} - - FastOS_UNIX_RealProcess::FastOS_UNIX_RealProcess(int streamMask) : _pid(-1), - _died(false), _terse(false), _streamMask(streamMask), _runDir(), @@ -426,8 +392,7 @@ bool FastOS_UNIX_RealProcess:: ForkAndExec(const char *command, char **environmentVariables, - FastOS_UNIX_Process *process, - FastOS_UNIX_ProcessStarter *processStarter) + FastOS_UNIX_Process *process) { bool rc = false; int numArguments = 0; @@ -441,8 +406,7 @@ ForkAndExec(const char *command, for(int i=0; ; i++) { int length; - const char *arg = NextArgument(nextArg, &nextArg, - &length); + const char *arg = NextArgument(nextArg, &nextArg, &length); if (arg == nullptr) { // printf("ARG nullptr\n"); @@ -458,13 +422,7 @@ ForkAndExec(const char *command, PrepareExecVPE(execArgs[0]); } } - if (process == nullptr) { - processStarter->CloseProxyDescs(IsStdinPiped() ? _stdinDes[0] : -1, - IsStdoutPiped() ? _stdoutDes[1] : -1, - IsStderrPiped() ? _stderrDes[1] : -1, - _handshakeDes[0], - _handshakeDes[1]); - } + _pid = safe_fork(); if (_pid == static_cast<pid_t>(0)) { // Fork success, child side. @@ -511,8 +469,6 @@ ForkAndExec(const char *command, if (fd != _handshakeDes[1]) CloseDescriptor(fd); } - } else { - processStarter->CloseProxiedChildDescs(); } if (fcntl(_handshakeDes[1], F_SETFD, FD_CLOEXEC) != 0) _exit(127); @@ -728,12 +684,10 @@ FastOS_UNIX_RealProcess::Setup() FastOS_UNIX_Process:: -FastOS_UNIX_Process (const char *cmdLine, bool pipeStdin, +FastOS_UNIX_Process (const char *cmdLine, FastOS_ProcessRedirectListener *stdoutListener, - FastOS_ProcessRedirectListener *stderrListener, - int bufferSize) : - FastOS_ProcessInterface(cmdLine, pipeStdin, stdoutListener, - stderrListener, bufferSize), + FastOS_ProcessRedirectListener *stderrListener) : + FastOS_ProcessInterface(cmdLine, stdoutListener, stderrListener), _pid(0), _died(false), _returnCode(-1), @@ -744,10 +698,11 @@ FastOS_UNIX_Process (const char *cmdLine, bool pipeStdin, _killed(false), _closing(nullptr) { + constexpr uint32_t RING_BUFFER_SIZE = 0x10000; if (stdoutListener != nullptr) - _descriptor[TYPE_STDOUT]._readBuffer.reset(new FastOS_RingBuffer(bufferSize)); + _descriptor[TYPE_STDOUT]._readBuffer = std::make_unique<FastOS_RingBuffer>(RING_BUFFER_SIZE); if (stderrListener != nullptr) - _descriptor[TYPE_STDERR]._readBuffer.reset(new FastOS_RingBuffer(bufferSize)); + _descriptor[TYPE_STDERR]._readBuffer = std::make_unique<FastOS_RingBuffer>(RING_BUFFER_SIZE); { auto guard = _app->getProcessGuard(); @@ -786,7 +741,6 @@ FastOS_UNIX_Process::~FastOS_UNIX_Process () bool FastOS_UNIX_Process::CreateInternal (bool useShell) { return GetProcessStarter()->CreateProcess(this, useShell, - _pipeStdin, _stdoutListener != nullptr, _stderrListener != nullptr); } @@ -866,9 +820,7 @@ bool FastOS_UNIX_Process::PollWait (int *returnCode, bool *stillRunning) int FastOS_UNIX_Process::BuildStreamMask (bool useShell) { - int streamMask = 0; - - if (_pipeStdin) streamMask |= FastOS_UNIX_RealProcess::STREAM_STDIN; + int streamMask = FastOS_UNIX_RealProcess::STREAM_STDIN; if (_stdoutListener) streamMask |= FastOS_UNIX_RealProcess::STREAM_STDOUT; if (_stderrListener) streamMask |= FastOS_UNIX_RealProcess::STREAM_STDERR; if (useShell) streamMask |= FastOS_UNIX_RealProcess::EXEC_SHELL; @@ -877,83 +829,17 @@ int FastOS_UNIX_Process::BuildStreamMask (bool useShell) } -void FastOS_UNIX_ProcessStarter:: -AddChildProcess (FastOS_UNIX_RealProcess *node) -{ - node->_prev = nullptr; - node->_next = _processList; - - if (_processList != nullptr) - _processList->_prev = node; - _processList = node; -} - -void FastOS_UNIX_ProcessStarter:: -RemoveChildProcess (FastOS_UNIX_RealProcess *node) -{ - if (node->_prev) - node->_prev->_next = node->_next; - else - _processList = node->_next; - - if (node->_next) { - node->_next->_prev = node->_prev; - node->_next = nullptr; - } - - if (node->_prev != nullptr) - node->_prev = nullptr; -} - - FastOS_UNIX_ProcessStarter::FastOS_UNIX_ProcessStarter (FastOS_ApplicationInterface *app) : _app(app), - _processList(nullptr), - _pid(-1), - _closedProxyProcessFiles(false), _hasDirectChildren(false) { } -FastOS_UNIX_ProcessStarter::~FastOS_UNIX_ProcessStarter () -{ -} - - -void -FastOS_UNIX_ProcessStarter::CloseProxiedChildDescs() -{ -} - - -void -FastOS_UNIX_ProcessStarter::CloseProxyDescs(int stdinPipedDes, int stdoutPipedDes, int stderrPipedDes, - int handshakeDes0, int handshakeDes1) -{ - return; - if (_closedProxyProcessFiles) - return; - int fdlimit = sysconf(_SC_OPEN_MAX); - for(int fd = STDERR_FILENO + 1; fd < fdlimit; fd++) - { - if (fd != stdinPipedDes && - fd != stdoutPipedDes && - fd != stderrPipedDes && - fd != handshakeDes0 && - fd != handshakeDes1) - close(fd); - } - _closedProxyProcessFiles = true; -} - +FastOS_UNIX_ProcessStarter::~FastOS_UNIX_ProcessStarter () = default; bool FastOS_UNIX_ProcessStarter:: -CreateProcess (FastOS_UNIX_Process *process, - bool useShell, - bool pipeStdin, - bool pipeStdout, - bool pipeStderr) +CreateProcess (FastOS_UNIX_Process *process, bool useShell, bool pipeStdout, bool pipeStderr) { bool rc = false; @@ -977,14 +863,13 @@ CreateProcess (FastOS_UNIX_Process *process, } rprocess->SetTerse(); rprocess->Setup(); - if (pipeStdin) - process->SetDescriptor(FastOS_UNIX_Process::TYPE_STDIN, rprocess->HandoverStdinDescriptor()); + process->SetDescriptor(FastOS_UNIX_Process::TYPE_STDIN, rprocess->HandoverStdinDescriptor()); if (pipeStdout) process->SetDescriptor(FastOS_UNIX_Process::TYPE_STDOUT, rprocess->HandoverStdoutDescriptor()); if (pipeStderr) process->SetDescriptor(FastOS_UNIX_Process::TYPE_STDERR, rprocess->HandoverStderrDescriptor()); pid_t processId = -1; - if (rprocess->ForkAndExec(cmdLine, environ, process, this)) { + if (rprocess->ForkAndExec(cmdLine, environ, process)) { processId = rprocess->GetProcessID(); } if (processId != -1) { @@ -1093,8 +978,8 @@ FastOS_UNIX_Process::DescriptorHandle::CloseHandle() close(_fd); _fd = -1; } - if (_readBuffer.get() != nullptr) + if (_readBuffer) _readBuffer->Close(); - if (_writeBuffer.get() != nullptr) + if (_writeBuffer) _writeBuffer->Close(); } diff --git a/fastos/src/vespa/fastos/unix_process.h b/fastos/src/vespa/fastos/unix_process.h index a86474eccf7..43b4388e0c3 100644 --- a/fastos/src/vespa/fastos/unix_process.h +++ b/fastos/src/vespa/fastos/unix_process.h @@ -103,10 +103,9 @@ public: } } - FastOS_UNIX_Process (const char *cmdLine, bool pipeStdin = false, + FastOS_UNIX_Process (const char *cmdLine, FastOS_ProcessRedirectListener *stdoutListener = nullptr, - FastOS_ProcessRedirectListener *stderrListener = nullptr, - int bufferSize = 65535); + FastOS_ProcessRedirectListener *stderrListener = nullptr); ~FastOS_UNIX_Process (); bool CreateInternal (bool useShell); bool Create () override { return CreateInternal(false); } @@ -140,7 +139,7 @@ public: return _descriptor[type]; } - bool GetKillFlag () {return _killed; } + bool GetKillFlag () { return _killed; } const char *GetRunDir() const { return _runDir.c_str(); } const char *GetStdoutRedirName() const { return _stdoutRedirName.c_str(); } @@ -157,28 +156,16 @@ private: protected: FastOS_ApplicationInterface *_app; - - FastOS_UNIX_RealProcess *_processList; - - pid_t _pid; - bool _closedProxyProcessFiles; bool _hasDirectChildren; - void AddChildProcess (FastOS_UNIX_RealProcess *node); - void RemoveChildProcess (FastOS_UNIX_RealProcess *node); - void PollReapDirectChildren(); public: FastOS_UNIX_ProcessStarter (FastOS_ApplicationInterface *app); ~FastOS_UNIX_ProcessStarter (); - void CloseProxiedChildDescs(); - void CloseProxyDescs(int stdinPipedDes, int stdoutPipedDes, int stderrPipedDes, - int handshakeDes0, int handshakeDes1); - bool CreateProcess (FastOS_UNIX_Process *process, bool useShell, - bool pipeStdin, bool pipeStdout, bool pipeStderr); + bool pipeStdout, bool pipeStderr); bool Wait (FastOS_UNIX_Process *process, int timeOutSeconds, bool *pollStillRunning); }; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 881b62d1e04..5376fa983af 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -390,6 +390,13 @@ public class Flags { "Takes effect immediately", TENANT_ID); + public static final UnboundBooleanFlag MERGE_GROUPING_RESULT_IN_SEARCH_INVOKER = defineFeatureFlag( + "merge-grouping-result-in-search-invoker", false, + List.of("bjorncs", "baldersheim"), "2022-02-23", "2022-08-01", + "Merge grouping results incrementally in interleaved search invoker", + "Takes effect at redeployment", + APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index 23daf3c4138..3ae85a54453 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -170,10 +170,10 @@ public class PermanentFlags { ); public static final UnboundStringFlag ENDPOINT_CERTIFICATE_ALGORITHM = defineStringFlag( - "endpoint-certificate-algorithm", "", - // Acceptable values are: "rsa_2048", "rsa_4096", "ecdsa_p256" - "Selects algorithm used for an applications endpoint certificate, or use provider default if blank", - "Takes effect when a new endpoint certificate is requested (first deployment of new application/instance)", + "endpoint-certificate-algorithm", "ecdsa_p256", + // Acceptable values are: "rsa_4096", "ecdsa_p256" + "Selects algorithm used for an applications endpoint certificate", + "Takes effect when a new endpoint certificate is requested (on first deployment or deployment adding new endpoints)", APPLICATION_ID); public static final UnboundDoubleFlag RESOURCE_LIMIT_DISK = defineDoubleFlag( diff --git a/flags/src/main/java/com/yahoo/vespa/flags/custom/HostResources.java b/flags/src/main/java/com/yahoo/vespa/flags/custom/HostResources.java index 31c3bdd8422..6b2cac39065 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/custom/HostResources.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/custom/HostResources.java @@ -21,6 +21,7 @@ public class HostResources { private static final Set<String> validDiskSpeeds = Set.of("slow", "fast"); private static final Set<String> validStorageTypes = Set.of("remote", "local"); private static final Set<String> validClusterTypes = Set.of("container", "content", "combined", "admin"); + private static final Set<String> validArchitectures = Set.of("arm64", "x86_64"); private final double vcpu; private final double memoryGb; @@ -33,6 +34,7 @@ public class HostResources { private final Optional<String> clusterType; private final int containers; + private final String architecture; @JsonCreator public HostResources(@JsonProperty("vcpu") Double vcpu, @@ -42,7 +44,8 @@ public class HostResources { @JsonProperty("diskSpeed") String diskSpeed, @JsonProperty("storageType") String storageType, @JsonProperty("clusterType") String clusterType, - @JsonProperty("containers") Integer containers) { + @JsonProperty("containers") Integer containers, + @JsonProperty("architecture") String architecture) { this.vcpu = requirePositive("vcpu", vcpu); this.memoryGb = requirePositive("memoryGb", memoryGb); this.diskGb = requirePositive("diskGb", diskGb); @@ -51,6 +54,7 @@ public class HostResources { this.storageType = validateEnum("storageType", validStorageTypes, storageType); this.clusterType = Optional.ofNullable(clusterType).map(cType -> validateEnum("clusterType", validClusterTypes, cType)); this.containers = requirePositive("containers", containers); + this.architecture = validateEnum("architecture", validArchitectures, architecture); } @JsonProperty("vcpu") @@ -80,6 +84,9 @@ public class HostResources { @JsonProperty("containers") public int containers() { return containers; } + @JsonProperty("architecture") + public String architecture() { return architecture; } + public boolean satisfiesClusterType(String clusterType) { return this.clusterType.map(clusterType::equalsIgnoreCase).orElse(true); } @@ -121,6 +128,7 @@ public class HostResources { ", storageType='" + storageType + '\'' + ", clusterType='" + clusterType + '\'' + ", containers=" + containers + + ", architecture=" + architecture + '}'; } @@ -136,11 +144,12 @@ public class HostResources { diskSpeed.equals(resources.diskSpeed) && storageType.equals(resources.storageType) && clusterType.equals(resources.clusterType) && - containers == resources.containers; + containers == resources.containers && + architecture.equals(resources.architecture); } @Override public int hashCode() { - return Objects.hash(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType, clusterType, containers); + return Objects.hash(vcpu, memoryGb, diskGb, bandwidthGbps, diskSpeed, storageType, clusterType, containers, architecture); } } diff --git a/flags/src/test/java/com/yahoo/vespa/flags/PermanentFlagsTest.java b/flags/src/test/java/com/yahoo/vespa/flags/PermanentFlagsTest.java index a4d8ecd0f29..8cac286be5e 100644 --- a/flags/src/test/java/com/yahoo/vespa/flags/PermanentFlagsTest.java +++ b/flags/src/test/java/com/yahoo/vespa/flags/PermanentFlagsTest.java @@ -15,11 +15,10 @@ import static com.yahoo.vespa.flags.FlagsTest.testGeneric; class PermanentFlagsTest { @Test public void testSharedHostFlag() { - SharedHost sharedHost = new SharedHost(List.of(new HostResources( - 4.0, 16.0, 50.0, 0.3, - "fast", "local", "admin", - 10)), - null); + SharedHost sharedHost = new SharedHost(List.of(new HostResources(4.0, 16.0, 50.0, 0.3, + "fast", "local", "admin", + 10, "x86_64")), + null); testGeneric(PermanentFlags.SHARED_HOST, sharedHost); } diff --git a/flags/src/test/java/com/yahoo/vespa/flags/custom/SharedHostTest.java b/flags/src/test/java/com/yahoo/vespa/flags/custom/SharedHostTest.java index 7bf39b884ef..852e1f9b7b1 100644 --- a/flags/src/test/java/com/yahoo/vespa/flags/custom/SharedHostTest.java +++ b/flags/src/test/java/com/yahoo/vespa/flags/custom/SharedHostTest.java @@ -12,8 +12,12 @@ import static org.junit.Assert.assertEquals; public class SharedHostTest { @Test public void serialization() throws IOException { - verifySerialization(new SharedHost(List.of(new HostResources(1.0, 2.0, 3.0, 4.0, "fast", "remote", "container", 5)), 6)); - verifySerialization(new SharedHost(List.of(new HostResources(1.0, 2.0, 3.0, 4.0, "fast", "remote", "admin", 5)), null)); + verifySerialization(new SharedHost(List.of( + new HostResources(1.0, 2.0, 3.0, 4.0, "fast", "remote", + "container", 5, "x86_64")), 6)); + verifySerialization(new SharedHost(List.of( + new HostResources(1.0, 2.0, 3.0, 4.0, "fast", "remote", + "admin", 5, "arm64")), null)); } private void verifySerialization(SharedHost sharedHost) throws IOException { diff --git a/fnet/src/vespa/fnet/scheduler.cpp b/fnet/src/vespa/fnet/scheduler.cpp index c6b1263bf61..f4e1e31219c 100644 --- a/fnet/src/vespa/fnet/scheduler.cpp +++ b/fnet/src/vespa/fnet/scheduler.cpp @@ -8,7 +8,7 @@ #include <vespa/log/log.h> LOG_SETUP(".fnet.scheduler"); -const vespalib::duration FNET_Scheduler::tick_ms = vespalib::from_s(10*1.0/vespalib::getVespaTimerHz()); +const vespalib::duration FNET_Scheduler::tick_ms = vespalib::adjustTimeoutByDetectedHz(10ms); FNET_Scheduler::FNET_Scheduler(vespalib::steady_time *sampler) : _cond(), diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/JvmDumper.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/JvmDumper.java index 8b6ca1384e9..c534c091a92 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/JvmDumper.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/JvmDumper.java @@ -22,7 +22,7 @@ class JvmDumper { @Override public List<Artifact> produceArtifacts(Context ctx) { - ContainerPath heapDumpFile = ctx.outputContainerPath().resolve("jvm-heap-dump.bin"); + ContainerPath heapDumpFile = ctx.outputContainerPath().resolve("jvm-heap-dump.hprof"); List<String> cmd = List.of( "jmap", "-dump:live,format=b,file=" + heapDumpFile.pathInContainer(), Integer.toString(ctx.servicePid())); ctx.executeCommandInNode(cmd, true); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java index 1d5c9f7a17d..ba8aa49d87c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepoStats.java @@ -68,9 +68,6 @@ public class NodeRepoStats { NodeResources resources = node.get().resources(); - // TODO: Skip arm64 for now, add() does not work with mix of x86_64 and arm64 nodes - if (resources.architecture() == NodeResources.Architecture.arm64) continue; - load = load.add(snapshot.get().load().multiply(resources)); totalActiveResources = totalActiveResources.add(resources.justNumbers()); } @@ -78,9 +75,6 @@ public class NodeRepoStats { NodeResources totalHostResources = NodeResources.zero(); for (var host : allNodes.hosts()) { - // TODO: Skip arm64 for now, add() does not work with mix of x86_64 and arm64 nodes - if (host.resources().architecture() == NodeResources.Architecture.arm64) continue; - totalHostResources = totalHostResources.add(host.resources().justNumbers()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java index 849ea03665b..5664b9bd7f4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java @@ -8,12 +8,10 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; -import com.yahoo.vespa.hosted.provision.provisioning.CapacityPolicies; import com.yahoo.vespa.hosted.provision.provisioning.NodeResourceLimits; import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; /** * @author bratseth diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index 9148a2165a5..763fe1cab6f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -104,7 +104,8 @@ public class GroupPreparer { .map(deficit -> hostProvisioner.get().provisionHosts(allocation.provisionIndices(deficit.count()), hostType, deficit.resources(), - application, osVersion, + application, + osVersion, sharing, Optional.of(cluster.type()))) .orElseGet(List::of); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacity.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacity.java index b112c4f6531..f29bf61149c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacity.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacity.java @@ -131,7 +131,8 @@ public class HostCapacity { .map(node -> node.flavor().resources().justNumbers()) .reduce(hostResources.justNumbers(), NodeResources::subtract) .with(host.flavor().resources().diskSpeed()) - .with(host.flavor().resources().storageType()); + .with(host.flavor().resources().storageType()) + .with(host.flavor().resources().architecture()); } private static boolean inactiveOrRetired(Node node) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index 7836ca3265e..67d89835c0d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -162,7 +162,7 @@ class NodeAllocation { if (violatesParentHostPolicy(candidate)) return Retirement.violatesParentHostPolicy; if ( ! hasCompatibleFlavor(candidate)) return Retirement.incompatibleFlavor; if (candidate.wantToRetire()) return Retirement.hardRequest; - if (candidate.preferToRetire() && candidate.replacableBy(candidates)) return Retirement.softRequest; + if (candidate.preferToRetire() && candidate.replaceableBy(candidates)) return Retirement.softRequest; if (violatesExclusivity(candidate)) return Retirement.violatesExclusivity; return Retirement.none; } @@ -277,7 +277,8 @@ class NodeAllocation { NodeResources hostResources = allNodesAndHosts.parentOf(node).get().flavor().resources(); return node.with(new Flavor(requestedNodes.resources().get() .with(hostResources.diskSpeed()) - .with(hostResources.storageType())), + .with(hostResources.storageType()) + .with(hostResources.architecture())), Agent.application, nodeRepository.clock().instant()); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java index 6f7eeb75c03..8e13ac65c21 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java @@ -102,7 +102,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat public abstract boolean isValid(); /** Returns whether this can be replaced by any of the reserved candidates */ - public boolean replacableBy(List<NodeCandidate> candidates) { + public boolean replaceableBy(List<NodeCandidate> candidates) { return candidates.stream() .filter(candidate -> candidate.state() == Node.State.reserved) .anyMatch(candidate -> { @@ -428,7 +428,8 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat allocation.get().hostname(), parentHostname().get(), resources.with(parent.get().resources().diskSpeed()) - .with(parent.get().resources().storageType()), + .with(parent.get().resources().storageType()) + .with(parent.get().resources().architecture()), NodeType.tenant).build(); return new ConcreteNodeCandidate(node, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, isSurplus, isNew, isResizable); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 4dc199dd7b2..97fe14caef6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -283,6 +283,7 @@ public class NodeRepositoryProvisioner implements Provisioner { .map(flavor -> nodeRepository.resourcesCalculator().advertisedResourcesOf(flavor)) .filter(resources -> resources.diskSpeed().compatibleWith(requestedResources.diskSpeed())) .filter(resources -> resources.storageType().compatibleWith(requestedResources.storageType())) + .filter(resources -> resources.architecture().compatibleWith(requestedResources.architecture())) .min(Comparator.comparingDouble(resources -> resources.distanceTo(requestedResources))) .orElseThrow() .withBandwidthGbps(requestedResources.bandwidthGbps()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index 1304b85be6b..f05492162db 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -289,7 +289,8 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { requiredField(resourcesInspector, "diskGb", Inspector::asDouble), requiredField(resourcesInspector, "bandwidthGbps", Inspector::asDouble), optionalString(resourcesInspector.field("diskSpeed")).map(NodeResourcesSerializer::diskSpeedFrom).orElse(NodeResources.DiskSpeed.getDefault()), - optionalString(resourcesInspector.field("storageType")).map(NodeResourcesSerializer::storageTypeFrom).orElse(NodeResources.StorageType.getDefault()))); + optionalString(resourcesInspector.field("storageType")).map(NodeResourcesSerializer::storageTypeFrom).orElse(NodeResources.StorageType.getDefault()), + optionalString(resourcesInspector.field("architecture")).map(NodeResourcesSerializer::architectureFrom).orElse(NodeResources.Architecture.getDefault()))); } Flavor flavor = nodeFlavors.getFlavorOrThrow(flavorInspector.asString()); @@ -306,6 +307,8 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { flavor = flavor.with(flavor.resources().with(NodeResourcesSerializer.diskSpeedFrom(resourcesInspector.field("diskSpeed").asString()))); if (resourcesInspector.field("storageType").valid()) flavor = flavor.with(flavor.resources().with(NodeResourcesSerializer.storageTypeFrom(resourcesInspector.field("storageType").asString()))); + if (resourcesInspector.field("architecture").valid()) + flavor = flavor.with(flavor.resources().with(NodeResourcesSerializer.architectureFrom(resourcesInspector.field("architecture").asString()))); } return flavor; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index ba9da20b615..94822c85a03 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterResources; @@ -13,15 +12,13 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.ParentHostUnavailableException; -import com.yahoo.config.provision.ProvisionLock; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; -import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeMutex; @@ -36,7 +33,6 @@ import com.yahoo.vespa.service.duper.InfraApplication; import org.junit.Test; import java.time.Duration; -import java.time.Instant; import java.util.Collection; import java.util.HashSet; import java.util.Iterator; @@ -1010,6 +1006,20 @@ public class ProvisioningTest { stateAsserter.accept(new Zone(SystemName.cd, Environment.prod, RegionName.from("us-east")), Node.State.dirty); } + @Test + public void arm64_architecture() { + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.dev, RegionName.from("us-east"))).build(); + + NodeResources nodeResources = new NodeResources(1, 4, 10, 4, NodeResources.DiskSpeed.any, NodeResources.StorageType.any, NodeResources.Architecture.arm64); + tester.makeReadyHosts(4, nodeResources); + tester.prepareAndActivateInfraApplication(ProvisioningTester.applicationId(), NodeType.host); + + ApplicationId application = ProvisioningTester.applicationId(); + SystemState state = prepare(application, 1, 1, 1, 1, nodeResources, tester); + assertEquals(4, state.allHosts.size()); + tester.activate(application, state.allHosts); + } + private SystemState prepare(ApplicationId application, int container0Size, int container1Size, int content0Size, int content1Size, NodeResources flavor, ProvisioningTester tester) { return prepare(application, tester, container0Size, container1Size, content0Size, content1Size, flavor, "6.42"); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index 7b988f83c01..9f722507ba2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -133,6 +133,7 @@ public class ProvisioningTester { b.addFlavor("dockerLarge", 2., 10., 20, 1, Flavor.Type.DOCKER_CONTAINER).cost(3); b.addFlavor("v-4-8-100", 4., 80., 100, 10, Flavor.Type.VIRTUAL_MACHINE).cost(4); b.addFlavor("large", 4., 80., 100, 10, Flavor.Type.BARE_METAL).cost(10); + b.addFlavor("arm64", 4., 80., 100, 10, Flavor.Type.BARE_METAL, NodeResources.Architecture.arm64).cost(15); return b.build(); } diff --git a/persistence/src/vespa/persistence/dummyimpl/dummy_bucket_executor.cpp b/persistence/src/vespa/persistence/dummyimpl/dummy_bucket_executor.cpp index 3eda5146af1..060215c4521 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummy_bucket_executor.cpp +++ b/persistence/src/vespa/persistence/dummyimpl/dummy_bucket_executor.cpp @@ -6,7 +6,7 @@ #include <vespa/vespalib/util/destructor_callbacks.h> using vespalib::makeLambdaTask; -using vespalib::makeLambdaCallback; +using vespalib::makeSharedLambdaCallback; namespace storage::spi::dummy { @@ -32,7 +32,7 @@ DummyBucketExecutor::execute(const Bucket & bucket, std::unique_ptr<BucketTask> } _inFlight.insert(bucket.getBucket()); } - bucketTask->run(bucket, makeLambdaCallback([this, bucket]() { + bucketTask->run(bucket, makeSharedLambdaCallback([this, bucket]() { std::unique_lock guard(_lock); assert(_inFlight.contains(bucket.getBucket())); _inFlight.erase(bucket.getBucket()); diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp index fd0ace21955..4369666591f 100644 --- a/searchcore/src/apps/tests/persistenceconformance_test.cpp +++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp @@ -171,7 +171,6 @@ private: DummyFileHeaderContext _fileHeaderContext; vespalib::string _tlsSpec; matching::QueryLimiter _queryLimiter; - vespalib::Clock _clock; mutable DummyWireService _metricsWireService; mutable MemoryConfigStores _config_stores; vespalib::ThreadStackExecutor _summaryExecutor; @@ -209,7 +208,7 @@ public: tuneFileDocDB, HwInfo()); mgr.forwardConfig(b); mgr.nextGeneration(_shared_service.transport(), 0ms); - return DocumentDB::create(_baseDir, mgr.getConfig(), _tlsSpec, _queryLimiter, _clock, docType, bucketSpace, + return DocumentDB::create(_baseDir, mgr.getConfig(), _tlsSpec, _queryLimiter, docType, bucketSpace, *b->getProtonConfigSP(), const_cast<DocumentDBFactory &>(*this), _shared_service, _bucketExecutor, _tls, _metricsWireService, _fileHeaderContext, _config_stores.getConfigStore(docType.toString()), @@ -223,7 +222,6 @@ DocumentDBFactory::DocumentDBFactory(const vespalib::string &baseDir, int tlsLis _fileHeaderContext(), _tlsSpec(vespalib::make_string("tcp/localhost:%d", tlsListenPort)), _queryLimiter(), - _clock(), _metricsWireService(), _summaryExecutor(8, 128_Ki), _shared_service(_summaryExecutor, _summaryExecutor), diff --git a/searchcore/src/tests/grouping/grouping.cpp b/searchcore/src/tests/grouping/grouping.cpp index 65c7361953e..a08d4703814 100644 --- a/searchcore/src/tests/grouping/grouping.cpp +++ b/searchcore/src/tests/grouping/grouping.cpp @@ -10,6 +10,7 @@ #include <vespa/searchcore/grouping/groupingsession.h> #include <vespa/searchcore/proton/matching/sessionmanager.h> #include <vespa/searchlib/test/mock_attribute_context.h> +#include <vespa/vespalib/util/testclock.h> #include <iostream> #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/log/log.h> @@ -111,7 +112,7 @@ private: }; struct DoomFixture { - vespalib::Clock clock; + vespalib::TestClock clock; steady_time timeOfDoom; DoomFixture() : clock(), timeOfDoom(steady_time::max()) {} }; @@ -168,7 +169,7 @@ TEST_F("testGroupingContextInitialization", DoomFixture()) { nos << (uint32_t)1; baseRequest.serialize(nos); - GroupingContext context(f1.clock, f1.timeOfDoom, os.data(), os.size()); + GroupingContext context(f1.clock.clock(), f1.timeOfDoom, os.data(), os.size()); ASSERT_TRUE(!context.empty()); GroupingContext::GroupingList list = context.getGroupingList(); ASSERT_TRUE(list.size() == 1); @@ -198,7 +199,7 @@ TEST_F("testGroupingContextUsage", DoomFixture()) { GroupingContext::GroupingPtr r1(new Grouping(request1)); GroupingContext::GroupingPtr r2(new Grouping(request2)); - GroupingContext context(f1.clock, f1.timeOfDoom); + GroupingContext context(f1.clock.clock(), f1.timeOfDoom); ASSERT_TRUE(context.empty()); context.addGrouping(r1); ASSERT_TRUE(context.getGroupingList().size() == 1); @@ -220,7 +221,7 @@ TEST_F("testGroupingContextSerializing", DoomFixture()) { nos << (uint32_t)1; baseRequest.serialize(nos); - GroupingContext context(f1.clock, f1.timeOfDoom); + GroupingContext context(f1.clock.clock(), f1.timeOfDoom); GroupingContext::GroupingPtr bp(new Grouping(baseRequest)); context.addGrouping(bp); context.serialize(); @@ -238,7 +239,7 @@ TEST_F("testGroupingManager", DoomFixture()) { .addLevel(createGL(MU<AttributeNode>("attr1"), MU<AttributeNode>("attr2"))) .addLevel(createGL(MU<AttributeNode>("attr2"), MU<AttributeNode>("attr3"))); - GroupingContext context(f1.clock, f1.timeOfDoom); + GroupingContext context(f1.clock.clock(), f1.timeOfDoom); GroupingContext::GroupingPtr bp(new Grouping(request1)); context.addGrouping(bp); GroupingManager manager(context); @@ -273,7 +274,7 @@ TEST_F("testGroupingSession", DoomFixture()) { GroupingContext::GroupingPtr r1(new Grouping(request1)); GroupingContext::GroupingPtr r2(new Grouping(request2)); - GroupingContext initContext(f1.clock, f1.timeOfDoom); + GroupingContext initContext(f1.clock.clock(), f1.timeOfDoom); initContext.addGrouping(r1); initContext.addGrouping(r2); SessionId id("foo"); @@ -305,7 +306,7 @@ TEST_F("testGroupingSession", DoomFixture()) { } // Test second pass { - GroupingContext context(f1.clock, f1.timeOfDoom); + GroupingContext context(f1.clock.clock(), f1.timeOfDoom); GroupingContext::GroupingPtr r(new Grouping(request1)); r->setFirstLevel(1); r->setLastLevel(1); @@ -316,7 +317,7 @@ TEST_F("testGroupingSession", DoomFixture()) { } // Test last pass. Session should be marked as finished { - GroupingContext context(f1.clock, f1.timeOfDoom); + GroupingContext context(f1.clock.clock(), f1.timeOfDoom); GroupingContext::GroupingPtr r(new Grouping(request1)); r->setFirstLevel(2); r->setLastLevel(2); @@ -340,7 +341,7 @@ TEST_F("testEmptySessionId", DoomFixture()) { .addLevel(createGL(MU<AttributeNode>("attr2"), MU<AttributeNode>("attr3"))); GroupingContext::GroupingPtr r1(new Grouping(request1)); - GroupingContext initContext(f1.clock, f1.timeOfDoom); + GroupingContext initContext(f1.clock.clock(), f1.timeOfDoom); initContext.addGrouping(r1); SessionId id; @@ -373,7 +374,7 @@ TEST_F("testSessionManager", DoomFixture()) { .setResult(Int64ResultNode(0)))); GroupingContext::GroupingPtr r1(new Grouping(request1)); - GroupingContext initContext(f1.clock, f1.timeOfDoom); + GroupingContext initContext(f1.clock.clock(), f1.timeOfDoom); initContext.addGrouping(r1); SessionManager mgr(2); @@ -431,7 +432,7 @@ TEST_F("test grouping fork/join", DoomFixture()) { .setLastLevel(1); GroupingContext::GroupingPtr g1(new Grouping(request)); - GroupingContext context(f1.clock, f1.timeOfDoom); + GroupingContext context(f1.clock.clock(), f1.timeOfDoom); context.addGrouping(g1); GroupingSession session(SessionId(), context, world.attributeContext); session.prepareThreadContextCreation(4); @@ -473,8 +474,8 @@ TEST_F("test session timeout", DoomFixture()) { SessionId id1("foo"); SessionId id2("bar"); - GroupingContext initContext1(f1.clock, steady_time(duration(10))); - GroupingContext initContext2(f1.clock, steady_time(duration(20))); + GroupingContext initContext1(f1.clock.clock(), steady_time(duration(10))); + GroupingContext initContext2(f1.clock.clock(), steady_time(duration(20))); GroupingSession::UP s1(new GroupingSession(id1, initContext1, world.attributeContext)); GroupingSession::UP s2(new GroupingSession(id2, initContext2, world.attributeContext)); mgr.insert(std::move(s1)); diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index faea1cc8b7c..ef8dc17dc0e 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -183,7 +183,6 @@ public: storage::spi::dummy::DummyBucketExecutor _bucketExecutor; bool _mkdirOk; matching::QueryLimiter _queryLimiter; - vespalib::Clock _clock; DummyWireService _dummy; ::config::DirSpec _spec; DocumentDBConfigHelper _configMgr; @@ -204,7 +203,6 @@ public: _bucketExecutor(2), _mkdirOk(FastOS_File::MakeDirectory("tmpdb")), _queryLimiter(), - _clock(), _dummy(), _spec(TEST_PATH("")), _configMgr(_spec, getDocTypeName()), @@ -227,7 +225,7 @@ public: if (! FastOS_File::MakeDirectory((std::string("tmpdb/") + docTypeName).c_str())) { LOG_ABORT("should not be reached"); } - _ddb = DocumentDB::create("tmpdb", _configMgr.getConfig(), "tcp/localhost:9013", _queryLimiter, _clock, + _ddb = DocumentDB::create("tmpdb", _configMgr.getConfig(), "tcp/localhost:9013", _queryLimiter, DocTypeName(docTypeName), makeBucketSpace(), *b->getProtonConfigSP(), *this, _shared_service, _bucketExecutor, _tls, _dummy, _fileHeaderContext, std::make_unique<MemoryConfigStore>(), diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp index 0e9eb926514..9e77047e578 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp +++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp @@ -28,6 +28,7 @@ #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/threadstackexecutor.h> +#include <vespa/vespalib/util/testclock.h> using namespace config; using namespace document; @@ -146,7 +147,7 @@ struct MyDocumentDBReferenceResolver : public IDocumentDBReferenceResolver { struct Fixture { - vespalib::Clock _clock; + vespalib::TestClock _clock; matching::QueryLimiter _queryLimiter; EmptyConstantValueFactory _constantValueFactory; ConstantValueRepo _constantValueRepo; @@ -175,7 +176,7 @@ Fixture::Fixture() vespalib::mkdir(BASE_DIR); initViewSet(_views); _configurer = std::make_unique<Configurer>(_views._summaryMgr, _views.searchView, _views.feedView, _queryLimiter, - _constantValueRepo, _clock, "test", 0); + _constantValueRepo, _clock.clock(), "test", 0); } Fixture::~Fixture() = default; @@ -184,7 +185,7 @@ Fixture::initViewSet(ViewSet &views) { using IndexManager = proton::index::IndexManager; using IndexConfig = proton::index::IndexConfig; - auto matchers = std::make_shared<Matchers>(_clock, _queryLimiter, _constantValueRepo); + auto matchers = std::make_shared<Matchers>(_clock.clock(), _queryLimiter, _constantValueRepo); auto indexMgr = make_shared<IndexManager>(BASE_DIR, IndexConfig(searchcorespi::index::WarmupConfig(), 2, 0), Schema(), 1, views._reconfigurer, views._service.write(), _summaryExecutor, TuneFileIndexManager(), TuneFileAttributes(), views._fileHeaderContext); diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index 27636324835..68a92c73b44 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -35,6 +35,7 @@ #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/destructor_callbacks.h> +#include <vespa/vespalib/util/testclock.h> #include <vespa/config/subscription/sourcespec.h> using namespace cloud::config::filedistribution; @@ -215,9 +216,9 @@ struct MySearchableConfig struct MySearchableContext { MyFastAccessContext _fastUpdCtx; - QueryLimiter _queryLimiter; - vespalib::Clock _clock; - SearchableContext _ctx; + QueryLimiter _queryLimiter; + vespalib::TestClock _clock; + SearchableContext _ctx; MySearchableContext(IThreadingService &writeService, std::shared_ptr<bucketdb::BucketDBOwner> bucketDB, IBucketDBHandlerInitializer & bucketDBHandlerInitializer); @@ -236,7 +237,7 @@ MySearchableContext::MySearchableContext(IThreadingService &writeService, IBucketDBHandlerInitializer & bucketDBHandlerInitializer) : _fastUpdCtx(writeService, bucketDB, bucketDBHandlerInitializer), _queryLimiter(), _clock(), - _ctx(_fastUpdCtx._ctx, _queryLimiter, _clock, writeService.shared()) + _ctx(_fastUpdCtx._ctx, _queryLimiter, _clock.clock(), writeService.shared()) {} MySearchableContext::~MySearchableContext() = default; diff --git a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp index c6a8df79a5e..1f3be7511da 100644 --- a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp @@ -127,7 +127,6 @@ struct Fixture : public FixtureBase { DummyFileHeaderContext _fileHeaderContext; TransLogServer _tls; matching::QueryLimiter _queryLimiter; - vespalib::Clock _clock; std::unique_ptr<ConfigStore> make_config_store(); Fixture(); @@ -151,8 +150,7 @@ Fixture::Fixture(bool file_config) _db(), _fileHeaderContext(), _tls(_shared_service.transport(), "tmp", 9014, ".", _fileHeaderContext), - _queryLimiter(), - _clock() + _queryLimiter() { auto documenttypesConfig = std::make_shared<DocumenttypesConfig>(); DocumentType docType("typea", 0); @@ -167,7 +165,7 @@ Fixture::Fixture(bool file_config) tuneFileDocumentDB, HwInfo()); mgr.forwardConfig(b); mgr.nextGeneration(_shared_service.transport(), 0ms); - _db = DocumentDB::create(".", mgr.getConfig(), "tcp/localhost:9014", _queryLimiter, _clock, DocTypeName("typea"), + _db = DocumentDB::create(".", mgr.getConfig(), "tcp/localhost:9014", _queryLimiter, DocTypeName("typea"), makeBucketSpace(), *b->getProtonConfigSP(), _myDBOwner, _shared_service, _bucketExecutor, _tls, _dummy, _fileHeaderContext, make_config_store(), diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index 09bb67dbac9..187c0463da3 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -8,7 +8,6 @@ #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> #include <vespa/searchcore/proton/matching/fakesearchcontext.h> #include <vespa/searchcore/proton/matching/i_constant_value_repo.h> -#include <vespa/searchcore/proton/matching/isearchcontext.h> #include <vespa/searchcore/proton/matching/matcher.h> #include <vespa/searchcore/proton/matching/querynodes.h> #include <vespa/searchcore/proton/matching/sessionmanager.h> @@ -37,6 +36,7 @@ #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/value_codec.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/util/testclock.h> #include <vespa/log/log.h> LOG_SETUP("matching_test"); @@ -120,7 +120,7 @@ struct MyWorld { SessionManager::SP sessionManager; DocumentMetaStore metaStore; MatchingStats matchingStats; - vespalib::Clock clock; + vespalib::TestClock clock; QueryLimiter queryLimiter; EmptyConstantValueRepo constantValueRepo; @@ -344,7 +344,7 @@ struct MyWorld { } Matcher::SP createMatcher() { - return std::make_shared<Matcher>(schema, config, clock, queryLimiter, constantValueRepo, RankingExpressions(), OnnxModels(), 0); + return std::make_shared<Matcher>(schema, config, clock.clock(), queryLimiter, constantValueRepo, RankingExpressions(), OnnxModels(), 0); } struct MySearchHandler : ISearchHandler { diff --git a/searchcore/src/tests/proton/matching/request_context/request_context_test.cpp b/searchcore/src/tests/proton/matching/request_context/request_context_test.cpp index 8ede5277d71..191c1718f61 100644 --- a/searchcore/src/tests/proton/matching/request_context/request_context_test.cpp +++ b/searchcore/src/tests/proton/matching/request_context/request_context_test.cpp @@ -6,6 +6,7 @@ #include <vespa/searchcore/proton/matching/requestcontext.h> #include <vespa/searchlib/attribute/attribute_blueprint_params.h> #include <vespa/searchlib/fef/properties.h> +#include <vespa/vespalib/util/testclock.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/objects/nbostream.h> @@ -29,7 +30,7 @@ public: class RequestContextTest : public ::testing::Test { private: - vespalib::Clock _clock; + vespalib::TestClock _clock; vespalib::Doom _doom; MyAttributeContext _attr_ctx; Properties _props; @@ -45,7 +46,7 @@ private: public: RequestContextTest() : _clock(), - _doom(_clock, vespalib::steady_time(), vespalib::steady_time(), false), + _doom(_clock.clock(), vespalib::steady_time(), vespalib::steady_time(), false), _attr_ctx(), _props(), _request_ctx(_doom, _attr_ctx, _props, AttributeBlueprintParams()), diff --git a/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp b/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp index 85fd304f395..86dc38159e8 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp +++ b/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp @@ -451,7 +451,6 @@ class MyBmNode : public BmNode int _distributor_status_port; vespalib::string _tls_spec; proton::matching::QueryLimiter _query_limiter; - vespalib::Clock _clock; proton::DummyWireService _metrics_wire_service; proton::MemoryConfigStores _config_stores; vespalib::ThreadStackExecutor _summary_executor; @@ -516,7 +515,6 @@ MyBmNode::MyBmNode(const vespalib::string& base_dir, int base_port, uint32_t nod _distributor_status_port(port_number(base_port, PortBias::DISTRIBUTOR_STATUS_PORT)), _tls_spec(vespalib::make_string("tcp/localhost:%d", _tls_listen_port)), _query_limiter(), - _clock(), _metrics_wire_service(), _config_stores(), _summary_executor(8, 128_Ki), @@ -591,7 +589,7 @@ MyBmNode::create_document_db(const BmClusterParams& params) tuneFileDocDB, HwInfo()); mgr.forwardConfig(bootstrap_config); mgr.nextGeneration(_shared_service.transport(), 0ms); - _document_db = DocumentDB::create(_base_dir, mgr.getConfig(), _tls_spec, _query_limiter, _clock, _doc_type_name, + _document_db = DocumentDB::create(_base_dir, mgr.getConfig(), _tls_spec, _query_limiter, _doc_type_name, _bucket_space, *bootstrap_config->getProtonConfigSP(), _document_db_owner, _shared_service, *_persistence_engine, _tls, _metrics_wire_service, _file_header_context, diff --git a/searchcore/src/vespa/searchcore/proton/attribute/imported_attributes_context.cpp b/searchcore/src/vespa/searchcore/proton/attribute/imported_attributes_context.cpp index 0069a61b818..7a86a4c5f31 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/imported_attributes_context.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/imported_attributes_context.cpp @@ -9,22 +9,34 @@ using search::attribute::IAttributeVector; using search::attribute::ImportedAttributeVector; -using LockGuard = std::lock_guard<std::mutex>; namespace proton { const IAttributeVector * ImportedAttributesContext::getOrCacheAttribute(const vespalib::string &name, AttributeCache &attributes, - bool stableEnumGuard, const LockGuard &) const + bool stableEnumGuard) const { + std::unique_lock guard(_cacheMutex); + std::shared_future<std::unique_ptr<AttributeReadGuard>> future_read_guard; auto itr = attributes.find(name); if (itr != attributes.end()) { - return itr->second->attribute(); + future_read_guard = itr->second; + guard.unlock(); + } else { + std::promise<std::unique_ptr<AttributeReadGuard>> promise; + future_read_guard = promise.get_future().share(); + attributes.emplace(name, future_read_guard); + guard.unlock(); + ImportedAttributeVector::SP result = _repo.get(name); + if (result) { + promise.set_value(result->makeReadGuard(stableEnumGuard)); + } else { + promise.set_value(std::unique_ptr<AttributeReadGuard>()); + } } - ImportedAttributeVector::SP result = _repo.get(name); - if (result) { - auto insRes = attributes.emplace(name, result->makeReadGuard(stableEnumGuard)); - return insRes.first->second->attribute(); + auto& read_guard = future_read_guard.get(); + if (read_guard) { + return read_guard->attribute(); } else { return nullptr; } @@ -43,15 +55,13 @@ ImportedAttributesContext::~ImportedAttributesContext() = default; const IAttributeVector * ImportedAttributesContext::getAttribute(const vespalib::string &name) const { - LockGuard guard(_cacheMutex); - return getOrCacheAttribute(name, _guardedAttributes, false, guard); + return getOrCacheAttribute(name, _guardedAttributes, false); } const IAttributeVector * ImportedAttributesContext::getAttributeStableEnum(const vespalib::string &name) const { - LockGuard guard(_cacheMutex); - return getOrCacheAttribute(name, _enumGuardedAttributes, true, guard); + return getOrCacheAttribute(name, _enumGuardedAttributes, true); } void @@ -67,7 +77,7 @@ ImportedAttributesContext::getAttributeList(std::vector<const IAttributeVector * void ImportedAttributesContext::releaseEnumGuards() { - LockGuard guard(_cacheMutex); + std::lock_guard guard(_cacheMutex); _enumGuardedAttributes.clear(); } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/imported_attributes_context.h b/searchcore/src/vespa/searchcore/proton/attribute/imported_attributes_context.h index 4b7058fdb83..33d6cef98ff 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/imported_attributes_context.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/imported_attributes_context.h @@ -4,6 +4,7 @@ #include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/vespalib/stllike/hash_fun.h> #include <vespa/vespalib/stllike/hash_map.h> +#include <future> #include <mutex> #include <unordered_map> @@ -31,8 +32,7 @@ private: using ImportedAttributeVector = search::attribute::ImportedAttributeVector; using IAttributeFunctor = search::attribute::IAttributeFunctor; - using AttributeCache = std::unordered_map<vespalib::string, std::unique_ptr<AttributeReadGuard>, vespalib::hash<vespalib::string>>; - using LockGuard = std::lock_guard<std::mutex>; + using AttributeCache = std::unordered_map<vespalib::string, std::shared_future<std::unique_ptr<AttributeReadGuard>>, vespalib::hash<vespalib::string>>; const ImportedAttributesRepo &_repo; mutable AttributeCache _guardedAttributes; @@ -40,7 +40,7 @@ private: mutable std::mutex _cacheMutex; const IAttributeVector *getOrCacheAttribute(const vespalib::string &name, AttributeCache &attributes, - bool stableEnumGuard, const LockGuard &) const; + bool stableEnumGuard) const; public: ImportedAttributesContext(const ImportedAttributesRepo &repo); diff --git a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt index 03bad4c7eaf..3f33871a0e2 100644 --- a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt @@ -18,6 +18,7 @@ vespa_add_library(searchcore_pcommon STATIC ipendinglidtracker.cpp operation_rate_tracker.cpp pendinglidtracker.cpp + replay_feed_token_factory.cpp replay_feedtoken_state.cpp select_utils.cpp selectcontext.cpp diff --git a/searchcore/src/vespa/searchcore/proton/common/replay_feed_token_factory.cpp b/searchcore/src/vespa/searchcore/proton/common/replay_feed_token_factory.cpp new file mode 100644 index 00000000000..18e6f415f16 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/common/replay_feed_token_factory.cpp @@ -0,0 +1,57 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "replay_feed_token_factory.h" +#include "replay_feedtoken_state.h" +#include <cassert> + +namespace proton::feedtoken { + +class ReplayFeedTokenFactory::Deleter { + ReplayFeedTokenFactory& _factory; +public: + Deleter(ReplayFeedTokenFactory& factory) + : _factory(factory) + { + } + void operator()(ReplayState* p) const noexcept { + _factory.on_delete(p); + delete p; + } +}; + +ReplayFeedTokenFactory::ReplayFeedTokenFactory(bool enable_tracking) + : _lock(), + _states(), + _enable_tracking(enable_tracking) +{ +} + +ReplayFeedTokenFactory::~ReplayFeedTokenFactory() +{ + std::lock_guard guard(_lock); + assert(_states.empty()); +} + +FeedToken +ReplayFeedTokenFactory::make_replay_feed_token(ThrottlerToken throttler_token, const FeedOperation& op) +{ + if (_enable_tracking) { + auto token = std::make_unique<ReplayState>(std::move(throttler_token), op); + std::lock_guard guard(_lock); + bool inserted = _states.insert(token.get()).second; + assert(inserted); + return std::shared_ptr<ReplayState>(token.release(), Deleter(*this)); + } else { + return std::make_shared<ReplayState>(std::move(throttler_token), op); + } +} + +void +ReplayFeedTokenFactory::on_delete(const ReplayState* state) noexcept +{ + std::lock_guard guard(_lock); + bool erased = _states.erase(state) > 0u; + assert(erased); +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/common/replay_feed_token_factory.h b/searchcore/src/vespa/searchcore/proton/common/replay_feed_token_factory.h new file mode 100644 index 00000000000..7e161ef1773 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/common/replay_feed_token_factory.h @@ -0,0 +1,33 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/util/shared_operation_throttler.h> +#include "feedtoken.h" +#include <mutex> +#include <unordered_set> + +namespace proton { class FeedOperation; } + +namespace proton::feedtoken { + +class ReplayState; + +/* + * A factory for replay feed tokens with optional tracking. + */ +class ReplayFeedTokenFactory { + using ThrottlerToken = vespalib::SharedOperationThrottler::Token; + class Deleter; + + std::mutex _lock; + std::unordered_set<const ReplayState*> _states; + bool _enable_tracking; + void on_delete(const ReplayState* state) noexcept; +public: + ReplayFeedTokenFactory(bool enable_tracking); + ~ReplayFeedTokenFactory(); + FeedToken make_replay_feed_token(ThrottlerToken throttler_token, const FeedOperation& op); +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/common/replay_feedtoken_state.cpp b/searchcore/src/vespa/searchcore/proton/common/replay_feedtoken_state.cpp index a3a473c9548..2e6469f870f 100644 --- a/searchcore/src/vespa/searchcore/proton/common/replay_feedtoken_state.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/replay_feedtoken_state.cpp @@ -4,9 +4,11 @@ namespace proton::feedtoken { -ReplayState::ReplayState(vespalib::SharedOperationThrottler::Token throttler_token) +ReplayState::ReplayState(vespalib::SharedOperationThrottler::Token throttler_token, const FeedOperation& op) : IState(), - _throttler_token(std::move(throttler_token)) + _throttler_token(std::move(throttler_token)), + _type(op.getType()), + _serial_num(op.getSerialNum()) { } diff --git a/searchcore/src/vespa/searchcore/proton/common/replay_feedtoken_state.h b/searchcore/src/vespa/searchcore/proton/common/replay_feedtoken_state.h index 512f12a50af..4ed0986b838 100644 --- a/searchcore/src/vespa/searchcore/proton/common/replay_feedtoken_state.h +++ b/searchcore/src/vespa/searchcore/proton/common/replay_feedtoken_state.h @@ -4,6 +4,7 @@ #include "feedtoken.h" #include <vespa/vespalib/util/shared_operation_throttler.h> +#include <vespa/searchcore/proton/feedoperation/feedoperation.h> namespace proton::feedtoken { @@ -13,10 +14,15 @@ namespace proton::feedtoken { * of the feed operation. */ class ReplayState : public IState { - vespalib::SharedOperationThrottler::Token _throttler_token; + using SerialNum = search::SerialNum; + using ThrottlerToken = vespalib::SharedOperationThrottler::Token; + + ThrottlerToken _throttler_token; + FeedOperation::Type _type; + SerialNum _serial_num; public: ~ReplayState() override; - ReplayState(vespalib::SharedOperationThrottler::Token throttler_token); + ReplayState(ThrottlerToken throttler_token, const FeedOperation& op); bool is_replay() const noexcept override; void fail() override; void setResult(ResultUP result, bool documentWasFound) override; diff --git a/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.cpp b/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.cpp index 435b6f718c7..56c3a1c761b 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.cpp @@ -1,12 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "fakesearchcontext.h" +#include <vespa/vespalib/util/testclock.h> + namespace proton::matching { FakeSearchContext::FakeSearchContext(size_t initialNumDocs) - : _clock(), - _doom(_clock, vespalib::steady_time()), + : _clock(std::make_unique<vespalib::TestClock>()), + _doom(_clock->clock(), vespalib::steady_time()), _selector(std::make_shared<search::FixedSourceSelector>(0, "fs", initialNumDocs)), _indexes(std::make_shared<IndexCollection>(_selector)), _attrSearchable(), diff --git a/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.h b/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.h index 254f473c4ae..659bfc6c6d8 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.h +++ b/searchcore/src/vespa/searchcore/proton/matching/fakesearchcontext.h @@ -11,6 +11,7 @@ #include <map> #include <vector> +namespace vespalib { class TestClock; } namespace proton::matching { using searchcorespi::FakeIndexSearchable; @@ -23,7 +24,7 @@ public: typedef search::queryeval::FakeSearchable FakeSearchable; private: - vespalib::Clock _clock; + std::unique_ptr<vespalib::TestClock> _clock; vespalib::Doom _doom; search::queryeval::ISourceSelector::SP _selector; IndexCollection::SP _indexes; diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp index 4d94d2629fe..e945bbb850b 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp @@ -150,7 +150,7 @@ Matcher::create_match_tools_factory(const search::engine::Request &request, ISea : _stats.softDoomFactor()) : 0.95; vespalib::duration safeLeft = std::chrono::duration_cast<vespalib::duration>(request.getTimeLeft() * factor); - vespalib::steady_time safeDoom(_clock.getTimeNSAssumeRunning() + safeLeft); + vespalib::steady_time safeDoom(_clock.getTimeNS() + safeLeft); if (softTimeoutEnabled) { LOG(debug, "Soft-timeout computed factor=%1.3f, used factor=%1.3f, userSupplied=%d, softTimeout=%" PRId64, _stats.softDoomFactor(), factor, hasFactorOverride, vespalib::count_ns(safeLeft)); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index cf6193555e5..37a17fecfc1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -140,7 +140,6 @@ DocumentDB::create(const vespalib::string &baseDir, DocumentDBConfig::SP currentSnapshot, const vespalib::string &tlsSpec, matching::QueryLimiter &queryLimiter, - const vespalib::Clock &clock, const DocTypeName &docTypeName, document::BucketSpace bucketSpace, const ProtonConfig &protonCfg, @@ -155,7 +154,7 @@ DocumentDB::create(const vespalib::string &baseDir, const HwInfo &hwInfo) { return DocumentDB::SP( - new DocumentDB(baseDir, std::move(currentSnapshot), tlsSpec, queryLimiter, clock, docTypeName, bucketSpace, + new DocumentDB(baseDir, std::move(currentSnapshot), tlsSpec, queryLimiter, docTypeName, bucketSpace, protonCfg, owner, shared_service, bucketExecutor, tlsWriterFactory, metricsWireService, fileHeaderContext, std::move(config_store), initializeThreads, hwInfo)); } @@ -163,7 +162,6 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, DocumentDBConfig::SP configSnapshot, const vespalib::string &tlsSpec, matching::QueryLimiter &queryLimiter, - const vespalib::Clock &clock, const DocTypeName &docTypeName, document::BucketSpace bucketSpace, const ProtonConfig &protonCfg, @@ -219,7 +217,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, _transient_usage_provider(std::make_shared<DocumentDBResourceUsageProvider>(*this)), _feedHandler(std::make_unique<FeedHandler>(_writeService, tlsSpec, docTypeName, *this, _writeFilter, *this, tlsWriterFactory)), _subDBs(*this, *this, *_feedHandler, _docTypeName, _writeService, shared_service.warmup(), fileHeaderContext, - metricsWireService, getMetrics(), queryLimiter, clock, _configMutex, _baseDir, hwInfo), + metricsWireService, getMetrics(), queryLimiter, shared_service.clock(), _configMutex, _baseDir, hwInfo), _maintenanceController(shared_service.transport(), _writeService.master(), shared_service.shared(), _refCount, _docTypeName), _jobTrackers(), _calc(), diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index 2030e6ffac9..e3d467fc3c1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -197,7 +197,6 @@ private: DocumentDBConfig::SP currentSnapshot, const vespalib::string &tlsSpec, matching::QueryLimiter &queryLimiter, - const vespalib::Clock &clock, const DocTypeName &docTypeName, document::BucketSpace bucketSpace, const ProtonConfig &protonCfg, @@ -228,7 +227,6 @@ public: DocumentDBConfig::SP currentSnapshot, const vespalib::string &tlsSpec, matching::QueryLimiter &queryLimiter, - const vespalib::Clock &clock, const DocTypeName &docTypeName, document::BucketSpace bucketSpace, const ProtonConfig &protonCfg, diff --git a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp index a268a6eac87..d6ecc6dd2d3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp @@ -13,6 +13,7 @@ using vespalib::CpuUsage; using vespalib::SequencedTaskExecutor; using vespalib::SingleExecutor; using vespalib::SyncableThreadExecutor; +using vespalib::steady_time; using OptimizeFor = vespalib::Executor::OptimizeFor; using SharedFieldWriterExecutor = proton::ThreadingServiceConfig::SharedFieldWriterExecutor; diff --git a/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp b/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp index d9fdee85e4a..c542437d440 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedstates.cpp @@ -9,7 +9,7 @@ #include <vespa/searchcore/proton/bucketdb/ibucketdbhandler.h> #include <vespa/searchcore/proton/feedoperation/operations.h> #include <vespa/searchcore/proton/common/eventlogger.h> -#include <vespa/searchcore/proton/common/replay_feedtoken_state.h> +#include <vespa/searchcore/proton/common/replay_feed_token_factory.h> #include <vespa/vespalib/util/idestructorcallback.h> #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/shared_operation_throttler.h> @@ -57,6 +57,7 @@ class TransactionLogReplayPacketHandler : public IReplayPacketHandler { IIncSerialNum &_inc_serial_num; CommitTimeTracker _commitTimeTracker; std::unique_ptr<SharedOperationThrottler> _throttler; + std::unique_ptr<feedtoken::ReplayFeedTokenFactory> _replay_feed_token_factory; static std::unique_ptr<SharedOperationThrottler> make_throttler(const ReplayThrottlingPolicy& replay_throttling_policy) { auto& params = replay_throttling_policy.get_params(); @@ -79,24 +80,25 @@ public: _config_store(config_store), _inc_serial_num(inc_serial_num), _commitTimeTracker(5ms), - _throttler(make_throttler(replay_throttling_policy)) + _throttler(make_throttler(replay_throttling_policy)), + _replay_feed_token_factory(std::make_unique<feedtoken::ReplayFeedTokenFactory>(true)) { } ~TransactionLogReplayPacketHandler() override = default; - FeedToken make_replay_feed_token() { + FeedToken make_replay_feed_token(const FeedOperation& op) { SharedOperationThrottler::Token throttler_token = _throttler->blocking_acquire_one(); - return std::make_shared<feedtoken::ReplayState>(std::move(throttler_token)); + return _replay_feed_token_factory->make_replay_feed_token(std::move(throttler_token), op); } void replay(const PutOperation &op) override { - _feed_view_ptr->handlePut(make_replay_feed_token(), op); + _feed_view_ptr->handlePut(make_replay_feed_token(op), op); } void replay(const RemoveOperation &op) override { - _feed_view_ptr->handleRemove(make_replay_feed_token(), op); + _feed_view_ptr->handleRemove(make_replay_feed_token(op), op); } void replay(const UpdateOperation &op) override { - _feed_view_ptr->handleUpdate(make_replay_feed_token(), op); + _feed_view_ptr->handleUpdate(make_replay_feed_token(op), op); } void replay(const NoopOperation &) override {} // ignored void replay(const NewConfigOperation &op) override { @@ -104,7 +106,7 @@ public: } void replay(const DeleteBucketOperation &op) override { - _feed_view_ptr->handleDeleteBucket(op, make_replay_feed_token()); + _feed_view_ptr->handleDeleteBucket(op, make_replay_feed_token(op)); } void replay(const SplitBucketOperation &op) override { _bucketDBHandler.handleSplit(op.getSerialNum(), op.getSource(), @@ -115,15 +117,15 @@ public: op.getSource2(), op.getTarget()); } void replay(const PruneRemovedDocumentsOperation &op) override { - _feed_view_ptr->handlePruneRemovedDocuments(op, make_replay_feed_token()); + _feed_view_ptr->handlePruneRemovedDocuments(op, make_replay_feed_token(op)); } void replay(const MoveOperation &op) override { - _feed_view_ptr->handleMove(op, make_replay_feed_token()); + _feed_view_ptr->handleMove(op, make_replay_feed_token(op)); } void replay(const CreateBucketOperation &) override { } void replay(const CompactLidSpaceOperation &op) override { - _feed_view_ptr->handleCompactLidSpace(op, make_replay_feed_token()); + _feed_view_ptr->handleCompactLidSpace(op, make_replay_feed_token(op)); } NewConfigOperation::IStreamHandler &getNewConfigStreamHandler() override { return _config_store; diff --git a/searchcore/src/vespa/searchcore/proton/server/i_shared_threading_service.h b/searchcore/src/vespa/searchcore/proton/server/i_shared_threading_service.h index a467f9bc320..12377fd25e0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_shared_threading_service.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_shared_threading_service.h @@ -7,6 +7,7 @@ namespace vespalib { class ISequencedTaskExecutor; class ThreadExecutor; class InvokeService; +class Clock; } namespace proton { @@ -51,6 +52,11 @@ public: * Returns a shared transport object that can be utilized by multiple services. */ virtual FNET_Transport & transport() = 0; + + /** + * Return a very cheap clock. + */ + virtual const vespalib::Clock & clock() const = 0; }; } diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 974c2522565..9bf6d19231e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -241,7 +241,6 @@ Proton::Proton(FastOS_ThreadPool & threadPool, FNET_Transport & transport, const _shared_service(), _compile_cache_executor_binding(), _queryLimiter(), - _clock(), _distributionKey(-1), _isInitializing(true), _abortInit(false), @@ -278,7 +277,6 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) setBucketCheckSumType(protonConfig); setFS4Compression(protonConfig); _shared_service = std::make_unique<SharedThreadingService>(SharedThreadingServiceConfig::make(protonConfig, hwInfo.cpu()), _transport); - _clock.start(_shared_service->invokeService()); _diskMemUsageSampler = std::make_unique<DiskMemUsageSampler>(_shared_service->transport(), protonConfig.basedir, diskMemUsageSamplerConfig(protonConfig, hwInfo)); @@ -473,7 +471,6 @@ Proton::~Proton() _persistenceEngine.reset(); _tls.reset(); _compile_cache_executor_binding.reset(); - _clock.stop(); _shared_service.reset(); LOG(debug, "Explicit destructor done"); } @@ -610,7 +607,6 @@ Proton::addDocumentDB(const document::DocumentType &docType, documentDBConfig, config.tlsspec, _queryLimiter, - _clock, docTypeName, bucketSpace, config, diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index 73b8ae83ef2..58c9a8bda3e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -111,7 +111,6 @@ private: std::unique_ptr<SharedThreadingService> _shared_service; vespalib::eval::CompileCache::ExecutorBinding::UP _compile_cache_executor_binding; matching::QueryLimiter _queryLimiter; - vespalib::Clock _clock; uint32_t _distributionKey; bool _isInitializing; bool _abortInit; diff --git a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.cpp b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.cpp index 1f06b29518b..e55282e31e8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.cpp @@ -3,13 +3,11 @@ #include "shared_threading_service.h" #include <vespa/vespalib/util/blockingthreadstackexecutor.h> #include <vespa/vespalib/util/cpu_usage.h> -#include <vespa/vespalib/util/isequencedtaskexecutor.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> #include <vespa/vespalib/util/size_literals.h> -#include <vespa/fnet/transport.h> -#include <vespa/fastos/thread.h> using vespalib::CpuUsage; +using vespalib::steady_time; VESPA_THREAD_STACK_TAG(proton_field_writer_executor) VESPA_THREAD_STACK_TAG(proton_shared_executor) @@ -26,9 +24,10 @@ SharedThreadingService::SharedThreadingService(const SharedThreadingServiceConfi _shared(std::make_shared<vespalib::BlockingThreadStackExecutor>(cfg.shared_threads(), 128_Ki, cfg.shared_task_limit(), proton_shared_executor)), _field_writer(), - _invokeService(std::max(vespalib::from_s(1.0/vespalib::getVespaTimerHz()), + _invokeService(std::max(vespalib::adjustTimeoutByDetectedHz(1ms), cfg.field_writer_config().reactionTime())), - _invokeRegistrations() + _invokeRegistrations(), + _clock(_invokeService.nowRef()) { const auto& fw_cfg = cfg.field_writer_config(); if (fw_cfg.shared_field_writer() == SharedFieldWriterExecutor::DOCUMENT_DB) { diff --git a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.h b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.h index 349dcc2d0ce..463823f10cb 100644 --- a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.h +++ b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.h @@ -5,6 +5,7 @@ #include "shared_threading_service_config.h" #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/syncable.h> +#include <vespa/vespalib/util/clock.h> #include <vespa/vespalib/util/invokeserviceimpl.h> #include <memory> @@ -22,7 +23,7 @@ private: std::unique_ptr<vespalib::ISequencedTaskExecutor> _field_writer; vespalib::InvokeServiceImpl _invokeService; std::vector<Registration> _invokeRegistrations; - + vespalib::Clock _clock; public: SharedThreadingService(const SharedThreadingServiceConfig& cfg, FNET_Transport & transport); ~SharedThreadingService() override; @@ -35,6 +36,7 @@ public: vespalib::ISequencedTaskExecutor* field_writer() override { return _field_writer.get(); } vespalib::InvokeService & invokeService() override { return _invokeService; } FNET_Transport & transport() override { return _transport; } + const vespalib::Clock & clock() const override { return _clock; } }; } diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_shared_threading_service.cpp b/searchcore/src/vespa/searchcore/proton/test/mock_shared_threading_service.cpp index d695f4d8dc7..ca9b89b1e60 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_shared_threading_service.cpp +++ b/searchcore/src/vespa/searchcore/proton/test/mock_shared_threading_service.cpp @@ -8,7 +8,8 @@ MockSharedThreadingService::MockSharedThreadingService(ThreadExecutor& warmup_in : _warmup(warmup_in), _shared(shared_in), _invokeService(10ms), - _transport() + _transport(), + _clock(_invokeService.nowRef()) { } diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_shared_threading_service.h b/searchcore/src/vespa/searchcore/proton/test/mock_shared_threading_service.h index 7db208fa3ef..35c8ee46de5 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_shared_threading_service.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_shared_threading_service.h @@ -4,6 +4,7 @@ #include "transport_helper.h" #include <vespa/searchcore/proton/server/i_shared_threading_service.h> #include <vespa/vespalib/util/invokeserviceimpl.h> +#include <vespa/vespalib/util/clock.h> namespace proton { @@ -14,7 +15,7 @@ private: ThreadExecutor & _shared; vespalib::InvokeServiceImpl _invokeService; Transport _transport; - + vespalib::Clock _clock; public: MockSharedThreadingService(ThreadExecutor& warmup_in, ThreadExecutor& shared_in); @@ -24,6 +25,7 @@ public: vespalib::ISequencedTaskExecutor* field_writer() override { return nullptr; } vespalib::InvokeService & invokeService() override { return _invokeService; } FNET_Transport & transport() override { return _transport.transport(); } + const vespalib::Clock & clock() const override { return _clock; } }; } diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp index d5297fe8f8c..1153a09d09f 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp @@ -37,7 +37,7 @@ using search::queryeval::ISourceSelector; using search::queryeval::Source; using search::SerialNum; using vespalib::makeLambdaTask; -using vespalib::makeLambdaCallback; +using vespalib::makeSharedLambdaCallback; using std::ostringstream; using vespalib::string; using vespalib::Executor; @@ -313,7 +313,7 @@ IndexMaintainer::loadDiskIndex(const string &indexDir) _disk_indexes->setActive(indexDir, stats.sizeOnDisk()); auto retval = std::make_shared<DiskIndexWithDestructorCallback>( std::move(index), - makeLambdaCallback([this, indexDir]() { deactivateDiskIndexes(indexDir); }), + makeSharedLambdaCallback([this, indexDir]() { deactivateDiskIndexes(indexDir); }), _layout, *_disk_indexes); if (LOG_WOULD_LOG(event)) { EventLogger::diskIndexLoadComplete(indexDir, vespalib::count_ms(timer.elapsed())); @@ -336,7 +336,7 @@ IndexMaintainer::reloadDiskIndex(const IDiskIndex &oldIndex) _disk_indexes->setActive(indexDir, stats.sizeOnDisk()); auto retval = std::make_shared<DiskIndexWithDestructorCallback>( std::move(index), - makeLambdaCallback([this, indexDir]() { deactivateDiskIndexes(indexDir); }), + makeSharedLambdaCallback([this, indexDir]() { deactivateDiskIndexes(indexDir); }), _layout, *_disk_indexes); if (LOG_WOULD_LOG(event)) { EventLogger::diskIndexLoadComplete(indexDir, vespalib::count_ms(timer.elapsed())); diff --git a/searchlib/abi-spec.json b/searchlib/abi-spec.json index 3213b1bb2b9..ced2517ff9f 100644 --- a/searchlib/abi-spec.json +++ b/searchlib/abi-spec.json @@ -1235,8 +1235,9 @@ "public java.lang.StringBuilder toString(java.lang.StringBuilder, com.yahoo.searchlib.rankingexpression.rule.SerializationContext, java.util.Deque, com.yahoo.searchlib.rankingexpression.rule.CompositeNode)", "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", "public com.yahoo.searchlib.rankingexpression.evaluation.Value evaluate(com.yahoo.searchlib.rankingexpression.evaluation.Context)", - "public static com.yahoo.searchlib.rankingexpression.rule.ArithmeticNode resolve(com.yahoo.searchlib.rankingexpression.rule.ExpressionNode, com.yahoo.searchlib.rankingexpression.rule.ArithmeticOperator, com.yahoo.searchlib.rankingexpression.rule.ExpressionNode)", - "public com.yahoo.searchlib.rankingexpression.rule.CompositeNode setChildren(java.util.List)" + "public com.yahoo.searchlib.rankingexpression.rule.CompositeNode setChildren(java.util.List)", + "public int hashCode()", + "public static com.yahoo.searchlib.rankingexpression.rule.ArithmeticNode resolve(com.yahoo.searchlib.rankingexpression.rule.ExpressionNode, com.yahoo.searchlib.rankingexpression.rule.ArithmeticOperator, com.yahoo.searchlib.rankingexpression.rule.ExpressionNode)" ], "fields": [] }, @@ -1295,6 +1296,7 @@ "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", "public com.yahoo.searchlib.rankingexpression.evaluation.Value evaluate(com.yahoo.searchlib.rankingexpression.evaluation.Context)", "public com.yahoo.searchlib.rankingexpression.rule.ComparisonNode setChildren(java.util.List)", + "public int hashCode()", "public bridge synthetic com.yahoo.searchlib.rankingexpression.rule.CompositeNode setChildren(java.util.List)" ], "fields": [] @@ -1327,7 +1329,8 @@ "public java.lang.StringBuilder toString(java.lang.StringBuilder, com.yahoo.searchlib.rankingexpression.rule.SerializationContext, java.util.Deque, com.yahoo.searchlib.rankingexpression.rule.CompositeNode)", "public java.lang.String sourceString()", "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", - "public com.yahoo.searchlib.rankingexpression.evaluation.Value evaluate(com.yahoo.searchlib.rankingexpression.evaluation.Context)" + "public com.yahoo.searchlib.rankingexpression.evaluation.Value evaluate(com.yahoo.searchlib.rankingexpression.evaluation.Context)", + "public int hashCode()" ], "fields": [] }, @@ -1345,7 +1348,8 @@ "public java.lang.StringBuilder toString(java.lang.StringBuilder, com.yahoo.searchlib.rankingexpression.rule.SerializationContext, java.util.Deque, com.yahoo.searchlib.rankingexpression.rule.CompositeNode)", "public com.yahoo.searchlib.rankingexpression.evaluation.Value evaluate(com.yahoo.searchlib.rankingexpression.evaluation.Context)", "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", - "public com.yahoo.searchlib.rankingexpression.rule.CompositeNode setChildren(java.util.List)" + "public com.yahoo.searchlib.rankingexpression.rule.CompositeNode setChildren(java.util.List)", + "public int hashCode()" ], "fields": [] }, @@ -1360,13 +1364,13 @@ ], "methods": [ "public void <init>()", - "public int hashCode()", - "public final boolean equals(java.lang.Object)", - "public final java.lang.String toString()", - "public final java.lang.StringBuilder toString(com.yahoo.searchlib.rankingexpression.rule.SerializationContext)", "public abstract java.lang.StringBuilder toString(java.lang.StringBuilder, com.yahoo.searchlib.rankingexpression.rule.SerializationContext, java.util.Deque, com.yahoo.searchlib.rankingexpression.rule.CompositeNode)", "public abstract com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", - "public abstract com.yahoo.searchlib.rankingexpression.evaluation.Value evaluate(com.yahoo.searchlib.rankingexpression.evaluation.Context)" + "public abstract com.yahoo.searchlib.rankingexpression.evaluation.Value evaluate(com.yahoo.searchlib.rankingexpression.evaluation.Context)", + "public final java.lang.StringBuilder toString(com.yahoo.searchlib.rankingexpression.rule.SerializationContext)", + "public final boolean equals(java.lang.Object)", + "public abstract int hashCode()", + "public final java.lang.String toString()" ], "fields": [] }, @@ -1438,6 +1442,7 @@ "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", "public com.yahoo.searchlib.rankingexpression.evaluation.Value evaluate(com.yahoo.searchlib.rankingexpression.evaluation.Context)", "public com.yahoo.searchlib.rankingexpression.rule.FunctionNode setChildren(java.util.List)", + "public int hashCode()", "public bridge synthetic com.yahoo.searchlib.rankingexpression.rule.CompositeNode setChildren(java.util.List)" ], "fields": [] @@ -1472,12 +1477,13 @@ ], "methods": [ "public void <init>(com.yahoo.tensor.TensorType, com.yahoo.searchlib.rankingexpression.rule.ExpressionNode)", + "public com.yahoo.searchlib.rankingexpression.rule.GeneratorLambdaFunctionNode$LongListToDoubleLambda asLongListToDoubleOperator()", "public java.util.List children()", "public com.yahoo.searchlib.rankingexpression.rule.CompositeNode setChildren(java.util.List)", "public java.lang.StringBuilder toString(java.lang.StringBuilder, com.yahoo.searchlib.rankingexpression.rule.SerializationContext, java.util.Deque, com.yahoo.searchlib.rankingexpression.rule.CompositeNode)", "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", "public com.yahoo.searchlib.rankingexpression.evaluation.Value evaluate(com.yahoo.searchlib.rankingexpression.evaluation.Context)", - "public com.yahoo.searchlib.rankingexpression.rule.GeneratorLambdaFunctionNode$LongListToDoubleLambda asLongListToDoubleOperator()" + "public int hashCode()" ], "fields": [] }, @@ -1500,6 +1506,7 @@ "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", "public com.yahoo.searchlib.rankingexpression.evaluation.Value evaluate(com.yahoo.searchlib.rankingexpression.evaluation.Context)", "public com.yahoo.searchlib.rankingexpression.rule.IfNode setChildren(java.util.List)", + "public int hashCode()", "public bridge synthetic com.yahoo.searchlib.rankingexpression.rule.CompositeNode setChildren(java.util.List)" ], "fields": [] @@ -1518,7 +1525,8 @@ "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", "public com.yahoo.searchlib.rankingexpression.evaluation.Value evaluate(com.yahoo.searchlib.rankingexpression.evaluation.Context)", "public java.util.function.DoubleUnaryOperator asDoubleUnaryOperator()", - "public java.util.function.DoubleBinaryOperator asDoubleBinaryOperator()" + "public java.util.function.DoubleBinaryOperator asDoubleBinaryOperator()", + "public int hashCode()" ], "fields": [] }, @@ -1534,7 +1542,8 @@ "public java.lang.String getValue()", "public java.lang.StringBuilder toString(java.lang.StringBuilder, com.yahoo.searchlib.rankingexpression.rule.SerializationContext, java.util.Deque, com.yahoo.searchlib.rankingexpression.rule.CompositeNode)", "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", - "public com.yahoo.searchlib.rankingexpression.evaluation.Value evaluate(com.yahoo.searchlib.rankingexpression.evaluation.Context)" + "public com.yahoo.searchlib.rankingexpression.evaluation.Value evaluate(com.yahoo.searchlib.rankingexpression.evaluation.Context)", + "public int hashCode()" ], "fields": [] }, @@ -1552,6 +1561,7 @@ "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", "public com.yahoo.searchlib.rankingexpression.evaluation.Value evaluate(com.yahoo.searchlib.rankingexpression.evaluation.Context)", "public com.yahoo.searchlib.rankingexpression.rule.NegativeNode setChildren(java.util.List)", + "public int hashCode()", "public bridge synthetic com.yahoo.searchlib.rankingexpression.rule.CompositeNode setChildren(java.util.List)" ], "fields": [] @@ -1570,6 +1580,7 @@ "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", "public com.yahoo.searchlib.rankingexpression.evaluation.Value evaluate(com.yahoo.searchlib.rankingexpression.evaluation.Context)", "public com.yahoo.searchlib.rankingexpression.rule.NotNode setChildren(java.util.List)", + "public int hashCode()", "public bridge synthetic com.yahoo.searchlib.rankingexpression.rule.CompositeNode setChildren(java.util.List)" ], "fields": [] @@ -1644,6 +1655,7 @@ "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", "public com.yahoo.searchlib.rankingexpression.evaluation.Value evaluate(com.yahoo.searchlib.rankingexpression.evaluation.Context)", "public com.yahoo.searchlib.rankingexpression.rule.SetMembershipNode setChildren(java.util.List)", + "public int hashCode()", "public bridge synthetic com.yahoo.searchlib.rankingexpression.rule.CompositeNode setChildren(java.util.List)" ], "fields": [] @@ -1663,6 +1675,7 @@ "public java.util.Optional asScalarFunction()", "public com.yahoo.tensor.Tensor evaluate(com.yahoo.tensor.evaluation.EvaluationContext)", "public java.lang.String toString()", + "public int hashCode()", "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)" ], "fields": [] @@ -1685,7 +1698,8 @@ "public static java.util.Map wrapScalars(java.util.Map)", "public static void wrapScalarBlock(com.yahoo.tensor.TensorType, java.util.List, java.lang.String, java.util.List, java.util.Map)", "public static java.util.List wrapScalars(com.yahoo.tensor.TensorType, java.util.List, java.util.List)", - "public static com.yahoo.tensor.functions.ScalarFunction wrapScalar(com.yahoo.searchlib.rankingexpression.rule.ExpressionNode)" + "public static com.yahoo.tensor.functions.ScalarFunction wrapScalar(com.yahoo.searchlib.rankingexpression.rule.ExpressionNode)", + "public int hashCode()" ], "fields": [] }, diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/FS4Hit.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/FS4Hit.java index 5d606493652..5bf161b380f 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/FS4Hit.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/FS4Hit.java @@ -18,14 +18,12 @@ public class FS4Hit extends Hit { private GlobalId globalId = new GlobalId(new byte[GlobalId.LENGTH]); private int distributionKey = -1; - /** - * Constructs an empty result node. - */ + /** Constructs an empty result node. */ public FS4Hit() { } /** - * Create a hit with the given path and document id. + * Creates a hit with the given path and document id. * * @param path The mangled search node path. * @param globalId The local document id. @@ -36,7 +34,7 @@ public class FS4Hit extends Hit { } /** - * Create a hit with the given path and document id. + * Creates a hit with the given path and document id. * * @param path The mangled search node path. * @param globalId The local document id. @@ -50,29 +48,17 @@ public class FS4Hit extends Hit { this.distributionKey = distributionKey; } - /** - * Obtain the (mangled) network path back to the search node returning this hit. - * - * @return The mangled search node path. - */ + /** Returns the (mangled) network path back to the search node returning this hit. */ public int getPath() { return path; } - /** - * Obtain the global document id on the search node returning this hit. - * - * @return The global document id. - */ + /** Returns the global document id on the search node returning this hit. */ public GlobalId getGlobalId() { return globalId; } - /** - * Obtain the distribution key for the node producing this hit. - * - * @return distribution key - */ + /** Returns the distribution key for the node producing this hit. */ public int getDistributionKey() { return distributionKey; } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/ForceLoad.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/ForceLoad.java index 729b070dd57..74d498d33f7 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/ForceLoad.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/ForceLoad.java @@ -28,11 +28,11 @@ public class ForceLoad { "hll.SparseSketch", "hll.NormalSketch" }; - com.yahoo.system.ForceLoad.forceLoad(pkg, classes, - ForceLoad.class.getClassLoader()); + com.yahoo.system.ForceLoad.forceLoad(pkg, classes, ForceLoad.class.getClassLoader()); } public static boolean forceLoad() { return true; } + } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Grouping.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Grouping.java index 25b3cb18ff9..dcbedf924b4 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Grouping.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Grouping.java @@ -23,10 +23,10 @@ public class Grouping extends Identifiable { // The client id for this grouping request. private int id = 0; - // Whether or not this grouping is valid. + // Whether this grouping is valid. private boolean valid = true; - // Whether or not to group all hits or only those with hits. Only applicable for streaming search. + // Whether to group all hits or only those with hits. Only applicable for streaming search. private boolean all = false; // How many hits to group per backend node. @@ -46,18 +46,17 @@ public class Grouping extends Identifiable { // Actual root group, does not require level details. private Group root = new Group(); - /** - * <p>Constructs an empty result node. <b>NOTE:</b> This instance is broken until non-optional member data is - * set.</p> - */ + private boolean postMergeCompleted = false; + + /** Constructs an empty result node. <b>NOTE:</b> This instance is broken until non-optional member data is set. */ public Grouping() { super(); } /** - * <p>Constructs an instance of this class with given client id.</p> + * Constructs an instance of this class with given client id. * - * @param id The client id for this grouping request. + * @param id the client id for this grouping request */ public Grouping(int id) { super(); @@ -65,98 +64,82 @@ public class Grouping extends Identifiable { } /** - * <p>Merges the content of the given grouping <b>into</b> this.</p> + * Merges the content of the given grouping <b>into</b> this. * - * @param rhs The grouping to merge with. + * @param rhs the grouping to merge with */ public void merge(Grouping rhs) { root.merge(firstLevel, 0, rhs.root); } /** - * <p>This method is invoked after merging is done. It is intended used for resolving any dependencies or derivates - * that might have changes due to the merge.</p> + * Invoked after merging is done. It is intended used for resolving any dependencies or derivates + * that might have changes due to the merge. */ public void postMerge() { + if (postMergeCompleted) return; root.postMerge(groupingLevels, firstLevel, 0); + postMergeCompleted = true; } - /** - * <p>Returns the client id of this grouping request.</p> - * - * @return The identifier. - */ + /** Returns the client id of this grouping request. */ public int getId() { return id; } /** - * <p>Sets the client id for this grouping request.</p> + * Sets the client id for this grouping request. * - * @param id The identifier to set. - * @return This, to allow chaining. + * @param id the identifier to set + * @return this, to allow chaining */ public Grouping setId(int id) { this.id = id; return this; } - /** - * <p>Returns whether or not this grouping request is valid.</p> - * - * @return True if valid. - */ + /** Returns whether this grouping request is valid. */ public boolean valid() { return valid; } /** - * <p>Returns whether or not to perform grouping on the entire document corpus instead of only those matching the - * search criteria. Please see note on {@link #setAll(boolean)}.</p> - * - * @return True if grouping all documents. + * Returns whether to perform grouping on the entire document corpus instead of only those matching the + * search criteria. Please see note on {@link #setAll(boolean)}. */ public boolean getAll() { return all; } /** - * <p>Sets whether or not to perform grouping on the entire document corpus instead of only those matching the - * search criteria. <b>NOTE:</b> This is only possible with streaming search.</p> + * Sets whether to perform grouping on the entire document corpus instead of only those matching the + * search criteria. <b>NOTE:</b> This is only possible with streaming search. * - * @param all True to group all documents. - * @return This, to allow chaining. + * @param all true to group all documents + * @return this, to allow chaining */ public Grouping setAll(boolean all) { this.all = all; return this; } - /** - * <p>Returns the number of candidate documents to group.</p> - * - * @return The number. - */ + /** Returns the number of candidate documents to group. */ public long getTopN() { return topN; } /** - * <p>Sets the number of candidate documents to group.</p> + * Sets the number of candidate documents to group. * - * @param topN The number to set. - * @return This, to allow chaining. + * @param topN the number to set + * @return this, to allow chaining */ public Grouping setTopN(long topN) { this.topN = topN; return this; } - /** - * <p>Returns the first level to start grouping work. See note on {@link #setFirstLevel(int)}.</p> - * - * @return The first level. - */ + /** Returns the first level to start grouping work. See note on {@link #setFirstLevel(int)}. */ public int getFirstLevel() { return firstLevel; } @@ -191,20 +174,16 @@ public class Grouping extends Identifiable { return this; } - /** - * <p>Returns the list of grouping levels that make up this grouping request.</p> - * - * @return The list. - */ + /** Returns the list of grouping levels that make up this grouping request. */ public List<GroupingLevel> getLevels() { return groupingLevels; } /** - * <p>Appends the given grouping level specification to the list of levels.</p> + * Appends the given grouping level specification to the list of levels. * - * @param level The level to add. - * @return This, to allow chaining. + * @param level the level to add + * @return this, to allow chaining * @throws NullPointerException If <code>level</code> argument is null. */ public Grouping addLevel(GroupingLevel level) { @@ -213,21 +192,17 @@ public class Grouping extends Identifiable { return this; } - /** - * <p>Returns the root group.</p> - * - * @return The root. - */ + /** Returns the root group. */ public Group getRoot() { return root; } /** - * <p>Sets the root group.</p> + * Sets the root group. * - * @param root The group to set as root. - * @return This, to allow chaining. - * @throws NullPointerException If <code>root</code> argument is null. + * @param root the group to set as root + * @return this, to allow chaining + * @throws NullPointerException If <code>root</code> argument is null */ public Grouping setRoot(Group root) { root.getClass(); // throws NullPointerException diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/GroupingLevel.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/GroupingLevel.java index c166aeb173f..0db933966a8 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/GroupingLevel.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/GroupingLevel.java @@ -15,7 +15,7 @@ public class GroupingLevel extends Identifiable { // The maximum number of groups allowed at this level. private long maxGroups = -1; - // The precsicion used for estimation. This is number of groups returned up when using orderby that need more info to get it correct. + // The precision used for estimation. This is number of groups returned up when using orderby that need more info to get it correct. private long precision = -1; // The classifier expression; the result of this is the group key. @@ -24,30 +24,17 @@ public class GroupingLevel extends Identifiable { // The prototype of the groups to create for each class. private Group collect = new Group(); - /** - * <p>Returns the presicion (i.e number of groups) returned up from this level.</p> - * - * @return The precision. - */ + /** Returns the precision (i.e number of groups) returned up from this level. */ public long getPrecision() { return precision; } - /** - * <p>Returns the maximum number of groups allowed at this level.</p> - * - * @return The maximum number. - */ + /** Returns the maximum number of groups allowed at this level. */ public long getMaxGroups() { return maxGroups; } - /** - * <p>Sets the maximum number of groups allowed at this level.</p> - * - * @param max The maximum number to set. - * @return This, to allow chaining. - */ + /** Sets the maximum number of groups allowed at this level. */ public GroupingLevel setMaxGroups(long max) { maxGroups = max; if (precision < maxGroups) { @@ -57,31 +44,22 @@ public class GroupingLevel extends Identifiable { } /** - * <p>Sets the presicion (i.e number of groups) returned up from this level.</p> + * Sets the precision (i.e number of groups) returned up from this level. * - * @param precision The precision to set. - * @return This, to allow chaining. + * @param precision the precision to set + * @return this, to allow chaining */ public GroupingLevel setPrecision(long precision) { this.precision = precision; return this; } - /** - * <p>Returns the expression used to classify hits into groups.</p> - * - * @return The classifier expression. - */ + /** Returns the expression used to classify hits into groups. */ public ExpressionNode getExpression() { return classify; } - /** - * <p>Sets the expression used to classify hits into groups.</p> - * - * @param exp The classifier expression to set. - * @return This, to allow chaining. - */ + /** Sets the expression used to classify hits into groups. */ public GroupingLevel setExpression(ExpressionNode exp) { classify = exp; return this; diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/HitsAggregationResult.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/HitsAggregationResult.java index b3c861cc140..ea39dde92e1 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/HitsAggregationResult.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/HitsAggregationResult.java @@ -23,7 +23,7 @@ public class HitsAggregationResult extends AggregationResult { public static final int classId = registerClass(0x4000 + 87, HitsAggregationResult.class); private String summaryClass = "default"; private int maxHits = -1; - private List<Hit> hits = new ArrayList<Hit>(); + private List<Hit> hits = new ArrayList<>(); /** * Constructs an empty result node. diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/RawData.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/RawData.java index 8c830fa4f25..7a343da0165 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/RawData.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/RawData.java @@ -7,8 +7,8 @@ import com.yahoo.vespa.objects.Serializer; import java.util.Arrays; /** - * <p>This class encapsulates a byte array into a cloneable and comparable object. It also implements a sane {@link - * #hashCode()} and {@link #toString()}.</p> + * A byte array a a cloneable and comparable object. It also implements a sane {@link + * #hashCode()} and {@link #toString()}. * * @author Simon Thoresen Hult */ @@ -16,26 +16,24 @@ public class RawData implements Cloneable, Comparable<RawData> { private byte[] data; - /** - * <p>Constructs an empty data object.</p> - */ + /** Constructs an empty data object. */ public RawData() { data = new byte[0]; } /** - * <p>Constructs a raw data object that holds the given byte array.</p> + * Constructs a raw data object that holds the given byte array. * - * @param data The rank to set. + * @param data the rank to set */ public RawData(byte[] data) { setData(data); } /** - * <p>Serializes the content of this data into the given byte buffer.</p> + * Serializes the content of this data into the given byte buffer. * - * @param buf The buffer to serialize to. + * @param buf the buffer to serialize to */ public void serialize(Serializer buf) { buf.putInt(null, data.length); @@ -43,9 +41,9 @@ public class RawData implements Cloneable, Comparable<RawData> { } /** - * <p>Deserializes the content for this data from the given byte buffer.</p> + * Deserializes the content for this data from the given byte buffer. * - * @param buf The buffer to deserialize from. + * @param buf the buffer to deserialize from */ public void deserialize(Deserializer buf) { int len = buf.getInt(null); @@ -53,20 +51,20 @@ public class RawData implements Cloneable, Comparable<RawData> { } /** - * <p>Returns the byte array that constitutes this data.</p> + * Returns the byte array that constitutes this data. * - * @return The byte array. + * @return the byte array */ public byte[] getData() { return data; } /** - * <p>Sets the byte array that constitutes this data. This does <b>not</b> copy the given array, it simply assigns - * it to this.</p> + * Sets the byte array that constitutes this data. This does <b>not</b> copy the given array, it simply assigns + * it to this. * - * @param data The data to set. - * @return This, to allow chaining. + * @param data the data to set + * @return this, to allow chaining */ public RawData setData(byte[] data) { if (data == null) { @@ -109,11 +107,11 @@ public class RawData implements Cloneable, Comparable<RawData> { } /** - * <p>Implements comparison of two byte arrays.</p> + * Implements comparison of two byte arrays. * - * @param lhs The left-hand-side of the comparison. - * @param rhs The right-hand-side of the comparison. - * @return The result of comparing the two byte arrays. + * @param lhs the left-hand-side of the comparison + * @param rhs the right-hand-side of the comparison + * @return the result of comparing the two byte arrays */ public static int compare(byte[] lhs, byte[] rhs) { int cmp = 0; @@ -127,4 +125,5 @@ public class RawData implements Cloneable, Comparable<RawData> { } return cmp; } + } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/BooleanValue.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/BooleanValue.java index 07710797ee2..770be98c739 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/BooleanValue.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/BooleanValue.java @@ -6,11 +6,10 @@ package com.yahoo.searchlib.rankingexpression.evaluation; * In numerical context true is interpreted as 1 and false as 0. * * @author bratseth - * @since 5.1.21 */ public class BooleanValue extends DoubleCompatibleValue { - private boolean value; + private final boolean value; /** * Create a boolean value which is frozen at the outset. diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java index 6490c69894f..207603c5038 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java @@ -97,6 +97,7 @@ public abstract class Value { @Override public abstract boolean equals(Object other); + /** Returns a hash which only depends on the content of this value. */ @Override public abstract int hashCode(); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTForestNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTForestNode.java index 13db51c1363..c8b20e774b5 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTForestNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTForestNode.java @@ -11,7 +11,9 @@ import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.TypeContext; +import java.util.Arrays; import java.util.Deque; +import java.util.Objects; /** * An optimized version of a sum of consecutive decision trees. @@ -42,8 +44,12 @@ public class GBDTForestNode extends ExpressionNode { } /** Returns (optimized sum of condition trees) */ + @Override public StringBuilder toString(StringBuilder string, SerializationContext context, Deque<String> path, CompositeNode parent) { return string.append("(optimized sum of condition trees of size ").append(values.length*8).append(" bytes)"); } + @Override + public int hashCode() { return Objects.hash("gbdtForest", Arrays.hashCode(values)); } + } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTNode.java index 6c6166c2869..949e1f026f7 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTNode.java @@ -11,7 +11,9 @@ import com.yahoo.searchlib.rankingexpression.rule.SerializationContext; import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.TypeContext; +import java.util.Arrays; import java.util.Deque; +import java.util.Objects; /** * An optimized version of a decision tree. @@ -105,4 +107,8 @@ public final class GBDTNode extends ExpressionNode { public StringBuilder toString(StringBuilder string, SerializationContext context, Deque<String> path, CompositeNode parent) { return string.append("(optimized condition tree)"); } + + @Override + public int hashCode() { return Objects.hash("gbdtNode", Arrays.hashCode(values)); } + } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ArithmeticNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ArithmeticNode.java index 55fe66be69e..580f42e67cb 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ArithmeticNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ArithmeticNode.java @@ -13,6 +13,7 @@ import java.util.ArrayList; import java.util.Deque; import java.util.Iterator; import java.util.List; +import java.util.Objects; /** * A binary mathematical operation @@ -115,6 +116,16 @@ public final class ArithmeticNode extends CompositeNode { lhs.value = rhs.op.evaluate(lhs.value, rhs.value); } + @Override + public CompositeNode setChildren(List<ExpressionNode> newChildren) { + if (children.size() != newChildren.size()) + throw new IllegalArgumentException("Expected " + children.size() + " children but got " + newChildren.size()); + return new ArithmeticNode(newChildren, operators); + } + + @Override + public int hashCode() { return Objects.hash(children, operators); } + public static ArithmeticNode resolve(ExpressionNode left, ArithmeticOperator op, ExpressionNode right) { if ( ! (left instanceof ArithmeticNode)) return new ArithmeticNode(left, op, right); @@ -140,12 +151,5 @@ public final class ArithmeticNode extends CompositeNode { } } - @Override - public CompositeNode setChildren(List<ExpressionNode> newChildren) { - if (children.size() != newChildren.size()) - throw new IllegalArgumentException("Expected " + children.size() + " children but got " + newChildren.size()); - return new ArithmeticNode(newChildren, operators); - } - } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ComparisonNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ComparisonNode.java index 600d3b8d408..e726a351f74 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ComparisonNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ComparisonNode.java @@ -9,6 +9,7 @@ import com.yahoo.tensor.evaluation.TypeContext; import java.util.Deque; import java.util.List; +import java.util.Objects; /** * A node which returns the outcome of a comparison. @@ -62,4 +63,7 @@ public class ComparisonNode extends BooleanNode { return new ComparisonNode(children.get(0), operator, children.get(1)); } + @Override + public int hashCode() { return Objects.hash(operator, conditions); } + } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ConstantNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ConstantNode.java index ffbeec37c78..46e833197f9 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ConstantNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ConstantNode.java @@ -8,6 +8,7 @@ import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.TypeContext; import java.util.Deque; +import java.util.Objects; /** * A node which holds a constant (frozen) value. @@ -55,4 +56,7 @@ public final class ConstantNode extends ExpressionNode { return value; } + @Override + public int hashCode() { return Objects.hash("constantNode", value); } + } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/EmbracedNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/EmbracedNode.java index 9d389a4f6e9..64a1f42a7ba 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/EmbracedNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/EmbracedNode.java @@ -10,6 +10,7 @@ import com.yahoo.tensor.evaluation.TypeContext; import java.util.Collections; import java.util.Deque; import java.util.List; +import java.util.Objects; /** * This class represents another expression enclosed in braces. @@ -60,4 +61,7 @@ public final class EmbracedNode extends CompositeNode { return new EmbracedNode(newChildren.get(0)); } + @Override + public int hashCode() { return Objects.hash("embraced", value); } + } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ExpressionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ExpressionNode.java index 8e00be3f056..51067930dd0 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ExpressionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/ExpressionNode.java @@ -18,31 +18,13 @@ import java.util.Deque; */ public abstract class ExpressionNode implements Serializable { - @Override - public int hashCode() { - return toString().hashCode(); - } - - @Override - public final boolean equals(Object obj) { - return obj instanceof ExpressionNode && toString().equals(obj.toString()); - } - - @Override - public final String toString() { - return toString(new SerializationContext()).toString(); - } - public final StringBuilder toString(SerializationContext context) { - return toString(new StringBuilder(), context, null, null); - } - /** - * Returns a script instance of this based on the supplied script functions. + * Returns this in serialized form. * * @param builder the StringBuilder that will be appended to * @param context the serialization context * @param path the call path to this, used for cycle detection, or null if this is a root - * @param parent the parent node of this, or null if it a root + * @param parent the parent node of this, or null if it is a root * @return the main script, referring to script instances. */ public abstract StringBuilder toString(StringBuilder builder, SerializationContext context, Deque<String> path, CompositeNode parent); @@ -63,4 +45,22 @@ public abstract class ExpressionNode implements Serializable { */ public abstract Value evaluate(Context context); + public final StringBuilder toString(SerializationContext context) { + return toString(new StringBuilder(), context, null, null); + } + + @Override + public final boolean equals(Object obj) { + return obj instanceof ExpressionNode && toString().equals(obj.toString()); + } + + /** Returns a hashcode computed from the data in this */ + @Override + public abstract int hashCode(); + + @Override + public final String toString() { + return toString(new SerializationContext()).toString(); + } + } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionNode.java index d32cfb51f95..5e8bfc245a7 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/FunctionNode.java @@ -14,6 +14,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.Deque; import java.util.List; +import java.util.Objects; /** * Invocation of a native function. @@ -108,4 +109,7 @@ public final class FunctionNode extends CompositeNode { return new FunctionNode(function, children.get(0), children.get(1)); } + @Override + public int hashCode() { return Objects.hash("functionNode", function, arguments); } + } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/GeneratorLambdaFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/GeneratorLambdaFunctionNode.java index 7ff3a71d036..8d858341976 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/GeneratorLambdaFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/GeneratorLambdaFunctionNode.java @@ -11,6 +11,7 @@ import com.yahoo.tensor.evaluation.TypeContext; import java.util.Collections; import java.util.Deque; import java.util.List; +import java.util.Objects; /** * A tensor generating function, whose arguments are determined by a tensor type @@ -31,6 +32,11 @@ public class GeneratorLambdaFunctionNode extends CompositeNode { this.generator = generator; } + /** Returns this as an operator which converts a list of integers into a double. */ + public LongListToDoubleLambda asLongListToDoubleOperator() { + return new LongListToDoubleLambda(); + } + @Override public List<ExpressionNode> children() { return Collections.singletonList(generator); @@ -57,12 +63,8 @@ public class GeneratorLambdaFunctionNode extends CompositeNode { return generator.evaluate(context); } - /** - * Returns this as an operator which converts a list of integers into a double - */ - public LongListToDoubleLambda asLongListToDoubleOperator() { - return new LongListToDoubleLambda(); - } + @Override + public int hashCode() { return Objects.hash("generator", type, generator); } private class LongListToDoubleLambda implements java.util.function.Function<List<Long>, Double> { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/IfNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/IfNode.java index 02d437e83bf..6f46222c1d8 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/IfNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/IfNode.java @@ -9,6 +9,7 @@ import com.yahoo.tensor.evaluation.TypeContext; import java.util.Deque; import java.util.List; +import java.util.Objects; /** * A conditional branch of a ranking expression. @@ -17,8 +18,9 @@ import java.util.List; * @author bratseth */ public final class IfNode extends CompositeNode { - /** [condition, trueExpression, falseExpression]*/ - private final List<ExpressionNode> asList; + + /** [condition, trueExpression, falseExpression] */ + private final List<ExpressionNode> arguments; private final Double trueProbability; public IfNode(ExpressionNode condition, ExpressionNode trueExpression, ExpressionNode falseExpression) { @@ -39,19 +41,19 @@ public final class IfNode extends CompositeNode { if (trueProbability != null && ( trueProbability < 0.0 || trueProbability > 1.0) ) throw new IllegalArgumentException("trueProbability must be a between 0.0 and 1.0, not " + trueProbability); this.trueProbability = trueProbability; - this.asList = List.of(condition, trueExpression, falseExpression); + this.arguments = List.of(condition, trueExpression, falseExpression); } @Override public List<ExpressionNode> children() { - return asList; + return arguments; } - public ExpressionNode getCondition() { return asList.get(0); } + public ExpressionNode getCondition() { return arguments.get(0); } - public ExpressionNode getTrueExpression() { return asList.get(1); } + public ExpressionNode getTrueExpression() { return arguments.get(1); } - public ExpressionNode getFalseExpression() { return asList.get(2); } + public ExpressionNode getFalseExpression() { return arguments.get(2); } /** The average probability that the condition of this node will evaluate to true, or null if not known */ public Double getTrueProbability() { return trueProbability; } @@ -95,4 +97,7 @@ public final class IfNode extends CompositeNode { return new IfNode(children.get(0), children.get(1), children.get(2)); } + @Override + public int hashCode() { return Objects.hash("if", arguments, trueProbability); } + } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/LambdaFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/LambdaFunctionNode.java index a2b86360923..9f07f146264 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/LambdaFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/LambdaFunctionNode.java @@ -13,6 +13,7 @@ import java.util.Collections; import java.util.Deque; import java.util.HashSet; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.DoubleBinaryOperator; @@ -162,6 +163,8 @@ public class LambdaFunctionNode extends CompositeNode { return Set.of(); } + @Override + public int hashCode() { return Objects.hash("lambdaFunction", arguments, functionExpression); } private class DoubleUnaryLambda implements DoubleUnaryOperator { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/NameNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/NameNode.java index 34c8664c0cf..fec643e81df 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/NameNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/NameNode.java @@ -8,6 +8,7 @@ import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.TypeContext; import java.util.Deque; +import java.util.Objects; /** * An opaque name in a ranking expression. This is used to represent names passed to the context @@ -41,4 +42,7 @@ public final class NameNode extends ExpressionNode { throw new RuntimeException("Name nodes should never be evaluated"); } + @Override + public int hashCode() { return Objects.hash("name", name); } + } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/NegativeNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/NegativeNode.java index 9516f38a155..8d2cf7b6387 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/NegativeNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/NegativeNode.java @@ -10,6 +10,7 @@ import com.yahoo.tensor.evaluation.TypeContext; import java.util.Collections; import java.util.Deque; import java.util.List; +import java.util.Objects; /** * A node which flips the sign of the value produced from the nested expression @@ -54,4 +55,7 @@ public class NegativeNode extends CompositeNode { return new NegativeNode(children.get(0)); } + @Override + public int hashCode() { return Objects.hash("negative", value); } + } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/NotNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/NotNode.java index 8b5ae256038..ac3566c26af 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/NotNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/NotNode.java @@ -10,6 +10,7 @@ import com.yahoo.tensor.evaluation.TypeContext; import java.util.Collections; import java.util.Deque; import java.util.List; +import java.util.Objects; /** * A node which flips the logical value produced from the nested expression. @@ -55,5 +56,8 @@ public class NotNode extends BooleanNode { return new NotNode(children.get(0)); } + @Override + public int hashCode() { return Objects.hash("not", value); } + } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SetMembershipNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SetMembershipNode.java index 31f3013b756..5da2fbfe624 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SetMembershipNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SetMembershipNode.java @@ -14,6 +14,7 @@ import com.yahoo.tensor.evaluation.TypeContext; import java.util.ArrayList; import java.util.Deque; import java.util.List; +import java.util.Objects; import java.util.function.Predicate; /** @@ -100,4 +101,7 @@ public class SetMembershipNode extends BooleanNode { return new SetMembershipNode(children.get(0), children.subList(1, children.size())); } + @Override + public int hashCode() { return Objects.hash("setMembership", testValue, setValues); } + } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java index ce5832027b7..7b68ad7e2af 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/TensorFunctionNode.java @@ -144,6 +144,9 @@ public class TensorFunctionNode extends CompositeNode { return new ExpressionScalarFunction(node); } + @Override + public int hashCode() { return function.hashCode(); } + private static class ExpressionScalarFunction implements ScalarFunction<Reference> { private final ExpressionNode expression; @@ -251,6 +254,9 @@ public class TensorFunctionNode extends CompositeNode { } @Override + public int hashCode() { return expression.hashCode(); } + + @Override public String toString(ToStringContext<Reference> c) { ToStringContext<Reference> outermost = c; while (outermost.parent() != null) diff --git a/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp b/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp index 8879d591f5a..4877072eacd 100644 --- a/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp +++ b/searchlib/src/tests/queryeval/simple_phrase/simple_phrase_test.cpp @@ -9,6 +9,7 @@ #include <vespa/searchlib/fef/matchdatalayout.h> #include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/query/weight.h> +#include <vespa/vespalib/util/testclock.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/log/log.h> @@ -211,8 +212,8 @@ void Test::requireThatIteratorHonorsFutureDoom() { test.addTerm("foo", 0).addTerm("bar", 1); test.fetchPostings(false); - vespalib::Clock clock; - vespalib::Doom futureDoom(clock, vespalib::steady_time::max()); + vespalib::TestClock clock; + vespalib::Doom futureDoom(clock.clock(), vespalib::steady_time::max()); unique_ptr<SearchIterator> search(test.createSearch(false)); static_cast<SimplePhraseSearch &>(*search).setDoom(&futureDoom); EXPECT_TRUE(!search->seek(1u)); @@ -225,8 +226,8 @@ void Test::requireThatIteratorHonorsDoom() { test.addTerm("foo", 0).addTerm("bar", 1); test.fetchPostings(false); - vespalib::Clock clock; - vespalib::Doom futureDoom(clock, vespalib::steady_time()); + vespalib::TestClock clock; + vespalib::Doom futureDoom(clock.clock(), vespalib::steady_time()); unique_ptr<SearchIterator> search(test.createSearch(false)); static_cast<SimplePhraseSearch &>(*search).setDoom(&futureDoom); EXPECT_TRUE(!search->seek(1u)); diff --git a/searchlib/src/tests/rankingexpression/rankingexpressionlist b/searchlib/src/tests/rankingexpression/rankingexpressionlist index def378069a8..c032ffd1a01 100644 --- a/searchlib/src/tests/rankingexpression/rankingexpressionlist +++ b/searchlib/src/tests/rankingexpression/rankingexpressionlist @@ -165,3 +165,5 @@ if(1.09999~=1.1,2,3); if (1.09999 ~= 1.1, 2, 3) 1 && 0 || 1 !a && (a || a) 10 ^ 3 +true +false diff --git a/searchlib/src/tests/sortspec/multilevelsort.cpp b/searchlib/src/tests/sortspec/multilevelsort.cpp index 576e1d1336c..88b105ec80c 100644 --- a/searchlib/src/tests/sortspec/multilevelsort.cpp +++ b/searchlib/src/tests/sortspec/multilevelsort.cpp @@ -7,6 +7,7 @@ #include <vespa/searchlib/attribute/attributecontext.h> #include <vespa/searchlib/attribute/attributemanager.h> #include <vespa/searchlib/uca/ucaconverter.h> +#include <vespa/vespalib/util/testclock.h> #include <vespa/vespalib/testkit/testapp.h> #include <vespa/log/log.h> LOG_SETUP("multilevelsort_test"); @@ -14,13 +15,9 @@ LOG_SETUP("multilevelsort_test"); using namespace search; typedef FastS_SortSpec::VectorRef VectorRef; -typedef IntegerAttributeTemplate<uint8_t> Uint8; typedef IntegerAttributeTemplate<int8_t> Int8; -typedef IntegerAttributeTemplate<uint16_t> Uint16; typedef IntegerAttributeTemplate<int16_t> Int16; -typedef IntegerAttributeTemplate<uint32_t> Uint32; typedef IntegerAttributeTemplate<int32_t> Int32; -typedef IntegerAttributeTemplate<uint64_t> Uint64; typedef IntegerAttributeTemplate<int64_t> Int64; typedef FloatingPointAttributeTemplate<float> Float; typedef FloatingPointAttributeTemplate<double> Double; @@ -239,8 +236,8 @@ MultilevelSortTest::sortAndCheck(const std::vector<Spec> &spec, uint32_t num, hits[i]._rankValue = getRandomValue<uint32_t>(); } - vespalib::Clock clock; - vespalib::Doom doom(clock, vespalib::steady_time::max()); + vespalib::TestClock clock; + vespalib::Doom doom(clock.clock(), vespalib::steady_time::max()); search::uca::UcaConverterFactory ucaFactory; FastS_SortSpec sorter(7, doom, ucaFactory, _sortMethod); // init sorter with sort data @@ -396,8 +393,8 @@ TEST("require that all sort methods behave the same") } TEST("test that [docid] translates to [lid][paritionid]") { - vespalib::Clock clock; - vespalib::Doom doom(clock, vespalib::steady_time::max()); + vespalib::TestClock clock; + vespalib::Doom doom(clock.clock(), vespalib::steady_time::max()); search::uca::UcaConverterFactory ucaFactory; FastS_SortSpec asc(7, doom, ucaFactory); RankedHit hits[2] = {RankedHit(91, 0.0), RankedHit(3, 2.0)}; diff --git a/searchlib/src/vespa/searchlib/attribute/attributecontext.cpp b/searchlib/src/vespa/searchlib/attribute/attributecontext.cpp index 598fdc7ac40..e9a9f83d5db 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributecontext.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributecontext.cpp @@ -12,21 +12,24 @@ namespace search { const IAttributeVector * AttributeContext::getAttribute(AttributeMap & map, const string & name, bool stableEnum) const { + std::unique_lock guard(_cacheLock); AttributeMap::const_iterator itr = map.find(name); + std::shared_future<std::unique_ptr<attribute::AttributeReadGuard>> future_read_guard; if (itr != map.end()) { - if (itr->second) { - return itr->second->attribute(); - } else { - return nullptr; - } + future_read_guard = itr->second; + guard.unlock(); } else { - auto readGuard = _manager.getAttributeReadGuard(name, stableEnum); - const IAttributeVector *attribute = nullptr; - if (readGuard) { - attribute = readGuard->attribute(); - } - map[name] = std::move(readGuard); - return attribute; + std::promise<std::unique_ptr<attribute::AttributeReadGuard>> promise; + future_read_guard = promise.get_future().share(); + map[name] = future_read_guard; + guard.unlock(); + promise.set_value(_manager.getAttributeReadGuard(name, stableEnum)); + } + auto& read_guard = future_read_guard.get(); + if (read_guard) { + return read_guard->attribute(); + } else { + return nullptr; } } @@ -42,14 +45,12 @@ AttributeContext::~AttributeContext() = default; const IAttributeVector * AttributeContext::getAttribute(const string & name) const { - std::lock_guard<std::mutex> guard(_cacheLock); return getAttribute(_attributes, name, false); } const IAttributeVector * AttributeContext::getAttributeStableEnum(const string & name) const { - std::lock_guard<std::mutex> guard(_cacheLock); return getAttribute(_enumAttributes, name, true); } diff --git a/searchlib/src/vespa/searchlib/attribute/attributecontext.h b/searchlib/src/vespa/searchlib/attribute/attributecontext.h index 4ba3d07ef74..08516009914 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributecontext.h +++ b/searchlib/src/vespa/searchlib/attribute/attributecontext.h @@ -5,6 +5,7 @@ #include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/vespalib/stllike/hash_map.h> #include "iattributemanager.h" +#include <future> #include <mutex> namespace search { @@ -16,7 +17,7 @@ namespace search { class AttributeContext : public attribute::IAttributeContext { private: - using AttributeMap = vespalib::hash_map<string, std::unique_ptr<attribute::AttributeReadGuard>>; + using AttributeMap = vespalib::hash_map<string, std::shared_future<std::unique_ptr<attribute::AttributeReadGuard>>>; using IAttributeVector = attribute::IAttributeVector; using IAttributeFunctor = attribute::IAttributeFunctor; diff --git a/searchlib/src/vespa/searchlib/queryeval/fake_requestcontext.cpp b/searchlib/src/vespa/searchlib/queryeval/fake_requestcontext.cpp index ea4cbf84550..a123d3f6cd1 100644 --- a/searchlib/src/vespa/searchlib/queryeval/fake_requestcontext.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/fake_requestcontext.cpp @@ -1,12 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "fake_requestcontext.h" +#include <vespa/vespalib/util/testclock.h> + namespace search::queryeval { FakeRequestContext::FakeRequestContext(attribute::IAttributeContext * context, vespalib::steady_time softDoom, vespalib::steady_time hardDoom) - : _clock(), - _doom(_clock, softDoom, hardDoom, false), + : _clock(std::make_unique<vespalib::TestClock>()), + _doom(_clock->clock(), softDoom, hardDoom, false), _attributeContext(context), _query_tensor_name(), _query_tensor(), diff --git a/searchlib/src/vespa/searchlib/queryeval/fake_requestcontext.h b/searchlib/src/vespa/searchlib/queryeval/fake_requestcontext.h index 25a4fa559e0..a492f055cdb 100644 --- a/searchlib/src/vespa/searchlib/queryeval/fake_requestcontext.h +++ b/searchlib/src/vespa/searchlib/queryeval/fake_requestcontext.h @@ -13,6 +13,7 @@ #include <vespa/vespalib/util/doom.h> #include <limits> +namespace vespalib { class TestClock; } namespace search::queryeval { class FakeRequestContext : public IRequestContext @@ -47,7 +48,7 @@ public: const search::attribute::AttributeBlueprintParams& get_attribute_blueprint_params() const override; private: - vespalib::Clock _clock; + std::unique_ptr<vespalib::TestClock> _clock; const vespalib::Doom _doom; attribute::IAttributeContext *_attributeContext; vespalib::string _query_tensor_name; diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index ff35847aa77..928497bf09d 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/lambdatask.h> +#include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/retain_guard.h> #include <vespa/fastos/file.h> @@ -43,14 +44,11 @@ Domain::Domain(const string &domainName, const string & baseDir, vespalib::Execu _singleCommitter(std::make_unique<vespalib::ThreadStackExecutor>(1, 128_Ki)), _executor(executor), _sessionId(1), - _syncMonitor(), - _pendingSync(false), - _done_sync_tasks(), _name(domainName), _parts(), - _lock(), - _currentChunkMonitor(), - _sessionLock(), + _partsMutex(), + _currentChunkMutex(), + _sessionMutex(), _sessions(), _maxSessionRunTime(), _baseDir(baseDir), @@ -102,7 +100,7 @@ Domain::addPart(SerialNum partId, bool isLastPart) { dp->erase(dp->range().to() + 1); } else { { - std::lock_guard guard(_lock); + std::lock_guard guard(_partsMutex); _parts[partId] = dp; } if (! isLastPart) { @@ -112,16 +110,19 @@ Domain::addPart(SerialNum partId, bool isLastPart) { } Domain::~Domain() { - std::unique_lock guard(_currentChunkMonitor); - _currentChunkCond.notify_all(); - commitChunk(grabCurrentChunk(guard), guard); - _singleCommitter->shutdown().sync(); + { + std::unique_lock guard(_currentChunkMutex); + commitChunk(grabCurrentChunk(guard), guard); + } + vespalib::Gate gate; + _singleCommitter->execute(makeLambdaTask([callback=std::make_unique<vespalib::GateCallback>(gate)]() { (void) callback;})); + gate.await(); } DomainInfo Domain::getDomainInfo() const { - std::unique_lock guard(_lock); + std::unique_lock guard(_partsMutex); DomainInfo info(SerialNumRange(begin(guard), end(guard)), size(guard), byteSize(guard), _maxSessionRunTime); for (const auto &entry: _parts) { const DomainPart &part = *entry.second; @@ -133,12 +134,12 @@ Domain::getDomainInfo() const SerialNum Domain::begin() const { - return begin(UniqueLock(_lock)); + return begin(UniqueLock(_partsMutex)); } void Domain::verifyLock(const UniqueLock & guard) const { - assert(guard.mutex() == &_lock); + assert(guard.mutex() == &_partsMutex); assert(guard.owns_lock()); } SerialNum @@ -155,7 +156,7 @@ Domain::begin(const UniqueLock & guard) const SerialNum Domain::end() const { - return end(UniqueLock(_lock)); + return end(UniqueLock(_partsMutex)); } SerialNum @@ -172,7 +173,7 @@ Domain::end(const UniqueLock & guard) const size_t Domain::byteSize() const { - return byteSize(UniqueLock(_lock)); + return byteSize(UniqueLock(_partsMutex)); } size_t @@ -191,7 +192,7 @@ SerialNum Domain::getSynced() const { SerialNum s(0); - UniqueLock guard(_lock); + UniqueLock guard(_partsMutex); if (_parts.empty()) { return s; } @@ -207,41 +208,22 @@ Domain::getSynced() const void -Domain::triggerSyncNow(std::unique_ptr<vespalib::Executor::Task> done_sync_task) +Domain::triggerSyncNow(std::unique_ptr<vespalib::IDestructorCallback> after_sync) { { - std::unique_lock guard(_currentChunkMonitor); + std::unique_lock guard(_currentChunkMutex); commitAndTransferResponses(guard); } - if (done_sync_task) { - // Need to protect against being called from the _singleCommitter as that will cause a deadlock - // That is done from Domain::commitChunk.lamdba->Domain::doCommit()->optionallyRotateFile->triggerSyncNow({}) - _singleCommitter->sync(); - } - std::unique_lock guard(_syncMonitor); - if (done_sync_task) { - _done_sync_tasks.push_back(std::move(done_sync_task)); - } - if (!_pendingSync) { - _pendingSync = true; - _executor.execute(makeLambdaTask([this, domainPart= getActivePart()]() { - domainPart->sync(); - std::lock_guard monitorGuard(_syncMonitor); - _pendingSync = false; - _syncCond.notify_all(); - for (auto &task : _done_sync_tasks) { - auto failed_task = _executor.execute(std::move(task)); - assert(!failed_task); - } - _done_sync_tasks.clear(); - })); - } + _singleCommitter->execute(makeLambdaTask([after_sync=std::move(after_sync), domainPart=getActivePart()]() { + (void) after_sync; + domainPart->sync(); + })); } DomainPart::SP Domain::findPart(SerialNum s) { - std::lock_guard guard(_lock); + std::lock_guard guard(_partsMutex); DomainPartList::iterator it(_parts.upper_bound(s)); if (!_parts.empty() && it != _parts.begin()) { DomainPartList::iterator prev(it); @@ -258,14 +240,14 @@ Domain::findPart(SerialNum s) DomainPart::SP Domain::getActivePart() { - std::lock_guard guard(_lock); + std::lock_guard guard(_partsMutex); return _parts.rbegin()->second; } uint64_t Domain::size() const { - return size(UniqueLock(_lock)); + return size(UniqueLock(_partsMutex)); } uint64_t @@ -283,7 +265,7 @@ SerialNum Domain::findOldestActiveVisit() const { SerialNum oldestActive(std::numeric_limits<SerialNum>::max()); - std::lock_guard guard(_sessionLock); + std::lock_guard guard(_sessionMutex); for (const auto & pair : _sessions) { Session * session(pair.second.get()); if (!session->inSync()) { @@ -299,7 +281,7 @@ Domain::cleanSessions() if ( _sessions.empty()) { return; } - std::lock_guard guard(_sessionLock); + std::lock_guard guard(_sessionMutex); for (SessionList::iterator it(_sessions.begin()), mt(_sessions.end()); it != mt; ) { Session * session(it->second.get()); if (session->inSync()) { @@ -312,30 +294,15 @@ Domain::cleanSessions() } } -namespace { - -void -waitPendingSync(std::mutex &syncMonitor, std::condition_variable & syncCond, bool &pendingSync) -{ - std::unique_lock guard(syncMonitor); - while (pendingSync) { - syncCond.wait(guard); - } -} - -} - DomainPart::SP Domain::optionallyRotateFile(SerialNum serialNum) { DomainPart::SP dp = getActivePart(); if (dp->byteSize() > _config.getPartSizeLimit()) { - waitPendingSync(_syncMonitor, _syncCond, _pendingSync); - triggerSyncNow({}); - waitPendingSync(_syncMonitor, _syncCond, _pendingSync); + dp->sync(); dp->close(); dp = std::make_shared<DomainPart>(_name, dir(), serialNum, _fileHeaderContext, false); { - std::lock_guard guard(_lock); + std::lock_guard guard(_partsMutex); _parts[serialNum] = dp; assert(_parts.rbegin()->first == serialNum); } @@ -346,7 +313,7 @@ Domain::optionallyRotateFile(SerialNum serialNum) { void Domain::append(const Packet & packet, Writer::DoneCallback onDone) { - std::unique_lock guard(_currentChunkMonitor); + std::unique_lock guard(_currentChunkMutex); if (_lastSerial >= packet.range().from()) { throw runtime_error(fmt("Incoming serial number(%" PRIu64 ") must be bigger than the last one (%" PRIu64 ").", packet.range().from(), _lastSerial)); @@ -359,7 +326,7 @@ Domain::append(const Packet & packet, Writer::DoneCallback onDone) { Domain::CommitResult Domain::startCommit(DoneCallback onDone) { - std::unique_lock guard(_currentChunkMonitor); + std::unique_lock guard(_currentChunkMutex); if ( !_currentChunk->empty() ) { auto completed = grabCurrentChunk(guard); completed->setCommitDoneCallback(std::move(onDone)); @@ -386,7 +353,7 @@ Domain::commitAndTransferResponses(const UniqueLock &guard) { std::unique_ptr<CommitChunk> Domain::grabCurrentChunk(const UniqueLock & guard) { - assert(guard.mutex() == &_currentChunkMonitor && guard.owns_lock()); + assert(guard.mutex() == &_currentChunkMutex && guard.owns_lock()); auto chunk = std::move(_currentChunk); _currentChunk = createCommitChunk(_config); return chunk; @@ -394,7 +361,7 @@ Domain::grabCurrentChunk(const UniqueLock & guard) { void Domain::commitChunk(std::unique_ptr<CommitChunk> chunk, const UniqueLock & chunkOrderGuard) { - assert(chunkOrderGuard.mutex() == &_currentChunkMonitor && chunkOrderGuard.owns_lock()); + assert(chunkOrderGuard.mutex() == &_currentChunkMutex && chunkOrderGuard.owns_lock()); if (chunk->getPacket().empty()) return; chunk->shrinkPayloadToFit(); std::promise<SerializedChunk> promise; @@ -429,7 +396,7 @@ Domain::erase(SerialNum to) { bool retval(true); /// Do not erase the last element - UniqueLock guard(_lock); + UniqueLock guard(_partsMutex); for (DomainPartList::iterator it(_parts.begin()); (_parts.size() > 1) && (it->second.get()->range().to() < to); it = _parts.begin()) { DomainPart::SP dp(it->second); _parts.erase(it); @@ -452,7 +419,7 @@ Domain::visit(const Domain::SP & domain, SerialNum from, SerialNum to, std::uniq SerialNumRange range(from, to); auto session = std::make_shared<Session>(_sessionId++, range, domain, std::move(dest)); int id = session->id(); - std::lock_guard guard(_sessionLock); + std::lock_guard guard(_sessionMutex); _sessions[id] = std::move(session); return id; } @@ -461,7 +428,7 @@ int Domain::startSession(int sessionId) { int retval(-1); - std::lock_guard guard(_sessionLock); + std::lock_guard guard(_sessionMutex); SessionList::iterator found = _sessions.find(sessionId); if (found != _sessions.end()) { found->second->setStartTime(vespalib::steady_clock::now()); @@ -480,7 +447,7 @@ Domain::closeSession(int sessionId) int retval(-1); DurationSeconds sessionRunTime(0); { - std::lock_guard guard(_sessionLock); + std::lock_guard guard(_sessionMutex); SessionList::iterator found = _sessions.find(sessionId); if (found != _sessions.end()) { sessionRunTime = (vespalib::steady_clock::now() - found->second->getStartTime()); @@ -489,7 +456,7 @@ Domain::closeSession(int sessionId) } while (retval == 1) { std::this_thread::sleep_for(10ms); - std::lock_guard guard(_sessionLock); + std::lock_guard guard(_sessionMutex); SessionList::iterator found = _sessions.find(sessionId); if (found != _sessions.end()) { if ( ! found->second->isVisitRunning()) { @@ -501,7 +468,7 @@ Domain::closeSession(int sessionId) } } { - std::lock_guard guard(_lock); + std::lock_guard guard(_partsMutex); if (sessionRunTime > _maxSessionRunTime) { _maxSessionRunTime = sessionRunTime; } diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.h b/searchlib/src/vespa/searchlib/transactionlog/domain.h index 8dd5164689e..d7b59d676dd 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.h +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.h @@ -35,7 +35,7 @@ public: SerialNum begin() const; SerialNum end() const; SerialNum getSynced() const; - void triggerSyncNow(std::unique_ptr<vespalib::Executor::Task> done_sync_task); + void triggerSyncNow(std::unique_ptr<vespalib::IDestructorCallback> after_sync); bool getMarkedDeleted() const { return _markedDeleted; } void markDeleted() { _markedDeleted = true; } @@ -80,23 +80,19 @@ private: using SessionList = std::map<int, std::shared_ptr<Session>>; using DomainPartList = std::map<SerialNum, DomainPartSP>; using DurationSeconds = std::chrono::duration<double>; + using Executor = vespalib::Executor; DomainConfig _config; std::unique_ptr<CommitChunk> _currentChunk; SerialNum _lastSerial; - std::unique_ptr<vespalib::SyncableThreadExecutor> _singleCommitter; - vespalib::Executor &_executor; + std::unique_ptr<Executor> _singleCommitter; + Executor &_executor; std::atomic<int> _sessionId; - std::mutex _syncMonitor; - std::condition_variable _syncCond; - bool _pendingSync; - std::vector<std::unique_ptr<vespalib::Executor::Task>> _done_sync_tasks; vespalib::string _name; DomainPartList _parts; - mutable std::mutex _lock; - std::mutex _currentChunkMonitor; - std::condition_variable _currentChunkCond; - mutable std::mutex _sessionLock; + mutable std::mutex _partsMutex; + std::mutex _currentChunkMutex; + mutable std::mutex _sessionMutex; SessionList _sessions; DurationSeconds _maxSessionRunTime; vespalib::string _baseDir; diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index a0853dcbd86..b5806e1f962 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -9,7 +9,6 @@ #include <vespa/vespalib/util/cpu_usage.h> #include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/vespalib/util/exceptions.h> -#include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/stringfmt.h> #include <fstream> @@ -42,7 +41,7 @@ public: SyncHandler(std::atomic<bool>& closed, FRT_RPCRequest *req, Domain::SP domain, SerialNum syncTo) noexcept; ~SyncHandler(); - void poll(); + bool poll(); }; SyncHandler::SyncHandler(std::atomic<bool>& closed, FRT_RPCRequest *req, Domain::SP domain, SerialNum syncTo) noexcept @@ -55,7 +54,7 @@ SyncHandler::SyncHandler(std::atomic<bool>& closed, FRT_RPCRequest *req, Domain: SyncHandler::~SyncHandler() = default; -void +bool SyncHandler::poll() { SerialNum synced(_domain->getSynced()); @@ -67,9 +66,13 @@ SyncHandler::poll() rvals.AddInt32(0); rvals.AddInt64(synced); _req.Return(); - } else { - _domain->triggerSyncNow(vespalib::makeLambdaTask([self = shared_from_this()]() { self->poll(); })); + return true; } + _domain->triggerSyncNow(vespalib::makeUniqueLambdaCallback([self = shared_from_this()]() { + bool completed = self->poll(); + assert(completed); + })); + return false; } VESPA_THREAD_STACK_TAG(tls_executor); diff --git a/slobrok/src/vespa/slobrok/sbregister.cpp b/slobrok/src/vespa/slobrok/sbregister.cpp index cdab4d4009d..f00da4a9515 100644 --- a/slobrok/src/vespa/slobrok/sbregister.cpp +++ b/slobrok/src/vespa/slobrok/sbregister.cpp @@ -67,8 +67,8 @@ RegisterAPI::RegisterAPI(FRT_Supervisor &orb, const ConfiguratorFactory & config _names(), _pending(), _unreg(), - _target(0), - _req(0) + _target(nullptr), + _req(nullptr) { _configurator->poll(); if ( ! _slobrokSpecs.ok()) { @@ -82,12 +82,12 @@ RegisterAPI::RegisterAPI(FRT_Supervisor &orb, const ConfiguratorFactory & config RegisterAPI::~RegisterAPI() { Kill(); - _configurator.reset(0); - if (_req != 0) { + _configurator.reset(); + if (_req != nullptr) { _req->Abort(); _req->SubRef(); } - if (_target != 0) { + if (_target != nullptr) { _target->SubRef(); } } @@ -96,15 +96,15 @@ RegisterAPI::~RegisterAPI() void RegisterAPI::registerName(vespalib::stringref name) { - std::lock_guard<std::mutex> guard(_lock); - for (uint32_t i = 0; i < _names.size(); ++i) { - if (_names[i] == name) { + std::lock_guard guard(_lock); + for (const auto& existing_name : _names) { + if (existing_name == name) { return; } } _busy.store(true, std::memory_order_relaxed); - _names.push_back(name); - _pending.push_back(name); + _names.emplace_back(name); + _pending.emplace_back(name); discard(_unreg, name); ScheduleNow(); } @@ -113,11 +113,11 @@ RegisterAPI::registerName(vespalib::stringref name) void RegisterAPI::unregisterName(vespalib::stringref name) { - std::lock_guard<std::mutex> guard(_lock); + std::lock_guard guard(_lock); _busy.store(true, std::memory_order_relaxed); discard(_names, name); discard(_pending, name); - _unreg.push_back(name); + _unreg.emplace_back(name); ScheduleNow(); } @@ -125,6 +125,7 @@ RegisterAPI::unregisterName(vespalib::stringref name) void RegisterAPI::handleReqDone() { + std::lock_guard guard(_lock); if (_reqDone) { _reqDone = false; if (_req->IsError()) { @@ -133,10 +134,10 @@ RegisterAPI::handleReqDone() _req->GetErrorMessage(), _req->GetErrorCode()); // unexpected error; close our connection to this // slobrok server and try again with a fresh slate - if (_target != 0) { + if (_target != nullptr) { _target->SubRef(); } - _target = 0; + _target = nullptr; _busy.store(true, std::memory_order_relaxed); } else { LOG(warning, "%s(%s -> %s) failed: %s", @@ -146,7 +147,7 @@ RegisterAPI::handleReqDone() _req->GetErrorMessage()); } } else { - if (_logOnSuccess && (_pending.size() == 0) && (_names.size() > 0)) { + if (_logOnSuccess && _pending.empty() && !_names.empty()) { LOG(info, "[RPC @ %s] registering %s with location broker %s completed successfully", createSpec(_orb).c_str(), _names[0].c_str(), _currSlobrok.c_str()); _logOnSuccess = false; @@ -155,7 +156,7 @@ RegisterAPI::handleReqDone() _backOff.reset(); } _req->SubRef(); - _req = 0; + _req = nullptr; } } @@ -163,29 +164,29 @@ RegisterAPI::handleReqDone() void RegisterAPI::handleReconnect() { - if (_configurator->poll() && _target != 0) { + if (_configurator->poll() && _target != nullptr) { if (! _slobrokSpecs.contains(_currSlobrok)) { vespalib::string cps = _slobrokSpecs.logString(); LOG(warning, "[RPC @ %s] location broker %s removed, will disconnect and use one of: %s", createSpec(_orb).c_str(), _currSlobrok.c_str(), cps.c_str()); _target->SubRef(); - _target = 0; + _target = nullptr; } } - if (_target == 0) { + if (_target == nullptr) { _logOnSuccess = true; _currSlobrok = _slobrokSpecs.nextSlobrokSpec(); - if (_currSlobrok.size() > 0) { + if (!_currSlobrok.empty()) { // try next possible server. _target = _orb.GetTarget(_currSlobrok.c_str()); } { - std::lock_guard<std::mutex> guard(_lock); + std::lock_guard guard(_lock); // now that we have a new connection, we need to // immediately re-register everything. _pending = _names; } - if (_target == 0) { + if (_target == nullptr) { // we have tried all possible servers. // start from the top after a delay, // possibly with a warning. @@ -211,13 +212,13 @@ RegisterAPI::handlePending() bool reg = false; vespalib::string name; { - std::lock_guard<std::mutex> guard(_lock); + std::lock_guard guard(_lock); // pop off the todo stack, unregister has priority - if (_unreg.size() > 0) { + if (!_unreg.empty()) { name = _unreg.back(); _unreg.pop_back(); unreg = true; - } else if (_pending.size() > 0) { + } else if (!_pending.empty()) { name = _pending.back(); _pending.pop_back(); reg = true; @@ -244,7 +245,7 @@ RegisterAPI::handlePending() } else { // nothing more to do right now; schedule to re-register all // names after a long delay. - std::lock_guard<std::mutex> guard(_lock); + std::lock_guard guard(_lock); _pending = _names; LOG(debug, "done, reschedule in 30s"); _busy.store(false, std::memory_order_relaxed); @@ -256,23 +257,25 @@ void RegisterAPI::PerformTask() { handleReqDone(); - if (_req != 0) { + if (_req != nullptr) { LOG(debug, "req in progress"); return; // current request still in progress, don't start anything new } handleReconnect(); // still no connection? - if (_target == 0) return; + if (_target == nullptr) return; handlePending(); } void -RegisterAPI::RequestDone(FRT_RPCRequest *req) +RegisterAPI::RequestDone([[maybe_unused]] FRT_RPCRequest *req) { - LOG_ASSERT(req == _req && !_reqDone); - (void) req; - _reqDone = true; + { + std::lock_guard guard(_lock); + LOG_ASSERT(req == _req && !_reqDone); + _reqDone = true; + } ScheduleNow(); } @@ -296,16 +299,13 @@ RegisterAPI::RPCHooks::RPCHooks(RegisterAPI &owner) } -RegisterAPI::RPCHooks::~RPCHooks() -{ -} - +RegisterAPI::RPCHooks::~RPCHooks() = default; void RegisterAPI::RPCHooks::rpc_listNamesServed(FRT_RPCRequest *req) { FRT_Values &dst = *req->GetReturn(); - std::lock_guard<std::mutex> guard(_owner._lock); + std::lock_guard guard(_owner._lock); FRT_StringValue *names = dst.AddStringArray(_owner._names.size()); for (uint32_t i = 0; i < _owner._names.size(); ++i) { dst.SetString(&names[i], _owner._names[i].c_str()); diff --git a/staging_vespalib/src/tests/clock/clock_benchmark.cpp b/staging_vespalib/src/tests/clock/clock_benchmark.cpp index c8d3e2a5aff..249add4bc1a 100644 --- a/staging_vespalib/src/tests/clock/clock_benchmark.cpp +++ b/staging_vespalib/src/tests/clock/clock_benchmark.cpp @@ -139,17 +139,16 @@ main(int , char *argv[]) NSVolatile nsVolatile; NSAtomic nsAtomic; vespalib::InvokeServiceImpl invoker(vespalib::from_s(1.0/frequency)); - Clock clock; + Clock clock(invoker.nowRef()); TestClock nsClock(nsValue, 1.0/frequency); TestClock nsVolatileClock(nsVolatile, 1.0/frequency); TestClock nsAtomicClock(nsAtomic, 1.0/frequency); - clock.start(invoker); assert(pool.NewThread(&nsClock, nullptr) != nullptr); assert(pool.NewThread(&nsVolatileClock, nullptr) != nullptr); assert(pool.NewThread(&nsAtomicClock, nullptr) != nullptr); benchmark("vespalib::Clock", pool, samples, numThreads, [&clock]() { - return clock.getTimeNSAssumeRunning(); + return clock.getTimeNS(); }); benchmark("uint64_t", pool, samples, numThreads, [&nsValue]() { return steady_time (duration(nsValue._value)); @@ -175,6 +174,5 @@ main(int , char *argv[]) }); pool.Close(); - clock.stop(); return 0; } diff --git a/staging_vespalib/src/tests/clock/clock_test.cpp b/staging_vespalib/src/tests/clock/clock_test.cpp index 2c6cbc0c876..f2de085da84 100644 --- a/staging_vespalib/src/tests/clock/clock_test.cpp +++ b/staging_vespalib/src/tests/clock/clock_test.cpp @@ -19,19 +19,11 @@ void waitForMovement(steady_time start, Clock & clock, vespalib::duration timeou TEST("Test that clock is ticking forward") { vespalib::InvokeServiceImpl invoker(50ms); - Clock clock; - clock.start(invoker); + Clock clock(invoker.nowRef()); steady_time start = clock.getTimeNS(); waitForMovement(start, clock, 10s); steady_time stop = clock.getTimeNS(); EXPECT_TRUE(stop > start); - std::this_thread::sleep_for(1s); - start = clock.getTimeNS(); - waitForMovement(start, clock, 10s); - clock.stop(); - steady_time stop2 = clock.getTimeNS(); - EXPECT_TRUE(stop2 > stop); - EXPECT_TRUE(vespalib::count_ms(stop2 - stop) > 1000); } TEST_MAIN() { TEST_RUN_ALL(); }
\ No newline at end of file diff --git a/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt index e69dd36d6f5..2912ac2397b 100644 --- a/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -23,6 +23,7 @@ vespa_add_library(staging_vespalib_vespalib_util OBJECT shutdownguard.cpp scheduledexecutor.cpp singleexecutor.cpp + testclock.cpp xmlserializable.cpp xmlstream.cpp DEPENDS diff --git a/staging_vespalib/src/vespa/vespalib/util/clock.cpp b/staging_vespalib/src/vespa/vespalib/util/clock.cpp index a4b9ed0fd5d..4b6e1e8ab85 100644 --- a/staging_vespalib/src/vespa/vespalib/util/clock.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/clock.cpp @@ -1,37 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "clock.h" -#include <vespa/vespalib/util/invokeservice.h> - +#include <cassert> namespace vespalib { -Clock::Clock() : - _timeNS(0u), - _running(false), - _invokeRegistration() +Clock::Clock(const std::atomic<steady_time> & source) noexcept + : _timeNS(source) { - setTime(); + static_assert(std::atomic<steady_time>::is_always_lock_free); } Clock::~Clock() = default; -void Clock::setTime() const -{ - _timeNS.store(count_ns(steady_clock::now().time_since_epoch()), std::memory_order_relaxed); -} - -void -Clock::start(InvokeService & invoker) -{ - _running.store(true, std::memory_order_relaxed); - _invokeRegistration = invoker.registerInvoke([this]() { setTime(); }); -} - -void -Clock::stop() -{ - _running.store(false, std::memory_order_relaxed); - _invokeRegistration.reset(); -} - } diff --git a/staging_vespalib/src/vespa/vespalib/util/clock.h b/staging_vespalib/src/vespa/vespalib/util/clock.h index d4cffc200fe..4cd579b8f69 100644 --- a/staging_vespalib/src/vespa/vespalib/util/clock.h +++ b/staging_vespalib/src/vespa/vespalib/util/clock.h @@ -7,9 +7,6 @@ namespace vespalib { -class IDestructorCallback; -class InvokeService; - /** * Clock is a clock that updates the time at defined intervals. * It is intended used where you want to check the time with low cost, but where @@ -19,28 +16,18 @@ class InvokeService; class Clock { private: - mutable std::atomic<int64_t> _timeNS; - std::atomic<bool> _running; - std::unique_ptr<IDestructorCallback> _invokeRegistration; - - void setTime() const; + const std::atomic<steady_time> &_timeNS; public: - Clock(); + Clock(const std::atomic<steady_time> & source) noexcept; + Clock(const Clock &) = delete; + Clock & operator =(const Clock &) = delete; + Clock(Clock &&) = delete; + Clock & operator =(Clock &&) = delete; ~Clock(); - vespalib::steady_time getTimeNS() const { - if (!_running) { - setTime(); - } - return getTimeNSAssumeRunning(); - } - vespalib::steady_time getTimeNSAssumeRunning() const { - return vespalib::steady_time(std::chrono::nanoseconds(_timeNS.load(std::memory_order_relaxed))); + vespalib::steady_time getTimeNS() const noexcept { + return vespalib::steady_time(_timeNS.load(std::memory_order_relaxed)); } - - void start(InvokeService & invoker); - void stop(); }; } - diff --git a/staging_vespalib/src/vespa/vespalib/util/doom.h b/staging_vespalib/src/vespa/vespalib/util/doom.h index b9a7e76b8af..e7db0795fb7 100644 --- a/staging_vespalib/src/vespa/vespalib/util/doom.h +++ b/staging_vespalib/src/vespa/vespalib/util/doom.h @@ -14,8 +14,8 @@ public: Doom(const Clock &clock, steady_time softDoom, steady_time hardDoom, bool explicitSoftDoom); - bool soft_doom() const { return (_clock.getTimeNSAssumeRunning() > _softDoom); } - bool hard_doom() const { return (_clock.getTimeNSAssumeRunning() > _hardDoom); } + bool soft_doom() const { return (_clock.getTimeNS() > _softDoom); } + bool hard_doom() const { return (_clock.getTimeNS() > _hardDoom); } duration soft_left() const { return _softDoom - _clock.getTimeNS(); } duration hard_left() const { return _hardDoom - _clock.getTimeNS(); } bool isExplicitSoftDoom() const { return _isExplicitSoftDoom; } diff --git a/staging_vespalib/src/vespa/vespalib/util/testclock.cpp b/staging_vespalib/src/vespa/vespalib/util/testclock.cpp new file mode 100644 index 00000000000..bc5d37ca437 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/testclock.cpp @@ -0,0 +1,16 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "testclock.h" +#include <vespa/vespalib/util/invokeserviceimpl.h> + +namespace vespalib { + +TestClock::TestClock() + : _ticker(std::make_unique<InvokeServiceImpl>(10ms)), + _clock(_ticker->nowRef()) +{ +} + +TestClock::~TestClock() = default; + +} diff --git a/staging_vespalib/src/vespa/vespalib/util/testclock.h b/staging_vespalib/src/vespa/vespalib/util/testclock.h new file mode 100644 index 00000000000..9446ff32cb2 --- /dev/null +++ b/staging_vespalib/src/vespa/vespalib/util/testclock.h @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include "clock.h" + +namespace vespalib { + +class InvokeServiceImpl; + +/** + * Self contained clock useable for testing that provides a backing for the vespalib::Clock interface. + */ + +class TestClock +{ +private: + std::unique_ptr<InvokeServiceImpl> _ticker; + Clock _clock; +public: + TestClock(); + TestClock(const TestClock &) = delete; + TestClock & operator =(const TestClock &) = delete; + TestClock(TestClock &&) = delete; + TestClock & operator =(TestClock &&) = delete; + ~TestClock(); + const Clock & clock() { return _clock; } +}; + +} + diff --git a/storage/src/vespa/storage/distributor/distributor_stripe_pool.cpp b/storage/src/vespa/storage/distributor/distributor_stripe_pool.cpp index d501172731a..8b172743d27 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe_pool.cpp +++ b/storage/src/vespa/storage/distributor/distributor_stripe_pool.cpp @@ -15,7 +15,7 @@ DistributorStripePool::DistributorStripePool(bool test_mode, PrivateCtorTag) _mutex(), _parker_cond(), _parked_threads(0), - _bootstrap_tick_wait_duration(vespalib::from_s(1.0/vespalib::getVespaTimerHz())), + _bootstrap_tick_wait_duration(vespalib::adjustTimeoutByDetectedHz(1ms)), _bootstrap_ticks_before_wait(10), _single_threaded_test_mode(test_mode), _stopped(false) diff --git a/storage/src/vespa/storage/distributor/distributor_stripe_thread.cpp b/storage/src/vespa/storage/distributor/distributor_stripe_thread.cpp index 94e4dc648cc..ae5445da620 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe_thread.cpp +++ b/storage/src/vespa/storage/distributor/distributor_stripe_thread.cpp @@ -11,7 +11,7 @@ DistributorStripeThread::DistributorStripeThread(TickableStripe& stripe, DistributorStripePool& stripe_pool) : _stripe(stripe), _stripe_pool(stripe_pool), - _tick_wait_duration(vespalib::from_s(1.0/vespalib::getVespaTimerHz())), + _tick_wait_duration(vespalib::adjustTimeoutByDetectedHz(1ms)), _mutex(), _event_cond(), _park_cond(), diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h index 5c243ea4af9..6d8cdc16743 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandler.h @@ -128,6 +128,7 @@ public: CLOSED }; + FileStorHandler() : _getNextMessageTimout(100ms) { } virtual ~FileStorHandler() = default; @@ -170,7 +171,12 @@ public: * * @param stripe The stripe to get messages for */ - virtual LockedMessage getNextMessage(uint32_t stripeId) = 0; + virtual LockedMessage getNextMessage(uint32_t stripeId, vespalib::steady_time deadline) = 0; + + /** Only used for testing, should be removed */ + LockedMessage getNextMessage(uint32_t stripeId) { + return getNextMessage(stripeId, vespalib::steady_clock::now() + _getNextMessageTimout); + } /** * Lock a bucket. By default, each file stor thread has the locks of all @@ -268,7 +274,7 @@ public: virtual uint32_t getQueueSize() const = 0; // Commands used by testing - virtual void setGetNextMessageTimeout(vespalib::duration timeout) = 0; + void setGetNextMessageTimeout(vespalib::duration timeout) { _getNextMessageTimout = timeout; } virtual std::string dumpQueue() const = 0; @@ -276,7 +282,13 @@ public: virtual vespalib::SharedOperationThrottler& operation_throttler() const noexcept = 0; + virtual void reconfigure_dynamic_throttler(const vespalib::SharedOperationThrottler::DynamicThrottleParams& params) = 0; + + virtual void use_dynamic_operation_throttling(bool use_dynamic) noexcept = 0; + virtual void set_throttle_apply_bucket_diff_ops(bool throttle_apply_bucket_diff) noexcept = 0; +private: + vespalib::duration _getNextMessageTimout; }; } // storage diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index b5de5a233cc..c44ae305fa2 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -5,7 +5,6 @@ #include "mergestatus.h" #include <vespa/storageapi/message/bucketsplitting.h> #include <vespa/storageapi/message/persistence.h> -#include <vespa/storageapi/message/removelocation.h> #include <vespa/storage/bucketdb/storbucketdb.h> #include <vespa/storage/common/bucketmessages.h> #include <vespa/storage/common/statusmessages.h> @@ -40,22 +39,23 @@ uint32_t per_stripe_merge_limit(uint32_t num_threads, uint32_t num_stripes) noex FileStorHandlerImpl::FileStorHandlerImpl(MessageSender& sender, FileStorMetrics& metrics, ServiceLayerComponentRegister& compReg) - : FileStorHandlerImpl(1, 1, sender, metrics, compReg, vespalib::SharedOperationThrottler::make_unlimited_throttler()) + : FileStorHandlerImpl(1, 1, sender, metrics, compReg, vespalib::SharedOperationThrottler::DynamicThrottleParams()) { } FileStorHandlerImpl::FileStorHandlerImpl(uint32_t numThreads, uint32_t numStripes, MessageSender& sender, FileStorMetrics& metrics, ServiceLayerComponentRegister& compReg, - std::unique_ptr<vespalib::SharedOperationThrottler> operation_throttler) + const vespalib::SharedOperationThrottler::DynamicThrottleParams& dyn_throttle_params) : _component(compReg, "filestorhandlerimpl"), _state(FileStorHandler::AVAILABLE), _metrics(nullptr), - _operation_throttler(std::move(operation_throttler)), + _dynamic_operation_throttler(vespalib::SharedOperationThrottler::make_dynamic_throttler(dyn_throttle_params)), + _unlimited_operation_throttler(vespalib::SharedOperationThrottler::make_unlimited_throttler()), + _active_throttler(_unlimited_operation_throttler.get()), // Will be set by FileStorManager _stripes(), _messageSender(sender), _bucketIdFactory(_component.getBucketIdFactory()), - _getNextMessageTimeout(100ms), _max_active_merges_per_stripe(per_stripe_merge_limit(numThreads, numStripes)), _paused(false), _throttle_apply_bucket_diff_ops(false), @@ -251,6 +251,22 @@ FileStorHandlerImpl::schedule_and_get_next_async_message(const std::shared_ptr<a return {}; } +void +FileStorHandlerImpl::reconfigure_dynamic_throttler(const vespalib::SharedOperationThrottler::DynamicThrottleParams& params) +{ + _dynamic_operation_throttler->reconfigure_dynamic_throttling(params); +} + +void +FileStorHandlerImpl::use_dynamic_operation_throttling(bool use_dynamic) noexcept +{ + // Use release semantics instead of relaxed to ensure transitive visibility even in + // non-persistence threads that try to invoke the throttler (i.e. RPC threads). + _active_throttler.store(use_dynamic ? _dynamic_operation_throttler.get() + : _unlimited_operation_throttler.get(), + std::memory_order_release); +} + bool FileStorHandlerImpl::messageMayBeAborted(const api::StorageMessage& msg) { @@ -333,9 +349,9 @@ FileStorHandlerImpl::updateMetrics(const MetricLockGuard &) std::lock_guard lockGuard(_mergeStatesLock); _metrics->pendingMerges.addValue(_mergeStates.size()); _metrics->queueSize.addValue(getQueueSize()); - _metrics->throttle_window_size.addValue(_operation_throttler->current_window_size()); - _metrics->throttle_waiting_threads.addValue(_operation_throttler->waiting_threads()); - _metrics->throttle_active_tokens.addValue(_operation_throttler->current_active_token_count()); + _metrics->throttle_window_size.addValue(operation_throttler().current_window_size()); + _metrics->throttle_waiting_threads.addValue(operation_throttler().waiting_threads()); + _metrics->throttle_active_tokens.addValue(operation_throttler().current_active_token_count()); for (const auto & stripe : _metrics->stripes) { const auto & m = stripe->averageQueueWaitingTime; @@ -377,13 +393,13 @@ FileStorHandlerImpl::makeQueueTimeoutReply(api::StorageMessage& msg) } FileStorHandler::LockedMessage -FileStorHandlerImpl::getNextMessage(uint32_t stripeId) +FileStorHandlerImpl::getNextMessage(uint32_t stripeId, vespalib::steady_time deadline) { if (!tryHandlePause()) { return {}; // Still paused, return to allow tick. } - return getNextMessage(stripeId, _getNextMessageTimeout); + return _stripes[stripeId].getNextMessage(deadline); } std::shared_ptr<FileStorHandler::BucketLockInterface> @@ -919,7 +935,7 @@ FileStorHandlerImpl::Stripe::operation_type_should_be_throttled(api::MessageType } FileStorHandler::LockedMessage -FileStorHandlerImpl::Stripe::getNextMessage(vespalib::duration timeout) +FileStorHandlerImpl::Stripe::getNextMessage(vespalib::steady_time deadline) { std::unique_lock guard(*_lock); ThrottleToken throttle_token; @@ -955,12 +971,12 @@ FileStorHandlerImpl::Stripe::getNextMessage(vespalib::duration timeout) // Depending on whether we were blocked due to no usable ops in queue or throttling, // wait for either the queue or throttler to (hopefully) have some fresh stuff for us. if (!was_throttled) { - _cond->wait_for(guard, timeout); + _cond->wait_until(guard, deadline); } else { // Have to release lock before doing a blocking throttle token fetch, since it // prevents RPC threads from pushing onto the queue. guard.unlock(); - throttle_token = _owner.operation_throttler().blocking_acquire_one(timeout); + throttle_token = _owner.operation_throttler().blocking_acquire_one(deadline); guard.lock(); if (!throttle_token.valid()) { _metrics->timeouts_waiting_for_throttle_token.inc(); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h index 1bc0ab87b1c..dbef1d06dad 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.h @@ -132,7 +132,7 @@ public: std::shared_ptr<FileStorHandler::BucketLockInterface> lock(const document::Bucket & bucket, api::LockingRequirements lockReq); void failOperations(const document::Bucket & bucket, const api::ReturnCode & code); - FileStorHandler::LockedMessage getNextMessage(vespalib::duration timeout); + FileStorHandler::LockedMessage getNextMessage(vespalib::steady_time deadline); void dumpQueue(std::ostream & os) const; void dumpActiveHtml(std::ostream & os) const; void dumpQueueHtml(std::ostream & os) const; @@ -192,10 +192,10 @@ public: FileStorHandlerImpl(MessageSender& sender, FileStorMetrics& metrics, ServiceLayerComponentRegister& compReg); FileStorHandlerImpl(uint32_t numThreads, uint32_t numStripes, MessageSender&, FileStorMetrics&, - ServiceLayerComponentRegister&, std::unique_ptr<vespalib::SharedOperationThrottler>); + ServiceLayerComponentRegister&, + const vespalib::SharedOperationThrottler::DynamicThrottleParams& dyn_throttle_params); ~FileStorHandlerImpl() override; - void setGetNextMessageTimeout(vespalib::duration timeout) override { _getNextMessageTimeout = timeout; } void flush(bool killPendingMerges) override; void setDiskState(DiskState state) override; @@ -204,7 +204,7 @@ public: bool schedule(const std::shared_ptr<api::StorageMessage>&) override; ScheduleAsyncResult schedule_and_get_next_async_message(const std::shared_ptr<api::StorageMessage>& msg) override; - FileStorHandler::LockedMessage getNextMessage(uint32_t stripeId) override; + FileStorHandler::LockedMessage getNextMessage(uint32_t stripeId, vespalib::steady_time deadline) override; void remapQueueAfterJoin(const RemapInfo& source, RemapInfo& target) override; void remapQueueAfterSplit(const RemapInfo& source, RemapInfo& target1, RemapInfo& target2) override; @@ -245,9 +245,18 @@ public: void abortQueuedOperations(const AbortBucketOperationsCommand& cmd) override; vespalib::SharedOperationThrottler& operation_throttler() const noexcept override { - return *_operation_throttler; + // It would be reasonable to assume that this could be a relaxed load since the set + // of possible throttlers is static and all _persistence_ thread creation is sequenced + // after throttler creation. But since the throttler may be invoked by RPC threads + // created in another context, use acquire semantics to ensure transitive visibility. + // TODO remove need for atomics once the throttler testing dust settles + return *_active_throttler.load(std::memory_order_acquire); } + void reconfigure_dynamic_throttler(const vespalib::SharedOperationThrottler::DynamicThrottleParams& params) override; + + void use_dynamic_operation_throttling(bool use_dynamic) noexcept override; + void set_throttle_apply_bucket_diff_ops(bool throttle_apply_bucket_diff) noexcept override { // Relaxed is fine, worst case from temporarily observing a stale value is that // an ApplyBucketDiff message is (or isn't) throttled at a high level. @@ -264,13 +273,14 @@ private: ServiceLayerComponent _component; std::atomic<DiskState> _state; FileStorDiskMetrics * _metrics; - std::unique_ptr<vespalib::SharedOperationThrottler> _operation_throttler; + std::unique_ptr<vespalib::SharedOperationThrottler> _dynamic_operation_throttler; + std::unique_ptr<vespalib::SharedOperationThrottler> _unlimited_operation_throttler; + std::atomic<vespalib::SharedOperationThrottler*> _active_throttler; std::vector<Stripe> _stripes; MessageSender& _messageSender; const document::BucketIdFactory& _bucketIdFactory; mutable std::mutex _mergeStatesLock; std::map<document::Bucket, std::shared_ptr<MergeStatus>> _mergeStates; - vespalib::duration _getNextMessageTimeout; const uint32_t _max_active_merges_per_stripe; // Read concurrently by stripes. mutable std::mutex _pauseMonitor; mutable std::condition_variable _pauseCond; @@ -355,9 +365,6 @@ private: Stripe & stripe(const document::Bucket & bucket) { return _stripes[stripe_index(bucket)]; } - FileStorHandler::LockedMessage getNextMessage(uint32_t stripeId, vespalib::duration timeout) { - return _stripes[stripeId].getNextMessage(timeout); - } ActiveOperationsStats get_active_operations_stats(bool reset_min_max) const override; }; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp index f4aff96b53c..09bd842c308 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.cpp @@ -164,20 +164,6 @@ dynamic_throttle_params_from_config(const StorFilestorConfig& config, uint32_t n return params; } -std::unique_ptr<vespalib::SharedOperationThrottler> -make_operation_throttler_from_config(const StorFilestorConfig& config, uint32_t num_threads) -{ - // TODO only use struct config field instead once config model is updated - const bool use_dynamic_throttling = ((config.asyncOperationThrottlerType == StorFilestorConfig::AsyncOperationThrottlerType::DYNAMIC) || - (config.asyncOperationThrottler.type == StorFilestorConfig::AsyncOperationThrottler::Type::DYNAMIC)); - if (use_dynamic_throttling) { - auto dyn_params = dynamic_throttle_params_from_config(config, num_threads); - return vespalib::SharedOperationThrottler::make_dynamic_throttler(dyn_params); - } else { - return vespalib::SharedOperationThrottler::make_unlimited_throttler(); - } -} - #ifdef __PIC__ #define TLS_LINKAGE __attribute__((visibility("hidden"), tls_model("initial-exec"))) #else @@ -216,18 +202,17 @@ FileStorManager::getThreadLocalHandler() { } return *_G_threadLocalHandler; } -/** - * If live configuration, assuming storageserver makes sure no messages are - * incoming during reconfiguration - */ + void FileStorManager::configure(std::unique_ptr<StorFilestorConfig> config) { // If true, this is not the first configure. - bool liveUpdate = ! _threads.empty(); + const bool liveUpdate = ! _threads.empty(); _use_async_message_handling_on_schedule = config->useAsyncMessageHandlingOnSchedule; _host_info_reporter.set_noise_level(config->resourceUsageReporterNoiseLevel); + const bool use_dynamic_throttling = ((config->asyncOperationThrottlerType == StorFilestorConfig::AsyncOperationThrottlerType::DYNAMIC) || + (config->asyncOperationThrottler.type == StorFilestorConfig::AsyncOperationThrottler::Type::DYNAMIC)); const bool throttle_merge_feed_ops = config->asyncOperationThrottler.throttleIndividualMergeFeedOps; if (!liveUpdate) { @@ -235,10 +220,10 @@ FileStorManager::configure(std::unique_ptr<StorFilestorConfig> config) uint32_t numThreads = std::max(1, _config->numThreads); uint32_t numStripes = std::max(1u, numThreads / 2); _metrics->initDiskMetrics(numStripes, computeAllPossibleHandlerThreads(*_config)); - auto operation_throttler = make_operation_throttler_from_config(*_config, numThreads); + auto dyn_params = dynamic_throttle_params_from_config(*_config, numThreads); _filestorHandler = std::make_unique<FileStorHandlerImpl>(numThreads, numStripes, *this, *_metrics, - _compReg, std::move(operation_throttler)); + _compReg, dyn_params); uint32_t numResponseThreads = computeNumResponseThreads(_config->numResponseThreads); _sequencedExecutor = vespalib::SequencedTaskExecutor::create(CpuUsage::wrap(response_executor, CpuUsage::Category::WRITE), numResponseThreads, 10000, @@ -253,10 +238,11 @@ FileStorManager::configure(std::unique_ptr<StorFilestorConfig> config) } else { assert(_filestorHandler); auto updated_dyn_throttle_params = dynamic_throttle_params_from_config(*config, _threads.size()); - _filestorHandler->operation_throttler().reconfigure_dynamic_throttling(updated_dyn_throttle_params); + _filestorHandler->reconfigure_dynamic_throttler(updated_dyn_throttle_params); } // TODO remove once desired dynamic throttling behavior is set in stone { + _filestorHandler->use_dynamic_operation_throttling(use_dynamic_throttling); _filestorHandler->set_throttle_apply_bucket_diff_ops(!throttle_merge_feed_ops); std::lock_guard guard(_lock); for (auto& ph : _persistenceHandlers) { diff --git a/storage/src/vespa/storage/persistence/mergehandler.cpp b/storage/src/vespa/storage/persistence/mergehandler.cpp index 78f53de46b3..8287fe27509 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.cpp +++ b/storage/src/vespa/storage/persistence/mergehandler.cpp @@ -32,7 +32,6 @@ MergeHandler::MergeHandler(PersistenceUtil& env, spi::PersistenceProvider& spi, _cluster_context(cluster_context), _env(env), _spi(spi), - _operation_throttler(_env._fileStorHandler.operation_throttler()), _monitored_ref_count(std::make_unique<MonitoredRefCount>()), _maxChunkSize(maxChunkSize), _commonMergeChainOptimalizationMinimumSize(commonMergeChainOptimalizationMinimumSize), @@ -515,7 +514,7 @@ MergeHandler::applyDiffEntry(std::shared_ptr<ApplyBucketDiffState> async_results spi::Context& context, const document::DocumentTypeRepo& repo) const { - auto throttle_token = throttle_merge_feed_ops() ? _operation_throttler.blocking_acquire_one() + auto throttle_token = throttle_merge_feed_ops() ? _env._fileStorHandler.operation_throttler().blocking_acquire_one() : vespalib::SharedOperationThrottler::Token(); spi::Timestamp timestamp(e._entry._timestamp); if (!(e._entry._flags & (DELETED | DELETED_IN_PLACE))) { diff --git a/storage/src/vespa/storage/persistence/mergehandler.h b/storage/src/vespa/storage/persistence/mergehandler.h index 93fb7efc8d0..1ed2fa878bc 100644 --- a/storage/src/vespa/storage/persistence/mergehandler.h +++ b/storage/src/vespa/storage/persistence/mergehandler.h @@ -24,7 +24,6 @@ namespace vespalib { class ISequencedTaskExecutor; -class SharedOperationThrottler; } namespace storage { @@ -96,7 +95,6 @@ private: const ClusterContext &_cluster_context; PersistenceUtil &_env; spi::PersistenceProvider &_spi; - vespalib::SharedOperationThrottler& _operation_throttler; std::unique_ptr<vespalib::MonitoredRefCount> _monitored_ref_count; const uint32_t _maxChunkSize; const uint32_t _commonMergeChainOptimalizationMinimumSize; diff --git a/storage/src/vespa/storage/persistence/persistencethread.cpp b/storage/src/vespa/storage/persistence/persistencethread.cpp index b89c60d4720..8e1fdb06ded 100644 --- a/storage/src/vespa/storage/persistence/persistencethread.cpp +++ b/storage/src/vespa/storage/persistence/persistencethread.cpp @@ -33,10 +33,13 @@ PersistenceThread::run(framework::ThreadHandle& thread) { LOG(debug, "Started persistence thread"); + vespalib::duration max_wait_time = vespalib::adjustTimeoutByDetectedHz(100ms); while (!thread.interrupted()) { - thread.registerTick(); + vespalib::steady_time now = vespalib::steady_clock::now(); + thread.registerTick(framework::UNKNOWN_CYCLE, now); - FileStorHandler::LockedMessage lock(_fileStorHandler.getNextMessage(_stripeId)); + vespalib::steady_time deadline = now + max_wait_time; + FileStorHandler::LockedMessage lock(_fileStorHandler.getNextMessage(_stripeId, deadline)); if (lock.lock) { _persistenceHandler.processLockedMessage(std::move(lock)); diff --git a/storage/src/vespa/storage/storageserver/bouncer.cpp b/storage/src/vespa/storage/storageserver/bouncer.cpp index 5241a1a88dd..2b02156367c 100644 --- a/storage/src/vespa/storage/storageserver/bouncer.cpp +++ b/storage/src/vespa/storage/storageserver/bouncer.cpp @@ -264,12 +264,14 @@ Bouncer::onDown(const std::shared_ptr<api::StorageMessage>& msg) int maxClockSkewInSeconds; bool isInAvailableState; bool abortLoadWhenClusterDown; + bool cluster_is_up; int feedPriorityLowerBound; { std::lock_guard lock(_lock); state = &getDerivedNodeState(msg->getBucket().getBucketSpace()).getState(); maxClockSkewInSeconds = _config->maxClockSkewSeconds; abortLoadWhenClusterDown = _config->stopExternalLoadWhenClusterDown; + cluster_is_up = clusterIsUp(); isInAvailableState = state->oneOf(_config->stopAllLoadWhenNodestateNotIn.c_str()); feedPriorityLowerBound = _config->feedRejectionPriorityThreshold; } @@ -317,7 +319,7 @@ Bouncer::onDown(const std::shared_ptr<api::StorageMessage>& msg) } // If cluster state is not up, fail external load - if (abortLoadWhenClusterDown && !clusterIsUp()) { + if (abortLoadWhenClusterDown && !cluster_is_up) { abortCommandDueToClusterDown(*msg); return true; } diff --git a/storageserver/src/apps/storaged/storage.cpp b/storageserver/src/apps/storaged/storage.cpp index aaa85f7f476..c8352354175 100644 --- a/storageserver/src/apps/storaged/storage.cpp +++ b/storageserver/src/apps/storaged/storage.cpp @@ -65,7 +65,6 @@ public: ~StorageApp() override; void handleSignal(int signal) { - LOG(info, "Got signal %d", signal); _lastSignal = signal; _signalCond.notify_one(); } diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java index c01286a8064..1ea7266ab12 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/VespaCliTestRunner.java @@ -26,7 +26,6 @@ import java.util.stream.Stream; import static com.yahoo.vespa.testrunner.TestRunner.Status.ERROR; import static com.yahoo.vespa.testrunner.TestRunner.Status.FAILURE; -import static com.yahoo.vespa.testrunner.TestRunner.Status.INCONCLUSIVE; import static com.yahoo.vespa.testrunner.TestRunner.Status.NO_TESTS; import static com.yahoo.vespa.testrunner.TestRunner.Status.RUNNING; import static com.yahoo.vespa.testrunner.TestRunner.Status.SUCCESS; @@ -119,6 +118,9 @@ public class VespaCliTestRunner implements TestRunner { "--zone", config.zone().value(), "--target", "cloud"); builder.redirectErrorStream(true); + // The CI environment variables tells Vespa CLI to omit certain warnings that do not apply to CI environments + builder.environment().put("CI", "true"); + builder.environment().put("VESPA_CLI_CLOUD_CI", "true"); builder.environment().put("VESPA_CLI_HOME", ensureHomeDirectoryForVespaCli().toString()); builder.environment().put("VESPA_CLI_ENDPOINTS", toEndpointsConfig(config)); builder.environment().put("VESPA_CLI_DATA_PLANE_KEY_FILE", artifactsPath.resolve("key").toAbsolutePath().toString()); diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/VespaCliTestRunnerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/VespaCliTestRunnerTest.java index 5fd73bb4494..8b02575b950 100644 --- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/VespaCliTestRunnerTest.java +++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/VespaCliTestRunnerTest.java @@ -12,6 +12,7 @@ import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author jonmv @@ -55,6 +56,8 @@ class VespaCliTestRunnerTest { "--zone", "dev.aws-us-east-1c", "--target", "cloud"), builder.command()); + assertTrue(builder.environment().containsKey("CI")); + assertTrue(builder.environment().containsKey("VESPA_CLI_CLOUD_CI")); assertEquals("{\"endpoints\":[{\"cluster\":\"default\",\"url\":\"https://dev.endpoint:443/\"}]}", builder.environment().get("VESPA_CLI_ENDPOINTS")); assertEquals(artifacts.resolve("key").toAbsolutePath().toString(), diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java index 10e55527a2a..7b3e488a5a5 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java @@ -214,7 +214,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { this.operations = new ConcurrentLinkedDeque<>(); long resendDelayMS = SystemTimer.adjustTimeoutByDetectedHz(Duration.ofMillis(executorConfig.resendDelayMillis())).toMillis(); - //TODO Here it would be better do have dedicated threads with different wait depending on blocked or empty. + // TODO: Here it would be better do have dedicated threads with different wait depending on blocked or empty. this.dispatcher.scheduleWithFixedDelay(this::dispatchEnqueued, resendDelayMS, resendDelayMS, MILLISECONDS); this.visitDispatcher.scheduleWithFixedDelay(this::dispatchVisitEnqueued, resendDelayMS, resendDelayMS, MILLISECONDS); } diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index a30ee055538..4e25d8ab0e0 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -1232,6 +1232,7 @@ "public static java.lang.String toStandardString(com.yahoo.tensor.Tensor)", "public static java.lang.String contentToString(com.yahoo.tensor.Tensor)", "public abstract boolean equals(java.lang.Object)", + "public abstract int hashCode()", "public static boolean equals(com.yahoo.tensor.Tensor, com.yahoo.tensor.Tensor)", "public static boolean approxEquals(double, double, double)", "public static boolean approxEquals(double, double)", @@ -1563,7 +1564,8 @@ "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", "public com.yahoo.tensor.Tensor evaluate(com.yahoo.tensor.evaluation.EvaluationContext)", - "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)" + "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", + "public int hashCode()" ], "fields": [] }, @@ -1580,7 +1582,8 @@ "public java.util.List arguments()", "public com.yahoo.tensor.functions.TensorFunction withArguments(java.util.List)", "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", - "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)" + "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", + "public int hashCode()" ], "fields": [] }, @@ -1597,7 +1600,8 @@ "public java.util.List arguments()", "public com.yahoo.tensor.functions.TensorFunction withArguments(java.util.List)", "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", - "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)" + "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", + "public int hashCode()" ], "fields": [] }, @@ -1614,7 +1618,8 @@ "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", "public com.yahoo.tensor.Tensor evaluate(com.yahoo.tensor.evaluation.EvaluationContext)", - "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)" + "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", + "public int hashCode()" ], "fields": [] }, @@ -1644,6 +1649,7 @@ "public com.yahoo.tensor.functions.TensorFunction withArguments(java.util.List)", "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", + "public int hashCode()", "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", "public com.yahoo.tensor.Tensor evaluate(com.yahoo.tensor.evaluation.EvaluationContext)" ], @@ -1663,7 +1669,8 @@ "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", "public com.yahoo.tensor.Tensor evaluate(com.yahoo.tensor.evaluation.EvaluationContext)", - "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)" + "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", + "public int hashCode()" ], "fields": [] }, @@ -1678,7 +1685,8 @@ "public java.util.List arguments()", "public com.yahoo.tensor.functions.TensorFunction withArguments(java.util.List)", "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", - "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)" + "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", + "public int hashCode()" ], "fields": [] }, @@ -1711,7 +1719,8 @@ "public java.util.List arguments()", "public com.yahoo.tensor.functions.TensorFunction withArguments(java.util.List)", "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", - "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)" + "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", + "public int hashCode()" ], "fields": [] }, @@ -1730,7 +1739,8 @@ "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", "public com.yahoo.tensor.Tensor evaluate(com.yahoo.tensor.evaluation.EvaluationContext)", - "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)" + "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", + "public int hashCode()" ], "fields": [] }, @@ -1748,6 +1758,7 @@ "public com.yahoo.tensor.functions.TensorFunction withArguments(java.util.List)", "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", + "public int hashCode()", "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", "public com.yahoo.tensor.Tensor evaluate(com.yahoo.tensor.evaluation.EvaluationContext)" ], @@ -1764,7 +1775,8 @@ "public java.util.List arguments()", "public com.yahoo.tensor.functions.TensorFunction withArguments(java.util.List)", "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", - "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)" + "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", + "public int hashCode()" ], "fields": [] }, @@ -1779,7 +1791,8 @@ "public java.util.List arguments()", "public com.yahoo.tensor.functions.TensorFunction withArguments(java.util.List)", "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", - "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)" + "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", + "public int hashCode()" ], "fields": [] }, @@ -1799,7 +1812,8 @@ "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", "public com.yahoo.tensor.Tensor evaluate(com.yahoo.tensor.evaluation.EvaluationContext)", - "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)" + "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", + "public int hashCode()" ], "fields": [] }, @@ -1815,7 +1829,8 @@ "public java.util.List arguments()", "public com.yahoo.tensor.functions.TensorFunction withArguments(java.util.List)", "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", - "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)" + "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", + "public int hashCode()" ], "fields": [] }, @@ -1832,9 +1847,10 @@ "public java.util.List arguments()", "public com.yahoo.tensor.functions.TensorFunction withArguments(java.util.List)", "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", - "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", - "public com.yahoo.tensor.Tensor evaluate(com.yahoo.tensor.evaluation.EvaluationContext)" + "public com.yahoo.tensor.Tensor evaluate(com.yahoo.tensor.evaluation.EvaluationContext)", + "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", + "public int hashCode()" ], "fields": [] }, @@ -1861,7 +1877,8 @@ "public java.util.List arguments()", "public com.yahoo.tensor.functions.TensorFunction withArguments(java.util.List)", "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", - "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)" + "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", + "public int hashCode()" ], "fields": [] }, @@ -1876,7 +1893,8 @@ "public java.util.List arguments()", "public com.yahoo.tensor.functions.TensorFunction withArguments(java.util.List)", "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", - "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)" + "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", + "public int hashCode()" ], "fields": [] }, @@ -1919,7 +1937,8 @@ "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", - "public com.yahoo.tensor.Tensor evaluate(com.yahoo.tensor.evaluation.EvaluationContext)" + "public com.yahoo.tensor.Tensor evaluate(com.yahoo.tensor.evaluation.EvaluationContext)", + "public int hashCode()" ], "fields": [] }, @@ -1937,7 +1956,8 @@ "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", "public final com.yahoo.tensor.Tensor evaluate(com.yahoo.tensor.evaluation.EvaluationContext)", "public boolean canOptimize(com.yahoo.tensor.Tensor, com.yahoo.tensor.Tensor)", - "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)" + "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", + "public int hashCode()" ], "fields": [] }, @@ -1957,7 +1977,8 @@ "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", "public com.yahoo.tensor.Tensor evaluate(com.yahoo.tensor.evaluation.EvaluationContext)", - "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)" + "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", + "public int hashCode()" ], "fields": [] }, @@ -1990,7 +2011,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2005,7 +2027,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2020,7 +2043,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double, double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2035,7 +2059,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2050,7 +2075,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2065,7 +2091,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2081,6 +2108,7 @@ "public void <init>(double)", "public java.lang.Double apply(java.util.List)", "public java.lang.String toString()", + "public int hashCode()", "public bridge synthetic java.lang.Object apply(java.lang.Object)" ], "fields": [] @@ -2096,7 +2124,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2111,7 +2140,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double, double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2127,7 +2157,8 @@ "public void <init>()", "public void <init>(double)", "public double applyAsDouble(double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2142,7 +2173,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double, double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2157,6 +2189,7 @@ "methods": [ "public java.lang.Double apply(java.util.List)", "public java.lang.String toString()", + "public int hashCode()", "public bridge synthetic java.lang.Object apply(java.lang.Object)" ], "fields": [] @@ -2173,6 +2206,7 @@ "public void <init>()", "public double applyAsDouble(double)", "public java.lang.String toString()", + "public int hashCode()", "public static double erf(double)" ], "fields": [] @@ -2188,7 +2222,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2203,7 +2238,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2218,7 +2254,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double, double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2234,7 +2271,8 @@ "public void <init>()", "public static double hamming(double, double)", "public double applyAsDouble(double, double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2250,7 +2288,8 @@ "public void <init>()", "public void <init>(double)", "public double applyAsDouble(double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2265,7 +2304,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double, double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2280,7 +2320,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2295,7 +2336,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double, double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2310,7 +2352,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double, double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2325,7 +2368,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double, double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2340,7 +2384,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double, double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2355,7 +2400,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2370,7 +2416,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double, double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2386,6 +2433,7 @@ "public void <init>()", "public java.lang.Double apply(java.util.List)", "public java.lang.String toString()", + "public int hashCode()", "public bridge synthetic java.lang.Object apply(java.lang.Object)" ], "fields": [] @@ -2401,7 +2449,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2416,7 +2465,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2431,7 +2481,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2447,7 +2498,8 @@ "public void <init>()", "public void <init>(double, double)", "public double applyAsDouble(double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2462,7 +2514,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2477,7 +2530,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2492,7 +2546,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2507,7 +2562,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2522,7 +2578,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double, double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2537,7 +2594,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double, double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2552,6 +2610,7 @@ "methods": [ "public java.lang.Double apply(java.util.List)", "public java.lang.String toString()", + "public int hashCode()", "public bridge synthetic java.lang.Object apply(java.lang.Object)" ], "fields": [] @@ -2567,7 +2626,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2582,7 +2642,8 @@ "methods": [ "public void <init>()", "public double applyAsDouble(double)", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2658,7 +2719,8 @@ "public java.util.Optional dimension()", "public java.util.Optional label()", "public java.util.Optional index()", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public int hashCode()" ], "fields": [] }, @@ -2676,6 +2738,7 @@ "public com.yahoo.tensor.Tensor evaluate(com.yahoo.tensor.evaluation.EvaluationContext)", "public com.yahoo.tensor.TensorType type(com.yahoo.tensor.evaluation.TypeContext)", "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", + "public int hashCode()", "public bridge synthetic com.yahoo.tensor.functions.TensorFunction withArguments(java.util.List)" ], "fields": [] @@ -2692,7 +2755,8 @@ "public java.util.List arguments()", "public com.yahoo.tensor.functions.TensorFunction withArguments(java.util.List)", "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", - "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)" + "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", + "public int hashCode()" ], "fields": [] }, @@ -2713,7 +2777,8 @@ "public final com.yahoo.tensor.Tensor evaluate()", "public abstract java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", "public java.util.Optional asScalarFunction()", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public abstract int hashCode()" ], "fields": [] }, @@ -2759,7 +2824,8 @@ "public java.util.List arguments()", "public com.yahoo.tensor.functions.TensorFunction withArguments(java.util.List)", "public com.yahoo.tensor.functions.PrimitiveTensorFunction toPrimitive()", - "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)" + "public java.lang.String toString(com.yahoo.tensor.functions.ToStringContext)", + "public int hashCode()" ], "fields": [] }, diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java index c4588b79fa9..ca396ae5bf2 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java @@ -355,6 +355,10 @@ public interface Tensor { @Override boolean equals(Object o); + /** Returns a hash computed deterministically from the content of this tensor */ + @Override + int hashCode(); + /** * Implement here to make this work across implementations. * Implementations must override equals and call this because this is an interface and cannot override equals. diff --git a/vespajlib/src/main/java/com/yahoo/tensor/evaluation/VariableTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/evaluation/VariableTensor.java index dbc8396d701..8a9a85d343c 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/evaluation/VariableTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/evaluation/VariableTensor.java @@ -9,6 +9,7 @@ import com.yahoo.tensor.functions.ToStringContext; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.Optional; /** @@ -62,6 +63,9 @@ public class VariableTensor<NAMETYPE extends Name> extends PrimitiveTensorFuncti return name; } + @Override + public int hashCode() { return Objects.hash("variableTensor", name, requiredType); } + private void verifyType(TensorType givenType) { if (requiredType.isPresent() && ! givenType.isAssignableTo(requiredType.get())) throw new IllegalArgumentException("Variable '" + name + "' must be compatible with " + diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmax.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmax.java index 55dd8a7bc8a..d2762ad762d 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmax.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmax.java @@ -52,4 +52,7 @@ public class Argmax<NAMETYPE extends Name> extends CompositeTensorFunction<NAMET return "argmax(" + argument.toString(context) + Reduce.commaSeparated(dimensions) + ")"; } + @Override + public int hashCode() { return Objects.hash("argmax", argument, dimensions); } + } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmin.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmin.java index f1f0b9d67b0..baedf41bcb8 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmin.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Argmin.java @@ -52,4 +52,7 @@ public class Argmin<NAMETYPE extends Name> extends CompositeTensorFunction<NAMET return "argmin(" + argument.toString(context) + Reduce.commaSeparated(dimensions) + ")"; } + @Override + public int hashCode() { return Objects.hash("argmin", argument, dimensions); } + } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/CellCast.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/CellCast.java index 09f84e6747e..176847cec93 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/CellCast.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/CellCast.java @@ -111,4 +111,7 @@ public class CellCast<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAM return "cell_cast(" + argument.toString(context) + ", " + valueType + ")"; } + @Override + public int hashCode() { return Objects.hash("cellcast", argument, valueType); } + } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Concat.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Concat.java index 6d4b15be991..abf0d89c2b7 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Concat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Concat.java @@ -31,6 +31,191 @@ import java.util.stream.Collectors; */ public class Concat<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMETYPE> { + enum DimType { common, separate, concat } + + private final TensorFunction<NAMETYPE> argumentA, argumentB; + private final String dimension; + + public Concat(TensorFunction<NAMETYPE> argumentA, TensorFunction<NAMETYPE> argumentB, String dimension) { + Objects.requireNonNull(argumentA, "The first argument tensor cannot be null"); + Objects.requireNonNull(argumentB, "The second argument tensor cannot be null"); + Objects.requireNonNull(dimension, "The dimension cannot be null"); + this.argumentA = argumentA; + this.argumentB = argumentB; + this.dimension = dimension; + } + + @Override + public List<TensorFunction<NAMETYPE>> arguments() { return ImmutableList.of(argumentA, argumentB); } + + @Override + public TensorFunction<NAMETYPE> withArguments(List<TensorFunction<NAMETYPE>> arguments) { + if (arguments.size() != 2) + throw new IllegalArgumentException("Concat must have 2 arguments, got " + arguments.size()); + return new Concat<>(arguments.get(0), arguments.get(1), dimension); + } + + @Override + public PrimitiveTensorFunction<NAMETYPE> toPrimitive() { + return new Concat<>(argumentA.toPrimitive(), argumentB.toPrimitive(), dimension); + } + + @Override + public String toString(ToStringContext<NAMETYPE> context) { + return "concat(" + argumentA.toString(context) + ", " + argumentB.toString(context) + ", " + dimension + ")"; + } + + @Override + public int hashCode() { return Objects.hash("concat", argumentA, argumentB, dimension); } + + @Override + public TensorType type(TypeContext<NAMETYPE> context) { + return TypeResolver.concat(argumentA.type(context), argumentB.type(context), dimension); + } + + @Override + public Tensor evaluate(EvaluationContext<NAMETYPE> context) { + Tensor a = argumentA.evaluate(context); + Tensor b = argumentB.evaluate(context); + if (a instanceof IndexedTensor && b instanceof IndexedTensor) { + return oldEvaluate(a, b); + } + var helper = new Helper(a, b, dimension); + return helper.result; + } + + private Tensor oldEvaluate(Tensor a, Tensor b) { + TensorType concatType = TypeResolver.concat(a.type(), b.type(), dimension); + + a = ensureIndexedDimension(dimension, a, concatType.valueType()); + b = ensureIndexedDimension(dimension, b, concatType.valueType()); + + IndexedTensor aIndexed = (IndexedTensor) a; // If you get an exception here you have implemented a mixed tensor + IndexedTensor bIndexed = (IndexedTensor) b; + DimensionSizes concatSize = concatSize(concatType, aIndexed, bIndexed, dimension); + + Tensor.Builder builder = Tensor.Builder.of(concatType, concatSize); + long aDimensionLength = aIndexed.type().indexOfDimension(dimension).map(d -> aIndexed.dimensionSizes().size(d)).orElseThrow(RuntimeException::new); + int[] aToIndexes = mapIndexes(a.type(), concatType); + int[] bToIndexes = mapIndexes(b.type(), concatType); + concatenateTo(aIndexed, bIndexed, aDimensionLength, concatType, aToIndexes, bToIndexes, builder); + concatenateTo(bIndexed, aIndexed, 0, concatType, bToIndexes, aToIndexes, builder); + return builder.build(); + } + + private void concatenateTo(IndexedTensor a, IndexedTensor b, long offset, TensorType concatType, + int[] aToIndexes, int[] bToIndexes, Tensor.Builder builder) { + Set<String> otherADimensions = a.type().dimensionNames().stream().filter(d -> !d.equals(dimension)).collect(Collectors.toSet()); + for (Iterator<IndexedTensor.SubspaceIterator> ia = a.subspaceIterator(otherADimensions); ia.hasNext();) { + IndexedTensor.SubspaceIterator iaSubspace = ia.next(); + TensorAddress aAddress = iaSubspace.address(); + for (Iterator<IndexedTensor.SubspaceIterator> ib = b.subspaceIterator(otherADimensions); ib.hasNext();) { + IndexedTensor.SubspaceIterator ibSubspace = ib.next(); + while (ibSubspace.hasNext()) { + Tensor.Cell bCell = ibSubspace.next(); + TensorAddress combinedAddress = combineAddresses(aAddress, aToIndexes, bCell.getKey(), bToIndexes, + concatType, offset, dimension); + if (combinedAddress == null) continue; // incompatible + + builder.cell(combinedAddress, bCell.getValue()); + } + iaSubspace.reset(); + } + } + } + + private Tensor ensureIndexedDimension(String dimensionName, Tensor tensor, TensorType.Value combinedValueType) { + Optional<TensorType.Dimension> dimension = tensor.type().dimension(dimensionName); + if ( dimension.isPresent() ) { + if ( ! dimension.get().isIndexed()) + throw new IllegalArgumentException("Concat in dimension '" + dimensionName + + "' requires that dimension to be indexed or absent, " + + "but got a tensor with type " + tensor.type()); + return tensor; + } + else { // extend tensor with this dimension + if (tensor.type().dimensions().stream().anyMatch(d -> ! d.isIndexed())) + throw new IllegalArgumentException("Concat requires an indexed tensor, " + + "but got a tensor with type " + tensor.type()); + Tensor unitTensor = Tensor.Builder.of(new TensorType.Builder(combinedValueType) + .indexed(dimensionName, 1) + .build()) + .cell(1,0) + .build(); + return tensor.multiply(unitTensor); + } + + } + + /** Returns the concrete (not type) dimension sizes resulting from combining a and b */ + private DimensionSizes concatSize(TensorType concatType, IndexedTensor a, IndexedTensor b, String concatDimension) { + DimensionSizes.Builder concatSizes = new DimensionSizes.Builder(concatType.dimensions().size()); + for (int i = 0; i < concatSizes.dimensions(); i++) { + String currentDimension = concatType.dimensions().get(i).name(); + long aSize = a.type().indexOfDimension(currentDimension).map(d -> a.dimensionSizes().size(d)).orElse(0L); + long bSize = b.type().indexOfDimension(currentDimension).map(d -> b.dimensionSizes().size(d)).orElse(0L); + if (currentDimension.equals(concatDimension)) + concatSizes.set(i, aSize + bSize); + else if (aSize != 0 && bSize != 0 && aSize!=bSize ) + concatSizes.set(i, Math.min(aSize, bSize)); + else + concatSizes.set(i, Math.max(aSize, bSize)); + } + return concatSizes.build(); + } + + /** + * Combine two addresses, adding the offset to the concat dimension + * + * @return the combined address or null if the addresses are incompatible + * (in some other dimension than the concat dimension) + */ + private TensorAddress combineAddresses(TensorAddress a, int[] aToIndexes, TensorAddress b, int[] bToIndexes, + TensorType concatType, long concatOffset, String concatDimension) { + long[] combinedLabels = new long[concatType.dimensions().size()]; + Arrays.fill(combinedLabels, -1); + int concatDimensionIndex = concatType.indexOfDimension(concatDimension).get(); + mapContent(a, combinedLabels, aToIndexes, concatDimensionIndex, concatOffset); // note: This sets a nonsensical value in the concat dimension + boolean compatible = mapContent(b, combinedLabels, bToIndexes, concatDimensionIndex, concatOffset); // ... which is overwritten by the right value here + if ( ! compatible) return null; + return TensorAddress.of(combinedLabels); + } + + /** + * Returns the an array having one entry in order for each dimension of fromType + * containing the index at which toType contains the same dimension name. + * That is, if the returned array contains n at index i then + * fromType.dimensions().get(i).name.equals(toType.dimensions().get(n).name()) + * If some dimension in fromType is not present in toType, the corresponding index will be -1 + */ + // TODO: Stolen from join + private int[] mapIndexes(TensorType fromType, TensorType toType) { + int[] toIndexes = new int[fromType.dimensions().size()]; + for (int i = 0; i < fromType.dimensions().size(); i++) + toIndexes[i] = toType.indexOfDimension(fromType.dimensions().get(i).name()).orElse(-1); + return toIndexes; + } + + /** + * Maps the content in the given list to the given array, using the given index map. + * + * @return true if the mapping was successful, false if one of the destination positions was + * occupied by a different value + */ + private boolean mapContent(TensorAddress from, long[] to, int[] indexMap, int concatDimension, long concatOffset) { + for (int i = 0; i < from.size(); i++) { + int toIndex = indexMap[i]; + if (concatDimension == toIndex) { + to[toIndex] = from.numericLabel(i) + concatOffset; + } + else { + if (to[toIndex] != -1 && to[toIndex] != from.numericLabel(i)) return false; + to[toIndex] = from.numericLabel(i); + } + } + return true; + } + static class CellVector { ArrayList<Double> values = new ArrayList<>(); void setValue(int ccDimIndex, double value) { @@ -57,8 +242,6 @@ public class Concat<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMET } - enum DimType { common, separate, concat } - static class SplitHow { List<DimType> handleDims = new ArrayList<>(); long numCommon() { return handleDims.stream().filter(t -> (t == DimType.common)).count(); } @@ -76,7 +259,7 @@ public class Concat<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMET enum CombineHow { left, right, both, concat } List<CombineHow> combineHow = new ArrayList<>(); - + void aOnly(String dimName) { if (dimName.equals(concatDimension)) { splitInfoA.handleDims.add(DimType.concat); @@ -160,8 +343,8 @@ public class Concat<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMET static int concatDimensionSize(CellVectorMapMap data) { Set<Integer> sizes = new HashSet<>(); data.map.forEach((m, cvmap) -> - cvmap.map.forEach((e, vector) -> - sizes.add(vector.values.size()))); + cvmap.map.forEach((e, vector) -> + sizes.add(vector.values.size()))); if (sizes.isEmpty()) { return 1; } @@ -207,17 +390,17 @@ public class Concat<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMET var lhs = entry.getValue(); var rhs = b.map.get(common); lhs.map.forEach((leftOnly, leftCells) -> { - rhs.map.forEach((rightOnly, rightCells) -> { - for (int i = 0; i < leftCells.values.size(); i++) { - TensorAddress addr = combine(common, leftOnly, rightOnly, i); - builder.cell(addr, leftCells.values.get(i)); - } - for (int i = 0; i < rightCells.values.size(); i++) { - TensorAddress addr = combine(common, leftOnly, rightOnly, i + aConcatSize); - builder.cell(addr, rightCells.values.get(i)); - } - }); + rhs.map.forEach((rightOnly, rightCells) -> { + for (int i = 0; i < leftCells.values.size(); i++) { + TensorAddress addr = combine(common, leftOnly, rightOnly, i); + builder.cell(addr, leftCells.values.get(i)); + } + for (int i = 0; i < rightCells.values.size(); i++) { + TensorAddress addr = combine(common, leftOnly, rightOnly, i + aConcatSize); + builder.cell(addr, rightCells.values.get(i)); + } }); + }); } } return builder.build(); @@ -240,7 +423,7 @@ public class Concat<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMET commonLabels[commonIdx++] = addr.label(i); break; case separate: - separateLabels[separateIdx++] = addr.label(i); + separateLabels[separateIdx++] = addr.label(i); break; case concat: ccDimIndex = addr.numericLabel(i); @@ -257,184 +440,4 @@ public class Concat<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMET } } - private final TensorFunction<NAMETYPE> argumentA, argumentB; - private final String dimension; - - public Concat(TensorFunction<NAMETYPE> argumentA, TensorFunction<NAMETYPE> argumentB, String dimension) { - Objects.requireNonNull(argumentA, "The first argument tensor cannot be null"); - Objects.requireNonNull(argumentB, "The second argument tensor cannot be null"); - Objects.requireNonNull(dimension, "The dimension cannot be null"); - this.argumentA = argumentA; - this.argumentB = argumentB; - this.dimension = dimension; - } - - @Override - public List<TensorFunction<NAMETYPE>> arguments() { return ImmutableList.of(argumentA, argumentB); } - - @Override - public TensorFunction<NAMETYPE> withArguments(List<TensorFunction<NAMETYPE>> arguments) { - if (arguments.size() != 2) - throw new IllegalArgumentException("Concat must have 2 arguments, got " + arguments.size()); - return new Concat<>(arguments.get(0), arguments.get(1), dimension); - } - - @Override - public PrimitiveTensorFunction<NAMETYPE> toPrimitive() { - return new Concat<>(argumentA.toPrimitive(), argumentB.toPrimitive(), dimension); - } - - @Override - public String toString(ToStringContext<NAMETYPE> context) { - return "concat(" + argumentA.toString(context) + ", " + argumentB.toString(context) + ", " + dimension + ")"; - } - - @Override - public TensorType type(TypeContext<NAMETYPE> context) { - return TypeResolver.concat(argumentA.type(context), argumentB.type(context), dimension); - } - - @Override - public Tensor evaluate(EvaluationContext<NAMETYPE> context) { - Tensor a = argumentA.evaluate(context); - Tensor b = argumentB.evaluate(context); - if (a instanceof IndexedTensor && b instanceof IndexedTensor) { - return oldEvaluate(a, b); - } - var helper = new Helper(a, b, dimension); - return helper.result; - } - - private Tensor oldEvaluate(Tensor a, Tensor b) { - TensorType concatType = TypeResolver.concat(a.type(), b.type(), dimension); - - a = ensureIndexedDimension(dimension, a, concatType.valueType()); - b = ensureIndexedDimension(dimension, b, concatType.valueType()); - - IndexedTensor aIndexed = (IndexedTensor) a; // If you get an exception here you have implemented a mixed tensor - IndexedTensor bIndexed = (IndexedTensor) b; - DimensionSizes concatSize = concatSize(concatType, aIndexed, bIndexed, dimension); - - Tensor.Builder builder = Tensor.Builder.of(concatType, concatSize); - long aDimensionLength = aIndexed.type().indexOfDimension(dimension).map(d -> aIndexed.dimensionSizes().size(d)).orElseThrow(RuntimeException::new); - int[] aToIndexes = mapIndexes(a.type(), concatType); - int[] bToIndexes = mapIndexes(b.type(), concatType); - concatenateTo(aIndexed, bIndexed, aDimensionLength, concatType, aToIndexes, bToIndexes, builder); - concatenateTo(bIndexed, aIndexed, 0, concatType, bToIndexes, aToIndexes, builder); - return builder.build(); - } - - private void concatenateTo(IndexedTensor a, IndexedTensor b, long offset, TensorType concatType, - int[] aToIndexes, int[] bToIndexes, Tensor.Builder builder) { - Set<String> otherADimensions = a.type().dimensionNames().stream().filter(d -> !d.equals(dimension)).collect(Collectors.toSet()); - for (Iterator<IndexedTensor.SubspaceIterator> ia = a.subspaceIterator(otherADimensions); ia.hasNext();) { - IndexedTensor.SubspaceIterator iaSubspace = ia.next(); - TensorAddress aAddress = iaSubspace.address(); - for (Iterator<IndexedTensor.SubspaceIterator> ib = b.subspaceIterator(otherADimensions); ib.hasNext();) { - IndexedTensor.SubspaceIterator ibSubspace = ib.next(); - while (ibSubspace.hasNext()) { - Tensor.Cell bCell = ibSubspace.next(); - TensorAddress combinedAddress = combineAddresses(aAddress, aToIndexes, bCell.getKey(), bToIndexes, - concatType, offset, dimension); - if (combinedAddress == null) continue; // incompatible - - builder.cell(combinedAddress, bCell.getValue()); - } - iaSubspace.reset(); - } - } - } - - private Tensor ensureIndexedDimension(String dimensionName, Tensor tensor, TensorType.Value combinedValueType) { - Optional<TensorType.Dimension> dimension = tensor.type().dimension(dimensionName); - if ( dimension.isPresent() ) { - if ( ! dimension.get().isIndexed()) - throw new IllegalArgumentException("Concat in dimension '" + dimensionName + - "' requires that dimension to be indexed or absent, " + - "but got a tensor with type " + tensor.type()); - return tensor; - } - else { // extend tensor with this dimension - if (tensor.type().dimensions().stream().anyMatch(d -> ! d.isIndexed())) - throw new IllegalArgumentException("Concat requires an indexed tensor, " + - "but got a tensor with type " + tensor.type()); - Tensor unitTensor = Tensor.Builder.of(new TensorType.Builder(combinedValueType) - .indexed(dimensionName, 1) - .build()) - .cell(1,0) - .build(); - return tensor.multiply(unitTensor); - } - - } - - /** Returns the concrete (not type) dimension sizes resulting from combining a and b */ - private DimensionSizes concatSize(TensorType concatType, IndexedTensor a, IndexedTensor b, String concatDimension) { - DimensionSizes.Builder concatSizes = new DimensionSizes.Builder(concatType.dimensions().size()); - for (int i = 0; i < concatSizes.dimensions(); i++) { - String currentDimension = concatType.dimensions().get(i).name(); - long aSize = a.type().indexOfDimension(currentDimension).map(d -> a.dimensionSizes().size(d)).orElse(0L); - long bSize = b.type().indexOfDimension(currentDimension).map(d -> b.dimensionSizes().size(d)).orElse(0L); - if (currentDimension.equals(concatDimension)) - concatSizes.set(i, aSize + bSize); - else if (aSize != 0 && bSize != 0 && aSize!=bSize ) - concatSizes.set(i, Math.min(aSize, bSize)); - else - concatSizes.set(i, Math.max(aSize, bSize)); - } - return concatSizes.build(); - } - - /** - * Combine two addresses, adding the offset to the concat dimension - * - * @return the combined address or null if the addresses are incompatible - * (in some other dimension than the concat dimension) - */ - private TensorAddress combineAddresses(TensorAddress a, int[] aToIndexes, TensorAddress b, int[] bToIndexes, - TensorType concatType, long concatOffset, String concatDimension) { - long[] combinedLabels = new long[concatType.dimensions().size()]; - Arrays.fill(combinedLabels, -1); - int concatDimensionIndex = concatType.indexOfDimension(concatDimension).get(); - mapContent(a, combinedLabels, aToIndexes, concatDimensionIndex, concatOffset); // note: This sets a nonsensical value in the concat dimension - boolean compatible = mapContent(b, combinedLabels, bToIndexes, concatDimensionIndex, concatOffset); // ... which is overwritten by the right value here - if ( ! compatible) return null; - return TensorAddress.of(combinedLabels); - } - - /** - * Returns the an array having one entry in order for each dimension of fromType - * containing the index at which toType contains the same dimension name. - * That is, if the returned array contains n at index i then - * fromType.dimensions().get(i).name.equals(toType.dimensions().get(n).name()) - * If some dimension in fromType is not present in toType, the corresponding index will be -1 - */ - // TODO: Stolen from join - private int[] mapIndexes(TensorType fromType, TensorType toType) { - int[] toIndexes = new int[fromType.dimensions().size()]; - for (int i = 0; i < fromType.dimensions().size(); i++) - toIndexes[i] = toType.indexOfDimension(fromType.dimensions().get(i).name()).orElse(-1); - return toIndexes; - } - - /** - * Maps the content in the given list to the given array, using the given index map. - * - * @return true if the mapping was successful, false if one of the destination positions was - * occupied by a different value - */ - private boolean mapContent(TensorAddress from, long[] to, int[] indexMap, int concatDimension, long concatOffset) { - for (int i = 0; i < from.size(); i++) { - int toIndex = indexMap[i]; - if (concatDimension == toIndex) { - to[toIndex] = from.numericLabel(i) + concatOffset; - } - else { - if (to[toIndex] != -1 && to[toIndex] != from.numericLabel(i)) return false; - to[toIndex] = from.numericLabel(i); - } - } - return true; - } - } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/ConstantTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/ConstantTensor.java index a0fd9272f54..92a72dfd280 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/ConstantTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/ConstantTensor.java @@ -9,6 +9,7 @@ import com.yahoo.tensor.evaluation.TypeContext; import java.util.Collections; import java.util.List; +import java.util.Objects; /** * A function which returns a constant tensor. @@ -49,4 +50,9 @@ public class ConstantTensor<NAMETYPE extends Name> extends PrimitiveTensorFuncti @Override public String toString(ToStringContext<NAMETYPE> context) { return constant.toString(); } + @Override + public int hashCode() { + return Objects.hash("constant", constant.hashCode()); + } + } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Diag.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Diag.java index 92d89ec68f7..7218375de89 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Diag.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Diag.java @@ -6,6 +6,7 @@ import com.yahoo.tensor.evaluation.Name; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -40,13 +41,16 @@ public class Diag<NAMETYPE extends Name> extends CompositeTensorFunction<NAMETYP return new Generate<>(type, diagFunction); } + private Stream<String> dimensionNames() { + return type.dimensions().stream().map(TensorType.Dimension::name); + } + @Override public String toString(ToStringContext<NAMETYPE> context) { return "diag(" + dimensionNames().collect(Collectors.joining(",")) + ")" + diagFunction; } - private Stream<String> dimensionNames() { - return type.dimensions().stream().map(TensorType.Dimension::name); - } + @Override + public int hashCode() { return Objects.hash("diag", type, diagFunction); } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/DynamicTensor.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/DynamicTensor.java index 46992115c23..c402a1bde5b 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/DynamicTensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/DynamicTensor.java @@ -13,6 +13,7 @@ import com.yahoo.tensor.evaluation.TypeContext; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Objects; /** * A function which is a tensor whose values are computed by individual lambda functions on evaluation. @@ -45,13 +46,13 @@ public abstract class DynamicTensor<NAMETYPE extends Name> extends PrimitiveTens TensorType type() { return type; } + abstract String contentToString(ToStringContext<NAMETYPE> context); + @Override public String toString(ToStringContext<NAMETYPE> context) { return type().toString() + ":" + contentToString(context); } - abstract String contentToString(ToStringContext<NAMETYPE> context); - /** Creates a dynamic tensor function. The cell addresses must match the type. */ public static <NAMETYPE extends Name> DynamicTensor<NAMETYPE> from(TensorType type, Map<TensorAddress, ScalarFunction<NAMETYPE>> cells) { return new MappedDynamicTensor<>(type, cells); @@ -98,6 +99,9 @@ public abstract class DynamicTensor<NAMETYPE extends Name> extends PrimitiveTens return b.toString(); } + @Override + public int hashCode() { return Objects.hash("mappedDynamicTensor", type(), cells); } + } private static class IndexedDynamicTensor<NAMETYPE extends Name> extends DynamicTensor<NAMETYPE> { @@ -141,6 +145,9 @@ public abstract class DynamicTensor<NAMETYPE extends Name> extends PrimitiveTens return b.toString(); } + @Override + public int hashCode() { return Objects.hash("indexedDynamicTensor", type(), cells); } + } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Expand.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Expand.java index c049e5d41da..eee037c8dba 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Expand.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Expand.java @@ -6,6 +6,7 @@ import com.yahoo.tensor.evaluation.Name; import java.util.Collections; import java.util.List; +import java.util.Objects; /** * The <i>expand</i> tensor function returns a tensor with a new dimension of @@ -45,4 +46,7 @@ public class Expand<NAMETYPE extends Name> extends CompositeTensorFunction<NAMET return "expand(" + argument.toString(context) + ", " + dimensionName + ")"; } + @Override + public int hashCode() { return Objects.hash("expand", argument, dimensionName); } + } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Generate.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Generate.java index 54e83fa472f..3ad3e1114cc 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Generate.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Generate.java @@ -126,6 +126,9 @@ public class Generate<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAM return boundGenerator.toString(new GenerateToStringContext(context)); } + @Override + public int hashCode() { return Objects.hash("generate", type, freeGenerator, boundGenerator); } + /** * A context for generating all the values of a tensor produced by evaluating Generate. * This returns all the current index values as variables and falls back to delivering from the given diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java index 52bef482fb4..4ec5b196dbc 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Join.java @@ -80,6 +80,9 @@ public class Join<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMETYP } @Override + public int hashCode() { return Objects.hash("join", argumentA, argumentB, combinator); } + + @Override public TensorType type(TypeContext<NAMETYPE> context) { return outputType(argumentA.type(context), argumentB.type(context)); } @@ -356,7 +359,6 @@ public class Join<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMETYP return builder.build(); } - /** * Returns the an array having one entry in order for each dimension of fromType * containing the index at which toType contains the same dimension name. diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/L1Normalize.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/L1Normalize.java index f47202d1b9f..38cc95ac6b2 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/L1Normalize.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/L1Normalize.java @@ -5,6 +5,7 @@ import com.yahoo.tensor.evaluation.Name; import java.util.Collections; import java.util.List; +import java.util.Objects; /** * @author bratseth @@ -43,4 +44,7 @@ public class L1Normalize<NAMETYPE extends Name> extends CompositeTensorFunction< return "l1_normalize(" + argument.toString(context) + ", " + dimension + ")"; } + @Override + public int hashCode() { return Objects.hash("l1_normalize", argument, dimension); } + } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/L2Normalize.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/L2Normalize.java index 8f4e2f466d4..4a676449657 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/L2Normalize.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/L2Normalize.java @@ -5,6 +5,7 @@ import com.yahoo.tensor.evaluation.Name; import java.util.Collections; import java.util.List; +import java.util.Objects; /** * @author bratseth @@ -45,4 +46,7 @@ public class L2Normalize<NAMETYPE extends Name> extends CompositeTensorFunction< return "l2_normalize(" + argument.toString(context) + ", " + dimension + ")"; } + @Override + public int hashCode() { return Objects.hash("l2_normalize", argument, dimension); } + } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Map.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Map.java index 46772d8cbff..68645546be9 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Map.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Map.java @@ -75,4 +75,7 @@ public class Map<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMETYPE return "map(" + argument.toString(context) + ", " + mapper + ")"; } + @Override + public int hashCode() { return Objects.hash("map", argument, mapper); } + } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Matmul.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Matmul.java index 8ac6d711c48..3239ab1a70c 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Matmul.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Matmul.java @@ -6,6 +6,7 @@ import com.yahoo.tensor.TensorType; import com.yahoo.tensor.evaluation.Name; import java.util.List; +import java.util.Objects; /** * @author bratseth @@ -49,4 +50,7 @@ public class Matmul<NAMETYPE extends Name> extends CompositeTensorFunction<NAMET return "matmul(" + argument1.toString(context) + ", " + argument2.toString(context) + ", " + dimension + ")"; } + @Override + public int hashCode() { return Objects.hash("matmul", argument1, argument2, dimension); } + } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Merge.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Merge.java index adc84225a63..2b9dc709e0e 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Merge.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Merge.java @@ -70,11 +70,6 @@ public class Merge<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMETY } @Override - public String toString(ToStringContext<NAMETYPE> context) { - return "merge(" + argumentA.toString(context) + ", " + argumentB.toString(context) + ", " + merger + ")"; - } - - @Override public TensorType type(TypeContext<NAMETYPE> context) { return outputType(argumentA.type(context), argumentB.type(context)); } @@ -87,6 +82,15 @@ public class Merge<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMETY return evaluate(a, b, mergedType, merger); } + + @Override + public String toString(ToStringContext<NAMETYPE> context) { + return "merge(" + argumentA.toString(context) + ", " + argumentB.toString(context) + ", " + merger + ")"; + } + + @Override + public int hashCode() { return Objects.hash("merge", argumentA, argumentB, merger); } + static Tensor evaluate(Tensor a, Tensor b, TensorType mergedType, DoubleBinaryOperator combinator) { // Choose merge algorithm if (hasSingleIndexedDimension(a) && hasSingleIndexedDimension(b) && a.type().dimensions().get(0).name().equals(b.type().dimensions().get(0).name())) diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Random.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Random.java index 18c5db8e3a7..34b8eba3e67 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Random.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Random.java @@ -6,6 +6,7 @@ import com.yahoo.tensor.evaluation.Name; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -42,6 +43,9 @@ public class Random<NAMETYPE extends Name> extends CompositeTensorFunction<NAMET return "random(" + dimensionNames().collect(Collectors.joining(",")) + ")"; } + @Override + public int hashCode() { return Objects.hash("random", type); } + private Stream<String> dimensionNames() { return type.dimensions().stream().map(TensorType.Dimension::toString); } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Range.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Range.java index 45b827db900..7053eeb0a1c 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Range.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Range.java @@ -6,6 +6,7 @@ import com.yahoo.tensor.evaluation.Name; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -50,4 +51,9 @@ public class Range<NAMETYPE extends Name> extends CompositeTensorFunction<NAMETY return type.dimensions().stream().map(TensorType.Dimension::toString); } + @Override + public int hashCode() { + return Objects.hash("range", type, rangeFunction); + } + } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Reduce.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Reduce.java index 8841cff15e9..96465de6c0f 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Reduce.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Reduce.java @@ -107,6 +107,11 @@ public class Reduce<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMET return evaluate(this.argument.evaluate(context), dimensions, aggregator); } + @Override + public int hashCode() { + return Objects.hash("reduce", argument, dimensions, aggregator); + } + static Tensor evaluate(Tensor argument, List<String> dimensions, Aggregator aggregator) { if ( ! dimensions.isEmpty() && ! argument.type().dimensionNames().containsAll(dimensions)) throw new IllegalArgumentException("Cannot reduce " + argument + " over dimensions " + @@ -191,6 +196,10 @@ public class Reduce<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMET /** Resets the aggregator */ public abstract void reset(); + /** Returns a hash of this aggregator which only depends on its identity */ + @Override + public abstract int hashCode(); + } private static class AvgAggregator extends ValueAggregator { @@ -214,6 +223,10 @@ public class Reduce<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMET valueCount = 0; valueSum = 0.0; } + + @Override + public int hashCode() { return "avgAggregator".hashCode(); } + } private static class CountAggregator extends ValueAggregator { @@ -234,6 +247,10 @@ public class Reduce<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMET public void reset() { valueCount = 0; } + + @Override + public int hashCode() { return "countAggregator".hashCode(); } + } private static class MaxAggregator extends ValueAggregator { @@ -255,6 +272,10 @@ public class Reduce<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMET public void reset() { maxValue = Double.NEGATIVE_INFINITY; } + + @Override + public int hashCode() { return "maxAggregator".hashCode(); } + } private static class MedianAggregator extends ValueAggregator { @@ -288,6 +309,9 @@ public class Reduce<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMET values = new ArrayList<>(); } + @Override + public int hashCode() { return "medianAggregator".hashCode(); } + } private static class MinAggregator extends ValueAggregator { @@ -310,6 +334,9 @@ public class Reduce<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMET minValue = Double.POSITIVE_INFINITY; } + @Override + public int hashCode() { return "minAggregator".hashCode(); } + } private static class ProdAggregator extends ValueAggregator { @@ -330,6 +357,10 @@ public class Reduce<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMET public void reset() { valueProd = 1.0; } + + @Override + public int hashCode() { return "prodAggregator".hashCode(); } + } private static class SumAggregator extends ValueAggregator { @@ -350,6 +381,10 @@ public class Reduce<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMET public void reset() { valueSum = 0.0; } + + @Override + public int hashCode() { return "sumAggregator".hashCode(); } + } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/ReduceJoin.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/ReduceJoin.java index 7505355beed..ccb437ef5a7 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/ReduceJoin.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/ReduceJoin.java @@ -11,6 +11,7 @@ import com.yahoo.tensor.evaluation.Name; import java.util.Arrays; import java.util.List; +import java.util.Objects; import java.util.function.DoubleBinaryOperator; import java.util.stream.Collectors; @@ -322,6 +323,11 @@ public class ReduceJoin<NAMETYPE extends Name> extends CompositeTensorFunction<N Reduce.commaSeparated(dimensions) + ")"; } + @Override + public int hashCode() { + return Objects.hash("reduce_join", argumentA, argumentB, combinator, aggregator, dimensions); + } + private static class MultiDimensionIterator { private final long[] bounds; diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Rename.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Rename.java index a434ecba5cc..023e91e424f 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Rename.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Rename.java @@ -127,12 +127,6 @@ public class Rename<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMET return TensorAddress.of(reorderedLabels); } - @Override - public String toString(ToStringContext<NAMETYPE> context) { - return "rename(" + argument.toString(context) + ", " + - toVectorString(fromDimensions) + ", " + toVectorString(toDimensions) + ")"; - } - private String toVectorString(List<String> elements) { if (elements.size() == 1) return elements.get(0); @@ -144,4 +138,13 @@ public class Rename<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMET return b.toString(); } + @Override + public String toString(ToStringContext<NAMETYPE> context) { + return "rename(" + argument.toString(context) + ", " + + toVectorString(fromDimensions) + ", " + toVectorString(toDimensions) + ")"; + } + + @Override + public int hashCode() { return Objects.hash("rename", argument, fromDimensions, toDimensions); } + } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/ScalarFunctions.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/ScalarFunctions.java index 517f6683cbf..2639e153923 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/ScalarFunctions.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/ScalarFunctions.java @@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableList; import java.util.Comparator; import java.util.List; +import java.util.Objects; import java.util.PriorityQueue; import java.util.concurrent.ThreadLocalRandom; import java.util.function.DoubleBinaryOperator; @@ -75,6 +76,8 @@ public class ScalarFunctions { public double applyAsDouble(double left, double right) { return left + right; } @Override public String toString() { return "f(a,b)(a + b)"; } + @Override + public int hashCode() { return "add".hashCode(); } } public static class Equal implements DoubleBinaryOperator { @@ -82,6 +85,8 @@ public class ScalarFunctions { public double applyAsDouble(double left, double right) { return left == right ? 1 : 0; } @Override public String toString() { return "f(a,b)(a==b)"; } + @Override + public int hashCode() { return "equal".hashCode(); } } public static class Greater implements DoubleBinaryOperator { @@ -89,6 +94,8 @@ public class ScalarFunctions { public double applyAsDouble(double left, double right) { return left > right ? 1 : 0; } @Override public String toString() { return "f(a,b)(a > b)"; } + @Override + public int hashCode() { return "greater".hashCode(); } } public static class Less implements DoubleBinaryOperator { @@ -96,6 +103,8 @@ public class ScalarFunctions { public double applyAsDouble(double left, double right) { return left < right ? 1 : 0; } @Override public String toString() { return "f(a,b)(a < b)"; } + @Override + public int hashCode() { return "less".hashCode(); } } public static class Max implements DoubleBinaryOperator { @@ -103,6 +112,8 @@ public class ScalarFunctions { public double applyAsDouble(double left, double right) { return Math.max(left, right); } @Override public String toString() { return "f(a,b)(max(a, b))"; } + @Override + public int hashCode() { return "max".hashCode(); } } public static class Min implements DoubleBinaryOperator { @@ -110,6 +121,8 @@ public class ScalarFunctions { public double applyAsDouble(double left, double right) { return Math.min(left, right); } @Override public String toString() { return "f(a,b)(min(a, b))"; } + @Override + public int hashCode() { return "min".hashCode(); } } public static class Mean implements DoubleBinaryOperator { @@ -117,6 +130,8 @@ public class ScalarFunctions { public double applyAsDouble(double left, double right) { return (left + right) / 2; } @Override public String toString() { return "f(a,b)((a + b) / 2)"; } + @Override + public int hashCode() { return "mean".hashCode(); } } public static class Multiply implements DoubleBinaryOperator { @@ -124,6 +139,8 @@ public class ScalarFunctions { public double applyAsDouble(double left, double right) { return left * right; } @Override public String toString() { return "f(a,b)(a * b)"; } + @Override + public int hashCode() { return "multiply".hashCode(); } } public static class Pow implements DoubleBinaryOperator { @@ -131,6 +148,8 @@ public class ScalarFunctions { public double applyAsDouble(double left, double right) { return Math.pow(left, right); } @Override public String toString() { return "f(a,b)(pow(a, b))"; } + @Override + public int hashCode() { return "pow".hashCode(); } } public static class Divide implements DoubleBinaryOperator { @@ -138,6 +157,8 @@ public class ScalarFunctions { public double applyAsDouble(double left, double right) { return left / right; } @Override public String toString() { return "f(a,b)(a / b)"; } + @Override + public int hashCode() { return "divide".hashCode(); } } public static class SquaredDifference implements DoubleBinaryOperator { @@ -145,6 +166,8 @@ public class ScalarFunctions { public double applyAsDouble(double left, double right) { return (left - right) * (left - right); } @Override public String toString() { return "f(a,b)((a-b) * (a-b))"; } + @Override + public int hashCode() { return "squareddifference".hashCode(); } } public static class Subtract implements DoubleBinaryOperator { @@ -152,6 +175,8 @@ public class ScalarFunctions { public double applyAsDouble(double left, double right) { return left - right; } @Override public String toString() { return "f(a,b)(a - b)"; } + @Override + public int hashCode() { return "subtract".hashCode(); } } @@ -172,6 +197,8 @@ public class ScalarFunctions { public double applyAsDouble(double left, double right) { return hamming(left, right); } @Override public String toString() { return "f(a,b)(hamming(a,b))"; } + @Override + public int hashCode() { return "hamming".hashCode(); } } @@ -182,6 +209,8 @@ public class ScalarFunctions { public double applyAsDouble(double operand) { return Math.abs(operand); } @Override public String toString() { return "f(a)(fabs(a))"; } + @Override + public int hashCode() { return "abs".hashCode(); } } public static class Acos implements DoubleUnaryOperator { @@ -189,6 +218,8 @@ public class ScalarFunctions { public double applyAsDouble(double operand) { return Math.acos(operand); } @Override public String toString() { return "f(a)(acos(a))"; } + @Override + public int hashCode() { return "acos".hashCode(); } } public static class Asin implements DoubleUnaryOperator { @@ -196,6 +227,8 @@ public class ScalarFunctions { public double applyAsDouble(double operand) { return Math.asin(operand); } @Override public String toString() { return "f(a)(asin(a))"; } + @Override + public int hashCode() { return "asin".hashCode(); } } public static class Atan implements DoubleUnaryOperator { @@ -203,6 +236,8 @@ public class ScalarFunctions { public double applyAsDouble(double operand) { return Math.atan(operand); } @Override public String toString() { return "f(a)(atan(a))"; } + @Override + public int hashCode() { return "atan".hashCode(); } } public static class Ceil implements DoubleUnaryOperator { @@ -210,6 +245,8 @@ public class ScalarFunctions { public double applyAsDouble(double operand) { return Math.ceil(operand); } @Override public String toString() { return "f(a)(ceil(a))"; } + @Override + public int hashCode() { return "ceil".hashCode(); } } public static class Cos implements DoubleUnaryOperator { @@ -217,6 +254,8 @@ public class ScalarFunctions { public double applyAsDouble(double operand) { return Math.cos(operand); } @Override public String toString() { return "f(a)(cos(a))"; } + @Override + public int hashCode() { return "cos".hashCode(); } } public static class Elu implements DoubleUnaryOperator { @@ -231,6 +270,8 @@ public class ScalarFunctions { public double applyAsDouble(double operand) { return operand < 0 ? alpha * (Math.exp(operand) - 1) : operand; } @Override public String toString() { return "f(a)(if(a < 0, " + alpha + " * (exp(a)-1), a))"; } + @Override + public int hashCode() { return Objects.hash("elu", alpha); } } public static class Exp implements DoubleUnaryOperator { @@ -238,6 +279,8 @@ public class ScalarFunctions { public double applyAsDouble(double operand) { return Math.exp(operand); } @Override public String toString() { return "f(a)(exp(a))"; } + @Override + public int hashCode() { return "exp".hashCode(); } } public static class Floor implements DoubleUnaryOperator { @@ -245,6 +288,8 @@ public class ScalarFunctions { public double applyAsDouble(double operand) { return Math.floor(operand); } @Override public String toString() { return "f(a)(floor(a))"; } + @Override + public int hashCode() { return "floor".hashCode(); } } public static class Log implements DoubleUnaryOperator { @@ -252,6 +297,8 @@ public class ScalarFunctions { public double applyAsDouble(double operand) { return Math.log(operand); } @Override public String toString() { return "f(a)(log(a))"; } + @Override + public int hashCode() { return "log".hashCode(); } } public static class Neg implements DoubleUnaryOperator { @@ -259,6 +306,8 @@ public class ScalarFunctions { public double applyAsDouble(double operand) { return -operand; } @Override public String toString() { return "f(a)(-a)"; } + @Override + public int hashCode() { return "neg".hashCode(); } } public static class Reciprocal implements DoubleUnaryOperator { @@ -266,6 +315,8 @@ public class ScalarFunctions { public double applyAsDouble(double operand) { return 1.0 / operand; } @Override public String toString() { return "f(a)(1 / a)"; } + @Override + public int hashCode() { return "reciprocal".hashCode(); } } public static class Relu implements DoubleUnaryOperator { @@ -273,6 +324,8 @@ public class ScalarFunctions { public double applyAsDouble(double operand) { return Math.max(operand, 0); } @Override public String toString() { return "f(a)(max(0, a))"; } + @Override + public int hashCode() { return "relu".hashCode(); } } public static class Selu implements DoubleUnaryOperator { @@ -290,6 +343,8 @@ public class ScalarFunctions { public double applyAsDouble(double operand) { return scale * (operand >= 0.0 ? operand : alpha * (Math.exp(operand)-1)); } @Override public String toString() { return "f(a)(" + scale + " * if(a >= 0, a, " + alpha + " * (exp(a) - 1)))"; } + @Override + public int hashCode() { return Objects.hash("selu", scale, alpha); } } public static class LeakyRelu implements DoubleUnaryOperator { @@ -304,6 +359,8 @@ public class ScalarFunctions { public double applyAsDouble(double operand) { return Math.max(alpha * operand, operand); } @Override public String toString() { return "f(a)(max(" + alpha + " * a, a))"; } + @Override + public int hashCode() { return Objects.hash("leakyrelu", alpha); } } public static class Sin implements DoubleUnaryOperator { @@ -311,6 +368,8 @@ public class ScalarFunctions { public double applyAsDouble(double operand) { return Math.sin(operand); } @Override public String toString() { return "f(a)(sin(a))"; } + @Override + public int hashCode() { return "sin".hashCode(); } } public static class Rsqrt implements DoubleUnaryOperator { @@ -318,6 +377,8 @@ public class ScalarFunctions { public double applyAsDouble(double operand) { return 1.0 / Math.sqrt(operand); } @Override public String toString() { return "f(a)(1.0 / sqrt(a))"; } + @Override + public int hashCode() { return "rsqrt".hashCode(); } } public static class Sigmoid implements DoubleUnaryOperator { @@ -325,6 +386,8 @@ public class ScalarFunctions { public double applyAsDouble(double operand) { return 1.0 / (1.0 + Math.exp(-operand)); } @Override public String toString() { return "f(a)(1 / (1 + exp(-a)))"; } + @Override + public int hashCode() { return "sigmoid".hashCode(); } } public static class Sqrt implements DoubleUnaryOperator { @@ -332,6 +395,8 @@ public class ScalarFunctions { public double applyAsDouble(double operand) { return Math.sqrt(operand); } @Override public String toString() { return "f(a)(sqrt(a))"; } + @Override + public int hashCode() { return "sqrt".hashCode(); } } public static class Square implements DoubleUnaryOperator { @@ -339,6 +404,8 @@ public class ScalarFunctions { public double applyAsDouble(double operand) { return operand * operand; } @Override public String toString() { return "f(a)(a * a)"; } + @Override + public int hashCode() { return "square".hashCode(); } } public static class Tan implements DoubleUnaryOperator { @@ -346,6 +413,8 @@ public class ScalarFunctions { public double applyAsDouble(double operand) { return Math.tan(operand); } @Override public String toString() { return "f(a)(tan(a))"; } + @Override + public int hashCode() { return "tan".hashCode(); } } public static class Tanh implements DoubleUnaryOperator { @@ -353,6 +422,8 @@ public class ScalarFunctions { public double applyAsDouble(double operand) { return Math.tanh(operand); } @Override public String toString() { return "f(a)(tanh(a))"; } + @Override + public int hashCode() { return "tanh".hashCode(); } } public static class Erf implements DoubleUnaryOperator { @@ -410,6 +481,8 @@ public class ScalarFunctions { public double applyAsDouble(double operand) { return erf(operand); } @Override public String toString() { return "f(a)(erf(a))"; } + @Override + public int hashCode() { return "erf".hashCode(); } static final double nearZeroMultiplier = 2.0 / Math.sqrt(Math.PI); @@ -464,6 +537,8 @@ public class ScalarFunctions { } return b.toString(); } + @Override + public int hashCode() { return Objects.hash("equal", argumentNames); } } public static class Random implements Function<List<Long>, Double> { @@ -473,6 +548,8 @@ public class ScalarFunctions { } @Override public String toString() { return "random"; } + @Override + public int hashCode() { return "random".hashCode(); } } public static class SumElements implements Function<List<Long>, Double> { @@ -492,6 +569,8 @@ public class ScalarFunctions { public String toString() { return argumentNames.stream().collect(Collectors.joining("+")); } + @Override + public int hashCode() { return Objects.hash("sum", argumentNames); } } public static class Constant implements Function<List<Long>, Double> { @@ -506,6 +585,8 @@ public class ScalarFunctions { } @Override public String toString() { return Double.toString(value); } + @Override + public int hashCode() { return Objects.hash("constant", value); } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Slice.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Slice.java index e3464255fac..39bddc3a3cd 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Slice.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Slice.java @@ -166,6 +166,9 @@ public class Slice<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMETY return b.toString(); } + @Override + public int hashCode() { return Objects.hash("slice", argument, subspaceAddress); } + public static class DimensionValue<NAMETYPE extends Name> { private final Optional<String> dimension; @@ -255,6 +258,10 @@ public class Slice<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMETY return index.toString(context); } + @Override + public int hashCode() { return Objects.hash(dimension, label, index); } + + } private static class ConstantIntegerFunction<NAMETYPE extends Name> implements ScalarFunction<NAMETYPE> { @@ -273,6 +280,9 @@ public class Slice<NAMETYPE extends Name> extends PrimitiveTensorFunction<NAMETY @Override public String toString() { return String.valueOf(value); } + @Override + public int hashCode() { return Objects.hash("constantIntegerFunction", value); } + } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/Softmax.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/Softmax.java index 9ea9040831b..df8cd6d39cd 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/Softmax.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/Softmax.java @@ -7,6 +7,7 @@ import com.yahoo.tensor.evaluation.Name; import java.util.Collections; import java.util.List; +import java.util.Objects; /** * @author bratseth @@ -50,4 +51,7 @@ public class Softmax<NAMETYPE extends Name> extends CompositeTensorFunction<NAME return "softmax(" + argument.toString(context) + ", " + dimension + ")"; } + @Override + public int hashCode() { return Objects.hash("softmax", argument, dimension); } + } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/TensorFunction.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/TensorFunction.java index 1e1d1d3b5b9..503f414d8eb 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/TensorFunction.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/TensorFunction.java @@ -68,4 +68,8 @@ public abstract class TensorFunction<NAMETYPE extends Name> { @Override public String toString() { return toString(ToStringContext.empty()); } + /** Returns a hashcode computed from the data in this */ + @Override + public abstract int hashCode(); + } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/XwPlusB.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/XwPlusB.java index 0223ad4d588..bd4fc7b8336 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/XwPlusB.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/XwPlusB.java @@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableList; import com.yahoo.tensor.evaluation.Name; import java.util.List; +import java.util.Objects; /** * @author bratseth @@ -51,4 +52,7 @@ public class XwPlusB<NAMETYPE extends Name> extends CompositeTensorFunction<NAME dimension + ")"; } + @Override + public int hashCode() { return Objects.hash("xwplusb", x, w, b, dimension); } + } diff --git a/vespalib/src/tests/btree/btree-stress/CMakeLists.txt b/vespalib/src/tests/btree/btree-stress/CMakeLists.txt index 29f3c0acc77..f7f682ddd17 100644 --- a/vespalib/src/tests/btree/btree-stress/CMakeLists.txt +++ b/vespalib/src/tests/btree/btree-stress/CMakeLists.txt @@ -6,4 +6,4 @@ vespa_add_executable(vespalib_btree_stress_test_app vespalib GTest::GTest ) -vespa_add_test(NAME vespalib_btree_stress_test_app COMMAND vespalib_btree_stress_test_app BENCHMARK) +vespa_add_test(NAME vespalib_btree_stress_test_app COMMAND vespalib_btree_stress_test_app --smoke-test) diff --git a/vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp b/vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp index 05f126839e9..7d11990577d 100644 --- a/vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp +++ b/vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp @@ -21,7 +21,11 @@ #include <vespa/vespalib/btree/btree.hpp> #include <vespa/vespalib/btree/btreestore.hpp> #include <vespa/vespalib/btree/btreeaggregator.hpp> +#include <vespa/vespalib/datastore/atomic_entry_ref.h> +#include <vespa/vespalib/datastore/buffer_type.hpp> +#include <vespa/vespalib/datastore/compaction_spec.h> #include <vespa/vespalib/datastore/compaction_strategy.h> +#include <vespa/vespalib/datastore/entry_ref_filter.h> #include <vespa/log/log.h> LOG_SETUP("btree_stress_test"); @@ -29,8 +33,11 @@ LOG_SETUP("btree_stress_test"); using GenerationHandler = vespalib::GenerationHandler; using RefType = vespalib::datastore::EntryRefT<22>; using vespalib::btree::NoAggregated; +using vespalib::datastore::AtomicEntryRef; +using vespalib::datastore::CompactionSpec; using vespalib::datastore::CompactionStrategy; using vespalib::datastore::EntryRef; +using vespalib::datastore::EntryRefFilter; using vespalib::makeLambdaTask; using generation_t = GenerationHandler::generation_t; @@ -38,17 +45,31 @@ namespace { constexpr uint32_t value_offset = 1000000000; +bool smoke_test = false; +const vespalib::string smoke_test_option = "--smoke-test"; + class RealIntStore { - vespalib::datastore::DataStore<uint32_t> _store; + using StoreType = vespalib::datastore::DataStore<uint32_t>; + using StoreRefType = StoreType::RefType; + StoreType _store; public: RealIntStore(); ~RealIntStore(); EntryRef add(uint32_t value) { return _store.addEntry(value); } - void hold(EntryRef ref) { _store.holdElem(ref, 1); } + AtomicEntryRef add_relaxed(uint32_t value) { return AtomicEntryRef(add(value)); } + void hold(const AtomicEntryRef& ref) { _store.holdElem(ref.load_relaxed(), 1); } + EntryRef move(EntryRef ref); void transfer_hold_lists(generation_t gen) { _store.transferHoldLists(gen); } void trim_hold_lists(generation_t gen) { _store.trimHoldLists(gen); } - uint32_t get(EntryRef ref) const { return _store.getEntry(ref); } + uint32_t get_acquire(const AtomicEntryRef& ref) const { return get(ref.load_acquire()); } + uint32_t get_relaxed(const AtomicEntryRef& ref) const { return get(ref.load_relaxed()); } + std::vector<uint32_t> start_compact(); + void finish_compact(std::vector<uint32_t> to_hold); + static constexpr bool is_indirect = true; + static uint32_t get_offset_bits() { return StoreRefType::offset_bits; } + static uint32_t get_num_buffers() { return StoreRefType::numBuffers(); } + bool has_held_buffers() const noexcept { return _store.has_held_buffers(); } }; RealIntStore::RealIntStore() @@ -58,6 +79,27 @@ RealIntStore::RealIntStore() RealIntStore::~RealIntStore() = default; +std::vector<uint32_t> +RealIntStore::start_compact() +{ + // Use a compaction strategy that will compact all active buffers + CompactionStrategy compaction_strategy(0.0, 0.0, get_num_buffers(), 1.0); + CompactionSpec compaction_spec(true, false); + return _store.startCompactWorstBuffers(compaction_spec, compaction_strategy); +} + +void +RealIntStore::finish_compact(std::vector<uint32_t> to_hold) +{ + _store.finishCompact(to_hold); +} + +EntryRef +RealIntStore::move(EntryRef ref) +{ + return add(get(ref)); +} + class RealIntStoreCompare { const RealIntStore& _store; @@ -71,10 +113,10 @@ public: uint32_t get(EntryRef ref) const { return (ref.valid() ? _store.get(ref) : _lookup_key); } - bool operator()(EntryRef lhs, EntryRef rhs) const { - return get(lhs) < get(rhs); + bool operator()(const AtomicEntryRef& lhs, const AtomicEntryRef& rhs) const { + return get(lhs.load_acquire()) < get(rhs.load_acquire()); } - static EntryRef lookup_key() noexcept { return EntryRef(); } + static AtomicEntryRef lookup_key() noexcept { return AtomicEntryRef(); } const RealIntStoreCompare& get_compare() const noexcept { return *this; } }; @@ -83,10 +125,14 @@ public: NoIntStore() = default; ~NoIntStore() = default; static uint32_t add(uint32_t value) noexcept { return value; } + static uint32_t add_relaxed(uint32_t value) noexcept { return value; } static void hold(uint32_t) noexcept { } static void transfer_hold_lists(generation_t) noexcept { } static void trim_hold_lists(generation_t) noexcept { } static uint32_t get(uint32_t value) noexcept { return value; } + static uint32_t get_acquire(uint32_t value) noexcept { return value; } + static uint32_t get_relaxed(uint32_t value) noexcept { return value; } + static constexpr bool is_indirect = false; }; class NoIntStoreCompare @@ -109,7 +155,7 @@ public: struct IndirectKeyValueParams { using IntStore = RealIntStore; using MyCompare = RealIntStoreCompare; - using MyTree = vespalib::btree::BTree<EntryRef, EntryRef, NoAggregated, RealIntStoreCompare>; + using MyTree = vespalib::btree::BTree<AtomicEntryRef, AtomicEntryRef, NoAggregated, RealIntStoreCompare>; }; struct DirectKeyValueParams { @@ -118,6 +164,29 @@ struct DirectKeyValueParams { using MyTree = vespalib::btree::BTree<uint32_t, uint32_t>; }; +template <uint32_t divisor, uint32_t remainder> +class ConsiderCompact { + uint32_t _count; + bool _want_compact; +public: + ConsiderCompact() + : _count(0u), + _want_compact(false) + { + } + bool consider(uint32_t idx) { + if ((idx % divisor) == remainder) { + _want_compact = true; + } + return _want_compact; + } + void track_compacted() { + ++_count; + _want_compact = false; + } + uint32_t get_count() const noexcept { return _count; } +}; + template <typename Params> class Fixture : public testing::Test { @@ -141,9 +210,9 @@ protected: std::atomic<long> _doneReadWork; std::atomic<bool> _stopRead; bool _reportWork; - bool _want_compact_tree; - uint32_t _consider_compact_tree_checks; - uint32_t _compact_tree_count; + ConsiderCompact<1000, 0> _compact_tree; + ConsiderCompact<1000, 300> _compact_keys; + ConsiderCompact<1000, 600> _compact_values; Fixture(); ~Fixture() override; @@ -152,7 +221,9 @@ protected: void insert(uint32_t key); void remove(uint32_t key); void compact_tree(); - void consider_compact_tree(); + void compact_keys(); + void compact_values(); + void consider_compact(uint32_t idx); void readWork(uint32_t cnt); void readWork(); @@ -180,9 +251,9 @@ Fixture<Params>::Fixture() _doneReadWork(0), _stopRead(false), _reportWork(false), - _want_compact_tree(false), - _consider_compact_tree_checks(0u), - _compact_tree_count(0u) + _compact_tree(), + _compact_keys(), + _compact_values() { _rnd.srand48(32); } @@ -226,13 +297,13 @@ bool Fixture<Params>::adjustWriteIterator(uint32_t key) { MyCompare compare(_keys, key); - if (_writeItr.valid() && _keys.get(_writeItr.getKey()) < key) { + if (_writeItr.valid() && _keys.get_relaxed(_writeItr.getKey()) < key) { _writeItr.binarySeek(compare.lookup_key(), compare.get_compare()); } else { _writeItr.lower_bound(compare.lookup_key(), compare.get_compare()); } - assert(!_writeItr.valid() || _keys.get(_writeItr.getKey()) >= key); - return (_writeItr.valid() && _keys.get(_writeItr.getKey()) == key); + assert(!_writeItr.valid() || _keys.get_relaxed(_writeItr.getKey()) >= key); + return (_writeItr.valid() && _keys.get_relaxed(_writeItr.getKey()) == key); } template <typename Params> @@ -240,9 +311,9 @@ void Fixture<Params>::insert(uint32_t key) { if (!adjustWriteIterator(key)) { - _tree.insert(_writeItr, _keys.add(key), _values.add(key + value_offset)); + _tree.insert(_writeItr, _keys.add_relaxed(key), _values.add_relaxed(key + value_offset)); } else { - EXPECT_EQ(key + value_offset, _values.get(_writeItr.getData())); + EXPECT_EQ(key + value_offset, _values.get_relaxed(_writeItr.getData())); } } @@ -251,7 +322,7 @@ void Fixture<Params>::remove(uint32_t key) { if (adjustWriteIterator(key)) { - EXPECT_EQ(key + value_offset, _values.get(_writeItr.getData())); + EXPECT_EQ(key + value_offset, _values.get_relaxed(_writeItr.getData())); _keys.hold(_writeItr.getKey()); _values.hold(_writeItr.getData()); _tree.remove(_writeItr); @@ -266,20 +337,69 @@ Fixture<Params>::compact_tree() CompactionStrategy compaction_strategy(0.0, 0.0, RefType::numBuffers(), 1.0); _tree.compact_worst(compaction_strategy); _writeItr = _tree.begin(); + _compact_tree.track_compacted(); +} + +template <typename Params> +void +Fixture<Params>::compact_keys() +{ + if constexpr (_keys.is_indirect) { + auto to_hold = _keys.start_compact(); + EntryRefFilter filter(_keys.get_num_buffers(), _keys.get_offset_bits()); + filter.add_buffers(to_hold); + auto itr = _tree.begin(); + while (itr.valid()) { + auto old_ref = itr.getKey().load_relaxed(); + if (filter.has(old_ref)) { + auto new_ref = _keys.move(old_ref); + itr.writeKey(AtomicEntryRef(new_ref)); + } + ++itr; + } + _keys.finish_compact(std::move(to_hold)); + } + _compact_keys.track_compacted(); } template <typename Params> void -Fixture<Params>::consider_compact_tree() +Fixture<Params>::compact_values() { - if ((_consider_compact_tree_checks % 1000) == 0) { - _want_compact_tree = true; + if constexpr (_values.is_indirect) { + auto to_hold = _values.start_compact(); + EntryRefFilter filter(_values.get_num_buffers(), _values.get_offset_bits()); + filter.add_buffers(to_hold); + auto itr = _tree.begin(); + while (itr.valid()) { + auto old_ref = itr.getData().load_relaxed(); + if (filter.has(old_ref)) { + auto new_ref = _values.move(old_ref); + itr.getWData().store_release(new_ref); + } + ++itr; + } + _values.finish_compact(std::move(to_hold)); } - ++_consider_compact_tree_checks; - if (_want_compact_tree && !_tree.getAllocator().getNodeStore().has_held_buffers()) { + _compact_values.track_compacted(); +} + +template <typename Params> +void +Fixture<Params>::consider_compact(uint32_t idx) +{ + if (_compact_tree.consider(idx) && !_tree.getAllocator().getNodeStore().has_held_buffers()) { compact_tree(); - _want_compact_tree = false; - ++_compact_tree_count; + } + if constexpr (_keys.is_indirect) { + if (_compact_keys.consider(idx) && !_keys.has_held_buffers()) { + compact_keys(); + } + } + if constexpr (_values.is_indirect) { + if (_compact_values.consider(idx) && !_values.has_held_buffers()) { + compact_values(); + } } } @@ -296,9 +416,9 @@ Fixture<Params>::readWork(uint32_t cnt) uint32_t key = rnd.lrand48() % (_keyLimit + 1); MyCompare compare(_keys, key); MyTreeConstIterator itr = _tree.getFrozenView().lowerBound(compare.lookup_key(), compare.get_compare()); - assert(!itr.valid() || _keys.get(itr.getKey()) >= key); - if (itr.valid() && _keys.get(itr.getKey()) == key) { - EXPECT_EQ(key + value_offset, _values.get(itr.getData())); + assert(!itr.valid() || _keys.get_acquire(itr.getKey()) >= key); + if (itr.valid() && _keys.get_acquire(itr.getKey()) == key) { + EXPECT_EQ(key + value_offset, _values.get_acquire(itr.getData())); ++hits; } } @@ -319,7 +439,7 @@ Fixture<Params>::writeWork(uint32_t cnt) { vespalib::Rand48 &rnd(_rnd); for (uint32_t i = 0; i < cnt; ++i) { - consider_compact_tree(); + consider_compact(i); uint32_t key = rnd.lrand48() % _keyLimit; if ((rnd.lrand48() & 1) == 0) { insert(key); @@ -330,7 +450,10 @@ Fixture<Params>::writeWork(uint32_t cnt) } _doneWriteWork += cnt; _stopRead = true; - LOG(info, "done %u write work, %u compact tree", cnt, _compact_tree_count); + LOG(info, "done %u write work, %u compact tree, %u compact keys, %u compact values", cnt, + _compact_tree.get_count(), + _compact_keys.get_count(), + _compact_values.get_count()); } template <typename Params> @@ -348,7 +471,7 @@ Fixture<Params>::basic_lower_bound() MyCompare compare(_keys, 3); auto itr = _tree.getFrozenView().lowerBound(compare.lookup_key(), compare.get_compare()); ASSERT_TRUE(itr.valid()); - EXPECT_EQ(4u, _keys.get(itr.getKey())); + EXPECT_EQ(4u, _keys.get_acquire(itr.getKey())); } template <typename Params> @@ -365,7 +488,7 @@ template <typename Params> void Fixture<Params>::single_lower_bound_reader_during_updates() { - uint32_t cnt = 1000000; + uint32_t cnt = smoke_test ? 10000 : 1000000; _reportWork = true; _writer.execute(makeLambdaTask([this, cnt]() { writeWork(cnt); })); _readers.execute(makeLambdaTask([this]() { readWork(); })); @@ -377,7 +500,7 @@ template <typename Params> void Fixture<Params>::multiple_lower_bound_readers_during_updates() { - uint32_t cnt = 1000000; + uint32_t cnt = smoke_test ? 10000 : 1000000; _reportWork = true; _writer.execute(makeLambdaTask([this, cnt]() { writeWork(cnt); })); _readers.execute(makeLambdaTask([this]() { readWork(); })); @@ -423,4 +546,12 @@ TYPED_TEST(BTreeStressTest, multiple_lower_bound_readers_during_updates) #pragma GCC diagnostic pop -GTEST_MAIN_RUN_ALL_TESTS() +int main(int argc, char **argv) { + if (argc > 1 && argv[1] == smoke_test_option) { + smoke_test = true; + ++argv; + --argc; + } + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/vespalib/src/tests/invokeservice/invokeservice_test.cpp b/vespalib/src/tests/invokeservice/invokeservice_test.cpp index 1d4791f4a33..ff84017d48b 100644 --- a/vespalib/src/tests/invokeservice/invokeservice_test.cpp +++ b/vespalib/src/tests/invokeservice/invokeservice_test.cpp @@ -29,6 +29,25 @@ TEST("require that wakeup is called") { EXPECT_EQUAL(countAtStop, a._count); } +TEST("require that now is moving forward") { + steady_time prev = steady_clock::now(); + InvokeCounter a; + InvokeServiceImpl service(1ms); + EXPECT_EQUAL(0u, a._count); + auto ra = service.registerInvoke([&prev, &a, &now= service.nowRef() ]() noexcept { + EXPECT_GREATER(now.load(), prev); + prev = now.load(); + a.inc(); + }); + EXPECT_TRUE(ra); + a.wait_for_atleast(100); + ra.reset(); + EXPECT_GREATER_EQUAL(a._count, 100u); + steady_time now = steady_clock::now(); + EXPECT_GREATER(now, prev); + EXPECT_LESS(now - prev, 5s); +} + TEST("require that same wakeup can be registered multiple times.") { InvokeCounter a; InvokeCounter b; diff --git a/vespalib/src/tests/shared_operation_throttler/shared_operation_throttler_test.cpp b/vespalib/src/tests/shared_operation_throttler/shared_operation_throttler_test.cpp index eefc0ca72c0..d6946905236 100644 --- a/vespalib/src/tests/shared_operation_throttler/shared_operation_throttler_test.cpp +++ b/vespalib/src/tests/shared_operation_throttler/shared_operation_throttler_test.cpp @@ -4,6 +4,8 @@ #include <vespa/vespalib/util/barrier.h> #include <thread> +using vespalib::steady_clock; + namespace vespalib { using ThrottleToken = SharedOperationThrottler::Token; @@ -47,7 +49,7 @@ TEST_F("blocking acquire returns immediately if slot available", DynamicThrottle auto token = f1._throttler->blocking_acquire_one(); EXPECT_TRUE(token.valid()); token.reset(); - token = f1._throttler->blocking_acquire_one(600s); // Should never block. + token = f1._throttler->blocking_acquire_one(steady_clock::now() + 600s); // Should never block. EXPECT_TRUE(token.valid()); } @@ -70,11 +72,11 @@ TEST_F("blocking call woken up if throttle slot available", DynamicThrottleFixtu TEST_F("time-bounded blocking acquire waits for timeout", DynamicThrottleFixture()) { auto window_filling_token = f1._throttler->try_acquire_one(); - auto before = std::chrono::steady_clock::now(); + auto before = steady_clock::now(); // Will block for at least 1ms. Since no window slot will be available by that time, // an invalid token should be returned. - auto token = f1._throttler->blocking_acquire_one(1ms); - auto after = std::chrono::steady_clock::now(); + auto token = f1._throttler->blocking_acquire_one(before + 1ms); + auto after = steady_clock::now(); EXPECT_TRUE((after - before) >= 1ms); EXPECT_FALSE(token.valid()); } diff --git a/vespalib/src/tests/time/time_test.cpp b/vespalib/src/tests/time/time_test.cpp index c2858308006..0ca583c8326 100644 --- a/vespalib/src/tests/time/time_test.cpp +++ b/vespalib/src/tests/time/time_test.cpp @@ -68,4 +68,19 @@ TEST(TimeTest, default_timer_frequency_is_1000_hz) { EXPECT_EQ(1000u, getVespaTimerHz()); } +TEST(TimeTest, timeout_is_relative_to_frequency) { + EXPECT_EQ(1000u, getVespaTimerHz()); + + EXPECT_EQ(1ms, adjustTimeoutByDetectedHz(1ms)); + EXPECT_EQ(20ms, adjustTimeoutByDetectedHz(20ms)); + + EXPECT_EQ(1ms, adjustTimeoutByHz(1ms, 1000)); + EXPECT_EQ(10ms, adjustTimeoutByHz(1ms, 100)); + EXPECT_EQ(100ms, adjustTimeoutByHz(1ms, 10)); + + EXPECT_EQ(20ms, adjustTimeoutByHz(20ms, 1000)); + EXPECT_EQ(200ms, adjustTimeoutByHz(20ms, 100)); + EXPECT_EQ(2000ms, adjustTimeoutByHz(20ms, 10)); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/util/generationhandler/CMakeLists.txt b/vespalib/src/tests/util/generationhandler/CMakeLists.txt index 677d5caa0e6..fdf54c59854 100644 --- a/vespalib/src/tests/util/generationhandler/CMakeLists.txt +++ b/vespalib/src/tests/util/generationhandler/CMakeLists.txt @@ -4,5 +4,6 @@ vespa_add_executable(vespalib_generationhandler_test_app TEST generationhandler_test.cpp DEPENDS vespalib + GTest::GTest ) vespa_add_test(NAME vespalib_generationhandler_test_app COMMAND vespalib_generationhandler_test_app) diff --git a/vespalib/src/tests/util/generationhandler/generationhandler_test.cpp b/vespalib/src/tests/util/generationhandler/generationhandler_test.cpp index f269fe729fa..00da752a749 100644 --- a/vespalib/src/tests/util/generationhandler/generationhandler_test.cpp +++ b/vespalib/src/tests/util/generationhandler/generationhandler_test.cpp @@ -1,157 +1,137 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/util/generationhandler.h> #include <deque> namespace vespalib { -typedef GenerationHandler::Guard GenGuard; +using GenGuard = GenerationHandler::Guard; -class Test : public vespalib::TestApp { -private: - void requireThatGenerationCanBeIncreased(); - void requireThatReadersCanTakeGuards(); - void requireThatGuardsCanBeCopied(); - void requireThatTheFirstUsedGenerationIsCorrect(); - void requireThatGenerationCanGrowLarge(); -public: - int Main() override; +class GenerationHandlerTest : public ::testing::Test { +protected: + GenerationHandler gh; + GenerationHandlerTest(); + ~GenerationHandlerTest() override; }; -void -Test::requireThatGenerationCanBeIncreased() +GenerationHandlerTest::GenerationHandlerTest() + : ::testing::Test(), + gh() { - GenerationHandler gh; - EXPECT_EQUAL(0u, gh.getCurrentGeneration()); - EXPECT_EQUAL(0u, gh.getFirstUsedGeneration()); +} + +GenerationHandlerTest::~GenerationHandlerTest() = default; + +TEST_F(GenerationHandlerTest, require_that_generation_can_be_increased) +{ + EXPECT_EQ(0u, gh.getCurrentGeneration()); + EXPECT_EQ(0u, gh.getFirstUsedGeneration()); gh.incGeneration(); - EXPECT_EQUAL(1u, gh.getCurrentGeneration()); - EXPECT_EQUAL(1u, gh.getFirstUsedGeneration()); + EXPECT_EQ(1u, gh.getCurrentGeneration()); + EXPECT_EQ(1u, gh.getFirstUsedGeneration()); } -void -Test::requireThatReadersCanTakeGuards() +TEST_F(GenerationHandlerTest, require_that_readers_can_take_guards) { - GenerationHandler gh; - EXPECT_EQUAL(0u, gh.getGenerationRefCount(0)); + EXPECT_EQ(0u, gh.getGenerationRefCount(0)); { GenGuard g1 = gh.takeGuard(); - EXPECT_EQUAL(1u, gh.getGenerationRefCount(0)); + EXPECT_EQ(1u, gh.getGenerationRefCount(0)); { GenGuard g2 = gh.takeGuard(); - EXPECT_EQUAL(2u, gh.getGenerationRefCount(0)); + EXPECT_EQ(2u, gh.getGenerationRefCount(0)); gh.incGeneration(); { GenGuard g3 = gh.takeGuard(); - EXPECT_EQUAL(2u, gh.getGenerationRefCount(0)); - EXPECT_EQUAL(1u, gh.getGenerationRefCount(1)); - EXPECT_EQUAL(3u, gh.getGenerationRefCount()); + EXPECT_EQ(2u, gh.getGenerationRefCount(0)); + EXPECT_EQ(1u, gh.getGenerationRefCount(1)); + EXPECT_EQ(3u, gh.getGenerationRefCount()); } - EXPECT_EQUAL(2u, gh.getGenerationRefCount(0)); - EXPECT_EQUAL(0u, gh.getGenerationRefCount(1)); + EXPECT_EQ(2u, gh.getGenerationRefCount(0)); + EXPECT_EQ(0u, gh.getGenerationRefCount(1)); gh.incGeneration(); { GenGuard g3 = gh.takeGuard(); - EXPECT_EQUAL(2u, gh.getGenerationRefCount(0)); - EXPECT_EQUAL(0u, gh.getGenerationRefCount(1)); - EXPECT_EQUAL(1u, gh.getGenerationRefCount(2)); + EXPECT_EQ(2u, gh.getGenerationRefCount(0)); + EXPECT_EQ(0u, gh.getGenerationRefCount(1)); + EXPECT_EQ(1u, gh.getGenerationRefCount(2)); } - EXPECT_EQUAL(2u, gh.getGenerationRefCount(0)); - EXPECT_EQUAL(0u, gh.getGenerationRefCount(1)); - EXPECT_EQUAL(0u, gh.getGenerationRefCount(2)); + EXPECT_EQ(2u, gh.getGenerationRefCount(0)); + EXPECT_EQ(0u, gh.getGenerationRefCount(1)); + EXPECT_EQ(0u, gh.getGenerationRefCount(2)); } - EXPECT_EQUAL(1u, gh.getGenerationRefCount(0)); - EXPECT_EQUAL(0u, gh.getGenerationRefCount(1)); - EXPECT_EQUAL(0u, gh.getGenerationRefCount(2)); + EXPECT_EQ(1u, gh.getGenerationRefCount(0)); + EXPECT_EQ(0u, gh.getGenerationRefCount(1)); + EXPECT_EQ(0u, gh.getGenerationRefCount(2)); } - EXPECT_EQUAL(0u, gh.getGenerationRefCount(0)); - EXPECT_EQUAL(0u, gh.getGenerationRefCount(1)); - EXPECT_EQUAL(0u, gh.getGenerationRefCount(2)); + EXPECT_EQ(0u, gh.getGenerationRefCount(0)); + EXPECT_EQ(0u, gh.getGenerationRefCount(1)); + EXPECT_EQ(0u, gh.getGenerationRefCount(2)); } -void -Test::requireThatGuardsCanBeCopied() +TEST_F(GenerationHandlerTest, require_that_guards_can_be_copied) { - GenerationHandler gh; GenGuard g1 = gh.takeGuard(); - EXPECT_EQUAL(1u, gh.getGenerationRefCount(0)); + EXPECT_EQ(1u, gh.getGenerationRefCount(0)); GenGuard g2(g1); - EXPECT_EQUAL(2u, gh.getGenerationRefCount(0)); + EXPECT_EQ(2u, gh.getGenerationRefCount(0)); gh.incGeneration(); GenGuard g3 = gh.takeGuard(); - EXPECT_EQUAL(2u, gh.getGenerationRefCount(0)); - EXPECT_EQUAL(1u, gh.getGenerationRefCount(1)); + EXPECT_EQ(2u, gh.getGenerationRefCount(0)); + EXPECT_EQ(1u, gh.getGenerationRefCount(1)); g3 = g2; - EXPECT_EQUAL(3u, gh.getGenerationRefCount(0)); - EXPECT_EQUAL(0u, gh.getGenerationRefCount(1)); + EXPECT_EQ(3u, gh.getGenerationRefCount(0)); + EXPECT_EQ(0u, gh.getGenerationRefCount(1)); } -void -Test::requireThatTheFirstUsedGenerationIsCorrect() +TEST_F(GenerationHandlerTest, require_that_the_first_used_generation_is_correct) { - GenerationHandler gh; - EXPECT_EQUAL(0u, gh.getFirstUsedGeneration()); + EXPECT_EQ(0u, gh.getFirstUsedGeneration()); gh.incGeneration(); - EXPECT_EQUAL(1u, gh.getFirstUsedGeneration()); + EXPECT_EQ(1u, gh.getFirstUsedGeneration()); { GenGuard g1 = gh.takeGuard(); gh.incGeneration(); - EXPECT_EQUAL(1u, gh.getGenerationRefCount()); - EXPECT_EQUAL(1u, gh.getFirstUsedGeneration()); + EXPECT_EQ(1u, gh.getGenerationRefCount()); + EXPECT_EQ(1u, gh.getFirstUsedGeneration()); } - EXPECT_EQUAL(1u, gh.getFirstUsedGeneration()); + EXPECT_EQ(1u, gh.getFirstUsedGeneration()); gh.updateFirstUsedGeneration(); // Only writer should call this - EXPECT_EQUAL(0u, gh.getGenerationRefCount()); - EXPECT_EQUAL(2u, gh.getFirstUsedGeneration()); + EXPECT_EQ(0u, gh.getGenerationRefCount()); + EXPECT_EQ(2u, gh.getFirstUsedGeneration()); { GenGuard g1 = gh.takeGuard(); gh.incGeneration(); gh.incGeneration(); - EXPECT_EQUAL(1u, gh.getGenerationRefCount()); - EXPECT_EQUAL(2u, gh.getFirstUsedGeneration()); + EXPECT_EQ(1u, gh.getGenerationRefCount()); + EXPECT_EQ(2u, gh.getFirstUsedGeneration()); { GenGuard g2 = gh.takeGuard(); - EXPECT_EQUAL(2u, gh.getFirstUsedGeneration()); + EXPECT_EQ(2u, gh.getFirstUsedGeneration()); } } - EXPECT_EQUAL(2u, gh.getFirstUsedGeneration()); + EXPECT_EQ(2u, gh.getFirstUsedGeneration()); gh.updateFirstUsedGeneration(); // Only writer should call this - EXPECT_EQUAL(0u, gh.getGenerationRefCount()); - EXPECT_EQUAL(4u, gh.getFirstUsedGeneration()); + EXPECT_EQ(0u, gh.getGenerationRefCount()); + EXPECT_EQ(4u, gh.getFirstUsedGeneration()); } -void -Test::requireThatGenerationCanGrowLarge() +TEST_F(GenerationHandlerTest, require_that_generation_can_grow_large) { - GenerationHandler gh; std::deque<GenGuard> guards; for (size_t i = 0; i < 10000; ++i) { - EXPECT_EQUAL(i, gh.getCurrentGeneration()); + EXPECT_EQ(i, gh.getCurrentGeneration()); guards.push_back(gh.takeGuard()); // take guard on current generation if (i >= 128) { - EXPECT_EQUAL(i - 128, gh.getFirstUsedGeneration()); + EXPECT_EQ(i - 128, gh.getFirstUsedGeneration()); guards.pop_front(); - EXPECT_EQUAL(128u, gh.getGenerationRefCount()); + EXPECT_EQ(128u, gh.getGenerationRefCount()); } gh.incGeneration(); } } -int -Test::Main() -{ - TEST_INIT("generationhandler_test"); - - TEST_DO(requireThatGenerationCanBeIncreased()); - TEST_DO(requireThatReadersCanTakeGuards()); - TEST_DO(requireThatGuardsCanBeCopied()); - TEST_DO(requireThatTheFirstUsedGenerationIsCorrect()); - TEST_DO(requireThatGenerationCanGrowLarge()); - - TEST_DONE(); -} - } -TEST_APPHOOK(vespalib::Test); +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/util/generationhandler_stress/CMakeLists.txt b/vespalib/src/tests/util/generationhandler_stress/CMakeLists.txt index 569db489e3c..7e5a5af79b5 100644 --- a/vespalib/src/tests/util/generationhandler_stress/CMakeLists.txt +++ b/vespalib/src/tests/util/generationhandler_stress/CMakeLists.txt @@ -4,5 +4,6 @@ vespa_add_executable(vespalib_generation_handler_stress_test_app generation_handler_stress_test.cpp DEPENDS vespalib + GTest::GTest ) -vespa_add_test(NAME vespalib_generation_handler_stress_test_app COMMAND vespalib_generation_handler_stress_test_app BENCHMARK) +vespa_add_test(NAME vespalib_generation_handler_stress_test_app COMMAND vespalib_generation_handler_stress_test_app --smoke-test) diff --git a/vespalib/src/tests/util/generationhandler_stress/generation_handler_stress_test.cpp b/vespalib/src/tests/util/generationhandler_stress/generation_handler_stress_test.cpp index fa2c525b518..57c765b8e44 100644 --- a/vespalib/src/tests/util/generationhandler_stress/generation_handler_stress_test.cpp +++ b/vespalib/src/tests/util/generationhandler_stress/generation_handler_stress_test.cpp @@ -1,15 +1,36 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/log.h> LOG_SETUP("generation_handler_stress_test"); -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/util/generationhandler.h> +#include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/size_literals.h> +#include <thread> using vespalib::Executor; using vespalib::GenerationHandler; +using vespalib::makeLambdaTask; using vespalib::ThreadStackExecutor; +namespace { + +bool smoke_test = false; +const vespalib::string smoke_test_option = "--smoke-test"; + +} + +class ReadStopper { + std::atomic<bool> &_stop_read; +public: + ReadStopper(std::atomic<bool>& stop_read) + : _stop_read(stop_read) + { + } + ~ReadStopper() { + _stop_read = true; + } +}; struct WorkContext { @@ -21,26 +42,49 @@ struct WorkContext } }; -struct Fixture { +struct IndirectContext { + std::atomic<uint64_t *> _value_ptr; + char _pad[256]; + static constexpr size_t values_size = 65536; + uint64_t _values[values_size]; + + IndirectContext(); + uint64_t* calc_value_ptr(uint64_t idx) { return &_values[(idx & (values_size - 1))]; } +}; + +IndirectContext::IndirectContext() + : _value_ptr(nullptr), + _pad(), + _values() +{ + _value_ptr = &_values[0]; +} + +class Fixture : public ::testing::Test { +protected: GenerationHandler _generationHandler; uint32_t _readThreads; ThreadStackExecutor _writer; // 1 write thread - ThreadStackExecutor _readers; // multiple reader threads + std::unique_ptr<ThreadStackExecutor> _readers; // multiple reader threads std::atomic<long> _readSeed; std::atomic<long> _doneWriteWork; std::atomic<long> _doneReadWork; - std::atomic<int> _stopRead; + std::atomic<bool> _stopRead; bool _reportWork; - Fixture(uint32_t readThreads = 1); - + Fixture(); ~Fixture(); - void readWork(const WorkContext &context); - void writeWork(uint32_t cnt, WorkContext &context); + void set_read_threads(uint32_t read_threads); + uint32_t getReadThreads() const { return _readThreads; } void stressTest(uint32_t writeCnt); - + void stress_test_indirect(uint64_t write_cnt); +public: + void readWork(const WorkContext &context); + void writeWork(uint32_t cnt, WorkContext &context); + void read_indirect_work(const IndirectContext& context); + void write_indirect_work(uint64_t cnt, IndirectContext& context); private: Fixture(const Fixture &index) = delete; Fixture(Fixture &&index) = delete; @@ -49,23 +93,27 @@ private: }; -Fixture::Fixture(uint32_t readThreads) - : _generationHandler(), - _readThreads(readThreads), +Fixture::Fixture() + : ::testing::Test(), + _generationHandler(), + _readThreads(1), _writer(1, 128_Ki), - _readers(readThreads, 128_Ki), + _readers(), _doneWriteWork(0), _doneReadWork(0), - _stopRead(0), + _stopRead(false), _reportWork(false) { + set_read_threads(1); } Fixture::~Fixture() { - _readers.sync(); - _readers.shutdown(); + if (_readers) { + _readers->sync(); + _readers->shutdown(); + } _writer.sync(); _writer.shutdown(); if (_reportWork) { @@ -75,6 +123,16 @@ Fixture::~Fixture() } } +void +Fixture::set_read_threads(uint32_t read_threads) +{ + if (_readers) { + _readers->sync(); + _readers->shutdown(); + } + _readThreads = read_threads; + _readers = std::make_unique<ThreadStackExecutor>(read_threads, 128_Ki); +} void Fixture::readWork(const WorkContext &context) @@ -82,10 +140,10 @@ Fixture::readWork(const WorkContext &context) uint32_t i; uint32_t cnt = std::numeric_limits<uint32_t>::max(); - for (i = 0; i < cnt && _stopRead.load() == 0; ++i) { + for (i = 0; i < cnt && !_stopRead.load(); ++i) { auto guard = _generationHandler.takeGuard(); auto generation = context._generation.load(std::memory_order_relaxed); - EXPECT_GREATER_EQUAL(generation, guard.getGeneration()); + EXPECT_GE(generation, guard.getGeneration()); } _doneReadWork += i; LOG(info, "done %u read work", i); @@ -95,12 +153,12 @@ Fixture::readWork(const WorkContext &context) void Fixture::writeWork(uint32_t cnt, WorkContext &context) { + ReadStopper read_stopper(_stopRead); for (uint32_t i = 0; i < cnt; ++i) { context._generation.store(_generationHandler.getNextGeneration(), std::memory_order_relaxed); _generationHandler.incGeneration(); } _doneWriteWork += cnt; - _stopRead = 1; LOG(info, "done %u write work", cnt); } @@ -150,19 +208,111 @@ Fixture::stressTest(uint32_t writeCnt) auto context = std::make_shared<WorkContext>(); _writer.execute(std::make_unique<WriteWorkTask>(*this, writeCnt, context)); for (uint32_t i = 0; i < readThreads; ++i) { - _readers.execute(std::make_unique<ReadWorkTask>(*this, context)); + _readers->execute(std::make_unique<ReadWorkTask>(*this, context)); } + _writer.sync(); + _readers->sync(); +} + +void +Fixture::read_indirect_work(const IndirectContext& context) +{ + uint64_t i; + uint64_t cnt = std::numeric_limits<uint32_t>::max(); + uint64_t old_value = 0; + for (i = 0; i < cnt && !_stopRead.load(); ++i) { + auto guard = _generationHandler.takeGuard(); + // Data referenced by pointer is protected by guard + auto v_ptr = context._value_ptr.load(std::memory_order_acquire); + EXPECT_GE(*v_ptr, old_value); + old_value = *v_ptr; + } + _doneReadWork += i; + LOG(info, "done %" PRIu64 " read work", i); } -TEST_F("stress test, 2 readers", Fixture(2)) +void +Fixture::write_indirect_work(uint64_t cnt, IndirectContext& context) { - f.stressTest(1000000); + ReadStopper read_stopper(_stopRead); + uint32_t sleep_cnt = 0; + ASSERT_EQ(0, _generationHandler.getCurrentGeneration()); + auto oldest_gen = _generationHandler.getFirstUsedGeneration(); + for (uint64_t i = 0; i < cnt; ++i) { + auto gen = _generationHandler.getCurrentGeneration(); + // Hold data for gen, write new data for next_gen + auto next_gen = gen + 1; + auto *v_ptr = context.calc_value_ptr(next_gen); + ASSERT_EQ(0u, *v_ptr); + *v_ptr = next_gen; + context._value_ptr.store(v_ptr, std::memory_order_release); + _generationHandler.incGeneration(); + auto first_used_gen = _generationHandler.getFirstUsedGeneration(); + while (oldest_gen < first_used_gen) { + // Clear data that readers should no longer have access to. + *context.calc_value_ptr(oldest_gen) = 0; + ++oldest_gen; + } + while ((next_gen - first_used_gen) >= context.values_size - 2) { + // Sleep if writer gets too much ahead of readers. + std::this_thread::sleep_for(1ms); + ++sleep_cnt; + _generationHandler.updateFirstUsedGeneration(); + first_used_gen = _generationHandler.getFirstUsedGeneration(); + } + } + _doneWriteWork += cnt; + LOG(info, "done %" PRIu64 " write work, %u sleeps", cnt, sleep_cnt); } -TEST_F("stress test, 4 readers", Fixture(4)) +void +Fixture::stress_test_indirect(uint64_t write_cnt) { - f.stressTest(1000000); + _reportWork = true; + uint32_t read_threads = getReadThreads(); + LOG(info, "starting stress test indirect, 1 write thread, %u read threads, %" PRIu64 " writes", read_threads, write_cnt); + auto context = std::make_shared<IndirectContext>(); + _writer.execute(makeLambdaTask([this, context, write_cnt]() { write_indirect_work(write_cnt, *context); })); + for (uint32_t i = 0; i < read_threads; ++i) { + _readers->execute(makeLambdaTask([this, context]() { read_indirect_work(*context); })); + } + _writer.sync(); + _readers->sync(); } -TEST_MAIN() { TEST_RUN_ALL(); } +using GenerationHandlerStressTest = Fixture; + +TEST_F(GenerationHandlerStressTest, stress_test_2_readers) +{ + set_read_threads(2); + stressTest(smoke_test ? 10000 : 1000000); +} + +TEST_F(GenerationHandlerStressTest, stress_test_4_readers) +{ + set_read_threads(4); + stressTest(smoke_test ? 10000 : 1000000); +} + +TEST_F(GenerationHandlerStressTest, stress_test_indirect_2_readers) +{ + set_read_threads(2); + stress_test_indirect(smoke_test ? 10000 : 1000000); +} + +TEST_F(GenerationHandlerStressTest, stress_test_indirect_4_readers) +{ + set_read_threads(4); + stress_test_indirect(smoke_test ? 10000 : 1000000); +} + +int main(int argc, char **argv) { + if (argc > 1 && argv[1] == smoke_test_option) { + smoke_test = true; + ++argv; + --argc; + } + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/vespalib/src/vespa/vespalib/btree/btreebuilder.hpp b/vespalib/src/vespa/vespalib/btree/btreebuilder.hpp index 3162dfb4820..c15190a895e 100644 --- a/vespalib/src/vespa/vespalib/btree/btreebuilder.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreebuilder.hpp @@ -191,11 +191,11 @@ normalize() if (pnode->validSlots() > 0) { uint32_t s = pnode->validSlots() - 1; LeafNodeType *l = _allocator.mapLeafRef(pnode->get_child_relaxed(s)); - pnode->writeKey(s, l->getLastKey()); + pnode->write_key_relaxed(s, l->getLastKey()); if (s > 0) { --s; l = _allocator.mapLeafRef(pnode->get_child_relaxed(s)); - pnode->writeKey(s, l->getLastKey()); + pnode->write_key_relaxed(s, l->getLastKey()); } } if (!leftInodes.empty() && _allocator.isValidRef(leftInodes[0])) { @@ -203,7 +203,7 @@ normalize() _allocator.mapInternalRef(leftInodes[0]); uint32_t s = lpnode->validSlots() - 1; LeafNodeType *l = _allocator.mapLeafRef(lpnode->get_child_relaxed(s)); - lpnode->writeKey(s, l->getLastKey()); + lpnode->write_key_relaxed(s, l->getLastKey()); } } @@ -257,11 +257,11 @@ normalize() uint32_t s = pnode->validSlots() - 1; InternalNodeType *n = _allocator.mapInternalRef(pnode->get_child_relaxed(s)); - pnode->writeKey(s, n->getLastKey()); + pnode->write_key_relaxed(s, n->getLastKey()); if (s > 0) { --s; n = _allocator.mapInternalRef(pnode->get_child_relaxed(s)); - pnode->writeKey(s, n->getLastKey()); + pnode->write_key_relaxed(s, n->getLastKey()); } } if (level + 1 < leftInodes.size() && @@ -271,7 +271,7 @@ normalize() uint32_t s = lpnode->validSlots() - 1; InternalNodeType *n = _allocator.mapInternalRef(lpnode->get_child_relaxed(s)); - lpnode->writeKey(s, n->getLastKey()); + lpnode->write_key_relaxed(s, n->getLastKey()); } } /* Check fanout on root node */ diff --git a/vespalib/src/vespa/vespalib/btree/btreenode.h b/vespalib/src/vespa/vespalib/btree/btreenode.h index e457af73654..65186d5c98a 100644 --- a/vespalib/src/vespa/vespalib/btree/btreenode.h +++ b/vespalib/src/vespa/vespalib/btree/btreenode.h @@ -8,6 +8,7 @@ #include <vespa/vespalib/datastore/atomic_entry_ref.h> #include <vespa/vespalib/datastore/handle.h> #include <cassert> +#include <type_traits> #include <utility> #include <cstddef> @@ -205,7 +206,14 @@ protected: public: const KeyT & getKey(uint32_t idx) const { return _keys[idx]; } const KeyT & getLastKey() const { return _keys[validSlots() - 1]; } - void writeKey(uint32_t idx, const KeyT & key) { _keys[idx] = key; } + void writeKey(uint32_t idx, const KeyT & key) { + if constexpr (std::is_same_v<KeyT, vespalib::datastore::AtomicEntryRef>) { + _keys[idx].store_release(key.load_relaxed()); + } else { + _keys[idx] = key; + } + } + void write_key_relaxed(uint32_t idx, const KeyT & key) { _keys[idx] = key; } template <typename CompareT> uint32_t lower_bound(uint32_t sidx, const KeyT & key, CompareT comp) const; diff --git a/vespalib/src/vespa/vespalib/portal/http_connection.cpp b/vespalib/src/vespa/vespalib/portal/http_connection.cpp index fc9e8f200cc..26c784af028 100644 --- a/vespalib/src/vespa/vespalib/portal/http_connection.cpp +++ b/vespalib/src/vespa/vespalib/portal/http_connection.cpp @@ -158,7 +158,7 @@ HttpConnection::do_dispatch() void HttpConnection::do_wait() { - if (_reply_ready) { + if (_reply_ready.load(std::memory_order_acquire)) { set_state(State::WRITE_REPLY, false, true); } } @@ -248,8 +248,8 @@ HttpConnection::respond_with_content(const vespalib::string &content_type, dst.printf("\r\n"); dst.write(content.data(), content.size()); } - _reply_ready = true; _token->update(false, true); + _reply_ready.store(true, std::memory_order_release); } void @@ -261,8 +261,8 @@ HttpConnection::respond_with_error(int code, const vespalib::string &msg) dst.printf("Connection: close\r\n"); dst.printf("\r\n"); } - _reply_ready = true; _token->update(false, true); + _reply_ready.store(true, std::memory_order_release); } } // namespace vespalib::portal diff --git a/vespalib/src/vespa/vespalib/util/alloc.cpp b/vespalib/src/vespa/vespalib/util/alloc.cpp index bf5f0200600..3f49fc4b427 100644 --- a/vespalib/src/vespa/vespalib/util/alloc.cpp +++ b/vespalib/src/vespa/vespalib/util/alloc.cpp @@ -1,5 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "alloc.h" +#include "atomic.h" #include "memory_allocator.h" #include "round_up_to_page_size.h" #include <sys/mman.h> @@ -17,11 +18,13 @@ #include <vespa/log/log.h> LOG_SETUP(".vespalib.alloc"); +using namespace vespalib::atomic; + namespace vespalib { namespace { -volatile bool _G_hasHugePageFailureJustHappened(false); +std::atomic<bool> _G_hasHugePageFailureJustHappened(false); bool _G_SilenceCoreOnOOM(false); int _G_HugeFlags = 0; size_t _G_MMapLogLimit = std::numeric_limits<size_t>::max(); @@ -328,8 +331,8 @@ MMapAllocator::salloc(size_t sz, void * wantedAddress) } buf = mmap(wantedAddress, sz, prot, flags | _G_HugeFlags, -1, 0); if (buf == MAP_FAILED) { - if ( ! _G_hasHugePageFailureJustHappened ) { - _G_hasHugePageFailureJustHappened = true; + if ( ! load_relaxed(_G_hasHugePageFailureJustHappened)) { + store_relaxed(_G_hasHugePageFailureJustHappened, true); LOG(debug, "Failed allocating %ld bytes with hugepages due too '%s'." " Will resort to ordinary mmap until it works again.", sz, FastOS_FileInterface::getLastErrorString().c_str()); @@ -347,9 +350,7 @@ MMapAllocator::salloc(size_t sz, void * wantedAddress) } } } else { - if (_G_hasHugePageFailureJustHappened) { - _G_hasHugePageFailureJustHappened = false; - } + store_relaxed(_G_hasHugePageFailureJustHappened, false); } #ifdef __linux__ if (sz >= _G_MMapNoCoreLimit) { diff --git a/vespalib/src/vespa/vespalib/util/child_process.cpp b/vespalib/src/vespa/vespalib/util/child_process.cpp index 694bed60f1b..93db56cdf67 100644 --- a/vespalib/src/vespa/vespalib/util/child_process.cpp +++ b/vespalib/src/vespa/vespalib/util/child_process.cpp @@ -207,7 +207,7 @@ ChildProcess::checkProc() ChildProcess::ChildProcess(const char *cmd) : _reader(1), - _proc(cmd, true, &_reader), + _proc(cmd, &_reader), _running(false), _failed(false), _exitCode(-918273645) @@ -218,7 +218,7 @@ ChildProcess::ChildProcess(const char *cmd) ChildProcess::ChildProcess(const char *cmd, capture_stderr_tag) : _reader(2), - _proc(cmd, true, &_reader, &_reader), + _proc(cmd, &_reader, &_reader), _running(false), _failed(false), _exitCode(-918273645) diff --git a/vespalib/src/vespa/vespalib/util/destructor_callbacks.h b/vespalib/src/vespa/vespalib/util/destructor_callbacks.h index 7fc15e0185e..7452d72ea44 100644 --- a/vespalib/src/vespa/vespalib/util/destructor_callbacks.h +++ b/vespalib/src/vespa/vespalib/util/destructor_callbacks.h @@ -44,10 +44,17 @@ private: template<class FunctionType> std::shared_ptr<IDestructorCallback> -makeLambdaCallback(FunctionType &&function) { +makeSharedLambdaCallback(FunctionType &&function) { return std::make_shared<LambdaCallback<std::decay_t<FunctionType>>> (std::forward<FunctionType>(function)); } +template<class FunctionType> +std::unique_ptr<IDestructorCallback> +makeUniqueLambdaCallback(FunctionType &&function) { + return std::make_unique<LambdaCallback<std::decay_t<FunctionType>>> + (std::forward<FunctionType>(function)); +} + } diff --git a/vespalib/src/vespa/vespalib/util/invokeservice.h b/vespalib/src/vespa/vespalib/util/invokeservice.h index 3e3973234d1..22bee0d4526 100644 --- a/vespalib/src/vespa/vespalib/util/invokeservice.h +++ b/vespalib/src/vespa/vespalib/util/invokeservice.h @@ -3,6 +3,7 @@ #pragma once #include "idestructorcallback.h" +#include "time.h" #include <functional> namespace vespalib { @@ -13,8 +14,9 @@ namespace vespalib { **/ class InvokeService { public: + using InvokeFunc = std::function<void()>; virtual ~InvokeService() = default; - virtual std::unique_ptr<IDestructorCallback> registerInvoke(std::function<void()> func) = 0; + virtual std::unique_ptr<IDestructorCallback> registerInvoke(InvokeFunc func) = 0; }; } diff --git a/vespalib/src/vespa/vespalib/util/invokeserviceimpl.cpp b/vespalib/src/vespa/vespalib/util/invokeserviceimpl.cpp index ffa0825c950..c10029b2f58 100644 --- a/vespalib/src/vespa/vespalib/util/invokeserviceimpl.cpp +++ b/vespalib/src/vespa/vespalib/util/invokeserviceimpl.cpp @@ -7,11 +7,12 @@ namespace vespalib { InvokeServiceImpl::InvokeServiceImpl(duration napTime) : _naptime(napTime), + _now(steady_clock::now()), _lock(), _currId(0), _closed(false), _toInvoke(), - _thread() + _thread(std::make_unique<std::thread>([this]() { runLoop(); })) { } @@ -22,9 +23,7 @@ InvokeServiceImpl::~InvokeServiceImpl() assert(_toInvoke.empty()); _closed = true; } - if (_thread) { - _thread->join(); - } + _thread->join(); } class InvokeServiceImpl::Registration : public IDestructorCallback { @@ -44,20 +43,17 @@ private: }; std::unique_ptr<IDestructorCallback> -InvokeServiceImpl::registerInvoke(VoidFunc func) { +InvokeServiceImpl::registerInvoke(InvokeFunc func) { std::lock_guard guard(_lock); uint64_t id = _currId++; _toInvoke.emplace_back(id, std::move(func)); - if ( ! _thread) { - _thread = std::make_unique<std::thread>([this]() { runLoop(); }); - } return std::make_unique<Registration>(this, id); } void InvokeServiceImpl::unregister(uint64_t id) { std::lock_guard guard(_lock); - auto found = std::find_if(_toInvoke.begin(), _toInvoke.end(), [id](const std::pair<uint64_t, VoidFunc> & a) { + auto found = std::find_if(_toInvoke.begin(), _toInvoke.end(), [id](const IdAndFunc & a) { return id == a.first; }); assert (found != _toInvoke.end()); @@ -68,6 +64,7 @@ void InvokeServiceImpl::runLoop() { bool done = false; while ( ! done ) { + _now.store(steady_clock::now(), std::memory_order_relaxed); { std::lock_guard guard(_lock); for (auto & func: _toInvoke) { diff --git a/vespalib/src/vespa/vespalib/util/invokeserviceimpl.h b/vespalib/src/vespa/vespalib/util/invokeserviceimpl.h index 3b0c7690731..beb57ca1ce0 100644 --- a/vespalib/src/vespa/vespalib/util/invokeserviceimpl.h +++ b/vespalib/src/vespa/vespalib/util/invokeserviceimpl.h @@ -14,22 +14,24 @@ namespace vespalib { * An invoke service what will invoke the given function with at specified frequency. */ class InvokeServiceImpl : public InvokeService { - using VoidFunc = std::function<void()>; public: InvokeServiceImpl(duration napTime); InvokeServiceImpl(const InvokeServiceImpl &) = delete; InvokeServiceImpl & operator=(const InvokeServiceImpl &) = delete; ~InvokeServiceImpl() override; - std::unique_ptr<IDestructorCallback> registerInvoke(VoidFunc func) override; + std::unique_ptr<IDestructorCallback> registerInvoke(InvokeFunc func) override; + const std::atomic<steady_time> & nowRef() const { return _now; } private: + using IdAndFunc = std::pair<uint64_t, InvokeFunc>; class Registration; void unregister(uint64_t id); void runLoop(); duration _naptime; + std::atomic<steady_time> _now; std::mutex _lock; uint64_t _currId; bool _closed; - std::vector<std::pair<uint64_t, VoidFunc>> _toInvoke; + std::vector<IdAndFunc> _toInvoke; std::unique_ptr<std::thread> _thread; }; diff --git a/vespalib/src/vespa/vespalib/util/shared_operation_throttler.cpp b/vespalib/src/vespa/vespalib/util/shared_operation_throttler.cpp index 6e273d1a7ea..478d8c1b9e9 100644 --- a/vespalib/src/vespa/vespalib/util/shared_operation_throttler.cpp +++ b/vespalib/src/vespa/vespalib/util/shared_operation_throttler.cpp @@ -24,7 +24,7 @@ public: internal_ref_count_increase(); return Token(this, TokenCtorTag{}); } - Token blocking_acquire_one(vespalib::duration) noexcept override { + Token blocking_acquire_one(vespalib::steady_time) noexcept override { internal_ref_count_increase(); return Token(this, TokenCtorTag{}); } @@ -267,7 +267,7 @@ public: ~DynamicOperationThrottler() override; Token blocking_acquire_one() noexcept override; - Token blocking_acquire_one(vespalib::duration timeout) noexcept override; + Token blocking_acquire_one(vespalib::steady_time deadline) noexcept override; Token try_acquire_one() noexcept override; uint32_t current_window_size() const noexcept override; uint32_t current_active_token_count() const noexcept override; @@ -334,12 +334,12 @@ DynamicOperationThrottler::blocking_acquire_one() noexcept } DynamicOperationThrottler::Token -DynamicOperationThrottler::blocking_acquire_one(vespalib::duration timeout) noexcept +DynamicOperationThrottler::blocking_acquire_one(vespalib::steady_time deadline) noexcept { std::unique_lock lock(_mutex); if (!has_spare_capacity_in_active_window()) { ++_waiting_threads; - const bool accepted = _cond.wait_for(lock, timeout, [&] { + const bool accepted = _cond.wait_until(lock, deadline, [&] { return has_spare_capacity_in_active_window(); }); --_waiting_threads; diff --git a/vespalib/src/vespa/vespalib/util/shared_operation_throttler.h b/vespalib/src/vespa/vespalib/util/shared_operation_throttler.h index b7913029c1e..95d6d361cb6 100644 --- a/vespalib/src/vespa/vespalib/util/shared_operation_throttler.h +++ b/vespalib/src/vespa/vespalib/util/shared_operation_throttler.h @@ -54,9 +54,9 @@ public: // Acquire a valid throttling token, uninterruptedly blocking until one can be obtained. [[nodiscard]] virtual Token blocking_acquire_one() noexcept = 0; // Attempt to acquire a valid throttling token, waiting up to `timeout` for one to be - // available. If the timeout is exceeded without any tokens becoming available, an + // available. If the deadline is reached without any tokens becoming available, an // invalid token will be returned. - [[nodiscard]] virtual Token blocking_acquire_one(vespalib::duration timeout) noexcept = 0; + [[nodiscard]] virtual Token blocking_acquire_one(vespalib::steady_time deadline) noexcept = 0; // Attempt to acquire a valid throttling token if one is immediately available. // An invalid token will be returned if none is available. Never blocks (other than // when contending for the internal throttler mutex). diff --git a/vespalib/src/vespa/vespalib/util/time.cpp b/vespalib/src/vespa/vespalib/util/time.cpp index bcc289489ea..3737a25f268 100644 --- a/vespalib/src/vespa/vespalib/util/time.cpp +++ b/vespalib/src/vespa/vespalib/util/time.cpp @@ -30,6 +30,16 @@ getVespaTimerHz() { return 1000u; } +duration +adjustTimeoutByHz(duration timeout, long hz) { + return (timeout * 1000) / hz; +} + +duration +adjustTimeoutByDetectedHz(duration timeout) { + return adjustTimeoutByHz(timeout, getVespaTimerHz()); +} + namespace { string diff --git a/vespalib/src/vespa/vespalib/util/time.h b/vespalib/src/vespa/vespalib/util/time.h index dbbae862a8d..783831b5171 100644 --- a/vespalib/src/vespa/vespalib/util/time.h +++ b/vespalib/src/vespa/vespalib/util/time.h @@ -92,5 +92,7 @@ public: * The default frequency (1000hz) for vespa timer, with environment override VESPA_TIMER_HZ capped to [1..1000] */ uint32_t getVespaTimerHz(); +duration adjustTimeoutByDetectedHz(duration timeout); +duration adjustTimeoutByHz(duration timeout, long hz); } |