diff options
475 files changed, 6260 insertions, 4458 deletions
diff --git a/airlift-zstd/src/test/java/ai/vespa/airlift/zstd/TestCompressor.java b/airlift-zstd/src/test/java/ai/vespa/airlift/zstd/TestCompressor.java index d6f13b98c71..4aa00f91ffc 100644 --- a/airlift-zstd/src/test/java/ai/vespa/airlift/zstd/TestCompressor.java +++ b/airlift-zstd/src/test/java/ai/vespa/airlift/zstd/TestCompressor.java @@ -20,6 +20,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static sun.misc.Unsafe.ARRAY_BYTE_BASE_OFFSET; +@SuppressWarnings("proprietary") public class TestCompressor { @Test diff --git a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/InfrastructureApplication.java b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/InfrastructureApplication.java index ad73905c395..e54f952c056 100644 --- a/application-model/src/main/java/com/yahoo/vespa/applicationmodel/InfrastructureApplication.java +++ b/application-model/src/main/java/com/yahoo/vespa/applicationmodel/InfrastructureApplication.java @@ -13,6 +13,7 @@ import java.util.stream.Stream; * @author hakonhall */ public enum InfrastructureApplication { + CONTROLLER_HOST("controller-host", NodeType.controllerhost), CONTROLLER("controller", NodeType.controller), CONFIG_SERVER_HOST("configserver-host", NodeType.confighost), diff --git a/client/go/internal/admin/prog/valgrind.go b/client/go/internal/admin/prog/valgrind.go index 7d3fb059f8f..2d7f0a597d9 100644 --- a/client/go/internal/admin/prog/valgrind.go +++ b/client/go/internal/admin/prog/valgrind.go @@ -22,9 +22,10 @@ func (p *Spec) ConfigureValgrind() { p.shouldUseValgrind = false p.shouldUseCallgrind = false env := p.Getenv(envvars.VESPA_USE_VALGRIND) + allValgrind := env == "all" parts := strings.Split(env, " ") for _, part := range parts { - if p.BaseName == part { + if p.BaseName == part || allValgrind { trace.Trace("using valgrind as", p.Program, "has basename in", envvars.VESPA_USE_VALGRIND, "=>", env) backticks := util.BackTicksWithStderr out, err := backticks.Run(VALGRIND_PROG, "--help") diff --git a/client/go/internal/admin/prog/valgrind_test.go b/client/go/internal/admin/prog/valgrind_test.go index 6ec622277c6..11d9424405f 100644 --- a/client/go/internal/admin/prog/valgrind_test.go +++ b/client/go/internal/admin/prog/valgrind_test.go @@ -52,6 +52,11 @@ func TestValgrindDetection(t *testing.T) { assert.Equal(t, false, spec.shouldUseValgrind) assert.Equal(t, false, spec.shouldUseCallgrind) + t.Setenv("VESPA_USE_VALGRIND", "all") + spec.ConfigureValgrind() + assert.Equal(t, true, spec.shouldUseValgrind) + assert.Equal(t, false, spec.shouldUseCallgrind) + t.Setenv("VESPA_USE_VALGRIND", "foo bar") spec.ConfigureValgrind() assert.Equal(t, false, spec.shouldUseValgrind) diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind.go b/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind.go index 43a1ed602bd..cccb37df8e5 100644 --- a/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind.go +++ b/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind.go @@ -18,9 +18,10 @@ func (p *ProgSpec) configureValgrind() { p.shouldUseValgrind = false p.shouldUseCallgrind = false env := p.getenv(envvars.VESPA_USE_VALGRIND) + allValgrind := env == "all" parts := strings.Split(env, " ") for _, part := range parts { - if p.BaseName == part { + if p.BaseName == part || allValgrind { trace.Trace("using valgrind as", p.Program, "has basename in", envvars.VESPA_USE_VALGRIND, "=>", env) backticks := util.BackTicksWithStderr out, err := backticks.Run("which", "valgrind") diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind_test.go b/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind_test.go index 48cc78474ed..1a105d66c4a 100644 --- a/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind_test.go +++ b/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind_test.go @@ -52,6 +52,11 @@ func TestValgrindDetection(t *testing.T) { assert.Equal(t, false, spec.shouldUseValgrind) assert.Equal(t, false, spec.shouldUseCallgrind) + t.Setenv("VESPA_USE_VALGRIND", "all") + spec.configureValgrind() + assert.Equal(t, true, spec.shouldUseValgrind) + assert.Equal(t, false, spec.shouldUseCallgrind) + t.Setenv("VESPA_USE_VALGRIND", "foo bar") spec.configureValgrind() assert.Equal(t, false, spec.shouldUseValgrind) diff --git a/client/go/internal/cli/auth/auth0/auth0.go b/client/go/internal/cli/auth/auth0/auth0.go index 5f7612d4d2e..6fcd3f7680e 100644 --- a/client/go/internal/cli/auth/auth0/auth0.go +++ b/client/go/internal/cli/auth/auth0/auth0.go @@ -110,28 +110,40 @@ func (a *Client) getDeviceFlowConfig() (flowConfig, error) { } r, err := a.httpClient.Do(req, time.Second*30) if err != nil { - return flowConfig{}, fmt.Errorf("failed to get device flow config: %w", err) + return flowConfig{}, fmt.Errorf("auth0: failed to get device flow config: %w", err) } defer r.Body.Close() if r.StatusCode/100 != 2 { - return flowConfig{}, fmt.Errorf("failed to get device flow config: got response code %d from %s", r.StatusCode, url) + return flowConfig{}, fmt.Errorf("auth0: failed to get device flow config: got response code %d from %s", r.StatusCode, url) } var cfg flowConfig if err := json.NewDecoder(r.Body).Decode(&cfg); err != nil { - return flowConfig{}, fmt.Errorf("failed to decode response: %w", err) + return flowConfig{}, fmt.Errorf("auth0: failed to decode response: %w", err) } return cfg, nil } +func (a *Client) Authenticate(request *http.Request) error { + accessToken, err := a.AccessToken() + if err != nil { + return err + } + if request.Header == nil { + request.Header = make(http.Header) + } + request.Header.Set("Authorization", "Bearer "+accessToken) + return nil +} + // AccessToken returns an access token for the configured system, refreshing it if necessary. func (a *Client) AccessToken() (string, error) { creds, ok := a.provider.Systems[a.options.SystemName] if !ok { - return "", fmt.Errorf("system %s is not configured", a.options.SystemName) + return "", fmt.Errorf("auth0: system %s is not configured: %s", a.options.SystemName, reauthMessage) } else if creds.AccessToken == "" { - return "", fmt.Errorf("access token missing: %s", reauthMessage) + return "", fmt.Errorf("auth0: access token missing: %s", reauthMessage) } else if scopesChanged(creds) { - return "", fmt.Errorf("authentication scopes changed: %s", reauthMessage) + return "", fmt.Errorf("auth0: authentication scopes changed: %s", reauthMessage) } else if isExpired(creds.ExpiresAt, accessTokenExpiry) { // check if the stored access token is expired: // use the refresh token to get a new access token: @@ -142,7 +154,7 @@ func (a *Client) AccessToken() (string, error) { } resp, err := tr.Refresh(cancelOnInterrupt(), a.options.SystemName) if err != nil { - return "", fmt.Errorf("failed to renew access token: %w: %s", err, reauthMessage) + return "", fmt.Errorf("auth0: failed to renew access token: %w: %s", err, reauthMessage) } else { // persist the updated system with renewed access token creds.AccessToken = resp.AccessToken @@ -173,12 +185,6 @@ func scopesChanged(s Credentials) bool { return false } -// HasCredentials returns true if this client has retrived credentials for the configured system. -func (a *Client) HasCredentials() bool { - _, ok := a.provider.Systems[a.options.SystemName] - return ok -} - // WriteCredentials writes given credentials to the configuration file. func (a *Client) WriteCredentials(credentials Credentials) error { if a.provider.Systems == nil { @@ -186,7 +192,7 @@ func (a *Client) WriteCredentials(credentials Credentials) error { } a.provider.Systems[a.options.SystemName] = credentials if err := writeConfig(a.provider, a.options.ConfigPath); err != nil { - return fmt.Errorf("failed to write config: %w", err) + return fmt.Errorf("auth0: failed to write config: %w", err) } return nil } @@ -195,11 +201,11 @@ func (a *Client) WriteCredentials(credentials Credentials) error { func (a *Client) RemoveCredentials() error { tr := &auth.TokenRetriever{Secrets: &auth.Keyring{}} if err := tr.Delete(a.options.SystemName); err != nil { - return fmt.Errorf("failed to remove system %s from secret storage: %w", a.options.SystemName, err) + return fmt.Errorf("auth0: failed to remove system %s from secret storage: %w", a.options.SystemName, err) } delete(a.provider.Systems, a.options.SystemName) if err := writeConfig(a.provider, a.options.ConfigPath); err != nil { - return fmt.Errorf("failed to write config: %w", err) + return fmt.Errorf("auth0: failed to write config: %w", err) } return nil } diff --git a/client/go/internal/cli/auth/zts/zts.go b/client/go/internal/cli/auth/zts/zts.go index 1e84912a271..2c66ff13e8b 100644 --- a/client/go/internal/cli/auth/zts/zts.go +++ b/client/go/internal/cli/auth/zts/zts.go @@ -1,7 +1,6 @@ package zts import ( - "crypto/tls" "encoding/json" "fmt" "net/http" @@ -18,26 +17,39 @@ const DefaultURL = "https://zts.athenz.ouroath.com:4443" type Client struct { client util.HTTPClient tokenURL *url.URL + domain string } // NewClient creates a new client for an Athenz ZTS service located at serviceURL. -func NewClient(client util.HTTPClient, serviceURL string) (*Client, error) { +func NewClient(client util.HTTPClient, domain, serviceURL string) (*Client, error) { tokenURL, err := url.Parse(serviceURL) if err != nil { return nil, err } tokenURL.Path = "/zts/v1/oauth2/token" - return &Client{tokenURL: tokenURL, client: client}, nil + return &Client{tokenURL: tokenURL, client: client, domain: domain}, nil } -// AccessToken returns an access token within the given domain, using certificate to authenticate with ZTS. -func (c *Client) AccessToken(domain string, certificate tls.Certificate) (string, error) { - data := fmt.Sprintf("grant_type=client_credentials&scope=%s:domain", domain) +func (c *Client) Authenticate(request *http.Request) error { + accessToken, err := c.AccessToken() + if err != nil { + return err + } + if request.Header == nil { + request.Header = make(http.Header) + } + request.Header.Add("Authorization", "Bearer "+accessToken) + return nil +} + +// AccessToken returns an access token within the domain configured in client c. +func (c *Client) AccessToken() (string, error) { + // TODO(mpolden): This should cache and re-use tokens until expiry + data := fmt.Sprintf("grant_type=client_credentials&scope=%s:domain", c.domain) req, err := http.NewRequest("POST", c.tokenURL.String(), strings.NewReader(data)) if err != nil { return "", err } - util.SetCertificate(c.client, []tls.Certificate{certificate}) response, err := c.client.Do(req, 10*time.Second) if err != nil { return "", err @@ -45,7 +57,7 @@ func (c *Client) AccessToken(domain string, certificate tls.Certificate) (string defer response.Body.Close() if response.StatusCode != http.StatusOK { - return "", fmt.Errorf("got status %d from %s", response.StatusCode, c.tokenURL.String()) + return "", fmt.Errorf("zts: got status %d from %s", response.StatusCode, c.tokenURL.String()) } var ztsResponse struct { AccessToken string `json:"access_token"` diff --git a/client/go/internal/cli/auth/zts/zts_test.go b/client/go/internal/cli/auth/zts/zts_test.go index d0cc7ea9f9d..1c75a94ee03 100644 --- a/client/go/internal/cli/auth/zts/zts_test.go +++ b/client/go/internal/cli/auth/zts/zts_test.go @@ -1,7 +1,6 @@ package zts import ( - "crypto/tls" "testing" "github.com/vespa-engine/vespa/client/go/internal/mock" @@ -9,17 +8,17 @@ import ( func TestAccessToken(t *testing.T) { httpClient := mock.HTTPClient{} - client, err := NewClient(&httpClient, "http://example.com") + client, err := NewClient(&httpClient, "vespa.vespa", "http://example.com") if err != nil { t.Fatal(err) } httpClient.NextResponseString(400, `{"message": "bad request"}`) - _, err = client.AccessToken("vespa.vespa", tls.Certificate{}) + _, err = client.AccessToken() if err == nil { t.Fatal("want error for non-ok response status") } httpClient.NextResponseString(200, `{"access_token": "foo bar"}`) - token, err := client.AccessToken("vespa.vespa", tls.Certificate{}) + token, err := client.AccessToken() if err != nil { t.Fatal(err) } diff --git a/client/go/internal/cli/cmd/cert.go b/client/go/internal/cli/cmd/cert.go index 7f79a9db358..48bad974c3f 100644 --- a/client/go/internal/cli/cmd/cert.go +++ b/client/go/internal/cli/cmd/cert.go @@ -34,13 +34,18 @@ package specified as an argument to this command (default '.'). It's possible to override the private key and certificate used through environment variables. This can be useful in continuous integration systems. -Example of setting the certificate and key in-line: +It's also possible override the CA certificate which can be useful when using self-signed certificates with a +self-hosted Vespa service. See https://docs.vespa.ai/en/mtls.html for more information. +Example of setting the CA certificate, certificate and key in-line: + + export VESPA_CLI_DATA_PLANE_CA_CERT="my CA cert" export VESPA_CLI_DATA_PLANE_CERT="my cert" export VESPA_CLI_DATA_PLANE_KEY="my private key" -Example of loading certificate and key from custom paths: +Example of loading CA certificate, certificate and key from custom paths: + export VESPA_CLI_DATA_PLANE_CA_CERT_FILE=/path/to/cacert export VESPA_CLI_DATA_PLANE_CERT_FILE=/path/to/cert export VESPA_CLI_DATA_PLANE_KEY_FILE=/path/to/key diff --git a/client/go/internal/cli/cmd/config.go b/client/go/internal/cli/cmd/config.go index fd049864096..e2132814386 100644 --- a/client/go/internal/cli/cmd/config.go +++ b/client/go/internal/cli/cmd/config.go @@ -19,7 +19,6 @@ import ( "github.com/fatih/color" "github.com/spf13/cobra" "github.com/spf13/pflag" - "github.com/vespa-engine/vespa/client/go/internal/cli/auth/auth0" "github.com/vespa-engine/vespa/client/go/internal/cli/config" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -250,9 +249,10 @@ type Config struct { } type KeyPair struct { - KeyPair tls.Certificate - CertificateFile string - PrivateKeyFile string + KeyPair tls.Certificate + RootCertificates []byte + CertificateFile string + PrivateKeyFile string } func loadConfig(environment map[string]string, flags map[string]*pflag.Flag) (*Config, error) { @@ -392,6 +392,10 @@ func (c *Config) deploymentIn(system vespa.System) (vespa.Deployment, error) { return vespa.Deployment{System: system, Application: app, Zone: zone}, nil } +func (c *Config) caCertificatePath() string { + return c.environment["VESPA_CLI_DATA_PLANE_CA_CERT_FILE"] +} + func (c *Config) certificatePath(app vespa.ApplicationID, targetType string) (string, error) { if override, ok := c.environment["VESPA_CLI_DATA_PLANE_CERT_FILE"]; ok { return override, nil @@ -412,50 +416,68 @@ func (c *Config) privateKeyPath(app vespa.ApplicationID, targetType string) (str return c.applicationFilePath(app, "data-plane-private-key.pem") } -func (c *Config) x509KeyPair(app vespa.ApplicationID, targetType string) (KeyPair, error) { +func (c *Config) readTLSOptions(app vespa.ApplicationID, targetType string) (vespa.TLSOptions, error) { + _, trustAll := c.environment["VESPA_CLI_DATA_PLANE_TRUST_ALL"] cert, certOk := c.environment["VESPA_CLI_DATA_PLANE_CERT"] key, keyOk := c.environment["VESPA_CLI_DATA_PLANE_KEY"] - var ( - kp tls.Certificate - err error - certFile string - keyFile string - ) + caCertText, caCertOk := c.environment["VESPA_CLI_DATA_PLANE_CA_CERT"] + options := vespa.TLSOptions{TrustAll: trustAll} + // CA certificate + if caCertOk { + options.CACertificate = []byte(caCertText) + } else { + caCertFile := c.caCertificatePath() + if caCertFile != "" { + b, err := os.ReadFile(caCertFile) + if err != nil { + return options, err + } + options.CACertificate = b + options.CACertificateFile = caCertFile + } + } + // Certificate and private key if certOk && keyOk { - // Use key pair from environment - kp, err = tls.X509KeyPair([]byte(cert), []byte(key)) + kp, err := tls.X509KeyPair([]byte(cert), []byte(key)) + if err != nil { + return vespa.TLSOptions{}, err + } + options.KeyPair = []tls.Certificate{kp} } else { - keyFile, err = c.privateKeyPath(app, targetType) + keyFile, err := c.privateKeyPath(app, targetType) if err != nil { - return KeyPair{}, err + return vespa.TLSOptions{}, err } - certFile, err = c.certificatePath(app, targetType) + certFile, err := c.certificatePath(app, targetType) if err != nil { - return KeyPair{}, err + return vespa.TLSOptions{}, err + } + kp, err := tls.LoadX509KeyPair(certFile, keyFile) + if err == nil { + options.KeyPair = []tls.Certificate{kp} + options.PrivateKeyFile = keyFile + options.CertificateFile = certFile + } else if err != nil && !os.IsNotExist(err) { + return vespa.TLSOptions{}, err } - kp, err = tls.LoadX509KeyPair(certFile, keyFile) - } - if err != nil { - return KeyPair{}, err } - if targetType == vespa.TargetHosted { - cert, err := x509.ParseCertificate(kp.Certificate[0]) + if options.KeyPair != nil { + cert, err := x509.ParseCertificate(options.KeyPair[0].Certificate[0]) if err != nil { - return KeyPair{}, err + return vespa.TLSOptions{}, err } now := time.Now() expiredAt := cert.NotAfter if expiredAt.Before(now) { delta := now.Sub(expiredAt).Truncate(time.Second) - return KeyPair{}, fmt.Errorf("certificate %s expired at %s (%s ago)", certFile, cert.NotAfter, delta) + source := options.CertificateFile + if source == "" { + source = "environment" + } + return vespa.TLSOptions{}, fmt.Errorf("certificate in %s expired at %s (%s ago)", source, cert.NotAfter, delta) } - return KeyPair{KeyPair: kp, CertificateFile: certFile, PrivateKeyFile: keyFile}, nil } - return KeyPair{ - KeyPair: kp, - CertificateFile: certFile, - PrivateKeyFile: keyFile, - }, nil + return options, nil } func (c *Config) apiKeyFileFromEnv() (string, bool) { @@ -490,11 +512,10 @@ func (c *Config) readAPIKey(cli *CLI, system vespa.System, tenantName string) ([ return nil, nil // Vespa Cloud CI only talks to data plane and does not have an API key } if !cli.isCI() { - client, err := cli.auth0Factory(cli.httpClient, auth0.Options{ConfigPath: c.authConfigPath(), SystemName: system.Name, SystemURL: system.URL}) - if err == nil && client.HasCredentials() { - return nil, nil // use Auth0 + if _, err := os.Stat(c.authConfigPath()); err == nil { + return nil, nil // We have auth config, so we should prefer Auth0 over API key } - cli.printWarning("Authenticating with API key. This is discouraged in non-CI environments", "Authenticate with 'vespa auth login'") + cli.printWarning("Authenticating with API key. This is discouraged in non-CI environments", "Authenticate with 'vespa auth login' instead") } return os.ReadFile(c.apiKeyPath(tenantName)) } @@ -544,12 +565,12 @@ func (c *Config) list(includeUnset bool) []string { } // flagValue returns the set value and default value of the named flag. -func (c *Config) flagValue(name string) (string, string) { +func (c *Config) flagValue(name string) (string, string, bool) { f, ok := c.flags[name] if !ok { - return "", "" + return "", "", ok } - return f.Value.String(), f.DefValue + return f.Value.String(), f.DefValue, f.Changed } // getNonEmpty returns value of given option, if that value is non-empty @@ -564,9 +585,9 @@ func (c *Config) getNonEmpty(option string) (string, bool) { // get returns the value associated with option, from the most preferred source in the following order: flag > local // config > global config. func (c *Config) get(option string) (string, bool) { - flagValue, flagDefault := c.flagValue(option) + flagValue, flagDefault, changed := c.flagValue(option) // explicit flag value always takes precedence over everything else - if flagValue != flagDefault { + if changed { return flagValue, true } // ... then local config, if option is explicitly defined there diff --git a/client/go/internal/cli/cmd/config_test.go b/client/go/internal/cli/cmd/config_test.go index 612904061de..66b65bf402b 100644 --- a/client/go/internal/cli/cmd/config_test.go +++ b/client/go/internal/cli/cmd/config_test.go @@ -2,15 +2,21 @@ package cmd import ( + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" "os" "path/filepath" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/vespa-engine/vespa/client/go/internal/cli/auth/auth0" "github.com/vespa-engine/vespa/client/go/internal/mock" - "github.com/vespa-engine/vespa/client/go/internal/util" "github.com/vespa-engine/vespa/client/go/internal/vespa" ) @@ -28,6 +34,7 @@ func TestConfig(t *testing.T) { assertConfigCommand(t, configHome, "", "config", "set", "target", "http://127.0.0.1:8080") assertConfigCommand(t, configHome, "", "config", "set", "target", "https://127.0.0.1") assertConfigCommand(t, configHome, "target = https://127.0.0.1\n", "config", "get", "target") + assertConfigCommand(t, configHome, "target = local\n", "config", "get", "-t", "local", "target") // application assertConfigCommandErr(t, configHome, "Error: invalid application: \"foo\"\n", "config", "set", "application", "foo") @@ -165,7 +172,7 @@ func TestReadAPIKey(t *testing.T) { require.Nil(t, err) assert.Equal(t, []byte("foo"), key) - // Cloud CI does not read key from disk as it's not expected to have any + // Cloud CI never reads key from disk as it's not expected to have any cli, _, _ = newTestCLI(t, "VESPA_CLI_CLOUD_CI=true") key, err = cli.config.readAPIKey(cli, vespa.PublicSystem, "t1") require.Nil(t, err) @@ -185,12 +192,111 @@ func TestReadAPIKey(t *testing.T) { require.Nil(t, err) assert.Equal(t, []byte("baz"), key) - // Auth0 is preferred when configured + // Prefer Auth0 if we have auth config cli, _, _ = newTestCLI(t) - cli.auth0Factory = func(httpClient util.HTTPClient, options auth0.Options) (auth0Client, error) { - return &mockAuth0{hasCredentials: true}, nil - } + require.Nil(t, os.WriteFile(filepath.Join(cli.config.homeDir, "auth.json"), []byte("foo"), 0600)) key, err = cli.config.readAPIKey(cli, vespa.PublicSystem, "t1") require.Nil(t, err) assert.Nil(t, key) } + +func TestConfigReadTLSOptions(t *testing.T) { + app := vespa.ApplicationID{Tenant: "t1", Application: "a1", Instance: "i1"} + homeDir := t.TempDir() + + // No environment variables, and no files on disk + assertTLSOptions(t, homeDir, app, vespa.TargetLocal, vespa.TLSOptions{}) + + // A single environment variable is set + assertTLSOptions(t, homeDir, app, vespa.TargetLocal, vespa.TLSOptions{TrustAll: true}, "VESPA_CLI_DATA_PLANE_TRUST_ALL=true") + + // Key pair is provided in-line in environment variables + pemCert, pemKey, keyPair := createKeyPair(t) + assertTLSOptions(t, homeDir, app, + vespa.TargetLocal, + vespa.TLSOptions{ + TrustAll: true, + CACertificate: []byte("cacert"), + KeyPair: []tls.Certificate{keyPair}, + }, + "VESPA_CLI_DATA_PLANE_TRUST_ALL=true", + "VESPA_CLI_DATA_PLANE_CA_CERT=cacert", + "VESPA_CLI_DATA_PLANE_CERT="+string(pemCert), + "VESPA_CLI_DATA_PLANE_KEY="+string(pemKey), + ) + + // Key pair is provided as file paths through environment variables + certFile := filepath.Join(homeDir, "cert") + keyFile := filepath.Join(homeDir, "key") + caCertFile := filepath.Join(homeDir, "cacert") + require.Nil(t, os.WriteFile(certFile, pemCert, 0600)) + require.Nil(t, os.WriteFile(keyFile, pemKey, 0600)) + require.Nil(t, os.WriteFile(caCertFile, []byte("cacert"), 0600)) + assertTLSOptions(t, homeDir, app, + vespa.TargetLocal, + vespa.TLSOptions{ + KeyPair: []tls.Certificate{keyPair}, + CACertificate: []byte("cacert"), + CACertificateFile: caCertFile, + CertificateFile: certFile, + PrivateKeyFile: keyFile, + }, + "VESPA_CLI_DATA_PLANE_CERT_FILE="+certFile, + "VESPA_CLI_DATA_PLANE_KEY_FILE="+keyFile, + "VESPA_CLI_DATA_PLANE_CA_CERT_FILE="+caCertFile, + ) + + // Key pair resides in default paths + defaultCertFile := filepath.Join(homeDir, app.String(), "data-plane-public-cert.pem") + defaultKeyFile := filepath.Join(homeDir, app.String(), "data-plane-private-key.pem") + require.Nil(t, os.WriteFile(defaultCertFile, pemCert, 0600)) + require.Nil(t, os.WriteFile(defaultKeyFile, pemKey, 0600)) + assertTLSOptions(t, homeDir, app, + vespa.TargetLocal, + vespa.TLSOptions{ + KeyPair: []tls.Certificate{keyPair}, + CertificateFile: defaultCertFile, + PrivateKeyFile: defaultKeyFile, + }, + ) +} + +func assertTLSOptions(t *testing.T, homeDir string, app vespa.ApplicationID, target string, want vespa.TLSOptions, envVars ...string) { + t.Helper() + envVars = append(envVars, "VESPA_CLI_HOME="+homeDir) + cli, _, _ := newTestCLI(t, envVars...) + require.Nil(t, cli.Run("config", "set", "application", app.String())) + config, err := cli.config.readTLSOptions(app, vespa.TargetLocal) + require.Nil(t, err) + assert.Equal(t, want, config) +} + +func createKeyPair(t *testing.T) ([]byte, []byte, tls.Certificate) { + privateKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatal(err) + } + notBefore := time.Now() + notAfter := notBefore.Add(24 * time.Hour) + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "example.com"}, + NotBefore: notBefore, + NotAfter: notAfter, + } + certificateDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey) + if err != nil { + t.Fatal(err) + } + privateKeyDER, err := x509.MarshalPKCS8PrivateKey(privateKey) + if err != nil { + t.Fatal(err) + } + pemCert := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certificateDER}) + pemKey := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: privateKeyDER}) + kp, err := tls.X509KeyPair(pemCert, pemKey) + if err != nil { + t.Fatal(err) + } + return pemCert, pemKey, kp +} diff --git a/client/go/internal/cli/cmd/curl.go b/client/go/internal/cli/cmd/curl.go index 8fcd1fa6ef7..3d5aaff24dc 100644 --- a/client/go/internal/cli/cmd/curl.go +++ b/client/go/internal/cli/cmd/curl.go @@ -4,7 +4,6 @@ package cmd import ( "fmt" "log" - "net/http" "os" "strings" @@ -54,6 +53,7 @@ $ vespa curl -- -v --data-urlencode "yql=select * from music where album contain return err } case vespa.DocumentService, vespa.QueryService: + c.CaCertificate = service.TLSOptions.CACertificateFile c.PrivateKey = service.TLSOptions.PrivateKeyFile c.Certificate = service.TLSOptions.CertificateFile default: @@ -79,15 +79,7 @@ func addAccessToken(cmd *curl.Command, target vespa.Target) error { if target.Type() != vespa.TargetCloud { return nil } - req := http.Request{} - if err := target.SignRequest(&req, ""); err != nil { - return err - } - headerValue := req.Header.Get("Authorization") - if headerValue == "" { - return fmt.Errorf("no authorization header added when signing request") - } - cmd.Header("Authorization", headerValue) + cmd.Header("Authorization", "secret") return nil } diff --git a/client/go/internal/cli/cmd/feed.go b/client/go/internal/cli/cmd/feed.go index c8e032929b8..f0f82dd80d1 100644 --- a/client/go/internal/cli/cmd/feed.go +++ b/client/go/internal/cli/cmd/feed.go @@ -9,60 +9,95 @@ import ( "time" "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" "github.com/vespa-engine/vespa/client/go/internal/vespa/document" ) -func addFeedFlags(cmd *cobra.Command, concurrency *int) { - // TOOD(mpolden): Remove this flag - cmd.PersistentFlags().IntVarP(concurrency, "concurrency", "T", 64, "Number of goroutines to use for dispatching") +func addFeedFlags(cmd *cobra.Command, options *feedOptions) { + cmd.PersistentFlags().IntVar(&options.connections, "connections", 8, "The number of connections to use") + cmd.PersistentFlags().StringVar(&options.route, "route", "", "Target Vespa route for feed operations") + cmd.PersistentFlags().IntVar(&options.traceLevel, "trace", 0, "The trace level of network traffic. 0 to disable") + cmd.PersistentFlags().IntVar(&options.timeoutSecs, "timeout", 0, "Feed operation timeout in seconds. 0 to disable") + cmd.PersistentFlags().BoolVar(&options.verbose, "verbose", false, "Verbose mode. Print successful operations in addition to errors") +} + +type feedOptions struct { + connections int + route string + verbose bool + traceLevel int + timeoutSecs int } func newFeedCmd(cli *CLI) *cobra.Command { - var ( - concurrency int - ) + var options feedOptions cmd := &cobra.Command{ Use: "feed FILE", Short: "Feed documents to a Vespa cluster", Long: `Feed documents to a Vespa cluster. A high performance feeding client. This can be used to feed large amounts of -documents to Vespa cluster efficiently. +documents to a Vespa cluster efficiently. The contents of FILE must be either a JSON array or JSON objects separated by newline (JSONL). + +If FILE is a single dash ('-'), documents will be read from standard input. `, Example: `$ vespa feed documents.jsonl +$ cat documents.jsonl | vespa feed - `, Args: cobra.ExactArgs(1), DisableAutoGenTag: true, SilenceUsage: true, Hidden: true, // TODO(mpolden): Remove when ready for public use RunE: func(cmd *cobra.Command, args []string) error { - f, err := os.Open(args[0]) - if err != nil { - return err + var r io.Reader + if args[0] == "-" { + r = cli.Stdin + } else { + f, err := os.Open(args[0]) + if err != nil { + return err + } + defer f.Close() + r = f } - defer f.Close() - return feed(f, cli, concurrency) + return feed(r, cli, options) }, } - addFeedFlags(cmd, &concurrency) + addFeedFlags(cmd, &options) return cmd } -func feed(r io.Reader, cli *CLI, concurrency int) error { +func createServiceClients(service *vespa.Service, n int) []util.HTTPClient { + clients := make([]util.HTTPClient, 0, n) + for i := 0; i < n; i++ { + client := service.Client().Clone() + // Feeding should always use HTTP/2 + util.ForceHTTP2(client, service.TLSOptions.KeyPair, service.TLSOptions.CACertificate, service.TLSOptions.TrustAll) + clients = append(clients, client) + } + return clients +} + +func feed(r io.Reader, cli *CLI, options feedOptions) error { service, err := documentService(cli) if err != nil { return err } + clients := createServiceClients(service, options.connections) client := document.NewClient(document.ClientOptions{ - BaseURL: service.BaseURL, - }, service) - throttler := document.NewThrottler() + Timeout: time.Duration(options.timeoutSecs) * time.Second, + Route: options.route, + TraceLevel: options.traceLevel, + BaseURL: service.BaseURL, + }, clients) + throttler := document.NewThrottler(options.connections) // TODO(mpolden): Make doom duration configurable circuitBreaker := document.NewCircuitBreaker(10*time.Second, 0) - dispatcher := document.NewDispatcher(client, throttler, circuitBreaker) + dispatcher := document.NewDispatcher(client, throttler, circuitBreaker, cli.Stderr, options.verbose) dec := document.NewDecoder(r) start := cli.now() diff --git a/client/go/internal/cli/cmd/feed_test.go b/client/go/internal/cli/cmd/feed_test.go index 1bf1ef6ab9b..521d2b2abd0 100644 --- a/client/go/internal/cli/cmd/feed_test.go +++ b/client/go/internal/cli/cmd/feed_test.go @@ -1,6 +1,7 @@ package cmd import ( + "bytes" "os" "path/filepath" "testing" @@ -42,7 +43,7 @@ func TestFeed(t *testing.T) { require.Nil(t, cli.Run("feed", jsonFile)) assert.Equal(t, "", stderr.String()) - assert.Equal(t, `{ + want := `{ "feeder.seconds": 1.000, "feeder.ok.count": 1, "feeder.ok.rate": 1.000, @@ -63,5 +64,15 @@ func TestFeed(t *testing.T) { "200": 1 } } -`, stdout.String()) +` + assert.Equal(t, want, stdout.String()) + + stdout.Reset() + cli.Stdin = bytes.NewBuffer([]byte(`{ + "put": "id:ns:type::doc1", + "fields": {"foo": "123"} +}`)) + httpClient.NextResponseString(200, `{"message":"OK"}`) + require.Nil(t, cli.Run("feed", "-")) + assert.Equal(t, want, stdout.String()) } diff --git a/client/go/internal/cli/cmd/prod.go b/client/go/internal/cli/cmd/prod.go index 0e00597221a..f322bd667df 100644 --- a/client/go/internal/cli/cmd/prod.go +++ b/client/go/internal/cli/cmd/prod.go @@ -105,21 +105,17 @@ https://cloud.vespa.ai/en/reference/deployment`, func newProdSubmitCmd(cli *CLI) *cobra.Command { return &cobra.Command{ Use: "submit", - Short: "Submit your application for production deployment", - Long: `Submit your application for production deployment. + Short: "Submit an application for production deployment", + Long: `Submit an application for production deployment. -This commands uploads your application package to Vespa Cloud and deploys it to +This commands uploads an application package to Vespa Cloud and deploys it to the production zones specified in deployment.xml. -Nodes are allocated to your application according to resources specified in +Nodes are allocated to the application according to resources specified in services.xml. -While submitting an application from a local development environment is -supported, it's strongly recommended that production deployments are performed -by a continuous build system. - For more information about production deployments in Vespa Cloud see: -https://cloud.vespa.ai/en/getting-to-production +https://cloud.vespa.ai/en/production-deployment https://cloud.vespa.ai/en/automated-deployments`, DisableAutoGenTag: true, SilenceUsage: true, diff --git a/client/go/internal/cli/cmd/root.go b/client/go/internal/cli/cmd/root.go index 58e940d59ef..88d43411983 100644 --- a/client/go/internal/cli/cmd/root.go +++ b/client/go/internal/cli/cmd/root.go @@ -2,7 +2,6 @@ package cmd import ( - "crypto/tls" "encoding/json" "fmt" "io" @@ -88,18 +87,9 @@ func (c *execSubprocess) Run(name string, args ...string) ([]byte, error) { return exec.Command(name, args...).Output() } -type ztsClient interface { - AccessToken(domain string, certficiate tls.Certificate) (string, error) -} - -type auth0Client interface { - AccessToken() (string, error) - HasCredentials() bool -} - -type auth0Factory func(httpClient util.HTTPClient, options auth0.Options) (auth0Client, error) +type auth0Factory func(httpClient util.HTTPClient, options auth0.Options) (vespa.Authenticator, error) -type ztsFactory func(httpClient util.HTTPClient, url string) (ztsClient, error) +type ztsFactory func(httpClient util.HTTPClient, domain, url string) (vespa.Authenticator, error) // New creates the Vespa CLI, writing output to stdout and stderr, and reading environment variables from environment. func New(stdout, stderr io.Writer, environment []string) (*CLI, error) { @@ -143,11 +133,11 @@ For detailed description of flags and configuration, see 'vespa help config'. httpClient: util.CreateClient(time.Second * 10), exec: &execSubprocess{}, now: time.Now, - auth0Factory: func(httpClient util.HTTPClient, options auth0.Options) (auth0Client, error) { + auth0Factory: func(httpClient util.HTTPClient, options auth0.Options) (vespa.Authenticator, error) { return auth0.NewClient(httpClient, options) }, - ztsFactory: func(httpClient util.HTTPClient, url string) (ztsClient, error) { - return zts.NewClient(httpClient, url) + ztsFactory: func(httpClient util.HTTPClient, domain, url string) (vespa.Authenticator, error) { + return zts.NewClient(httpClient, domain, url) }, } cli.isTerminal = func() bool { return isTerminal(cli.Stdout) && isTerminal(cli.Stderr) } @@ -321,16 +311,34 @@ func (c *CLI) createTarget(opts targetOptions) (vespa.Target, error) { if err != nil { return nil, err } + customURL := "" if strings.HasPrefix(targetType, "http") { - return vespa.CustomTarget(c.httpClient, targetType), nil + customURL = targetType + targetType = vespa.TargetCustom } switch targetType { - case vespa.TargetLocal: - return vespa.LocalTarget(c.httpClient), nil + case vespa.TargetLocal, vespa.TargetCustom: + return c.createCustomTarget(targetType, customURL) case vespa.TargetCloud, vespa.TargetHosted: return c.createCloudTarget(targetType, opts) + default: + return nil, errHint(fmt.Errorf("invalid target: %s", targetType), "Valid targets are 'local', 'cloud', 'hosted' or an URL") + } +} + +func (c *CLI) createCustomTarget(targetType, customURL string) (vespa.Target, error) { + tlsOptions, err := c.config.readTLSOptions(vespa.DefaultApplication, targetType) + if err != nil { + return nil, err + } + switch targetType { + case vespa.TargetLocal: + return vespa.LocalTarget(c.httpClient, tlsOptions), nil + case vespa.TargetCustom: + return vespa.CustomTarget(c.httpClient, customURL, tlsOptions), nil + default: + return nil, fmt.Errorf("invalid custom target: %s", targetType) } - return nil, errHint(fmt.Errorf("invalid target: %s", targetType), "Valid targets are 'local', 'cloud', 'hosted' or an URL") } func (c *CLI) createCloudTarget(targetType string, opts targetOptions) (vespa.Target, error) { @@ -347,48 +355,53 @@ func (c *CLI) createCloudTarget(targetType string, opts targetOptions) (vespa.Ta return nil, err } var ( - apiKey []byte - authConfigPath string + apiAuth vespa.Authenticator + deploymentAuth vespa.Authenticator apiTLSOptions vespa.TLSOptions deploymentTLSOptions vespa.TLSOptions ) switch targetType { case vespa.TargetCloud: - apiKey, err = c.config.readAPIKey(c, system, deployment.Application.Tenant) + apiKey, err := c.config.readAPIKey(c, system, deployment.Application.Tenant) if err != nil { return nil, err } - authConfigPath = c.config.authConfigPath() + if apiKey == nil { + authConfigPath := c.config.authConfigPath() + auth0, err := c.auth0Factory(c.httpClient, auth0.Options{ConfigPath: authConfigPath, SystemName: system.Name, SystemURL: system.URL}) + if err != nil { + return nil, err + } + apiAuth = auth0 + } else { + apiAuth = vespa.NewRequestSigner(deployment.Application.SerializedForm(), apiKey) + } deploymentTLSOptions = vespa.TLSOptions{} if !opts.noCertificate { - kp, err := c.config.x509KeyPair(deployment.Application, targetType) + kp, err := c.config.readTLSOptions(deployment.Application, targetType) if err != nil { - return nil, errHint(err, "Deployment to cloud requires a certificate. Try 'vespa auth cert'") - } - deploymentTLSOptions = vespa.TLSOptions{ - KeyPair: &kp.KeyPair, - CertificateFile: kp.CertificateFile, - PrivateKeyFile: kp.PrivateKeyFile, + return nil, errHint(err, "Deployment to cloud requires a certificate", "Try 'vespa auth cert' to create a self-signed certificate") } + deploymentTLSOptions = kp } case vespa.TargetHosted: - kp, err := c.config.x509KeyPair(deployment.Application, targetType) + kp, err := c.config.readTLSOptions(deployment.Application, targetType) if err != nil { return nil, errHint(err, "Deployment to hosted requires an Athenz certificate", "Try renewing certificate with 'athenz-user-cert'") } - apiTLSOptions = vespa.TLSOptions{ - KeyPair: &kp.KeyPair, - CertificateFile: kp.CertificateFile, - PrivateKeyFile: kp.PrivateKeyFile, + zts, err := c.ztsFactory(c.httpClient, system.AthenzDomain, zts.DefaultURL) + if err != nil { + return nil, err } - deploymentTLSOptions = apiTLSOptions + deploymentAuth = zts + apiTLSOptions = kp + deploymentTLSOptions = kp default: return nil, fmt.Errorf("invalid cloud target: %s", targetType) } apiOptions := vespa.APIOptions{ System: system, TLSOptions: apiTLSOptions, - APIKey: apiKey, } deploymentOptions := vespa.CloudDeploymentOptions{ Deployment: deployment, @@ -403,15 +416,7 @@ func (c *CLI) createCloudTarget(targetType string, opts targetOptions) (vespa.Ta Writer: c.Stdout, Level: vespa.LogLevel(logLevel), } - auth0, err := c.auth0Factory(c.httpClient, auth0.Options{ConfigPath: authConfigPath, SystemName: apiOptions.System.Name, SystemURL: apiOptions.System.URL}) - if err != nil { - return nil, err - } - zts, err := c.ztsFactory(c.httpClient, zts.DefaultURL) - if err != nil { - return nil, err - } - return vespa.CloudTarget(c.httpClient, zts, auth0, apiOptions, deploymentOptions, logOptions) + return vespa.CloudTarget(c.httpClient, apiAuth, deploymentAuth, apiOptions, deploymentOptions, logOptions) } // system returns the appropiate system for the target configured in this CLI. @@ -460,7 +465,6 @@ func (c *CLI) createDeploymentOptions(pkg vespa.ApplicationPackage, target vespa ApplicationPackage: pkg, Target: target, Timeout: timeout, - HTTPClient: c.httpClient, }, nil } diff --git a/client/go/internal/cli/cmd/test.go b/client/go/internal/cli/cmd/test.go index 4a53fe6bed3..8c4501e2870 100644 --- a/client/go/internal/cli/cmd/test.go +++ b/client/go/internal/cli/cmd/test.go @@ -263,7 +263,7 @@ func verify(step step, defaultCluster string, defaultParameters map[string]strin var response *http.Response if externalEndpoint { - util.SetCertificate(context.cli.httpClient, []tls.Certificate{}) + util.ConfigureTLS(context.cli.httpClient, []tls.Certificate{}, nil, false) response, err = context.cli.httpClient.Do(request, 60*time.Second) } else { response, err = service.Do(request, 600*time.Second) // Vespa should provide a response within the given request timeout diff --git a/client/go/internal/cli/cmd/testutil_test.go b/client/go/internal/cli/cmd/testutil_test.go index 61f8dab2264..492e40d8855 100644 --- a/client/go/internal/cli/cmd/testutil_test.go +++ b/client/go/internal/cli/cmd/testutil_test.go @@ -3,13 +3,14 @@ package cmd import ( "bytes" - "crypto/tls" + "net/http" "path/filepath" "testing" "github.com/vespa-engine/vespa/client/go/internal/cli/auth/auth0" "github.com/vespa-engine/vespa/client/go/internal/mock" "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" ) func newTestCLI(t *testing.T, envVars ...string) (*CLI, *bytes.Buffer, *bytes.Buffer) { @@ -29,21 +30,15 @@ func newTestCLI(t *testing.T, envVars ...string) (*CLI, *bytes.Buffer, *bytes.Bu httpClient := &mock.HTTPClient{} cli.httpClient = httpClient cli.exec = &mock.Exec{} - cli.auth0Factory = func(httpClient util.HTTPClient, options auth0.Options) (auth0Client, error) { - return &mockAuth0{}, nil + cli.auth0Factory = func(httpClient util.HTTPClient, options auth0.Options) (vespa.Authenticator, error) { + return &mockAuthenticator{}, nil } - cli.ztsFactory = func(httpClient util.HTTPClient, url string) (ztsClient, error) { - return &mockZTS{}, nil + cli.ztsFactory = func(httpClient util.HTTPClient, domain, url string) (vespa.Authenticator, error) { + return &mockAuthenticator{}, nil } return cli, &stdout, &stderr } -type mockZTS struct{} +type mockAuthenticator struct{} -func (z *mockZTS) AccessToken(domain string, cert tls.Certificate) (string, error) { return "", nil } - -type mockAuth0 struct{ hasCredentials bool } - -func (a *mockAuth0) AccessToken() (string, error) { return "", nil } - -func (a *mockAuth0) HasCredentials() bool { return a.hasCredentials } +func (a *mockAuthenticator) Authenticate(request *http.Request) error { return nil } diff --git a/client/go/internal/cli/cmd/visit_test.go b/client/go/internal/cli/cmd/visit_test.go index 4302680b9d9..b6e5b893e0b 100644 --- a/client/go/internal/cli/cmd/visit_test.go +++ b/client/go/internal/cli/cmd/visit_test.go @@ -47,7 +47,7 @@ func TestQuoteFunc(t *testing.T) { res := quoteArgForUrl(s) if i < 32 || i > 127 { assert.Equal(t, "a+z", res) - } else { + } else if testing.Verbose() { // go test -v fmt.Printf("res %3d => '%s'\n", i, res) } } diff --git a/client/go/internal/mock/http.go b/client/go/internal/mock/http.go index 9c55f2e79bf..58614d7e5bd 100644 --- a/client/go/internal/mock/http.go +++ b/client/go/internal/mock/http.go @@ -6,6 +6,8 @@ import ( "net/http" "strconv" "time" + + "github.com/vespa-engine/vespa/client/go/internal/util" ) type HTTPClient struct { @@ -58,3 +60,5 @@ func (c *HTTPClient) Do(request *http.Request, timeout time.Duration) (*http.Res }, nil } + +func (c *HTTPClient) Clone() util.HTTPClient { return c } diff --git a/client/go/internal/util/http.go b/client/go/internal/util/http.go index cb35932c8e7..8a67b24dffb 100644 --- a/client/go/internal/util/http.go +++ b/client/go/internal/util/http.go @@ -2,9 +2,11 @@ package util import ( - "bytes" + "context" "crypto/tls" + "crypto/x509" "fmt" + "net" "net/http" "time" @@ -14,6 +16,7 @@ import ( type HTTPClient interface { Do(request *http.Request, timeout time.Duration) (response *http.Response, error error) + Clone() HTTPClient } type defaultHTTPClient struct { @@ -31,50 +34,57 @@ func (c *defaultHTTPClient) Do(request *http.Request, timeout time.Duration) (re return c.client.Do(request) } -func SetCertificate(client HTTPClient, certificates []tls.Certificate) { +func (c *defaultHTTPClient) Clone() HTTPClient { return CreateClient(c.client.Timeout) } + +func ConfigureTLS(client HTTPClient, certificates []tls.Certificate, caCertificate []byte, trustAll bool) { c, ok := client.(*defaultHTTPClient) if !ok { return } - // Use HTTP/2 transport explicitly. Connection reuse does not work properly when using regular http.Transport, even - // though it upgrades to HTTP/2 automatically - // https://github.com/golang/go/issues/16582 - // https://github.com/golang/go/issues/22091 - var transport *http2.Transport - if _, ok := c.client.Transport.(*http.Transport); ok { - transport = &http2.Transport{} - c.client.Transport = transport - } else if t, ok := c.client.Transport.(*http2.Transport); ok { - transport = t + var tlsConfig *tls.Config = nil + if certificates != nil { + tlsConfig = &tls.Config{ + Certificates: certificates, + MinVersion: tls.VersionTLS12, + InsecureSkipVerify: trustAll, + } + if caCertificate != nil { + certs := x509.NewCertPool() + certs.AppendCertsFromPEM(caCertificate) + tlsConfig.RootCAs = certs + } + } + if tr, ok := c.client.Transport.(*http.Transport); ok { + tr.TLSClientConfig = tlsConfig + } else if tr, ok := c.client.Transport.(*http2.Transport); ok { + tr.TLSClientConfig = tlsConfig } else { panic(fmt.Sprintf("unknown transport type: %T", c.client.Transport)) } - if ok && !c.hasCertificates(transport.TLSClientConfig, certificates) { - transport.TLSClientConfig = &tls.Config{ - Certificates: certificates, - MinVersion: tls.VersionTLS12, - } - } } -func (c *defaultHTTPClient) hasCertificates(tlsConfig *tls.Config, certs []tls.Certificate) bool { - if tlsConfig == nil { - return false - } - if len(tlsConfig.Certificates) != len(certs) { - return false +func ForceHTTP2(client HTTPClient, certificates []tls.Certificate, caCertificate []byte, trustAll bool) { + c, ok := client.(*defaultHTTPClient) + if !ok { + return } - for i := 0; i < len(certs); i++ { - if len(tlsConfig.Certificates[i].Certificate) != len(certs[i].Certificate) { - return false - } - for j := 0; j < len(certs[i].Certificate); j++ { - if !bytes.Equal(tlsConfig.Certificates[i].Certificate[j], certs[i].Certificate[j]) { - return false - } + var dialFunc func(ctx context.Context, network, addr string, cfg *tls.Config) (net.Conn, error) + if certificates == nil { + // No certificate, so force H2C (HTTP/2 over clear-text) by using a non-TLS Dialer + dialer := net.Dialer{} + dialFunc = func(ctx context.Context, network, addr string, cfg *tls.Config) (net.Conn, error) { + return dialer.DialContext(ctx, network, addr) } } - return true + // Use HTTP/2 transport explicitly. Connection reuse does not work properly when using regular http.Transport, even + // though it upgrades to HTTP/2 automatically + // https://github.com/golang/go/issues/16582 + // https://github.com/golang/go/issues/22091 + c.client.Transport = &http2.Transport{ + AllowHTTP: true, + DialTLSContext: dialFunc, + } + ConfigureTLS(client, certificates, caCertificate, trustAll) } func CreateClient(timeout time.Duration) HTTPClient { diff --git a/client/go/internal/vespa/crypto.go b/client/go/internal/vespa/crypto.go index 9621d0c1180..5e273538869 100644 --- a/client/go/internal/vespa/crypto.go +++ b/client/go/internal/vespa/crypto.go @@ -111,6 +111,8 @@ func NewRequestSigner(keyID string, pemPrivateKey []byte) *RequestSigner { } } +func (rs *RequestSigner) Authenticate(request *http.Request) error { return rs.SignRequest(request) } + // SignRequest signs the given HTTP request using the private key in rs func (rs *RequestSigner) SignRequest(request *http.Request) error { timestamp := rs.now().UTC().Format(time.RFC3339) diff --git a/client/go/internal/vespa/deploy.go b/client/go/internal/vespa/deploy.go index 687bfc46124..f633c8ed9ee 100644 --- a/client/go/internal/vespa/deploy.go +++ b/client/go/internal/vespa/deploy.go @@ -45,7 +45,6 @@ type DeploymentOptions struct { ApplicationPackage ApplicationPackage Timeout time.Duration Version version.Version - HTTPClient util.HTTPClient } type LogLinePrepareResponse struct { @@ -130,7 +129,7 @@ func Prepare(deployment DeploymentOptions) (PrepareResult, error) { return PrepareResult{}, err } serviceDescription := "Deploy service" - response, err := deployment.HTTPClient.Do(req, time.Second*30) + response, err := deployServiceDo(req, time.Second*30, deployment) if err != nil { return PrepareResult{}, err } @@ -171,7 +170,7 @@ func Activate(sessionID int64, deployment DeploymentOptions) error { return err } serviceDescription := "Deploy service" - response, err := deployment.HTTPClient.Do(req, time.Second*30) + response, err := deployServiceDo(req, time.Second*30, deployment) if err != nil { return err } @@ -263,11 +262,7 @@ func Submit(opts DeploymentOptions) error { } request.Header.Set("Content-Type", writer.FormDataContentType()) serviceDescription := "Submit service" - sigKeyId := opts.Target.Deployment().Application.SerializedForm() - if err := opts.Target.SignRequest(request, sigKeyId); err != nil { - return fmt.Errorf("failed to sign api request: %w", err) - } - response, err := opts.HTTPClient.Do(request, time.Minute*10) + response, err := deployServiceDo(request, time.Minute*10, opts) if err != nil { return err } @@ -275,6 +270,14 @@ func Submit(opts DeploymentOptions) error { return checkResponse(request, response, serviceDescription) } +func deployServiceDo(request *http.Request, timeout time.Duration, opts DeploymentOptions) (*http.Response, error) { + s, err := opts.Target.Service(DeployService, 0, 0, "") + if err != nil { + return nil, err + } + return s.Do(request, timeout) +} + func checkDeploymentOpts(opts DeploymentOptions) error { if opts.Target.Type() == TargetCloud && !opts.ApplicationPackage.HasCertificate() { return fmt.Errorf("%s: missing certificate in package", opts) @@ -334,11 +337,6 @@ func uploadApplicationPackage(url *url.URL, opts DeploymentOptions) (PrepareResu if err != nil { return PrepareResult{}, err } - - keyID := opts.Target.Deployment().Application.SerializedForm() - if err := opts.Target.SignRequest(request, keyID); err != nil { - return PrepareResult{}, err - } response, err := service.Do(request, time.Minute*10) if err != nil { return PrepareResult{}, err diff --git a/client/go/internal/vespa/deploy_test.go b/client/go/internal/vespa/deploy_test.go index 3e74e9ab3b6..da2604282c0 100644 --- a/client/go/internal/vespa/deploy_test.go +++ b/client/go/internal/vespa/deploy_test.go @@ -19,12 +19,11 @@ import ( func TestDeploy(t *testing.T) { httpClient := mock.HTTPClient{} - target := LocalTarget(&httpClient) + target := LocalTarget(&httpClient, TLSOptions{}) appDir, _ := mock.ApplicationPackageDir(t, false, false) opts := DeploymentOptions{ Target: target, ApplicationPackage: ApplicationPackage{Path: appDir}, - HTTPClient: &httpClient, } _, err := Deploy(opts) assert.Nil(t, err) @@ -47,7 +46,6 @@ func TestDeployCloud(t *testing.T) { opts := DeploymentOptions{ Target: target, ApplicationPackage: ApplicationPackage{Path: appDir}, - HTTPClient: &httpClient, } _, err := Deploy(opts) require.Nil(t, err) diff --git a/client/go/internal/vespa/document/dispatcher.go b/client/go/internal/vespa/document/dispatcher.go index 7011ae7a9b6..533ca7a0019 100644 --- a/client/go/internal/vespa/document/dispatcher.go +++ b/client/go/internal/vespa/document/dispatcher.go @@ -1,7 +1,10 @@ package document import ( + "container/list" "fmt" + "io" + "strings" "sync" "sync/atomic" "time" @@ -16,79 +19,109 @@ type Dispatcher struct { circuitBreaker CircuitBreaker stats Stats - closed bool - ready chan Id - results chan Result + started bool + ready chan Id + results chan Result + msgs chan string + inflight map[string]*documentGroup inflightCount int64 + output io.Writer + verbose bool mu sync.RWMutex wg sync.WaitGroup resultWg sync.WaitGroup } -// documentGroup holds document operations which share their ID, and must be dispatched in order. -type documentGroup struct { - id Id - operations []documentOp - mu sync.Mutex -} - +// documentOp represents a document operation and the number of times it has been attempted. type documentOp struct { document Document attempts int } -func (g *documentGroup) append(op documentOp) { +// documentGroup holds document operations which share an ID, and must be dispatched in order. +type documentGroup struct { + ops *list.List + mu sync.Mutex +} + +func (g *documentGroup) add(op documentOp, first bool) { g.mu.Lock() defer g.mu.Unlock() - g.operations = append(g.operations, op) + if g.ops == nil { + g.ops = list.New() + } + if first { + g.ops.PushFront(op) + } else { + g.ops.PushBack(op) + } } -func NewDispatcher(feeder Feeder, throttler Throttler, breaker CircuitBreaker) *Dispatcher { +func NewDispatcher(feeder Feeder, throttler Throttler, breaker CircuitBreaker, output io.Writer, verbose bool) *Dispatcher { d := &Dispatcher{ feeder: feeder, throttler: throttler, circuitBreaker: breaker, inflight: make(map[string]*documentGroup), + output: output, + verbose: verbose, } d.start() return d } -func (d *Dispatcher) dispatchAll(g *documentGroup) { - g.mu.Lock() - defer g.mu.Unlock() - for i := 0; i < len(g.operations); i++ { - op := g.operations[i] - ok := false - for !ok { - op.attempts++ - result := d.feeder.Send(op.document) - d.results <- result - ok = result.Status.Success() - if !d.shouldRetry(op, result) { - break - } - } - d.releaseSlot() +func (d *Dispatcher) sendDocumentIn(group *documentGroup) { + group.mu.Lock() + first := group.ops.Front() + if first == nil { + panic("sending from empty document group, this should not happen") + } + op := group.ops.Remove(first).(documentOp) + op.attempts++ + result := d.feeder.Send(op.document) + d.results <- result + d.releaseSlot() + group.mu.Unlock() + if d.shouldRetry(op, result) { + d.enqueue(op) } - g.operations = nil } func (d *Dispatcher) shouldRetry(op documentOp, result Result) bool { if result.HTTPStatus/100 == 2 || result.HTTPStatus == 404 || result.HTTPStatus == 412 { + if d.verbose { + d.msgs <- fmt.Sprintf("feed: successfully fed %s with status %d", op.document.Id, result.HTTPStatus) + } d.throttler.Success() d.circuitBreaker.Success() return false } if result.HTTPStatus == 429 || result.HTTPStatus == 503 { + d.msgs <- fmt.Sprintf("feed: %s was throttled with status %d: retrying\n", op.document, result.HTTPStatus) d.throttler.Throttled(atomic.LoadInt64(&d.inflightCount)) return true } - if result.HTTPStatus == 500 || result.HTTPStatus == 502 || result.HTTPStatus == 504 { + if result.Err != nil || result.HTTPStatus == 500 || result.HTTPStatus == 502 || result.HTTPStatus == 504 { + retry := op.attempts <= maxAttempts + var msg strings.Builder + msg.WriteString("feed: ") + msg.WriteString(op.document.String()) + if result.Err != nil { + msg.WriteString("error ") + msg.WriteString(result.Err.Error()) + } else { + msg.WriteString(fmt.Sprintf("status %d", result.HTTPStatus)) + } + if retry { + msg.WriteString(": retrying") + } else { + msg.WriteString(fmt.Sprintf(": giving up after %d attempts", maxAttempts)) + } + d.msgs <- msg.String() d.circuitBreaker.Error(fmt.Errorf("request failed with status %d", result.HTTPStatus)) - if op.attempts <= maxAttempts { + if retry { return true } } @@ -98,19 +131,27 @@ func (d *Dispatcher) shouldRetry(op documentOp, result Result) bool { func (d *Dispatcher) start() { d.mu.Lock() defer d.mu.Unlock() + if d.started { + return + } d.ready = make(chan Id, 4096) d.results = make(chan Result, 4096) - d.closed = false + d.msgs = make(chan string, 4096) + d.started = true d.wg.Add(1) go func() { defer d.wg.Done() d.readDocuments() }() - d.resultWg.Add(1) + d.resultWg.Add(2) go func() { defer d.resultWg.Done() d.readResults() }() + go func() { + defer d.resultWg.Done() + d.readMessages() + }() } func (d *Dispatcher) readDocuments() { @@ -118,13 +159,11 @@ func (d *Dispatcher) readDocuments() { d.mu.RLock() group := d.inflight[id.String()] d.mu.RUnlock() - if group != nil { - d.wg.Add(1) - go func() { - defer d.wg.Done() - d.dispatchAll(group) - }() - } + d.wg.Add(1) + go func() { + defer d.wg.Done() + d.sendDocumentIn(group) + }() } } @@ -134,28 +173,29 @@ func (d *Dispatcher) readResults() { } } -func (d *Dispatcher) Enqueue(doc Document) error { +func (d *Dispatcher) readMessages() { + for msg := range d.msgs { + fmt.Fprintln(d.output, msg) + } +} + +func (d *Dispatcher) enqueue(op documentOp) error { d.mu.Lock() - if d.closed { + if !d.started { return fmt.Errorf("dispatcher is closed") } - group, ok := d.inflight[doc.Id.String()] - if ok { - group.append(documentOp{document: doc}) - } else { - group = &documentGroup{ - id: doc.Id, - operations: []documentOp{{document: doc}}, - } - d.inflight[doc.Id.String()] = group + key := op.document.Id.String() + group, ok := d.inflight[key] + if !ok { + group = &documentGroup{} + d.inflight[key] = group } d.mu.Unlock() - d.enqueueWithSlot(doc.Id) + group.add(op, op.attempts > 0) + d.enqueueWithSlot(op.document.Id) return nil } -func (d *Dispatcher) Stats() Stats { return d.stats } - func (d *Dispatcher) enqueueWithSlot(id Id) { d.acquireSlot() d.ready <- id @@ -171,21 +211,26 @@ func (d *Dispatcher) acquireSlot() { func (d *Dispatcher) releaseSlot() { atomic.AddInt64(&d.inflightCount, -1) } -func closeAndWait[T any](ch chan T, wg *sync.WaitGroup, d *Dispatcher, markClosed bool) { +func (d *Dispatcher) Enqueue(doc Document) error { return d.enqueue(documentOp{document: doc}) } + +func (d *Dispatcher) Stats() Stats { return d.stats } + +// Close closes the dispatcher and waits for all inflight operations to complete. +func (d *Dispatcher) Close() error { d.mu.Lock() - if !d.closed { - close(ch) - if markClosed { - d.closed = true - } + if d.started { + close(d.ready) } d.mu.Unlock() - wg.Wait() -} + d.wg.Wait() // Wait for inflight operations to complete -// Close closes the dispatcher and waits for all inflight operations to complete. -func (d *Dispatcher) Close() error { - closeAndWait(d.ready, &d.wg, d, false) - closeAndWait(d.results, &d.resultWg, d, true) + d.mu.Lock() + if d.started { + close(d.results) + close(d.msgs) + d.started = false + } + d.mu.Unlock() + d.resultWg.Wait() // Wait for results return nil } diff --git a/client/go/internal/vespa/document/dispatcher_test.go b/client/go/internal/vespa/document/dispatcher_test.go index 8a6d8c6117c..d066f5bc9ae 100644 --- a/client/go/internal/vespa/document/dispatcher_test.go +++ b/client/go/internal/vespa/document/dispatcher_test.go @@ -1,6 +1,7 @@ package document import ( + "io" "sync" "testing" "time" @@ -29,7 +30,7 @@ func (f *mockFeeder) Send(doc Document) Result { } else { f.documents = append(f.documents, doc) } - if !result.Status.Success() { + if !result.Success() { result.Stats.Errors = 1 } return result @@ -38,9 +39,9 @@ func (f *mockFeeder) Send(doc Document) Result { func TestDispatcher(t *testing.T) { feeder := &mockFeeder{} clock := &manualClock{tick: time.Second} - throttler := newThrottler(clock.now) + throttler := newThrottler(8, clock.now) breaker := NewCircuitBreaker(time.Second, 0) - dispatcher := NewDispatcher(feeder, throttler, breaker) + dispatcher := NewDispatcher(feeder, throttler, breaker, io.Discard, false) docs := []Document{ {Id: mustParseId("id:ns:type::doc1"), Operation: OperationPut, Body: []byte(`{"fields":{"foo": "123"}}`)}, {Id: mustParseId("id:ns:type::doc2"), Operation: OperationPut, Body: []byte(`{"fields":{"bar": "456"}}`)}, @@ -71,9 +72,9 @@ func TestDispatcherOrdering(t *testing.T) { {Id: mustParseId("id:ns:type::doc9"), Operation: OperationPut}, } clock := &manualClock{tick: time.Second} - throttler := newThrottler(clock.now) + throttler := newThrottler(8, clock.now) breaker := NewCircuitBreaker(time.Second, 0) - dispatcher := NewDispatcher(feeder, throttler, breaker) + dispatcher := NewDispatcher(feeder, throttler, breaker, io.Discard, false) for _, d := range docs { dispatcher.Enqueue(d) } @@ -107,9 +108,9 @@ func TestDispatcherOrderingWithFailures(t *testing.T) { } feeder.failAfterN(2) clock := &manualClock{tick: time.Second} - throttler := newThrottler(clock.now) + throttler := newThrottler(8, clock.now) breaker := NewCircuitBreaker(time.Second, 0) - dispatcher := NewDispatcher(feeder, throttler, breaker) + dispatcher := NewDispatcher(feeder, throttler, breaker, io.Discard, false) for _, d := range docs { dispatcher.Enqueue(d) } diff --git a/client/go/internal/vespa/document/document.go b/client/go/internal/vespa/document/document.go index 98cb2d1b6c6..efb60ad8c0a 100644 --- a/client/go/internal/vespa/document/document.go +++ b/client/go/internal/vespa/document/document.go @@ -130,6 +130,27 @@ type Decoder struct { jsonl bool } +func (d Document) String() string { + var sb strings.Builder + switch d.Operation { + case OperationPut: + sb.WriteString("put ") + case OperationUpdate: + sb.WriteString("update ") + case OperationRemove: + sb.WriteString("remove ") + } + sb.WriteString(d.Id.String()) + if d.Condition != "" { + sb.WriteString(", condition=") + sb.WriteString(d.Condition) + } + if d.Create { + sb.WriteString(", create=true") + } + return sb.String() +} + func (d *Decoder) guessMode() error { for !d.array && !d.jsonl { b, err := d.buf.ReadByte() diff --git a/client/go/internal/vespa/document/feeder.go b/client/go/internal/vespa/document/feeder.go index 8bdd5bca5ba..9e6768d0eb4 100644 --- a/client/go/internal/vespa/document/feeder.go +++ b/client/go/internal/vespa/document/feeder.go @@ -17,8 +17,6 @@ const ( // StatusTransportFailure indicates that there was failure in the transport layer error while sending the document // operation to Vespa. StatusTransportFailure - // StatusError is a catch-all status for any other error that might occur. - StatusError ) // Result represents the result of a feeding operation. @@ -32,8 +30,9 @@ type Result struct { Stats Stats } -// Success returns whether status s is considered a success. -func (s Status) Success() bool { return s == StatusSuccess || s == StatusConditionNotMet } +func (r Result) Success() bool { + return r.Err == nil && (r.Status == StatusSuccess || r.Status == StatusConditionNotMet) +} // Stats represents feeding operation statistics. type Stats struct { diff --git a/client/go/internal/vespa/document/http.go b/client/go/internal/vespa/document/http.go index 4dadcd1d05c..1bcd7eff39e 100644 --- a/client/go/internal/vespa/document/http.go +++ b/client/go/internal/vespa/document/http.go @@ -5,10 +5,12 @@ import ( "encoding/json" "fmt" "io" + "math" "net/http" "net/url" "strconv" "strings" + "sync/atomic" "time" "github.com/vespa-engine/vespa/client/go/internal/util" @@ -16,9 +18,10 @@ import ( // Client represents a HTTP client for the /document/v1/ API. type Client struct { - options ClientOptions - httpClient util.HTTPClient - now func() time.Time + options ClientOptions + httpClients []countingHTTPClient + now func() time.Time + sendCount int32 } // ClientOptions specifices the configuration options of a feed client. @@ -26,7 +29,19 @@ type ClientOptions struct { BaseURL string Timeout time.Duration Route string - TraceLevel *int + TraceLevel int +} + +type countingHTTPClient struct { + client util.HTTPClient + inflight int64 +} + +func (c *countingHTTPClient) addInflight(n int64) { atomic.AddInt64(&c.inflight, n) } + +func (c *countingHTTPClient) Do(req *http.Request, timeout time.Duration) (*http.Response, error) { + defer c.addInflight(-1) + return c.client.Do(req, timeout) } type countingReader struct { @@ -40,25 +55,35 @@ func (r *countingReader) Read(p []byte) (int, error) { return n, err } -func NewClient(options ClientOptions, httpClient util.HTTPClient) *Client { - c := &Client{ - options: options, - httpClient: httpClient, - now: time.Now, +func NewClient(options ClientOptions, httpClients []util.HTTPClient) *Client { + if len(httpClients) < 1 { + panic("need at least one HTTP client") + } + countingClients := make([]countingHTTPClient, 0, len(httpClients)) + for _, client := range httpClients { + countingClients = append(countingClients, countingHTTPClient{client: client}) + } + return &Client{ + options: options, + httpClients: countingClients, + now: time.Now, } - return c } func (c *Client) queryParams() url.Values { params := url.Values{} - if c.options.Timeout > 0 { - params.Set("timeout", strconv.FormatInt(c.options.Timeout.Milliseconds(), 10)+"ms") + timeout := c.options.Timeout + if timeout == 0 { + timeout = 200 * time.Second + } else { + timeout = timeout*11/10 + 1000 } + params.Set("timeout", strconv.FormatInt(timeout.Milliseconds(), 10)+"ms") if c.options.Route != "" { params.Set("route", c.options.Route) } - if c.options.TraceLevel != nil { - params.Set("tracelevel", strconv.Itoa(*c.options.TraceLevel)) + if c.options.TraceLevel > 0 { + params.Set("tracelevel", strconv.Itoa(c.options.TraceLevel)) } return params } @@ -109,45 +134,56 @@ func (c *Client) feedURL(d Document, queryParams url.Values) (string, *url.URL, return httpMethod, u, nil } -// Send given document the URL configured in this client. +func (c *Client) leastBusyClient() *countingHTTPClient { + leastBusy := c.httpClients[0] + min := int64(math.MaxInt64) + next := atomic.AddInt32(&c.sendCount, 1) + start := int(next) % len(c.httpClients) + for i := range c.httpClients { + j := (i + start) % len(c.httpClients) + client := c.httpClients[j] + inflight := atomic.LoadInt64(&client.inflight) + if inflight < min { + leastBusy = client + min = inflight + } + } + leastBusy.addInflight(1) + return &leastBusy +} + +// Send given document to the endpoint configured in this client. func (c *Client) Send(document Document) Result { start := c.now() - result := Result{Id: document.Id} - result.Stats.Requests = 1 + result := Result{Id: document.Id, Stats: Stats{Requests: 1}} method, url, err := c.feedURL(document, c.queryParams()) if err != nil { - result.Stats.Errors = 1 - result.Err = err - return result + return resultWithErr(result, err) } req, err := http.NewRequest(method, url.String(), bytes.NewReader(document.Body)) if err != nil { - result.Stats.Errors = 1 - result.Status = StatusError - result.Err = err - return result + return resultWithErr(result, err) } - resp, err := c.httpClient.Do(req, 190*time.Second) + resp, err := c.leastBusyClient().Do(req, 190*time.Second) if err != nil { - result.Stats.Errors = 1 - result.Status = StatusTransportFailure - result.Err = err - return result + return resultWithErr(result, err) } defer resp.Body.Close() - result.Stats.Responses = 1 - result.Stats.ResponsesByCode = map[int]int64{ - resp.StatusCode: 1, - } - result.Stats.BytesSent = int64(len(document.Body)) elapsed := c.now().Sub(start) - result.Stats.TotalLatency = elapsed - result.Stats.MinLatency = elapsed - result.Stats.MaxLatency = elapsed - return c.resultWithResponse(resp, result) + return resultWithResponse(resp, result, document, elapsed) +} + +func resultWithErr(result Result, err error) Result { + result.Stats.Errors++ + result.Status = StatusTransportFailure + result.Err = err + return result } -func (c *Client) resultWithResponse(resp *http.Response, result Result) Result { +func resultWithResponse(resp *http.Response, result Result, document Document, elapsed time.Duration) Result { + result.HTTPStatus = resp.StatusCode + result.Stats.Responses++ + result.Stats.ResponsesByCode = map[int]int64{resp.StatusCode: 1} switch resp.StatusCode { case 200: result.Status = StatusSuccess @@ -165,14 +201,18 @@ func (c *Client) resultWithResponse(resp *http.Response, result Result) Result { cr := countingReader{reader: resp.Body} jsonDec := json.NewDecoder(&cr) if err := jsonDec.Decode(&body); err != nil { - result.Status = StatusError + result.Status = StatusVespaFailure result.Err = fmt.Errorf("failed to decode json response: %w", err) } result.Message = body.Message result.Trace = string(body.Trace) + result.Stats.BytesSent = int64(len(document.Body)) result.Stats.BytesRecv = cr.bytesRead - if !result.Status.Success() { - result.Stats.Errors = 1 + if !result.Success() { + result.Stats.Errors++ } + result.Stats.TotalLatency = elapsed + result.Stats.MinLatency = elapsed + result.Stats.MaxLatency = elapsed return result } diff --git a/client/go/internal/vespa/document/http_test.go b/client/go/internal/vespa/document/http_test.go index 311668fa16e..8f8394a5d4e 100644 --- a/client/go/internal/vespa/document/http_test.go +++ b/client/go/internal/vespa/document/http_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/vespa-engine/vespa/client/go/internal/mock" + "github.com/vespa-engine/vespa/client/go/internal/util" ) type manualClock struct { @@ -25,6 +26,37 @@ func (c *manualClock) now() time.Time { func (c *manualClock) advance(d time.Duration) { c.t = c.t.Add(d) } +type mockHTTPClient struct { + id int + *mock.HTTPClient +} + +func TestLeastBusyClient(t *testing.T) { + httpClient := mock.HTTPClient{} + var httpClients []util.HTTPClient + for i := 0; i < 4; i++ { + httpClients = append(httpClients, &mockHTTPClient{i, &httpClient}) + } + client := NewClient(ClientOptions{}, httpClients) + client.httpClients[0].addInflight(1) + client.httpClients[1].addInflight(1) + assertLeastBusy(t, 2, client) + assertLeastBusy(t, 2, client) + assertLeastBusy(t, 3, client) + client.httpClients[3].addInflight(1) + client.httpClients[1].addInflight(-1) + assertLeastBusy(t, 1, client) +} + +func assertLeastBusy(t *testing.T, id int, client *Client) { + t.Helper() + leastBusy := client.leastBusyClient() + got := leastBusy.client.(*mockHTTPClient).id + if got != id { + t.Errorf("got client.id=%d, want %d", got, id) + } +} + func TestClientSend(t *testing.T) { docs := []Document{ {Create: true, Id: mustParseId("id:ns:type::doc1"), Operation: OperationUpdate, Body: []byte(`{"fields":{"foo": "123"}}`)}, @@ -35,26 +67,48 @@ func TestClientSend(t *testing.T) { client := NewClient(ClientOptions{ BaseURL: "https://example.com:1337", Timeout: time.Duration(5 * time.Second), - }, &httpClient) + }, []util.HTTPClient{&httpClient}) clock := manualClock{t: time.Now(), tick: time.Second} client.now = clock.now var stats Stats for i, doc := range docs { + wantRes := Result{ + Id: doc.Id, + Stats: Stats{ + Requests: 1, + Responses: 1, + TotalLatency: time.Second, + MinLatency: time.Second, + MaxLatency: time.Second, + BytesSent: 25, + }, + } if i < 2 { httpClient.NextResponseString(200, `{"message":"All good!"}`) + wantRes.Status = StatusSuccess + wantRes.HTTPStatus = 200 + wantRes.Message = "All good!" + wantRes.Stats.ResponsesByCode = map[int]int64{200: 1} + wantRes.Stats.BytesRecv = 23 } else { httpClient.NextResponseString(502, `{"message":"Good bye, cruel world!"}`) + wantRes.Status = StatusVespaFailure + wantRes.HTTPStatus = 502 + wantRes.Message = "Good bye, cruel world!" + wantRes.Stats.ResponsesByCode = map[int]int64{502: 1} + wantRes.Stats.Errors = 1 + wantRes.Stats.BytesRecv = 36 } res := client.Send(doc) - stats.Add(res.Stats) - if res.Err != nil { - t.Fatalf("got unexpected error %q", res.Err) + if !reflect.DeepEqual(res, wantRes) { + t.Fatalf("got result %+v, want %+v", res, wantRes) } + stats.Add(res.Stats) r := httpClient.LastRequest if r.Method != http.MethodPut { t.Errorf("got r.Method = %q, want %q", r.Method, http.MethodPut) } - wantURL := fmt.Sprintf("https://example.com:1337/document/v1/ns/type/docid/%s?create=true&timeout=5000ms", doc.Id.UserSpecific) + wantURL := fmt.Sprintf("https://example.com:1337/document/v1/ns/type/docid/%s?create=true&timeout=5500ms", doc.Id.UserSpecific) if r.URL.String() != wantURL { t.Errorf("got r.URL = %q, want %q", r.URL, wantURL) } @@ -176,7 +230,7 @@ func TestClientFeedURL(t *testing.T) { httpClient := mock.HTTPClient{} client := NewClient(ClientOptions{ BaseURL: "https://example.com", - }, &httpClient) + }, []util.HTTPClient{&httpClient}) for i, tt := range tests { moreParams := url.Values{} moreParams.Set("foo", "ba/r") diff --git a/client/go/internal/vespa/document/throttler.go b/client/go/internal/vespa/document/throttler.go index 69bb7c8d7ac..5b0aab6174e 100644 --- a/client/go/internal/vespa/document/throttler.go +++ b/client/go/internal/vespa/document/throttler.go @@ -7,13 +7,7 @@ import ( "time" ) -const ( - throttlerWeight = 0.7 - // TODO(mpolden): Multiply this by connections per endpoint, and number of endpoints when this becomes configurable - // for local target - throttlerMinInflight = 16 - throttlerMaxInflight = 256 * throttlerMinInflight // 4096 max streams per connection on the server side -) +const throttlerWeight = 0.7 type Throttler interface { // Sent notifies the the throttler that a document has been sent. @@ -27,29 +21,38 @@ type Throttler interface { } type dynamicThrottler struct { - ok int64 + minInflight int64 + maxInflight int64 targetInflight int64 targetTimesTen int64 throughputs []float64 + ok int64 sent int64 start time.Time now func() time.Time } -func newThrottler(nowFunc func() time.Time) *dynamicThrottler { +func newThrottler(connections int, nowFunc func() time.Time) *dynamicThrottler { + var ( + minInflight = 16 * int64(connections) + maxInflight = 256 * minInflight // 4096 max streams per connection on the server side + ) return &dynamicThrottler{ + minInflight: minInflight, + maxInflight: maxInflight, + targetInflight: 8 * minInflight, + targetTimesTen: 10 * maxInflight, + throughputs: make([]float64, 128), - start: nowFunc(), - now: nowFunc, - targetInflight: 8 * throttlerMinInflight, - targetTimesTen: 10 * throttlerMaxInflight, + start: nowFunc(), + now: nowFunc, } } -func NewThrottler() Throttler { return newThrottler(time.Now) } +func NewThrottler(connections int) Throttler { return newThrottler(connections, time.Now) } func (t *dynamicThrottler) Sent() { currentInflight := atomic.LoadInt64(&t.targetInflight) @@ -64,7 +67,7 @@ func (t *dynamicThrottler) Sent() { currentThroughput := float64(atomic.SwapInt64(&t.ok, 0)) / float64(elapsed) // Use buckets for throughput over inflight, along the log-scale, in [minInflight, maxInflight). - index := int(float64(len(t.throughputs)) * math.Log(max(1, min(255, float64(currentInflight)/throttlerMinInflight))) / math.Log(256)) + index := int(float64(len(t.throughputs)) * math.Log(max(1, min(255, float64(currentInflight)/float64(t.minInflight)))) / math.Log(256)) t.throughputs[index] = currentThroughput // Loop over throughput measurements and pick the one which optimises throughput and latency. @@ -74,7 +77,7 @@ func (t *dynamicThrottler) Sent() { if t.throughputs[i] == 0 { continue // Skip unknown values } - inflight := float64(throttlerMinInflight) * math.Pow(256, (float64(i)+0.5)/float64(len(t.throughputs))) + inflight := float64(t.minInflight) * math.Pow(256, (float64(i)+0.5)/float64(len(t.throughputs))) objective := t.throughputs[i] * math.Pow(inflight, throttlerWeight-1) // Optimise throughput (weight), but also latency (1 - weight) if objective > maxObjective { maxObjective = objective @@ -82,7 +85,7 @@ func (t *dynamicThrottler) Sent() { } } target := int64((rand.Float64()*0.20 + 0.92) * choice) // Random walk, skewed towards increase - atomic.StoreInt64(&t.targetInflight, max(throttlerMinInflight, min(throttlerMaxInflight, target))) + atomic.StoreInt64(&t.targetInflight, max(t.minInflight, min(t.maxInflight, target))) } func (t *dynamicThrottler) Success() { @@ -91,11 +94,11 @@ func (t *dynamicThrottler) Success() { } func (t *dynamicThrottler) Throttled(inflight int64) { - atomic.StoreInt64(&t.targetTimesTen, max(inflight*5, throttlerMinInflight*10)) + atomic.StoreInt64(&t.targetTimesTen, max(inflight*5, t.minInflight*10)) } func (t *dynamicThrottler) TargetInflight() int64 { - staticTargetInflight := min(throttlerMaxInflight, atomic.LoadInt64(&t.targetTimesTen)/10) + staticTargetInflight := min(t.maxInflight, atomic.LoadInt64(&t.targetTimesTen)/10) targetInflight := atomic.LoadInt64(&t.targetInflight) return min(staticTargetInflight, targetInflight) } diff --git a/client/go/internal/vespa/document/throttler_test.go b/client/go/internal/vespa/document/throttler_test.go index 2fd1e73a45a..a22f059207f 100644 --- a/client/go/internal/vespa/document/throttler_test.go +++ b/client/go/internal/vespa/document/throttler_test.go @@ -7,15 +7,15 @@ import ( func TestThrottler(t *testing.T) { clock := &manualClock{tick: time.Second} - tr := newThrottler(clock.now) + tr := newThrottler(8, clock.now) for i := 0; i < 100; i++ { tr.Sent() } - if got, want := tr.TargetInflight(), int64(128); got != want { + if got, want := tr.TargetInflight(), int64(1024); got != want { t.Errorf("got TargetInflight() = %d, but want %d", got, want) } tr.Throttled(5) - if got, want := tr.TargetInflight(), int64(16); got != want { + if got, want := tr.TargetInflight(), int64(128); got != want { t.Errorf("got TargetInflight() = %d, but want %d", got, want) } } diff --git a/client/go/internal/vespa/target.go b/client/go/internal/vespa/target.go index 51861eb12ab..9f3fd7f5c65 100644 --- a/client/go/internal/vespa/target.go +++ b/client/go/internal/vespa/target.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "sync" "time" "github.com/vespa-engine/vespa/client/go/internal/util" @@ -17,7 +18,7 @@ const ( // A target for a local Vespa service TargetLocal = "local" - // A target for a custom URL + // A target for a Vespa service at a custom URL TargetCustom = "custom" // A Vespa Cloud target @@ -38,13 +39,19 @@ const ( retryInterval = 2 * time.Second ) +// Authenticator authenticates the given HTTP request. +type Authenticator interface { + Authenticate(request *http.Request) error +} + // Service represents a Vespa service. type Service struct { BaseURL string Name string TLSOptions TLSOptions - zts zts + once sync.Once + auth Authenticator httpClient util.HTTPClient } @@ -65,19 +72,19 @@ type Target interface { // PrintLog writes the logs of this deployment using given options to control output. PrintLog(options LogOptions) error - // SignRequest signs request with given keyID as required by the implementation of this target. - SignRequest(request *http.Request, keyID string) error - // CheckVersion verifies whether clientVersion is compatible with this target. CheckVersion(clientVersion version.Version) error } -// TLSOptions configures the client certificate to use for cloud API or service requests. +// TLSOptions holds the client certificate to use for cloud API or service requests. type TLSOptions struct { - KeyPair *tls.Certificate - CertificateFile string - PrivateKeyFile string - AthenzDomain string + CACertificate []byte + KeyPair []tls.Certificate + TrustAll bool + + CACertificateFile string + CertificateFile string + PrivateKeyFile string } // LogOptions configures the log output to produce when writing log messages. @@ -90,21 +97,21 @@ type LogOptions struct { Level int } -// Do sends request to this service. Any required authentication happens automatically. +// Do sends request to this service. Authentication of the request happens automatically. func (s *Service) Do(request *http.Request, timeout time.Duration) (*http.Response, error) { - if s.TLSOptions.AthenzDomain != "" && s.TLSOptions.KeyPair != nil { - accessToken, err := s.zts.AccessToken(s.TLSOptions.AthenzDomain, *s.TLSOptions.KeyPair) - if err != nil { + s.once.Do(func() { + util.ConfigureTLS(s.httpClient, s.TLSOptions.KeyPair, s.TLSOptions.CACertificate, s.TLSOptions.TrustAll) + }) + if s.auth != nil { + if err := s.auth.Authenticate(request); err != nil { return nil, err } - if request.Header == nil { - request.Header = make(http.Header) - } - request.Header.Add("Authorization", "Bearer "+accessToken) } return s.httpClient.Do(request, timeout) } +func (s *Service) Client() util.HTTPClient { return s.httpClient } + // Wait polls the health check of this service until it succeeds or timeout passes. func (s *Service) Wait(timeout time.Duration) (int, error) { url := s.BaseURL @@ -116,7 +123,7 @@ func (s *Service) Wait(timeout time.Duration) (int, error) { default: return 0, fmt.Errorf("invalid service: %s", s.Name) } - return waitForOK(s.httpClient, url, s.TLSOptions.KeyPair, timeout) + return waitForOK(s, url, timeout) } func (s *Service) Description() string { @@ -131,27 +138,40 @@ func (s *Service) Description() string { return fmt.Sprintf("No description of service %s", s.Name) } -func isOK(status int) bool { return status/100 == 2 } +func isOK(status int) (bool, error) { + class := status / 100 + switch class { + case 2: // success + return true, nil + case 4: // client error + return false, fmt.Errorf("request failed with status %d", status) + default: // retry + return false, nil + } +} type responseFunc func(status int, response []byte) (bool, error) type requestFunc func() *http.Request -// waitForOK queries url and returns its status code. If the url returns a non-200 status code, it is repeatedly queried +// waitForOK queries url and returns its status code. If response status is not 2xx or 4xx, it is repeatedly queried // until timeout elapses. -func waitForOK(client util.HTTPClient, url string, certificate *tls.Certificate, timeout time.Duration) (int, error) { +func waitForOK(service *Service, url string, timeout time.Duration) (int, error) { req, err := http.NewRequest("GET", url, nil) if err != nil { return 0, err } - okFunc := func(status int, response []byte) (bool, error) { return isOK(status), nil } - return wait(client, okFunc, func() *http.Request { return req }, certificate, timeout) + okFunc := func(status int, response []byte) (bool, error) { + ok, err := isOK(status) + if err != nil { + return false, fmt.Errorf("failed to query %s at %s: %w", service.Description(), url, err) + } + return ok, err + } + return wait(service, okFunc, func() *http.Request { return req }, timeout) } -func wait(client util.HTTPClient, fn responseFunc, reqFn requestFunc, certificate *tls.Certificate, timeout time.Duration) (int, error) { - if certificate != nil { - util.SetCertificate(client, []tls.Certificate{*certificate}) - } +func wait(service *Service, fn responseFunc, reqFn requestFunc, timeout time.Duration) (int, error) { var ( httpErr error response *http.Response @@ -161,7 +181,7 @@ func wait(client util.HTTPClient, fn responseFunc, reqFn requestFunc, certificat loopOnce := timeout == 0 for time.Now().Before(deadline) || loopOnce { req := reqFn() - response, httpErr = client.Do(req, 10*time.Second) + response, httpErr = service.Do(req, 10*time.Second) if httpErr == nil { statusCode = response.StatusCode body, err := io.ReadAll(response.Body) diff --git a/client/go/internal/vespa/target_cloud.go b/client/go/internal/vespa/target_cloud.go index 2335d4f3432..928bb788494 100644 --- a/client/go/internal/vespa/target_cloud.go +++ b/client/go/internal/vespa/target_cloud.go @@ -2,7 +2,6 @@ package vespa import ( "bytes" - "crypto/tls" "encoding/json" "fmt" "math" @@ -35,8 +34,8 @@ type cloudTarget struct { deploymentOptions CloudDeploymentOptions logOptions LogOptions httpClient util.HTTPClient - zts zts - auth0 auth0 + apiAuth Authenticator + deploymentAuth Authenticator } type deploymentEndpoint struct { @@ -62,23 +61,15 @@ type logMessage struct { Message string `json:"message"` } -type zts interface { - AccessToken(domain string, certficiate tls.Certificate) (string, error) -} - -type auth0 interface { - AccessToken() (string, error) -} - // CloudTarget creates a Target for the Vespa Cloud or hosted Vespa platform. -func CloudTarget(httpClient util.HTTPClient, ztsClient zts, auth0Client auth0, apiOptions APIOptions, deploymentOptions CloudDeploymentOptions, logOptions LogOptions) (Target, error) { +func CloudTarget(httpClient util.HTTPClient, apiAuth Authenticator, deploymentAuth Authenticator, apiOptions APIOptions, deploymentOptions CloudDeploymentOptions, logOptions LogOptions) (Target, error) { return &cloudTarget{ httpClient: httpClient, apiOptions: apiOptions, deploymentOptions: deploymentOptions, logOptions: logOptions, - zts: ztsClient, - auth0: auth0Client, + apiAuth: apiAuth, + deploymentAuth: deploymentAuth, }, nil } @@ -118,25 +109,25 @@ func (t *cloudTarget) IsCloud() bool { return true } func (t *cloudTarget) Deployment() Deployment { return t.deploymentOptions.Deployment } func (t *cloudTarget) Service(name string, timeout time.Duration, runID int64, cluster string) (*Service, error) { - var service *Service switch name { case DeployService: - service = &Service{ + service := &Service{ Name: name, BaseURL: t.apiOptions.System.URL, TLSOptions: t.apiOptions.TLSOptions, - zts: t.zts, httpClient: t.httpClient, + auth: t.apiAuth, } if timeout > 0 { status, err := service.Wait(timeout) if err != nil { return nil, err } - if !isOK(status) { + if ok, _ := isOK(status); !ok { return nil, fmt.Errorf("got status %d from deploy service at %s", status, service.BaseURL) } } + return service, nil case QueryService, DocumentService: if t.deploymentOptions.ClusterURLs == nil { if err := t.waitForEndpoints(timeout, runID); err != nil { @@ -147,38 +138,15 @@ func (t *cloudTarget) Service(name string, timeout time.Duration, runID int64, c if err != nil { return nil, err } - t.deploymentOptions.TLSOptions.AthenzDomain = t.apiOptions.System.AthenzDomain - service = &Service{ + return &Service{ Name: name, BaseURL: url, TLSOptions: t.deploymentOptions.TLSOptions, - zts: t.zts, httpClient: t.httpClient, - } - + auth: t.deploymentAuth, + }, nil default: return nil, fmt.Errorf("unknown service: %s", name) - - } - if service.TLSOptions.KeyPair != nil { - util.SetCertificate(service.httpClient, []tls.Certificate{*service.TLSOptions.KeyPair}) - } - return service, nil -} - -func (t *cloudTarget) SignRequest(req *http.Request, keyID string) error { - if t.apiOptions.System.IsPublic() { - if t.apiOptions.APIKey != nil { - signer := NewRequestSigner(keyID, t.apiOptions.APIKey) - return signer.SignRequest(req) - } else { - return t.addAuth0AccessToken(req) - } - } else { - if t.apiOptions.TLSOptions.KeyPair.Certificate == nil { - return fmt.Errorf("system %s requires a certificate for authentication", t.apiOptions.System.Name) - } - return nil } } @@ -190,7 +158,11 @@ func (t *cloudTarget) CheckVersion(clientVersion version.Version) error { if err != nil { return err } - response, err := t.httpClient.Do(req, 10*time.Second) + deployService, err := t.Service(DeployService, 0, 0, "") + if err != nil { + return err + } + response, err := deployService.Do(req, 10*time.Second) if err != nil { return err } @@ -212,18 +184,6 @@ func (t *cloudTarget) CheckVersion(clientVersion version.Version) error { return nil } -func (t *cloudTarget) addAuth0AccessToken(request *http.Request) error { - accessToken, err := t.auth0.AccessToken() - if err != nil { - return err - } - if request.Header == nil { - request.Header = make(http.Header) - } - request.Header.Set("Authorization", "Bearer "+accessToken) - return nil -} - func (t *cloudTarget) logsURL() string { return fmt.Sprintf("%s/application/v4/tenant/%s/application/%s/instance/%s/environment/%s/region/%s/logs", t.apiOptions.System.URL, @@ -246,11 +206,10 @@ func (t *cloudTarget) PrintLog(options LogOptions) error { q.Set("to", strconv.FormatInt(toMillis, 10)) } req.URL.RawQuery = q.Encode() - t.SignRequest(req, t.deploymentOptions.Deployment.Application.SerializedForm()) return req } logFunc := func(status int, response []byte) (bool, error) { - if ok, err := isCloudOK(status); !ok { + if ok, err := isOK(status); !ok { return ok, err } logEntries, err := ReadLogEntries(bytes.NewReader(response)) @@ -275,10 +234,18 @@ func (t *cloudTarget) PrintLog(options LogOptions) error { if options.Follow { timeout = math.MaxInt64 // No timeout } - _, err = wait(t.httpClient, logFunc, requestFunc, t.apiOptions.TLSOptions.KeyPair, timeout) + _, err = t.deployServiceWait(logFunc, requestFunc, timeout) return err } +func (t *cloudTarget) deployServiceWait(fn responseFunc, reqFn requestFunc, timeout time.Duration) (int, error) { + deployService, err := t.Service(DeployService, 0, 0, "") + if err != nil { + return 0, err + } + return wait(deployService, fn, reqFn, timeout) +} + func (t *cloudTarget) waitForEndpoints(timeout time.Duration, runID int64) error { if runID > 0 { if err := t.waitForRun(runID, timeout); err != nil { @@ -302,13 +269,10 @@ func (t *cloudTarget) waitForRun(runID int64, timeout time.Duration) error { q := req.URL.Query() q.Set("after", strconv.FormatInt(lastID, 10)) req.URL.RawQuery = q.Encode() - if err := t.SignRequest(req, t.deploymentOptions.Deployment.Application.SerializedForm()); err != nil { - util.JustExitWith(err) - } return req } jobSuccessFunc := func(status int, response []byte) (bool, error) { - if ok, err := isCloudOK(status); !ok { + if ok, err := isOK(status); !ok { return ok, err } var resp jobResponse @@ -326,7 +290,7 @@ func (t *cloudTarget) waitForRun(runID int64, timeout time.Duration) error { } return true, nil } - _, err = wait(t.httpClient, jobSuccessFunc, requestFunc, t.apiOptions.TLSOptions.KeyPair, timeout) + _, err = t.deployServiceWait(jobSuccessFunc, requestFunc, timeout) return err } @@ -361,12 +325,9 @@ func (t *cloudTarget) discoverEndpoints(timeout time.Duration) error { if err != nil { return err } - if err := t.SignRequest(req, t.deploymentOptions.Deployment.Application.SerializedForm()); err != nil { - return err - } urlsByCluster := make(map[string]string) endpointFunc := func(status int, response []byte) (bool, error) { - if ok, err := isCloudOK(status); !ok { + if ok, err := isOK(status); !ok { return ok, err } var resp deploymentResponse @@ -384,7 +345,7 @@ func (t *cloudTarget) discoverEndpoints(timeout time.Duration) error { } return true, nil } - if _, err = wait(t.httpClient, endpointFunc, func() *http.Request { return req }, t.apiOptions.TLSOptions.KeyPair, timeout); err != nil { + if _, err := t.deployServiceWait(endpointFunc, func() *http.Request { return req }, timeout); err != nil { return err } if len(urlsByCluster) == 0 { @@ -393,11 +354,3 @@ func (t *cloudTarget) discoverEndpoints(timeout time.Duration) error { t.deploymentOptions.ClusterURLs = urlsByCluster return nil } - -func isCloudOK(status int) (bool, error) { - if status == 401 { - // when retrying we should give up immediately if we're not authorized - return false, fmt.Errorf("status %d: invalid credentials", status) - } - return isOK(status), nil -} diff --git a/client/go/internal/vespa/target_custom.go b/client/go/internal/vespa/target_custom.go index 848d19f0a90..0a3a9d48fed 100644 --- a/client/go/internal/vespa/target_custom.go +++ b/client/go/internal/vespa/target_custom.go @@ -15,6 +15,7 @@ type customTarget struct { targetType string baseURL string httpClient util.HTTPClient + tlsOptions TLSOptions } type serviceConvergeResponse struct { @@ -22,13 +23,13 @@ type serviceConvergeResponse struct { } // LocalTarget creates a target for a Vespa platform running locally. -func LocalTarget(httpClient util.HTTPClient) Target { - return &customTarget{targetType: TargetLocal, baseURL: "http://127.0.0.1", httpClient: httpClient} +func LocalTarget(httpClient util.HTTPClient, tlsOptions TLSOptions) Target { + return &customTarget{targetType: TargetLocal, baseURL: "http://127.0.0.1", httpClient: httpClient, tlsOptions: tlsOptions} } // CustomTarget creates a Target for a Vespa platform running at baseURL. -func CustomTarget(httpClient util.HTTPClient, baseURL string) Target { - return &customTarget{targetType: TargetCustom, baseURL: baseURL, httpClient: httpClient} +func CustomTarget(httpClient util.HTTPClient, baseURL string, tlsOptions TLSOptions) Target { + return &customTarget{targetType: TargetCustom, baseURL: baseURL, httpClient: httpClient, tlsOptions: tlsOptions} } func (t *customTarget) Type() string { return t.targetType } @@ -44,7 +45,7 @@ func (t *customTarget) createService(name string) (*Service, error) { if err != nil { return nil, err } - return &Service{BaseURL: url, Name: name, httpClient: t.httpClient}, nil + return &Service{BaseURL: url, Name: name, httpClient: t.httpClient, TLSOptions: t.tlsOptions}, nil } return nil, fmt.Errorf("unknown service: %s", name) } @@ -60,7 +61,7 @@ func (t *customTarget) Service(name string, timeout time.Duration, sessionOrRunI if err != nil { return nil, err } - if !isOK(status) { + if ok, _ := isOK(status); !ok { return nil, fmt.Errorf("got status %d from deploy service at %s", status, service.BaseURL) } } else { @@ -76,8 +77,6 @@ func (t *customTarget) PrintLog(options LogOptions) error { return fmt.Errorf("log access is only supported on cloud: run vespa-logfmt on the admin node instead") } -func (t *customTarget) SignRequest(req *http.Request, sigKeyId string) error { return nil } - func (t *customTarget) CheckVersion(version version.Version) error { return nil } func (t *customTarget) urlWithPort(serviceName string) (string, error) { @@ -101,19 +100,19 @@ func (t *customTarget) urlWithPort(serviceName string) (string, error) { } func (t *customTarget) waitForConvergence(timeout time.Duration) error { - deployURL, err := t.urlWithPort(DeployService) + deployService, err := t.createService(DeployService) if err != nil { return err } - url := fmt.Sprintf("%s/application/v2/tenant/default/application/default/environment/prod/region/default/instance/default/serviceconverge", deployURL) + url := fmt.Sprintf("%s/application/v2/tenant/default/application/default/environment/prod/region/default/instance/default/serviceconverge", deployService.BaseURL) req, err := http.NewRequest("GET", url, nil) if err != nil { return err } converged := false convergedFunc := func(status int, response []byte) (bool, error) { - if !isOK(status) { - return false, nil + if ok, err := isOK(status); !ok { + return ok, err } var resp serviceConvergeResponse if err := json.Unmarshal(response, &resp); err != nil { @@ -122,7 +121,7 @@ func (t *customTarget) waitForConvergence(timeout time.Duration) error { converged = resp.Converged return converged, nil } - if _, err := wait(t.httpClient, convergedFunc, func() *http.Request { return req }, nil, timeout); err != nil { + if _, err := wait(deployService, convergedFunc, func() *http.Request { return req }, timeout); err != nil { return err } if !converged { diff --git a/client/go/internal/vespa/target_test.go b/client/go/internal/vespa/target_test.go index b9d65f3d8a4..bf266e8f9ec 100644 --- a/client/go/internal/vespa/target_test.go +++ b/client/go/internal/vespa/target_test.go @@ -3,7 +3,6 @@ package vespa import ( "bytes" - "crypto/tls" "fmt" "io" "net/http" @@ -19,10 +18,16 @@ import ( type mockVespaApi struct { deploymentConverged bool + authFailure bool serverURL string } func (v *mockVespaApi) mockVespaHandler(w http.ResponseWriter, req *http.Request) { + if v.authFailure { + response := `{"message":"unauthorized"}` + w.WriteHeader(401) + w.Write([]byte(response)) + } switch req.URL.Path { case "/cli/v1/": response := `{"minVersion":"8.0.0"}` @@ -65,17 +70,17 @@ func (v *mockVespaApi) mockVespaHandler(w http.ResponseWriter, req *http.Request } func TestCustomTarget(t *testing.T) { - lt := LocalTarget(&mock.HTTPClient{}) + lt := LocalTarget(&mock.HTTPClient{}, TLSOptions{}) assertServiceURL(t, "http://127.0.0.1:19071", lt, "deploy") assertServiceURL(t, "http://127.0.0.1:8080", lt, "query") assertServiceURL(t, "http://127.0.0.1:8080", lt, "document") - ct := CustomTarget(&mock.HTTPClient{}, "http://192.0.2.42") + ct := CustomTarget(&mock.HTTPClient{}, "http://192.0.2.42", TLSOptions{}) assertServiceURL(t, "http://192.0.2.42:19071", ct, "deploy") assertServiceURL(t, "http://192.0.2.42:8080", ct, "query") assertServiceURL(t, "http://192.0.2.42:8080", ct, "document") - ct2 := CustomTarget(&mock.HTTPClient{}, "http://192.0.2.42:60000") + ct2 := CustomTarget(&mock.HTTPClient{}, "http://192.0.2.42:60000", TLSOptions{}) assertServiceURL(t, "http://192.0.2.42:60000", ct2, "deploy") assertServiceURL(t, "http://192.0.2.42:60000", ct2, "query") assertServiceURL(t, "http://192.0.2.42:60000", ct2, "document") @@ -85,7 +90,7 @@ func TestCustomTargetWait(t *testing.T) { vc := mockVespaApi{} srv := httptest.NewServer(http.HandlerFunc(vc.mockVespaHandler)) defer srv.Close() - target := CustomTarget(util.CreateClient(time.Second*10), srv.URL) + target := CustomTarget(util.CreateClient(time.Second*10), srv.URL, TLSOptions{}) _, err := target.Service("query", time.Millisecond, 42, "") assert.NotNil(t, err) @@ -107,6 +112,9 @@ func TestCloudTargetWait(t *testing.T) { var logWriter bytes.Buffer target := createCloudTarget(t, srv.URL, &logWriter) + vc.authFailure = true + assertServiceWaitErr(t, 401, true, target, "deploy") + vc.authFailure = false assertServiceWait(t, 200, target, "deploy") _, err := target.Service("query", time.Millisecond, 42, "") @@ -157,10 +165,11 @@ func createCloudTarget(t *testing.T, url string, logWriter io.Writer) Target { apiKey, err := CreateAPIKey() assert.Nil(t, err) + auth := &mockAuthenticator{} target, err := CloudTarget( util.CreateClient(time.Second*10), - &mockZTS{}, - &mockAuth0{}, + auth, + auth, APIOptions{APIKey: apiKey, System: PublicSystem}, CloudDeploymentOptions{ Deployment: Deployment{ @@ -175,7 +184,6 @@ func createCloudTarget(t *testing.T, url string, logWriter io.Writer) Target { } if ct, ok := target.(*cloudTarget); ok { ct.apiOptions.System.URL = url - ct.zts = &mockZTS{token: "foo bar"} } else { t.Fatalf("Wrong target type %T", ct) } @@ -189,22 +197,22 @@ func assertServiceURL(t *testing.T, url string, target Target, service string) { } func assertServiceWait(t *testing.T, expectedStatus int, target Target, service string) { + assertServiceWaitErr(t, expectedStatus, false, target, service) +} + +func assertServiceWaitErr(t *testing.T, expectedStatus int, expectErr bool, target Target, service string) { s, err := target.Service(service, 0, 42, "") assert.Nil(t, err) status, err := s.Wait(0) - assert.Nil(t, err) + if expectErr { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + } assert.Equal(t, expectedStatus, status) } -type mockZTS struct{ token string } - -func (c *mockZTS) AccessToken(domain string, certificate tls.Certificate) (string, error) { - return c.token, nil -} - -type mockAuth0 struct{} - -func (a *mockAuth0) AccessToken() (string, error) { return "", nil } +type mockAuthenticator struct{} -func (a *mockAuth0) HasCredentials() bool { return true } +func (a *mockAuthenticator) Authenticate(request *http.Request) error { return nil } diff --git a/client/pom.xml b/client/pom.xml index a310e7d6feb..6da6dc74a82 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -35,6 +35,12 @@ <artifactId>maven-compiler-plugin</artifactId> <configuration> <release>${vespaClients.jdk.releaseVersion}</release> + <compilerArgs> <!-- Remove (to use default) when not compiling for 8 --> + <arg>-Xlint:all</arg> + <arg>-Xlint:-rawtypes</arg> + <arg>-Xlint:-unchecked</arg> + <arg>-Xlint:-serial</arg> + </compilerArgs> </configuration> </plugin> <plugin> diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index daf3683ad26..4d5d801e0e3 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -46,7 +46,7 @@ <jaxb.version>2.3.0</jaxb.version> <jetty.version>11.0.14</jetty.version> <org.lz4.version>1.8.0</org.lz4.version> - <org.json.version>20220320</org.json.version> <!-- TODO: Remove on Vespa 9 --> + <org.json.version>20230227</org.json.version> <!-- TODO: Remove on Vespa 9 --> <slf4j.version>1.7.32</slf4j.version> <!-- WARNING: when updated, also update c.y.v.tenant:base pom --> <xml-apis.version>1.4.01</xml-apis.version> </properties> diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterInfo.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterInfo.java index 2cfaf64fe83..0f119d8de50 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterInfo.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterInfo.java @@ -15,7 +15,7 @@ import java.util.Set; import java.util.TreeMap; /** - * Detail information about the current state of all the distributor and storage nodes of the cluster. + * Detailed information about the current state of all the distributor and storage nodes of the cluster. * * @author hakonhall * @author bratseth @@ -127,11 +127,10 @@ public class ClusterInfo { /** Returns the node info object for a given node identifier */ private NodeInfo getInfo(Node node) { - switch (node.getType()) { - case DISTRIBUTOR : return getDistributorNodeInfo(node.getIndex()); - case STORAGE : return getStorageNodeInfo(node.getIndex()); - default : throw new IllegalArgumentException("No node type " + node.getType().toString()); - } + return switch (node.getType()) { + case DISTRIBUTOR -> getDistributorNodeInfo(node.getIndex()); + case STORAGE -> getStorageNodeInfo(node.getIndex()); + }; } } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateVersionSpecificRequest.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateVersionSpecificRequest.java index 70fbbb60e26..6855859c96f 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateVersionSpecificRequest.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ClusterStateVersionSpecificRequest.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.clustercontroller.core; /** - * Base class for distributor/content node node RPC requests that are bound + * Base class for distributor/content node RPC requests that are bound * to a particular cluster state version. */ public abstract class ClusterStateVersionSpecificRequest { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java index 9538167c6de..b83d70b8656 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ContentCluster.java @@ -20,16 +20,13 @@ import static com.yahoo.vdslib.state.NodeState.ORCHESTRATOR_RESERVED_DESCRIPTION public class ContentCluster { - private final String clusterName; + private static final int pollingFrequency = 5000; + private final String clusterName; private final ClusterInfo clusterInfo = new ClusterInfo(); - private final Map<Node, Long> nodeStartTimestamps = new TreeMap<>(); private int slobrokGenerationCount = 0; - - private int pollingFrequency = 5000; - private Distribution distribution; private final int maxNumberOfGroupsAllowedToBeDown; @@ -42,7 +39,7 @@ public class ContentCluster { this(options.clusterName(), options.nodes(), options.storageDistribution(), options.maxNumberOfGroupsAllowedToBeDown()); } - private ContentCluster(String clusterName, + ContentCluster(String clusterName, Collection<ConfiguredNode> configuredNodes, Distribution distribution, int maxNumberOfGroupsAllowedToBeDown) { @@ -91,7 +88,6 @@ public class ContentCluster { } public int getPollingFrequency() { return pollingFrequency; } - public void setPollingFrequency(int millisecs) { pollingFrequency = millisecs; } /** Returns the configured nodes of this as a read-only map indexed on node index (distribution key) */ public Map<Integer, ConfiguredNode> getConfiguredNodes() { @@ -131,7 +127,7 @@ public class ContentCluster { * @param clusterState the current cluster state version * @param condition the upgrade condition * @param oldState the old/current wanted state - * @param newState state wanted to be set @return NodeUpgradePrechecker.Response + * @param newState state wanted to be set * @param inMoratorium whether the CC is in moratorium */ public NodeStateChangeChecker.Result calculateEffectOfNewState( @@ -144,22 +140,22 @@ public class ContentCluster { /** Returns the indices of the nodes that have been safely set to the given state by the Orchestrator (best guess). */ public List<Integer> nodesSafelySetTo(State state) { - switch (state) { - case MAINTENANCE: // Orchestrator's ALLOWED_TO_BE_DOWN - case DOWN: // Orchestrator's PERMANENTLY_DOWN - return clusterInfo.getStorageNodeInfos().stream() - .filter(storageNodeInfo -> { - NodeState userWantedState = storageNodeInfo.getUserWantedState(); - return userWantedState.getState() == state && - Objects.equals(userWantedState.getDescription(), ORCHESTRATOR_RESERVED_DESCRIPTION); - }) - .map(NodeInfo::getNodeIndex) - .toList(); - default: - // Note: There is no trace left if the Orchestrator set the state to UP, so that's handled - // like any other state: - return List.of(); - } + return switch (state) { + // Orchestrator's ALLOWED_TO_BE_DOWN or PERMANENTLY_DOWN, respectively + case MAINTENANCE, DOWN -> + clusterInfo.getStorageNodeInfos().stream() + .filter(storageNodeInfo -> { + NodeState userWantedState = storageNodeInfo.getUserWantedState(); + return userWantedState.getState() == state && + Objects.equals(userWantedState.getDescription(), ORCHESTRATOR_RESERVED_DESCRIPTION); + }) + .map(NodeInfo::getNodeIndex) + .toList(); + default -> + // Note: There is no trace left if the Orchestrator sets the state to UP, so that's handled + // like any other state: + List.of(); + }; } public boolean hasConfiguredNode(int index) { diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/HierarchicalGroupVisiting.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/HierarchicalGroupVisiting.java index 19ff51f4cc4..09f1824824c 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/HierarchicalGroupVisiting.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/HierarchicalGroupVisiting.java @@ -1,16 +1,38 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// package com.yahoo.vespa.clustercontroller.core; +import com.yahoo.vdslib.distribution.Distribution; import com.yahoo.vdslib.distribution.GroupVisitor; -public interface HierarchicalGroupVisiting { - /** Returns true if the group contains more than one (leaf) group. */ - boolean isHierarchical(); +class HierarchicalGroupVisiting { + + private final Distribution distribution; + + public HierarchicalGroupVisiting(Distribution distribution) { + this.distribution = distribution; + } + + /** + * Returns true if the group contains more than one (leaf) group. + */ + public boolean isHierarchical() { + return !distribution.getRootGroup().isLeafGroup(); + } /** * Invoke the visitor for each leaf group of an implied group. If the group is non-hierarchical * (flat), the visitor will not be invoked. */ - void visit(GroupVisitor visitor); + public void visit(GroupVisitor visitor) { + if (isHierarchical()) { + distribution.visitGroups(group -> { + if (group.isLeafGroup()) { + return visitor.visitGroup(group); + } + + return true; + }); + } + } + } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/HierarchicalGroupVisitingAdapter.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/HierarchicalGroupVisitingAdapter.java deleted file mode 100644 index 4bc487bfa7f..00000000000 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/HierarchicalGroupVisitingAdapter.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// -package com.yahoo.vespa.clustercontroller.core; - -import com.yahoo.vdslib.distribution.Distribution; -import com.yahoo.vdslib.distribution.GroupVisitor; - -/** - * Exposes {@link Distribution} as a {@link HierarchicalGroupVisiting}. - * - * @author hakon - */ -public class HierarchicalGroupVisitingAdapter implements HierarchicalGroupVisiting { - private final Distribution distribution; - - public HierarchicalGroupVisitingAdapter(Distribution distribution) { - this.distribution = distribution; - } - - @Override - public boolean isHierarchical() { - return !distribution.getRootGroup().isLeafGroup(); - } - - @Override - public void visit(GroupVisitor visitor) { - if (isHierarchical()) { - distribution.visitGroups(group -> { - if (group.isLeafGroup()) { - return visitor.visitGroup(group); - } - - return true; - }); - } - } -} diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java index 3f1a7ab5d7b..d7aac1c26fa 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeInfo.java @@ -55,7 +55,10 @@ abstract public class NodeInfo implements Comparable<NodeInfo> { private long nextAttemptTime; /** Cached connection to this node. */ private Target connection; - /** We cache last connection we did request info on, as we want to report appropriate error for node regardless of whether other commands have created new connection. */ + /** + * We cache last connection we did request info on, as we want to report appropriate error for + * node regardless of whether other commands have created new connection. + */ public Target lastRequestInfoConnection; /** * Counts the number of attempts we have tried since last time we had @@ -163,7 +166,7 @@ abstract public class NodeInfo implements Comparable<NodeInfo> { } if (prematureCrashCount != count) { prematureCrashCount = count; - log.log(Level.FINE, () -> "Premature crash count on " + toString() + " set to " + count); + log.log(Level.FINE, () -> "Premature crash count on " + this + " set to " + count); } } public int getPrematureCrashCount() { return prematureCrashCount; } @@ -311,13 +314,13 @@ abstract public class NodeInfo implements Comparable<NodeInfo> { } if (state.getState().equals(State.DOWN) && !reportedState.getState().oneOf("d")) { downStableStateTime = time; - log.log(Level.FINE, () -> "Down stable state on " + toString() + " altered to " + time); + log.log(Level.FINE, () -> "Down stable state on " + this + " altered to " + time); if (reportedState.getState() == State.INITIALIZING) { recentlyObservedUnstableDuringInit = true; } } else if (state.getState().equals(State.UP) && !reportedState.getState().oneOf("u")) { upStableStateTime = time; - log.log(Level.FINE, () -> "Up stable state on " + toString() + " altered to " + time); + log.log(Level.FINE, () -> "Up stable state on " + this + " altered to " + time); } if (!state.getState().validReportedNodeState(node.getType())) { throw new IllegalStateException("Trying to set illegal reported node state: " + state); @@ -340,14 +343,14 @@ abstract public class NodeInfo implements Comparable<NodeInfo> { } else { nextAttemptTime = time + 5000; } - log.log(Level.FINEST, () -> "Failed to get state from node " + toString() + ", scheduling next attempt in " + (nextAttemptTime - time) + " ms."); + log.log(Level.FINEST, () -> "Failed to get state from node " + this + ", scheduling next attempt in " + (nextAttemptTime - time) + " ms."); } else { connectionAttemptCount = 0; timeOfFirstFailingConnectionAttempt = 0; reportedState = state; if (version == 0 || state.getState().equals(State.STOPPING)) { nextAttemptTime = time + cluster.getPollingFrequency(); - log.log(Level.FINEST, () -> "Scheduling next attempt to get state from " + toString() + " in " + (nextAttemptTime - time) + " ms (polling freq)."); + log.log(Level.FINEST, () -> "Scheduling next attempt to get state from " + this + " in " + (nextAttemptTime - time) + " ms (polling freq)."); } else { nextAttemptTime = time; } @@ -368,7 +371,7 @@ abstract public class NodeInfo implements Comparable<NodeInfo> { } catch (Exception e) { StringWriter sw = new StringWriter(); e.printStackTrace(new PrintWriter(sw)); - log.warning("Attempted to set wanted state with more than just a main state. Extra data stripped. Original data '" + state.serialize(true) + ":\n" + sw.toString()); + log.warning("Attempted to set wanted state with more than just a main state. Extra data stripped. Original data '" + state.serialize(true) + ":\n" + sw); } } wantedState = newWanted; diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeChecker.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeChecker.java index 2025dfef562..e242833fd0c 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeChecker.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeChecker.java @@ -46,7 +46,7 @@ public class NodeStateChangeChecker { public NodeStateChangeChecker(ContentCluster cluster, boolean inMoratorium) { this.requiredRedundancy = cluster.getDistribution().getRedundancy(); - this.groupVisiting = new HierarchicalGroupVisitingAdapter(cluster.getDistribution()); + this.groupVisiting = new HierarchicalGroupVisiting(cluster.getDistribution()); this.clusterInfo = cluster.clusterInfo(); this.inMoratorium = inMoratorium; this.maxNumberOfGroupsAllowedToBeDown = cluster.maxNumberOfGroupsAllowedToBeDown(); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java index 68e46414c22..6f4d0749f3f 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/NodeStateGatherer.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.clustercontroller.core; import com.yahoo.jrt.ErrorCode; import com.yahoo.jrt.Target; import com.yahoo.vdslib.state.NodeState; -import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.hostinfo.HostInfo; import com.yahoo.vespa.clustercontroller.core.listeners.NodeListener; import java.util.LinkedList; @@ -12,6 +11,9 @@ import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; +import static com.yahoo.vdslib.state.State.DOWN; +import static com.yahoo.vdslib.state.State.STOPPING; + /** * Collects the state of all nodes by making remote requests and handling the replies. */ @@ -65,20 +67,20 @@ public class NodeStateGatherer { if (info.getRpcAddress() == null || info.isNotInSlobrok()) { // Cannot query state of node without RPC address or not in slobrok log.log(Level.FINE, () -> "Not sending getNodeState request to node " + info.getNode() + ": Not in slobrok"); NodeState reportedState = info.getReportedState().clone(); - if (( ! reportedState.getState().equals(State.DOWN) && currentTime - info.lastSeenInSlobrok() > maxSlobrokDisconnectGracePeriod) - || reportedState.getState().equals(State.STOPPING)) // Don't wait for grace period if we expect node to be stopping + if (( ! reportedState.getState().equals(DOWN) && currentTime - info.lastSeenInSlobrok() > maxSlobrokDisconnectGracePeriod) + || reportedState.getState().equals(STOPPING)) // Don't wait for grace period if we expect node to be stopping { log.log(Level.FINE, () -> "Setting reported state to DOWN " - + (reportedState.getState().equals(State.STOPPING) + + (reportedState.getState().equals(STOPPING) ? "as node completed stopping." - : "as node has been out of slobrok longer than " + maxSlobrokDisconnectGracePeriod + ".")); + : "as node has been out of slobrok longer than " + maxSlobrokDisconnectGracePeriod + " ms.")); if (reportedState.getState().oneOf("iur") || ! reportedState.hasDescription()) { - StringBuilder sb = new StringBuilder().append("Set node down as it has been out of slobrok for ") - .append(currentTime - info.lastSeenInSlobrok()).append(" ms which is more than the max limit of ") - .append(maxSlobrokDisconnectGracePeriod).append(" ms."); - reportedState.setDescription(sb.toString()); + reportedState.setDescription("Set node down as it has been out of slobrok for " + + (currentTime - info.lastSeenInSlobrok()) + + " ms which is more than the max limit of " + + maxSlobrokDisconnectGracePeriod + " ms."); } - reportedState.setState(State.DOWN); + reportedState.setState(DOWN); listener.handleNewNodeState(info, reportedState.clone()); } info.setReportedState(reportedState, currentTime); // Must reset it to null to get connection attempts counted @@ -135,7 +137,7 @@ public class NodeStateGatherer { info.setReportedState(state, currentTime); } catch (Exception e) { log.log(Level.WARNING, "Failed to process get node state response", e); - info.setReportedState(new NodeState(info.getNode().getType(), State.DOWN), currentTime); + info.setReportedState(new NodeState(info.getNode().getType(), DOWN), currentTime); } // Important: The old host info should be accessible in info.getHostInfo(), see interface. @@ -152,7 +154,7 @@ public class NodeStateGatherer { private NodeState handleError(GetNodeStateRequest req, NodeInfo info, long currentTime) { String prefix = "Failed get node state request: "; - NodeState newState = new NodeState(info.getNode().getType(), State.DOWN); + NodeState newState = new NodeState(info.getNode().getType(), DOWN); if (req.getReply().getReturnCode() == ErrorCode.TIMEOUT) { String msg = "RPC timeout"; if (info.getReportedState().getState().oneOf("ui")) { @@ -177,7 +179,7 @@ public class NodeStateGatherer { log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } - newState.setState(State.DOWN); + newState.setState(DOWN); } else if (msg.equals("jrt: Connection closed by peer") || msg.equals("Connection reset by peer")) { msg = "Connection error: Closed at other end. (Node or switch likely shut down)"; if (info.isNotInSlobrok()) { @@ -189,7 +191,7 @@ public class NodeStateGatherer { if (log.isLoggable(Level.FINE)) log.log(Level.FINE, "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } - newState.setState(State.DOWN).setDescription(msg); + newState.setState(DOWN).setDescription(msg); } else if (msg.equals("Connection timed out")) { if (info.getReportedState().getState().oneOf("ui")) { msg = "Connection error: Timeout"; @@ -228,11 +230,11 @@ public class NodeStateGatherer { } else if (!info.getReportedState().hasDescription() || !info.getReportedState().getDescription().equals(msg)) { log.log(Level.FINE, () -> "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } - newState.setState(State.DOWN).setDescription(msg + ": get node state"); + newState.setState(DOWN).setDescription(msg + ": get node state"); } else if (req.getReply().getReturnCode() == 75004) { String msg = "Node refused to answer RPC request and is likely stopping: " + req.getReply().getReturnMessage(); // The node is shutting down and is not accepting requests from anyone - if (info.getReportedState().getState().equals(State.STOPPING)) { + if (info.getReportedState().getState().equals(STOPPING)) { log.log(Level.FINE, () -> "Failed to get node state from " + info + " because it is still shutting down."); } else { if (info.getReportedState().getState().oneOf("ui")) { @@ -241,7 +243,7 @@ public class NodeStateGatherer { log.log(Level.FINE, () -> "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } } - newState.setState(State.STOPPING).setDescription(msg); + newState.setState(STOPPING).setDescription(msg); } else { String msg = "Got unexpected error, assumed to be node issue " + req.getReply().getReturnCode() + ": " + req.getReply().getReturnMessage(); if (info.getReportedState().getState().oneOf("ui")) { @@ -249,7 +251,7 @@ public class NodeStateGatherer { } else if (!info.getReportedState().hasDescription() || !info.getReportedState().getDescription().equals(msg)) { log.log(Level.FINE, () -> "Failed to talk to node " + info + ": " + req.getReply().getReturnCode() + " " + req.getReply().getReturnMessage() + ": " + msg); } - newState.setState(State.DOWN).setDescription(msg); + newState.setState(DOWN).setDescription(msg); } return newState; } diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java index 4ab80ec6d7a..28149477e36 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/StateChangeHandler.java @@ -9,13 +9,19 @@ import com.yahoo.vdslib.state.NodeType; import com.yahoo.vdslib.state.State; import com.yahoo.vespa.clustercontroller.core.database.DatabaseHandler; import com.yahoo.vespa.clustercontroller.core.listeners.NodeListener; - +import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; import java.util.logging.Level; import java.util.logging.Logger; +import static com.yahoo.vdslib.state.State.DOWN; +import static com.yahoo.vdslib.state.State.INITIALIZING; +import static com.yahoo.vdslib.state.State.STOPPING; +import static java.util.logging.Level.FINE; +import static java.util.logging.Level.FINEST; + /** * This class gets node state updates and timer events and uses these to decide * whether a new cluster state should be generated. @@ -52,9 +58,9 @@ public class StateChangeHandler { public void handleAllDistributorsInSync(final ClusterState currentState, final Set<ConfiguredNode> nodes, final DatabaseHandler database, - final DatabaseHandler.DatabaseContext dbContext) throws InterruptedException { + final DatabaseHandler.DatabaseContext dbContext) { int startTimestampsReset = 0; - context.log(log, Level.FINE, "handleAllDistributorsInSync invoked for state version %d", currentState.getVersion()); + context.log(log, FINE, "handleAllDistributorsInSync invoked for state version %d", currentState.getVersion()); for (NodeType nodeType : NodeType.getTypes()) { for (ConfiguredNode configuredNode : nodes) { final Node node = new Node(nodeType, configuredNode.index()); @@ -62,15 +68,15 @@ public class StateChangeHandler { final NodeState nodeState = currentState.getNodeState(node); if (nodeInfo != null && nodeState != null) { if (nodeState.getStartTimestamp() > nodeInfo.getStartTimestamp()) { - log.log(Level.FINE, () -> String.format("Storing away new start timestamp for node %s (%d)", node, nodeState.getStartTimestamp())); + log.log(FINE, () -> String.format("Storing away new start timestamp for node %s (%d)", node, nodeState.getStartTimestamp())); nodeInfo.setStartTimestamp(nodeState.getStartTimestamp()); } if (nodeState.getStartTimestamp() > 0) { - log.log(Level.FINE, "Resetting timestamp in cluster state for node %s", node); + log.log(FINE, "Resetting timestamp in cluster state for node %s", node); ++startTimestampsReset; } - } else if (log.isLoggable(Level.FINE)) { - log.log(Level.FINE, node + ": " + + } else if (log.isLoggable(FINE)) { + log.log(FINE, node + ": " + (nodeInfo == null ? "null" : nodeInfo.getStartTimestamp()) + ", " + (nodeState == null ? "null" : nodeState.getStartTimestamp())); } @@ -83,7 +89,7 @@ public class StateChangeHandler { stateMayHaveChanged = true; database.saveStartTimestamps(dbContext); } else { - log.log(Level.FINE, "Found no start timestamps to reset in cluster state."); + log.log(FINE, "Found no start timestamps to reset in cluster state."); } } @@ -110,48 +116,45 @@ public class StateChangeHandler { // TODO nodeListener is only used via updateNodeInfoFromReportedState -> handlePrematureCrash // TODO this will recursively invoke proposeNewNodeState, which will presumably (i.e. hopefully) be a no-op... - public void handleNewReportedNodeState(final ClusterState currentClusterState, - final NodeInfo node, - final NodeState reportedState, - final NodeListener nodeListener) - { - final NodeState currentState = currentClusterState.getNodeState(node.getNode()); - final Level level = (currentState.equals(reportedState) && node.getVersion() == 0) ? Level.FINEST : Level.FINE; - if (log.isLoggable(level)) { - log.log(level, String.format("Got nodestate reply from %s: %s (Current state is %s)", - node, node.getReportedState().getTextualDifference(reportedState), currentState.toString(true))); - } - final long currentTime = timer.getCurrentTimeInMillis(); - - if (reportedState.getState().equals(State.DOWN)) { + public void handleNewReportedNodeState(ClusterState currentClusterState, + NodeInfo node, + NodeState reportedState, + NodeListener nodeListener) { + NodeState currentState = currentClusterState.getNodeState(node.getNode()); + Level level = (currentState.equals(reportedState) && node.getVersion() == 0) ? FINEST : FINE; + log.log(level, () -> String.format("Got nodestate reply from %s: %s (Current state is %s)", + node, node.getReportedState().getTextualDifference(reportedState), currentState.toString(true))); + long currentTime = timer.getCurrentTimeInMillis(); + + if (reportedState.getState().equals(DOWN)) { node.setTimeOfFirstFailingConnectionAttempt(currentTime); } // *** LOGGING ONLY if ( ! reportedState.similarTo(node.getReportedState())) { - if (reportedState.getState().equals(State.DOWN)) { + if (reportedState.getState().equals(DOWN)) { eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(node, "Failed to get node state: " + reportedState.toString(true), NodeEvent.Type.REPORTED, currentTime), Level.INFO); } else { - eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(node, "Now reporting state " + reportedState.toString(true), NodeEvent.Type.REPORTED, currentTime), Level.FINE); + eventLog.addNodeOnlyEvent(NodeEvent.forBaseline(node, "Now reporting state " + reportedState.toString(true), NodeEvent.Type.REPORTED, currentTime), FINE); } } - if (reportedState.equals(node.getReportedState()) && ! reportedState.getState().equals(State.INITIALIZING)) { + if (reportedState.equals(node.getReportedState()) && ! reportedState.getState().equals(INITIALIZING)) { return; } updateNodeInfoFromReportedState(node, currentState, reportedState, nodeListener); if (reportedState.getMinUsedBits() != currentState.getMinUsedBits()) { - final int oldCount = currentState.getMinUsedBits(); - final int newCount = reportedState.getMinUsedBits(); - log.log(Level.FINE, + int oldCount = currentState.getMinUsedBits(); + int newCount = reportedState.getMinUsedBits(); + log.log(FINE, () -> String.format("Altering node state to reflect that min distribution bit count has changed from %d to %d", oldCount, newCount)); eventLog.add(NodeEvent.forBaseline(node, String.format("Altered min distribution bit count from %d to %d", oldCount, newCount), NodeEvent.Type.CURRENT, currentTime), isMaster); } else { - log.log(Level.FINE, () -> String.format("Not altering state of %s in cluster state because new state is too similar: %s", - node, currentState.getTextualDifference(reportedState))); + log.log(FINE, () -> String.format("Not altering state of %s in cluster state because new state is too similar: %s", + node, currentState.getTextualDifference(reportedState))); } stateMayHaveChanged = true; @@ -162,10 +165,8 @@ public class StateChangeHandler { eventLog.add(NodeEvent.forBaseline(node, message, NodeEvent.Type.REPORTED, timer.getCurrentTimeInMillis()), isMaster); } - public void handleMissingNode(final ClusterState currentClusterState, - final NodeInfo node, - final NodeListener nodeListener) { - final long timeNow = timer.getCurrentTimeInMillis(); + public void handleMissingNode(ClusterState currentClusterState, NodeInfo node, NodeListener nodeListener) { + long timeNow = timer.getCurrentTimeInMillis(); if (node.getLatestNodeStateRequestTime() != null) { eventLog.add(NodeEvent.forBaseline(node, "Node is no longer in slobrok, but we still have a pending state request.", NodeEvent.Type.REPORTED, timeNow), isMaster); @@ -173,13 +174,13 @@ public class StateChangeHandler { eventLog.add(NodeEvent.forBaseline(node, "Node is no longer in slobrok. No pending state request to node.", NodeEvent.Type.REPORTED, timeNow), isMaster); } - if (node.getReportedState().getState().equals(State.STOPPING)) { - log.log(Level.FINE, () -> "Node " + node.getNode() + " is no longer in slobrok. Was in stopping state, so assuming it has shut down normally. Setting node down"); + if (node.getReportedState().getState().equals(STOPPING)) { + log.log(FINE, () -> "Node " + node.getNode() + " is no longer in slobrok. Was in stopping state, so assuming it has shut down normally. Setting node down"); NodeState ns = node.getReportedState().clone(); - ns.setState(State.DOWN); + ns.setState(DOWN); handleNewReportedNodeState(currentClusterState, node, ns.clone(), nodeListener); } else { - log.log(Level.FINE, () -> "Node " + node.getNode() + " no longer in slobrok was in state " + node.getReportedState() + ". Waiting to see if it reappears in slobrok"); + log.log(FINE, () -> "Node " + node.getNode() + " no longer in slobrok was in state " + node.getReportedState() + ". Waiting to see if it reappears in slobrok"); } stateMayHaveChanged = true; @@ -192,19 +193,19 @@ public class StateChangeHandler { * If the newly proposed state differs from the state the node currently has in the system, * a cluster state regeneration will be triggered. */ - public void proposeNewNodeState(final ClusterState currentClusterState, final NodeInfo node, final NodeState proposedState) { - final NodeState currentState = currentClusterState.getNodeState(node.getNode()); - final NodeState currentReported = node.getReportedState(); + public void proposeNewNodeState(ClusterState currentClusterState, NodeInfo node, NodeState proposedState) { + NodeState currentState = currentClusterState.getNodeState(node.getNode()); - if (currentState.getState().equals(proposedState.getState())) { + if (currentState.getState().equals(proposedState.getState())) return; - } + stateMayHaveChanged = true; - log.log(Level.FINE, () -> String.format("Got new wanted nodestate for %s: %s", node, currentState.getTextualDifference(proposedState))); + log.log(FINE, () -> String.format("Got new wanted nodestate for %s: %s", node, currentState.getTextualDifference(proposedState))); // Should be checked earlier before state was set in cluster assert(proposedState.getState().validWantedNodeState(node.getNode().getType())); long timeNow = timer.getCurrentTimeInMillis(); + NodeState currentReported = node.getReportedState(); if (proposedState.above(currentReported)) { eventLog.add(NodeEvent.forBaseline(node, String.format("Wanted state %s, but we cannot force node into that " + "state yet as it is currently in %s", proposedState, currentReported), @@ -239,12 +240,9 @@ public class StateChangeHandler { // generated cluster state. Still a bit of a mine field... // TODO remove all node state mutation from this function entirely in favor of ClusterStateGenerator! // `--> this will require adding more event edges and premature crash handling to it. Which is fine. - public boolean watchTimers(final ContentCluster cluster, - final ClusterState currentClusterState, - final NodeListener nodeListener) - { + public boolean watchTimers(ContentCluster cluster, ClusterState currentClusterState, NodeListener nodeListener) { boolean triggeredAnyTimers = false; - final long currentTime = timer.getCurrentTimeInMillis(); + long currentTime = timer.getCurrentTimeInMillis(); for(NodeInfo node : cluster.getNodeInfos()) { triggeredAnyTimers |= handleTimeDependentOpsForNode(currentClusterState, nodeListener, currentTime, node); @@ -256,23 +254,17 @@ public class StateChangeHandler { return triggeredAnyTimers; } - private boolean handleTimeDependentOpsForNode(final ClusterState currentClusterState, - final NodeListener nodeListener, - final long currentTime, - final NodeInfo node) - { - final NodeState currentStateInSystem = currentClusterState.getNodeState(node.getNode()); - final NodeState lastReportedState = node.getReportedState(); - boolean triggeredAnyTimers = false; - - triggeredAnyTimers = reportDownIfOutdatedSlobrokNode( - currentClusterState, nodeListener, currentTime, node, lastReportedState); + private boolean handleTimeDependentOpsForNode(ClusterState currentClusterState, + NodeListener nodeListener, + long currentTime, + NodeInfo node) { + NodeState currentStateInSystem = currentClusterState.getNodeState(node.getNode()); + NodeState lastReportedState = node.getReportedState(); + boolean triggeredAnyTimers = + reportDownIfOutdatedSlobrokNode(currentClusterState, nodeListener, currentTime, node, lastReportedState); - if (nodeStillUnavailableAfterTransitionTimeExceeded( - currentTime, node, currentStateInSystem, lastReportedState)) - { + if (nodeStillUnavailableAfterTransitionTimeExceeded(currentTime, node, currentStateInSystem, lastReportedState)) triggeredAnyTimers = true; - } if (nodeInitProgressHasTimedOut(currentTime, node, currentStateInSystem, lastReportedState)) { eventLog.add(NodeEvent.forBaseline(node, String.format( @@ -287,11 +279,11 @@ public class StateChangeHandler { if (mayResetCrashCounterOnStableUpNode(currentTime, node, lastReportedState)) { node.setPrematureCrashCount(0); - log.log(Level.FINE, () -> "Resetting premature crash count on node " + node + " as it has been up for a long time."); + log.log(FINE, () -> "Resetting premature crash count on node " + node + " as it has been up for a long time."); triggeredAnyTimers = true; } else if (mayResetCrashCounterOnStableDownNode(currentTime, node, lastReportedState)) { node.setPrematureCrashCount(0); - log.log(Level.FINE, () -> "Resetting premature crash count on node " + node + " as it has been down for a long time."); + log.log(FINE, () -> "Resetting premature crash count on node " + node + " as it has been down for a long time."); triggeredAnyTimers = true; } @@ -299,17 +291,18 @@ public class StateChangeHandler { } private boolean nodeInitProgressHasTimedOut(long currentTime, NodeInfo node, NodeState currentStateInSystem, NodeState lastReportedState) { - return !currentStateInSystem.getState().equals(State.DOWN) - && node.getWantedState().above(new NodeState(node.getNode().getType(), State.DOWN)) - && lastReportedState.getState().equals(State.INITIALIZING) + return !currentStateInSystem.getState().equals(DOWN) + && node.getWantedState().above(new NodeState(node.getNode().getType(), DOWN)) + && lastReportedState.getState().equals(INITIALIZING) && maxInitProgressTime != 0 && node.getInitProgressTime() + maxInitProgressTime <= currentTime && node.getNode().getType().equals(NodeType.STORAGE); } + // TODO: Merge this and the below method private boolean mayResetCrashCounterOnStableDownNode(long currentTime, NodeInfo node, NodeState lastReportedState) { return node.getDownStableStateTime() + stableStateTimePeriod <= currentTime - && lastReportedState.getState().equals(State.DOWN) + && lastReportedState.getState().equals(DOWN) && node.getPrematureCrashCount() <= maxPrematureCrashes && node.getPrematureCrashCount() != 0; } @@ -328,8 +321,8 @@ public class StateChangeHandler { NodeState lastReportedState) { return currentStateInSystem.getState().equals(State.MAINTENANCE) - && node.getWantedState().above(new NodeState(node.getNode().getType(), State.DOWN)) - && (lastReportedState.getState().equals(State.DOWN) || node.isNotInSlobrok()) + && node.getWantedState().above(new NodeState(node.getNode().getType(), DOWN)) + && (lastReportedState.getState().equals(DOWN) || node.isNotInSlobrok()) && node.getTransitionTime() + maxTransitionTime.get(node.getNode().getType()) < currentTime; } @@ -340,7 +333,7 @@ public class StateChangeHandler { NodeState lastReportedState) { if (node.isNotInSlobrok() - && !lastReportedState.getState().equals(State.DOWN) + && !lastReportedState.getState().equals(DOWN) && node.lastSeenInSlobrok() + maxSlobrokDisconnectGracePeriod <= currentTime) { final String desc = String.format( @@ -350,7 +343,7 @@ public class StateChangeHandler { maxSlobrokDisconnectGracePeriod); node.abortCurrentNodeStateRequests(); NodeState state = lastReportedState.clone(); - state.setState(State.DOWN); + state.setState(DOWN); if (!state.hasDescription()) { state.setDescription(desc); } @@ -362,10 +355,12 @@ public class StateChangeHandler { return false; } + private boolean isNotControlledShutdown(NodeState state) { return ! isControlledShutdown(state); } + private boolean isControlledShutdown(NodeState state) { - return (state.getState() == State.STOPPING - && (state.getDescription().contains("Received signal 15 (SIGTERM - Termination signal)") - || state.getDescription().contains("controlled shutdown"))); + return state.getState() == State.STOPPING + && List.of("Received signal 15 (SIGTERM - Termination signal)", "controlled shutdown") + .contains(state.getDescription()); } /** @@ -381,14 +376,14 @@ public class StateChangeHandler { final NodeState reportedState, final NodeListener nodeListener) { final long timeNow = timer.getCurrentTimeInMillis(); - log.log(Level.FINE, () -> String.format("Finding new cluster state entry for %s switching state %s", node, currentState.getTextualDifference(reportedState))); + log.log(FINE, () -> String.format("Finding new cluster state entry for %s switching state %s", node, currentState.getTextualDifference(reportedState))); if (handleReportedNodeCrashEdge(node, currentState, reportedState, nodeListener, timeNow)) { return; } if (initializationProgressHasIncreased(currentState, reportedState)) { node.setInitProgressTime(timeNow); - log.log(Level.FINEST, () -> "Reset initialize timer on " + node + " to " + node.getInitProgressTime()); + log.log(FINEST, () -> "Reset initialize timer on " + node + " to " + node.getInitProgressTime()); } if (handleImplicitCrashEdgeFromReverseInitProgress(node, currentState, reportedState, nodeListener, timeNow)) { return; @@ -402,9 +397,9 @@ public class StateChangeHandler { final NodeState reportedState, final NodeListener nodeListener, final long timeNow) { - if (currentState.getState().equals(State.INITIALIZING) + if (currentState.getState().equals(INITIALIZING) && reportedState.getState().oneOf("ds") - && !isControlledShutdown(reportedState)) + && isNotControlledShutdown(reportedState)) { eventLog.add(NodeEvent.forBaseline(node, String.format("Stop or crash during initialization. " + "Premature crash count is now %d.", node.getPrematureCrashCount() + 1), @@ -421,8 +416,8 @@ public class StateChangeHandler { final NodeState reportedState, final NodeListener nodeListener, final long timeNow) { - if (currentState.getState().equals(State.INITIALIZING) && - (reportedState.getState().equals(State.INITIALIZING) && reportedState.getInitProgress() < currentState.getInitProgress())) + if (currentState.getState().equals(INITIALIZING) && + (reportedState.getState().equals(INITIALIZING) && reportedState.getInitProgress() < currentState.getInitProgress())) { eventLog.add(NodeEvent.forBaseline(node, String.format( "Stop or crash during initialization detected from reverse initializing progress." + @@ -442,8 +437,8 @@ public class StateChangeHandler { long timeNow) { if (nodeUpToDownEdge(node, currentState, reportedState)) { node.setTransitionTime(timeNow); - if (node.getUpStableStateTime() + stableStateTimePeriod > timeNow && !isControlledShutdown(reportedState)) { - log.log(Level.FINE, () -> "Stable state: " + node.getUpStableStateTime() + " + " + stableStateTimePeriod + " > " + timeNow); + if (node.getUpStableStateTime() + stableStateTimePeriod > timeNow && isNotControlledShutdown(reportedState)) { + log.log(FINE, () -> "Stable state: " + node.getUpStableStateTime() + " + " + stableStateTimePeriod + " > " + timeNow); eventLog.add(NodeEvent.forBaseline(node, String.format("Stopped or possibly crashed after %d ms, which is before " + "stable state time period. Premature crash count is now %d.", @@ -457,20 +452,20 @@ public class StateChangeHandler { } private boolean initializationProgressHasIncreased(NodeState currentState, NodeState reportedState) { - return reportedState.getState().equals(State.INITIALIZING) && - (!currentState.getState().equals(State.INITIALIZING) || + return reportedState.getState().equals(INITIALIZING) && + (!currentState.getState().equals(INITIALIZING) || reportedState.getInitProgress() > currentState.getInitProgress()); } private boolean nodeUpToDownEdge(NodeInfo node, NodeState currentState, NodeState reportedState) { return currentState.getState().oneOf("ur") && reportedState.getState().oneOf("dis") - && (node.getWantedState().getState().equals(State.RETIRED) || !reportedState.getState().equals(State.INITIALIZING)); + && (node.getWantedState().getState().equals(State.RETIRED) || !reportedState.getState().equals(INITIALIZING)); } private boolean handlePrematureCrash(NodeInfo node, NodeListener changeListener) { node.setPrematureCrashCount(node.getPrematureCrashCount() + 1); if (disableUnstableNodes && node.getPrematureCrashCount() > maxPrematureCrashes) { - NodeState wantedState = new NodeState(node.getNode().getType(), State.DOWN) + NodeState wantedState = new NodeState(node.getNode().getType(), DOWN) .setDescription("Disabled by fleet controller as it prematurely shut down " + node.getPrematureCrashCount() + " times in a row"); NodeState oldState = node.getWantedState(); node.setWantedState(wantedState); diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java index 1c72594377a..01a75034ddf 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/restapiv2/requests/SetNodeStateRequest.java @@ -72,14 +72,13 @@ public class SetNodeStateRequest extends Request<SetResponse> { static NodeState getRequestedNodeState(Map<String, UnitState> newStates, Node n) throws StateRestApiException { UnitState newState = newStates.get("user"); if (newState == null) throw new InvalidContentException("No new user state given in request"); - State state; - switch (newState.getId().toLowerCase()) { - case "up": state = State.UP; break; - case "retired": state = State.RETIRED; break; - case "maintenance": state = State.MAINTENANCE; break; - case "down": state = State.DOWN; break; - default: throw new InvalidContentException("Invalid user state '" + newState.getId() + "' given."); - } + State state = switch (newState.getId().toLowerCase()) { + case "up" -> State.UP; + case "retired" -> State.RETIRED; + case "maintenance" -> State.MAINTENANCE; + case "down" -> State.DOWN; + default -> throw new InvalidContentException("Invalid user state '" + newState.getId() + "' given."); + }; return new NodeState(n.getType(), state).setDescription(newState.getReason()); } @@ -191,25 +190,18 @@ public class SetNodeStateRequest extends Request<SetResponse> { boolean probe) { Node distributorNode = new Node(NodeType.DISTRIBUTOR, index); NodeInfo nodeInfo = cluster.getNodeInfo(distributorNode); - if (nodeInfo == null) { - throw new IllegalStateException("Missing distributor at index " + - distributorNode.getIndex()); - } + if (nodeInfo == null) + throw new IllegalStateException("Missing distributor at index " + distributorNode.getIndex()); State newState; switch (newStorageWantedState.getState()) { - case MAINTENANCE: - newState = State.DOWN; - break; - case RETIRED: - newState = State.UP; - break; - default: + case MAINTENANCE -> newState = State.DOWN; + case RETIRED -> newState = State.UP; + default -> { newState = newStorageWantedState.getState(); - if (!newState.validWantedNodeState(distributorNode.getType())) { - throw new IllegalStateException("Distributor cannot be set to wanted state " + - newState); - } + if (!newState.validWantedNodeState(distributorNode.getType())) + throw new IllegalStateException("Distributor cannot be set to wanted state " + newState); + } } NodeState newWantedState = new NodeState(distributorNode.getType(), newState); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeCheckerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeCheckerTest.java index e789a3cc6a6..45ca07c88e4 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeCheckerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeCheckerTest.java @@ -19,6 +19,7 @@ import static com.yahoo.vdslib.state.NodeType.DISTRIBUTOR; import static com.yahoo.vdslib.state.NodeType.STORAGE; import static com.yahoo.vdslib.state.State.DOWN; import static com.yahoo.vdslib.state.State.INITIALIZING; +import static com.yahoo.vdslib.state.State.MAINTENANCE; import static com.yahoo.vdslib.state.State.UP; import static com.yahoo.vespa.clustercontroller.core.NodeStateChangeChecker.Result; import static com.yahoo.vespa.clustercontroller.utils.staterestapi.requests.SetUnitStateRequest.Condition.FORCE; @@ -37,7 +38,7 @@ public class NodeStateChangeCheckerTest { private static final Node nodeStorage = new Node(STORAGE, 1); private static final NodeState UP_NODE_STATE = new NodeState(STORAGE, UP); - private static final NodeState MAINTENANCE_NODE_STATE = createNodeState(State.MAINTENANCE, "Orchestrator"); + private static final NodeState MAINTENANCE_NODE_STATE = createNodeState(MAINTENANCE, "Orchestrator"); private static final NodeState DOWN_NODE_STATE = createNodeState(DOWN, "RetireEarlyExpirer"); private static NodeState createNodeState(State state, String description) { @@ -53,7 +54,14 @@ public class NodeStateChangeCheckerTest { } private static ClusterState defaultAllUpClusterState() { - return clusterState(String.format("version:%d distributor:4 storage:4", currentClusterStateVersion)); + return defaultAllUpClusterState(4); + } + + private static ClusterState defaultAllUpClusterState(int nodeCount) { + return clusterState(String.format("version:%d distributor:%d storage:%d", + currentClusterStateVersion, + nodeCount , + nodeCount)); } private NodeStateChangeChecker createChangeChecker(ContentCluster cluster) { @@ -144,7 +152,7 @@ public class NodeStateChangeCheckerTest { void testSafeMaintenanceDisallowedWhenOtherStorageNodeInFlatClusterIsSuspended() { // Nodes 0-3, storage node 0 being in maintenance with "Orchestrator" description. ContentCluster cluster = createCluster(4); - cluster.clusterInfo().getStorageNodeInfo(0).setWantedState(new NodeState(STORAGE, State.MAINTENANCE).setDescription("Orchestrator")); + cluster.clusterInfo().getStorageNodeInfo(0).setWantedState(new NodeState(STORAGE, MAINTENANCE).setDescription("Orchestrator")); var nodeStateChangeChecker = createChangeChecker(cluster); ClusterState clusterStateWith0InMaintenance = clusterState(String.format( "version:%d distributor:4 storage:4 .0.s:m", @@ -163,8 +171,7 @@ public class NodeStateChangeCheckerTest { void testSafeMaintenanceDisallowedWhenOtherDistributorInFlatClusterIsSuspended() { // Nodes 0-3, distributor 0 being down with "Orchestrator" description. ContentCluster cluster = createCluster(4); - cluster.clusterInfo().getDistributorNodeInfo(0) - .setWantedState(new NodeState(DISTRIBUTOR, DOWN).setDescription("Orchestrator")); + setDistributorNodeWantedState(cluster, 0, DOWN, "Orchestrator"); var nodeStateChangeChecker = createChangeChecker(cluster); ClusterState clusterStateWith0InMaintenance = clusterState(String.format( "version:%d distributor:4 .0.s:d storage:4", @@ -184,8 +191,7 @@ public class NodeStateChangeCheckerTest { // Nodes 0-3, distributor 0 being in maintenance with "Orchestrator" description. // 2 groups: nodes 0-1 is group 0, 2-3 is group 1. ContentCluster cluster = createCluster(4, 2); - cluster.clusterInfo().getDistributorNodeInfo(0) - .setWantedState(new NodeState(STORAGE, DOWN).setDescription("Orchestrator")); + setDistributorNodeWantedState(cluster, 0, DOWN, "Orchestrator"); var nodeStateChangeChecker = new NodeStateChangeChecker(cluster, false); ClusterState clusterStateWith0InMaintenance = clusterState(String.format( "version:%d distributor:4 .0.s:d storage:4", @@ -217,7 +223,7 @@ public class NodeStateChangeCheckerTest { // Nodes 0-3, storage node 0 being in maintenance with "Orchestrator" description. // 2 groups: nodes 0-1 is group 0, 2-3 is group 1. ContentCluster cluster = createCluster(4, 2); - cluster.clusterInfo().getStorageNodeInfo(0).setWantedState(new NodeState(STORAGE, State.MAINTENANCE).setDescription("Orchestrator")); + setStorageNodeWantedState(cluster, 0, MAINTENANCE, "Orchestrator"); var nodeStateChangeChecker = new NodeStateChangeChecker(cluster, false); ClusterState clusterStateWith0InMaintenance = clusterState(String.format( "version:%d distributor:4 storage:4 .0.s:m", @@ -406,7 +412,7 @@ public class NodeStateChangeCheckerTest { } private Result transitionToSameState(String oldDescription, String newDescription) { - return transitionToSameState(State.MAINTENANCE, oldDescription, newDescription); + return transitionToSameState(MAINTENANCE, oldDescription, newDescription); } @Test @@ -741,13 +747,14 @@ public class NodeStateChangeCheckerTest { .capacity(nodes) .partitions("1|*")); + int nodeIndex = 0; for (int i = 0; i < groups; ++i) { var groupBuilder = new StorDistributionConfig.Group.Builder() .index(String.valueOf(i)) .name(String.valueOf(i)) .capacity(nodesPerGroup) .partitions(""); - for (int nodeIndex = 0; nodeIndex < nodesPerGroup; ++nodeIndex) { + for (int j = 0; j < nodesPerGroup; ++j, ++nodeIndex) { groupBuilder.nodes(new StorDistributionConfig.Group.Nodes.Builder() .index(nodeIndex)); } @@ -756,4 +763,14 @@ public class NodeStateChangeCheckerTest { return configBuilder.build(); } + private void setStorageNodeWantedState(ContentCluster cluster, int nodeIndex, State state, String description) { + NodeState nodeState = new NodeState(STORAGE, state); + cluster.clusterInfo().getStorageNodeInfo(nodeIndex).setWantedState(nodeState.setDescription(description)); + } + + private void setDistributorNodeWantedState(ContentCluster cluster, int nodeIndex, State state, String description) { + NodeState nodeState = new NodeState(DISTRIBUTOR, state); + cluster.clusterInfo().getDistributorNodeInfo(nodeIndex).setWantedState(nodeState.setDescription(description)); + } + } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index fef2354c452..7f2dd4b6acd 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -113,6 +113,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"tokle, bjorncs"}, removeAfter = "8.108") default boolean enableDataPlaneFilter() { return true; } @ModelFeatureFlag(owners = {"arnej, bjorncs"}) default boolean enableGlobalPhase() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Select summary decode type") default String summaryDecodePolicy() { return "eager"; } + @ModelFeatureFlag(owners = {"hmusum"}) default boolean allowMoreThanOneContentGroupDown(ClusterSpec.Id id) { return false; } //Below are all flags that must be kept until 7 is out of the door @ModelFeatureFlag(owners = {"arnej"}, removeAfter="7.last") default boolean ignoreThreadStackSizes() { return false; } diff --git a/config-model/src/main/java/com/yahoo/schema/OnnxModel.java b/config-model/src/main/java/com/yahoo/schema/OnnxModel.java index 272b668b5fb..90a27d1f036 100644 --- a/config-model/src/main/java/com/yahoo/schema/OnnxModel.java +++ b/config-model/src/main/java/com/yahoo/schema/OnnxModel.java @@ -1,15 +1,17 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.schema; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.model.ml.OnnxModelInfo; -import com.yahoo.searchlib.rankingexpression.Reference; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; /** * A global ONNX model distributed using file distribution, similar to ranking constants. @@ -21,6 +23,7 @@ public class OnnxModel extends DistributableResource { private OnnxModelInfo modelInfo = null; private final Map<String, String> inputMap = new HashMap<>(); private final Map<String, String> outputMap = new HashMap<>(); + private final Set<String> initializers = new HashSet<>(); private String statelessExecutionMode = null; private Integer statelessInterOpThreads = null; @@ -101,11 +104,13 @@ public class OnnxModel extends DistributableResource { for (String onnxName : modelInfo.getOutputs()) { addOutputNameMapping(onnxName, OnnxModelInfo.asValidIdentifier(onnxName), false); } + initializers.addAll(modelInfo.getInitializers()); this.modelInfo = modelInfo; } public Map<String, String> getInputMap() { return Collections.unmodifiableMap(inputMap); } public Map<String, String> getOutputMap() { return Collections.unmodifiableMap(outputMap); } + public Set<String> getInitializers() { return Set.copyOf(initializers); } public String getDefaultOutput() { return modelInfo != null ? modelInfo.getDefaultOutput() : ""; diff --git a/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java b/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java index c8679b6166c..c032a7155b2 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java @@ -13,12 +13,14 @@ import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.schema.FieldSets; import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; import com.yahoo.schema.document.FieldSet; import com.yahoo.schema.document.GeoPos; import com.yahoo.schema.document.Matching; import com.yahoo.schema.document.MatchType; import com.yahoo.schema.document.SDDocumentType; import com.yahoo.schema.document.SDField; +import com.yahoo.schema.processing.TensorFieldProcessor; import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig; import java.util.LinkedHashMap; @@ -124,63 +126,68 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { private final Type type; private final boolean isAttribute; + private final Attribute.DistanceMetric distanceMetric; /** The streaming field type enumeration */ public static class Type { - public static Type INT8 = new Type("int8","INT8"); - public static Type INT16 = new Type("int16","INT16"); - public static Type INT32 = new Type("int32","INT32"); - public static Type INT64 = new Type("int64","INT64"); - public static Type FLOAT16 = new Type("float16", "FLOAT16"); - public static Type FLOAT = new Type("float","FLOAT"); - public static Type DOUBLE = new Type("double","DOUBLE"); - public static Type STRING = new Type("string","AUTOUTF8"); - public static Type BOOL = new Type("bool","BOOL"); - public static Type UNSEARCHABLESTRING = new Type("string","NONE"); - public static Type GEO_POSITION = new Type("position", "GEOPOS"); - - private String name; + public static Type INT8 = new Type("INT8"); + public static Type INT16 = new Type("INT16"); + public static Type INT32 = new Type("INT32"); + public static Type INT64 = new Type("INT64"); + public static Type FLOAT16 = new Type("FLOAT16"); + public static Type FLOAT = new Type("FLOAT"); + public static Type DOUBLE = new Type("DOUBLE"); + public static Type STRING = new Type("AUTOUTF8"); + public static Type BOOL = new Type("BOOL"); + public static Type UNSEARCHABLESTRING = new Type("NONE"); + public static Type GEO_POSITION = new Type("GEOPOS"); + public static Type NEAREST_NEIGHBOR = new Type("NEAREST_NEIGHBOR"); private String searchMethod; - private Type(String name, String searchMethod) { - this.name = name; + private Type(String searchMethod) { this.searchMethod = searchMethod; } @Override public int hashCode() { - return name.hashCode(); + return searchMethod.hashCode(); } - /** Returns the name of this type */ - public String getName() { return name; } - public String getSearchMethod() { return searchMethod; } @Override public boolean equals(Object other) { if ( ! (other instanceof Type)) return false; - return this.name.equals(((Type)other).name); + return this.searchMethod.equals(((Type)other).searchMethod); } @Override public String toString() { - return "type: " + name; + return "method: " + searchMethod; } } public StreamingField(SDField field) { - this(field.getName(), field.getDataType(), field.getMatching(), field.doesAttributing()); + this(field.getName(), field.getDataType(), field.getMatching(), field.doesAttributing(), getDistanceMetric(field)); } - private StreamingField(String name, DataType sourceType, Matching matching, boolean isAttribute) { + private StreamingField(String name, DataType sourceType, Matching matching, boolean isAttribute, Attribute.DistanceMetric distanceMetric) { this.name = name; this.type = convertType(sourceType); this.matching = matching; this.isAttribute = isAttribute; + this.distanceMetric = distanceMetric; + } + + private static Attribute.DistanceMetric getDistanceMetric(SDField field) { + var attr = field.getAttribute(); + if (attr != null) { + return attr.distanceMetric(); + } + return Attribute.DEFAULT_DISTANCE_METRIC; } /** Converts to the right index type from a field datatype */ @@ -211,6 +218,10 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { } else if (fval instanceof PredicateFieldValue) { return Type.UNSEARCHABLESTRING; } else if (fval instanceof TensorFieldValue) { + var tensorType = ((TensorFieldValue) fval).getDataType().getTensorType(); + if (TensorFieldProcessor.isTensorTypeThatSupportsHnswIndex(tensorType)) { + return Type.NEAREST_NEIGHBOR; + } return Type.UNSEARCHABLESTRING; } else if (fieldType instanceof CollectionDataType) { return convertType(((CollectionDataType) fieldType).getNestedType()); @@ -224,8 +235,7 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { public String getName() { return name; } - public VsmfieldsConfig.Fieldspec.Builder getFieldSpecConfig() { - VsmfieldsConfig.Fieldspec.Builder fB = new VsmfieldsConfig.Fieldspec.Builder(); + public String getMatchingName() { String matchingName = matching.getType().getName(); if (matching.getType().equals(MatchType.TEXT)) matchingName = ""; @@ -241,9 +251,21 @@ public class VsmFields extends Derived implements VsmfieldsConfig.Producer { if (type != Type.STRING) { matchingName = ""; } + return matchingName; + } + + public String getArg1() { + if (type == Type.NEAREST_NEIGHBOR) { + return distanceMetric.name(); + } + return getMatchingName(); + } + + public VsmfieldsConfig.Fieldspec.Builder getFieldSpecConfig() { + var fB = new VsmfieldsConfig.Fieldspec.Builder(); fB.name(getName()) .searchmethod(VsmfieldsConfig.Fieldspec.Searchmethod.Enum.valueOf(type.getSearchMethod())) - .arg1(matchingName) + .arg1(getArg1()) .fieldtype(isAttribute ? VsmfieldsConfig.Fieldspec.Fieldtype.ATTRIBUTE : VsmfieldsConfig.Fieldspec.Fieldtype.INDEX); diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java index af072c5b59a..5d3624cd3d3 100644 --- a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java @@ -2,7 +2,6 @@ package com.yahoo.schema.expressiontransforms; import com.yahoo.schema.FeatureNames; -import com.yahoo.schema.RankProfile; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.parser.ParseException; @@ -12,13 +11,12 @@ import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode; import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; -import com.yahoo.tensor.functions.DynamicTensor; import com.yahoo.tensor.functions.Generate; -import com.yahoo.tensor.functions.Slice; import java.io.StringReader; import java.util.HashSet; import java.util.Set; +import java.util.logging.Logger; /** * Analyzes expression to figure out what inputs it needs @@ -27,6 +25,8 @@ import java.util.Set; */ public class InputRecorder extends ExpressionTransformer<InputRecorderContext> { + private static final Logger log = Logger.getLogger(InputRecorder.class.getName()); + private final Set<String> neededInputs; private final Set<String> handled = new HashSet<>(); @@ -120,7 +120,7 @@ public class InputRecorder extends ExpressionTransformer<InputRecorderContext> { if (model == null) { throw new IllegalArgumentException("missing onnx model: " + arg); } - for (String onnxInput : model.getInputMap().values()) { + model.getInputMap().forEach((__, onnxInput) -> { var reader = new StringReader(onnxInput); try { var asExpression = new RankingExpression(reader); @@ -128,7 +128,7 @@ public class InputRecorder extends ExpressionTransformer<InputRecorderContext> { } catch (ParseException e) { throw new IllegalArgumentException("illegal onnx input '" + onnxInput + "': " + e.getMessage()); } - } + }); return; } neededInputs.add(feature.toString()); diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertSchemaCollection.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertSchemaCollection.java index 5509d11885c..b414d3757e2 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertSchemaCollection.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertSchemaCollection.java @@ -20,7 +20,7 @@ import java.util.List; * Class converting a collection of schemas from the intermediate format. * * @author arnej27959 - **/ + */ public class ConvertSchemaCollection { private final IntermediateCollection input; diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java index d8c1fb3125f..3c7e9b4066f 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java @@ -62,7 +62,7 @@ public class IndexingValidation extends Processor { final Set<String> prevNames = new HashSet<>(); @Override - protected ExpressionConverter branch() { + public ExpressionConverter branch() { MyConverter ret = new MyConverter(); ret.outputs.addAll(outputs); ret.prevNames.addAll(prevNames); diff --git a/config-model/src/main/java/com/yahoo/schema/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/TensorFieldProcessor.java index 37da07f8227..227054d9800 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/TensorFieldProcessor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/TensorFieldProcessor.java @@ -9,6 +9,7 @@ import com.yahoo.schema.Schema; import com.yahoo.schema.document.HnswIndexParams; import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.schema.document.SDField; +import com.yahoo.tensor.TensorType; import com.yahoo.vespa.model.container.search.QueryProfiles; /** @@ -50,6 +51,14 @@ public class TensorFieldProcessor extends Processor { private boolean isTensorTypeThatSupportsHnswIndex(ImmutableSDField field) { var type = ((TensorDataType)field.getDataType()).getTensorType(); + return isTensorTypeThatSupportsHnswIndex(type); + } + + /** + * Returns whether the given tensor type supports using HNSW index and + * nearest neighbor search. + */ + public static boolean isTensorTypeThatSupportsHnswIndex(TensorType type) { // Tensors with 1 indexed dimension support hnsw index (used for approximate nearest neighbor search). if ((type.dimensions().size() == 1) && type.dimensions().get(0).isIndexed()) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java deleted file mode 100644 index 9d4e732d3f8..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.admin.monitoring; - -import java.util.Optional; - -/** - * Helper object for CloudWatch configuration. - * - * @author gjoranv - */ -public class CloudWatch { - - private final String region; - private final String namespace; - private final MetricsConsumer consumer; - - private HostedAuth hostedAuth; - private SharedCredentials sharedCredentials; - - public CloudWatch(String region, String namespace, MetricsConsumer consumer) { - this.region = region; - this.namespace = namespace; - this.consumer = consumer; - } - - public String region() { return region; } - public String namespace() { return namespace; } - public String consumer() { return consumer.id(); } - - public Optional<HostedAuth> hostedAuth() {return Optional.ofNullable(hostedAuth); } - public Optional<SharedCredentials> sharedCredentials() {return Optional.ofNullable(sharedCredentials); } - - public void setHostedAuth(String accessKeyName, String secretKeyName) { - hostedAuth = new HostedAuth(accessKeyName, secretKeyName); - } - - public void setSharedCredentials(String file, Optional<String> profile) { - sharedCredentials = new SharedCredentials(file, profile); - } - - public static class HostedAuth { - public final String accessKeyName; - public final String secretKeyName; - - HostedAuth(String accessKeyName, String secretKeyName) { - this.accessKeyName = accessKeyName; - this.secretKeyName = secretKeyName; - } - } - - public static class SharedCredentials { - public final String file; - public final Optional<String> profile; - - SharedCredentials(String file, Optional<String> profile) { - this.file = file; - this.profile = profile; - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java index 671a90a0fee..c6bd4c6e295 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java @@ -41,9 +41,6 @@ public class MetricsConsumer { private final String id; private final MetricSet metricSet; - // TODO: This shouldn't be here - private final List<CloudWatch> cloudWatches = new ArrayList<>(); - /** * @param id the consumer * @param metricSet the metrics for this consumer @@ -66,14 +63,6 @@ public class MetricsConsumer { return metricSet.getMetrics(); } - public void addCloudWatch(CloudWatch cloudWatch) { - cloudWatches.add(cloudWatch); - } - - public List<CloudWatch> cloudWatches() { - return unmodifiableList(cloudWatches); - } - private static MetricsConsumer consumer(String id, MetricSet ... metricSets) { return new MetricsConsumer(id, new MetricSet(id + "-consumer-metrics", List.of(), Arrays.asList(metricSets))); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java index 5e76311dce3..aa61d2ffb74 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java @@ -29,12 +29,4 @@ public class Metrics { .anyMatch(existing -> existing.equalsIgnoreCase(id)); } - /** - * Returns true if any of the consumers have specified external metric systems. - */ - public boolean usesExternalMetricSystems() { - return consumers.values().stream() - .anyMatch(consumer -> ! consumer.cloudWatches().isEmpty()); - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java deleted file mode 100644 index ada4cb76d8e..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.admin.monitoring.builder.xml; - -import com.yahoo.vespa.model.admin.monitoring.CloudWatch; -import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; -import org.w3c.dom.Element; - -import static com.yahoo.config.model.builder.xml.XmlHelper.getOptionalAttribute; -import static com.yahoo.config.model.builder.xml.XmlHelper.getOptionalChild; - -/** - * @author gjoranv - */ -public class CloudWatchBuilder { - - private static final String REGION_ATTRIBUTE = "region"; - private static final String NAMESPACE_ATTRIBUTE = "namespace"; - private static final String CREDENTIALS_ELEMENT = "credentials"; - private static final String ACCESS_KEY_ATTRIBUTE = "access-key-name"; - private static final String SECRET_KEY_ATTRIBUTE = "secret-key-name"; - private static final String SHARED_CREDENTIALS_ELEMENT = "shared-credentials"; - private static final String PROFILE_ATTRIBUTE = "profile"; - private static final String FILE_ATTRIBUTE = "file"; - - public static CloudWatch buildCloudWatch(Element cloudwatchElement, MetricsConsumer consumer) { - CloudWatch cloudWatch = new CloudWatch(cloudwatchElement.getAttribute(REGION_ATTRIBUTE), - cloudwatchElement.getAttribute(NAMESPACE_ATTRIBUTE), - consumer); - - getOptionalChild(cloudwatchElement, CREDENTIALS_ELEMENT) - .ifPresent(elem -> cloudWatch.setHostedAuth(elem.getAttribute(ACCESS_KEY_ATTRIBUTE), - elem.getAttribute(SECRET_KEY_ATTRIBUTE))); - - getOptionalChild(cloudwatchElement, SHARED_CREDENTIALS_ELEMENT) - .ifPresent(elem -> cloudWatch.setSharedCredentials(elem.getAttribute(FILE_ATTRIBUTE), - getOptionalAttribute(elem, PROFILE_ATTRIBUTE))); - - return cloudWatch; - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java index cfd2ebb2fd1..f75b2a864f9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java @@ -41,9 +41,6 @@ public class MetricsBuilder { MetricSet metricSet = buildMetricSet(consumerId, consumerElement); var consumer = new MetricsConsumer(consumerId, metricSet); - for (Element cloudwatchElement : XML.getChildren(consumerElement, "cloudwatch")) { - consumer.addCloudWatch(CloudWatchBuilder.buildCloudWatch(cloudwatchElement, consumer)); - } metrics.addConsumer(consumer); } return metrics; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java deleted file mode 100644 index 847a3402dc5..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.application.validation; - -import com.yahoo.config.model.ConfigModelContext; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; - -import java.util.List; - -/** - * @author gjoranv - */ -public class CloudWatchValidator extends Validator { - - @Override - public void validate(VespaModel model, DeployState deployState) { - if (!deployState.isHosted()) return; - if (deployState.zone().system().isPublic()) return; - if (model.getAdmin().getApplicationType() != ConfigModelContext.ApplicationType.DEFAULT) return; - - var offendingConsumers = model.getAdmin().getUserMetrics().getConsumers().values().stream() - .filter(consumer -> !consumer.cloudWatches().isEmpty()) - .toList(); - - if (! offendingConsumers.isEmpty()) { - throw new IllegalArgumentException("CloudWatch cannot be set up for non-public hosted Vespa and must " + - "be removed for consumers: " + consumerIds(offendingConsumers)); - } - } - - private List<String> consumerIds(List<MetricsConsumer> offendingConsumers) { - return offendingConsumers.stream().map(MetricsConsumer::id).toList(); - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java index 773d696f3e8..ad126cfa22b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java @@ -5,6 +5,7 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.document.DataType; import com.yahoo.document.NumericDataType; +import com.yahoo.document.TensorDataType; import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.schema.document.Attribute; import com.yahoo.schema.document.ImmutableSDField; @@ -63,6 +64,8 @@ public class StreamingValidator extends Validator { // If the field is numeric, we can't print this, because we may have converted the field to // attribute indexing ourselves (IntegerIndex2Attribute) if (sd.getDataType() instanceof NumericDataType) return; + // Tensor fields are only searchable via nearest neighbor search, and match semantics are irrelevant. + if (sd.getDataType() instanceof TensorDataType) return; logger.logApplicationPackage(Level.WARNING, "For streaming search cluster '" + sc.getClusterName() + "', SD field '" + sd.getName() + "': 'attribute' has same match semantics as 'index'."); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index c7a363010b7..2576d9cb392 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -82,7 +82,6 @@ public class Validation { new SecretStoreValidator().validate(model, deployState); new EndpointCertificateSecretsValidator().validate(model, deployState); new AccessControlFilterValidator().validate(model, deployState); - new CloudWatchValidator().validate(model, deployState); new QuotaValidator().validate(model, deployState); new UriBindingsValidator().validate(model, deployState); new CloudDataPlaneFilterValidator().validate(model, deployState); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java index 0cc52edf3cc..b1eace947cc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java @@ -2,11 +2,15 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.vespa.model.VespaModel; import com.yahoo.config.application.api.ValidationId; +import com.yahoo.vespa.model.container.ApplicationContainer; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.content.cluster.ContentCluster; +import java.util.ArrayList; import java.util.List; /** @@ -19,16 +23,23 @@ public class ContentClusterRemovalValidator implements ChangeValidator { @Override public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, DeployState deployState) { + List<ConfigChangeAction> actions = new ArrayList<>(); for (String currentClusterId : current.getContentClusters().keySet()) { ContentCluster nextCluster = next.getContentClusters().get(currentClusterId); - if (nextCluster == null) + if (nextCluster == null) { deployState.validationOverrides().invalid(ValidationId.contentClusterRemoval, - "Content cluster '" + currentClusterId + "' is removed. " + - "This will cause loss of all data in this cluster", - deployState.now()); - } + "Content cluster '" + currentClusterId + "' is removed. " + + "This will cause loss of all data in this cluster", + deployState.now()); - return List.of(); + // If we allow the removal, we must restart all containers to ensure mbus is OK. + for (ApplicationContainerCluster cluster : next.getContainerClusters().values()) { + actions.add(new VespaRestartAction(cluster.id(), + "Content cluster '" + currentClusterId + "' has been removed", + cluster.getContainers().stream().map(ApplicationContainer::getServiceInfo).toList())); + } + } + } + return actions; } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java index 5e8bb85c29d..2b5dadf5512 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java @@ -51,15 +51,18 @@ public class IdentityProvider extends SimpleComponent implements IdentityConfig. builder.loadBalancerAddress(loadBalancerName.value()); builder.ztsUrl(ztsUrl != null ? ztsUrl.toString() : ""); builder.athenzDnsSuffix(athenzDnsSuffix != null ? athenzDnsSuffix : ""); - builder.nodeIdentityName("vespa.vespa.tenant"); // TODO Move to Oath configmodel amender + builder.nodeIdentityName(configServerDomain() + ".tenant"); // TODO Move to Oath configmodel amender builder.configserverIdentityName(getConfigserverIdentityName()); } // TODO Move to Oath configmodel amender private String getConfigserverIdentityName() { return String.format("%s.provider_%s_%s", - zone.system() == SystemName.main ? "vespa.vespa" : "vespa.vespa.cd", + configServerDomain(), zone.environment().value(), zone.region().value()); } + private String configServerDomain() { + return zone.system() == SystemName.main ? "vespa.vespa" : "vespa.vespa.cd"; + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java index 8ec4ae35658..201e0b5693a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java @@ -14,8 +14,6 @@ import org.w3c.dom.Element; /** * Config generation for common parameters for all fleet controllers. - * - * TODO: Author */ public class ClusterControllerConfig extends AnyConfigProducer implements FleetcontrollerConfig.Producer { @@ -23,11 +21,16 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc private final String clusterName; private final ModelElement clusterElement; private final ResourceLimits resourceLimits; + private final boolean allowMoreThanOneContentGroupDown; - public Builder(String clusterName, ModelElement clusterElement, ResourceLimits resourceLimits) { + public Builder(String clusterName, + ModelElement clusterElement, + ResourceLimits resourceLimits, + boolean allowMoreThanOneContentGroupDown) { this.clusterName = clusterName; this.clusterElement = clusterElement; this.resourceLimits = resourceLimits; + this.allowMoreThanOneContentGroupDown = allowMoreThanOneContentGroupDown; } @Override @@ -53,13 +56,15 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc tuning.childAsDouble("min-storage-up-ratio"), bucketSplittingMinimumBits, minNodeRatioPerGroup, - resourceLimits); + resourceLimits, + allowMoreThanOneContentGroupDown); } else { return new ClusterControllerConfig(ancestor, clusterName, null, null, null, null, null, null, bucketSplittingMinimumBits, minNodeRatioPerGroup, - resourceLimits); + resourceLimits, + allowMoreThanOneContentGroupDown); } } } @@ -74,6 +79,7 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc private final Integer minSplitBits; private final Double minNodeRatioPerGroup; private final ResourceLimits resourceLimits; + private final boolean allowMoreThanOneContentGroupDown; // TODO refactor; too many args private ClusterControllerConfig(TreeConfigProducer<?> parent, @@ -86,7 +92,8 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc Double minStorageUpRatio, Integer minSplitBits, Double minNodeRatioPerGroup, - ResourceLimits resourceLimits) { + ResourceLimits resourceLimits, + boolean allowMoreThanOneContentGroupDown) { super(parent, "fleetcontroller"); this.clusterName = clusterName; @@ -99,6 +106,7 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc this.minSplitBits = minSplitBits; this.minNodeRatioPerGroup = minNodeRatioPerGroup; this.resourceLimits = resourceLimits; + this.allowMoreThanOneContentGroupDown = allowMoreThanOneContentGroupDown; } @Override @@ -141,6 +149,7 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc builder.min_node_ratio_per_group(minNodeRatioPerGroup); } resourceLimits.getConfig(builder); + builder.max_number_of_groups_allowed_to_be_down(allowMoreThanOneContentGroupDown ? 1 : -1); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 7f4fc4cd89d..217c26516a9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -127,7 +127,8 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem .build(contentElement); c.clusterControllerConfig = new ClusterControllerConfig.Builder(clusterId, contentElement, - resourceLimits.getClusterControllerLimits()) + resourceLimits.getClusterControllerLimits(), + deployState.featureFlags().allowMoreThanOneContentGroupDown(new ClusterSpec.Id(clusterId))) .build(deployState, c, contentElement.getXml()); c.search = new ContentSearchCluster.Builder(documentDefinitions, globallyDistributedDocuments, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java index 2742dc59fcd..1984ceadac6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java @@ -23,6 +23,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.logging.Logger; import java.util.stream.Collectors; /** @@ -36,19 +37,24 @@ import java.util.stream.Collectors; */ public class OnnxModelInfo { + private static final Logger log = Logger.getLogger(OnnxModelInfo.class.getName()); + private final ApplicationPackage app; private final String modelPath; private final String defaultOutput; private final Map<String, OnnxTypeInfo> inputs; private final Map<String, OnnxTypeInfo> outputs; private final Map<String, TensorType> vespaTypes = new HashMap<>(); + private final Set<String> initializers; - private OnnxModelInfo(ApplicationPackage app, String path, Map<String, OnnxTypeInfo> inputs, Map<String, OnnxTypeInfo> outputs, String defaultOutput) { + private OnnxModelInfo(ApplicationPackage app, String path, Map<String, OnnxTypeInfo> inputs, + Map<String, OnnxTypeInfo> outputs, Set<String> initializers, String defaultOutput) { this.app = app; this.modelPath = path; this.inputs = Collections.unmodifiableMap(inputs); this.outputs = Collections.unmodifiableMap(outputs); this.defaultOutput = defaultOutput; + this.initializers = Set.copyOf(initializers); } public String getModelPath() { @@ -63,6 +69,8 @@ public class OnnxModelInfo { return outputs.keySet(); } + public Set<String> getInitializers() { return initializers; } + public String getDefaultOutput() { return defaultOutput; } @@ -191,15 +199,27 @@ public class OnnxModelInfo { } static private String onnxModelToJson(Onnx.ModelProto model, Path path) throws IOException { + var initializerNames = model.getGraph().getInitializerList().stream() + .map(Onnx.TensorProto::getName).collect(Collectors.toSet()); ByteArrayOutputStream out = new ByteArrayOutputStream(); JsonGenerator g = new JsonFactory().createGenerator(out, JsonEncoding.UTF8); g.writeStartObject(); g.writeStringField("path", path.toString()); g.writeArrayFieldStart("inputs"); + int skippedInput = 0; for (Onnx.ValueInfoProto valueInfo : model.getGraph().getInputList()) { + if (initializerNames.contains(valueInfo.getName())) { + log.fine(() -> "For '%s': skipping name '%s' as it's an initializer" + .formatted(path.getName(), valueInfo.getName())); + ++skippedInput; + continue; + } onnxTypeToJson(g, valueInfo); } + if (skippedInput > 0) + log.info("For '%s': skipped %d inputs that were also listed in initializers" + .formatted(path.getName(), skippedInput)); g.writeEndArray(); g.writeArrayFieldStart("outputs"); @@ -208,6 +228,14 @@ public class OnnxModelInfo { } g.writeEndArray(); + g.writeArrayFieldStart("initializers"); + for (Onnx.TensorProto initializers : model.getGraph().getInitializerList()) { + g.writeStartObject(); + g.writeStringField("name", initializers.getName()); + g.writeEndObject(); + } + g.writeEndArray(); + g.writeEndObject(); g.close(); return out.toString(); @@ -218,6 +246,7 @@ public class OnnxModelInfo { JsonNode root = m.readTree(json); Map<String, OnnxTypeInfo> inputs = new HashMap<>(); Map<String, OnnxTypeInfo> outputs = new HashMap<>(); + Set<String> initializers = new HashSet<>(); String defaultOutput = ""; String path = null; @@ -233,7 +262,13 @@ public class OnnxModelInfo { if (root.get("outputs").has(0)) { defaultOutput = root.get("outputs").get(0).get("name").textValue(); } - return new OnnxModelInfo(app, path, inputs, outputs, defaultOutput); + var initializerRoot = root.get("initializers"); + if (initializerRoot != null) { + for (JsonNode initializer : initializerRoot) { + initializers.add(initializer.get("name").textValue()); + } + } + return new OnnxModelInfo(app, path, inputs, outputs, initializers, defaultOutput); } static private void onnxTypeToJson(JsonGenerator g, Onnx.ValueInfoProto valueInfo) throws IOException { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java index ee18eceb719..2de06e2053a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java @@ -18,7 +18,6 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { private final static double SUMMARY_CACHE_SIZE_AS_FRACTION_OF_MEMORY = 0.04; private final static double MEMORY_GAIN_AS_FRACTION_OF_MEMORY = 0.08; private final static double MIN_MEMORY_PER_FLUSH_THREAD_GB = 16.0; - private final static double MAX_FLUSH_THREAD_RATIO = 1.0/8; private final static double TLS_SIZE_FRACTION = 0.02; final static long MB = 1024 * 1024; public final static long GB = MB * 1024; @@ -94,13 +93,12 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { } private void tuneFlushConcurrentThreads(ProtonConfig.Flush.Builder builder) { + int max_concurrent = 2; // TODO bring slowly up towards 4 if (usableMemoryGb() < MIN_MEMORY_PER_FLUSH_THREAD_GB) { - builder.maxconcurrent(1); + max_concurrent = 1; } - double min_concurrent_mem = usableMemoryGb() / (2*MIN_MEMORY_PER_FLUSH_THREAD_GB); - double min_concurrent_cpu = resources.vcpu() * MAX_FLUSH_THREAD_RATIO; - builder.maxconcurrent(Math.min(builder.build().maxconcurrent(), - (int)Math.ceil(Math.max(min_concurrent_mem, min_concurrent_cpu)))); + double min_concurrent_mem = usableMemoryGb() / MIN_MEMORY_PER_FLUSH_THREAD_GB; + builder.maxconcurrent(Math.min(max_concurrent, (int)Math.ceil(min_concurrent_mem))); } private void tuneFlushStrategyTlsSize(ProtonConfig.Flush.Memory.Builder builder) { diff --git a/config-model/src/test/derived/indexswitches/ilscripts.cfg b/config-model/src/test/derived/indexswitches/ilscripts.cfg index 77ac18e3261..5cda0a9fdc7 100644 --- a/config-model/src/test/derived/indexswitches/ilscripts.cfg +++ b/config-model/src/test/derived/indexswitches/ilscripts.cfg @@ -4,7 +4,7 @@ ilscript[].doctype "indexswitches" ilscript[].docfield[] "title" ilscript[].docfield[] "descr" ilscript[].docfield[] "source_src" -ilscript[].content[] "clear_state | guard { input source_src | switch { case \"theweb\": input source_src | tokenize normalize | summary source | index source; case \"amg\": input source_src | tokenize normalize | summary source; default: input source_src . \" partner\" | tokenize normalize | summary source | index source; }; }" +ilscript[].content[] "clear_state | guard { input source_src | switch { case \"amg\": input source_src | tokenize normalize | summary source; case \"theweb\": input source_src | tokenize normalize | summary source | index source; default: input source_src . \" partner\" | tokenize normalize | summary source | index source; }; }" ilscript[].content[] "clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }" ilscript[].content[] "clear_state | guard { input descr | tokenize normalize stem:\"BEST\" | summary descr | index descr; }" ilscript[].content[] "input source_src | passthrough source_src" diff --git a/config-model/src/test/derived/nearestneighbor_streaming/test.sd b/config-model/src/test/derived/nearestneighbor_streaming/test.sd new file mode 100644 index 00000000000..4427fa08ab6 --- /dev/null +++ b/config-model/src/test/derived/nearestneighbor_streaming/test.sd @@ -0,0 +1,24 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +schema test { + document test { + field vec_a type tensor<float>(x[16]) { + indexing: attribute + } + field vec_b type tensor<float>(x[16]) { + indexing: attribute + attribute { + distance-metric: angular + } + } + field vec_c type tensor<float>(m{},x[16]) { + indexing: attribute + attribute { + distance-metric: innerproduct + } + } + # This tensor field can not be used with nearest neighbor search. + field vec_d type tensor<float>(x{}) { + indexing: attribute + } + } +} diff --git a/config-model/src/test/derived/nearestneighbor_streaming/vsmfields.cfg b/config-model/src/test/derived/nearestneighbor_streaming/vsmfields.cfg new file mode 100644 index 00000000000..f8b1cf62048 --- /dev/null +++ b/config-model/src/test/derived/nearestneighbor_streaming/vsmfields.cfg @@ -0,0 +1,31 @@ +documentverificationlevel 0 +searchall 1 +fieldspec[].name "vec_a" +fieldspec[].searchmethod NEAREST_NEIGHBOR +fieldspec[].arg1 "EUCLIDEAN" +fieldspec[].maxlength 1048576 +fieldspec[].fieldtype ATTRIBUTE +fieldspec[].name "vec_b" +fieldspec[].searchmethod NEAREST_NEIGHBOR +fieldspec[].arg1 "ANGULAR" +fieldspec[].maxlength 1048576 +fieldspec[].fieldtype ATTRIBUTE +fieldspec[].name "vec_c" +fieldspec[].searchmethod NEAREST_NEIGHBOR +fieldspec[].arg1 "INNERPRODUCT" +fieldspec[].maxlength 1048576 +fieldspec[].fieldtype ATTRIBUTE +fieldspec[].name "vec_d" +fieldspec[].searchmethod NONE +fieldspec[].arg1 "" +fieldspec[].maxlength 1048576 +fieldspec[].fieldtype ATTRIBUTE +documenttype[].name "test" +documenttype[].index[].name "vec_a" +documenttype[].index[].field[].name "vec_a" +documenttype[].index[].name "vec_b" +documenttype[].index[].field[].name "vec_b" +documenttype[].index[].name "vec_c" +documenttype[].index[].field[].name "vec_c" +documenttype[].index[].name "vec_d" +documenttype[].index[].field[].name "vec_d" diff --git a/config-model/src/test/java/com/yahoo/schema/derived/NearestNeighborTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/NearestNeighborTestCase.java index b3a0b8d4558..713da6f5cbe 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/NearestNeighborTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/NearestNeighborTestCase.java @@ -35,4 +35,9 @@ public class NearestNeighborTestCase extends AbstractExportingTestCase { } } + @Test + void test_nearest_neighbor_streaming() throws IOException, ParseException { + assertCorrectDeriving("nearestneighbor_streaming"); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudWatchValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudWatchValidatorTest.java deleted file mode 100644 index f600900d3cd..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudWatchValidatorTest.java +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.application.validation; - -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.model.NullConfigModelRegistry; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.model.VespaModel; -import org.junit.jupiter.api.Test; -import org.xml.sax.SAXException; - -import java.io.IOException; - -import static com.yahoo.config.provision.Environment.prod; -import static org.junit.jupiter.api.Assertions.*; - - -/** - * @author gjoranv - */ -public class CloudWatchValidatorTest { - - @Test - void cloudwatch_in_public_zones_passes_validation() throws IOException, SAXException { - DeployState deployState = deployState(servicesWithCloudwatch(), true, true); - VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - - new CloudWatchValidator().validate(model, deployState); - } - - @Test - void cloudwatch_passes_validation_for_self_hosted_vespa() throws IOException, SAXException { - DeployState deployState = deployState(servicesWithCloudwatch(), false, false); - VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - - new CloudWatchValidator().validate(model, deployState); - } - - @Test - void cloudwatch_in_non_public_zones_fails_validation() throws IOException, SAXException { - DeployState deployState = deployState(servicesWithCloudwatch(), true, false); - VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - - try { - new CloudWatchValidator().validate(model, deployState); - fail(); - } catch (IllegalArgumentException e) { - assertEquals("CloudWatch cannot be set up for non-public hosted Vespa and must be removed for consumers: [cloudwatch-consumer]", - e.getMessage()); - } - } - - private static DeployState deployState(String servicesXml, boolean isHosted, boolean isPublic) { - ApplicationPackage app = new MockApplicationPackage.Builder() - .withServices(servicesXml) - .build(); - - DeployState.Builder builder = new DeployState.Builder() - .applicationPackage(app) - .properties(new TestProperties().setHostedVespa(isHosted)); - if (isHosted) { - var system = isPublic ? SystemName.Public : SystemName.main; - builder.zone(new Zone(system, Environment.prod, RegionName.from("foo"))); - } - final DeployState deployState = builder.build(); - - if (isHosted) { - assertTrue(deployState.isHosted(), "Test must emulate a hosted deployment."); - assertEquals(prod, deployState.zone().environment(), "Test must emulate a prod environment."); - } - return deployState; - } - - private String servicesWithCloudwatch() { - return String.join("\n", - "<services>", - " <admin version='2.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='cloudwatch-consumer'>", - " <metric id='my-metric'/>", - " <cloudwatch region='us-east-1' namespace='my-namespace' >", - " <credentials access-key-name='my-access-key' ", - " secret-key-name='my-secret-key' />", - " </cloudwatch>", - " </consumer>", - " </metrics>", - " </admin>", - "</services>" - ); - } - -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java index 5c360a9343f..65dfce8ff6c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java @@ -1,14 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change; +import com.yahoo.collections.Pair; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.provision.Environment; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.ValidationTester; import com.yahoo.yolean.Exceptions; import org.junit.jupiter.api.Test; +import java.util.List; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.fail; @@ -36,7 +40,12 @@ public class ContentClusterRemovalValidatorTest { @Test void testOverridingContentRemovalValidation() { VespaModel previous = tester.deploy(null, getServices("contentClusterId"), Environment.prod, null).getFirst(); - tester.deploy(previous, getServices("newContentClusterId"), Environment.prod, removalOverride); // Allowed due to override + var result = tester.deploy(previous, getServices("newContentClusterId"), Environment.prod, removalOverride); // Allowed due to override + assertEquals(result.getFirst().getContainerClusters().values().stream() + .flatMap(cluster -> cluster.getContainers().stream()) + .map(container -> container.getServiceInfo()) + .toList(), + result.getSecond().stream().flatMap(action -> action.getServices().stream()).toList()); } private static String getServices(String contentClusterId) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java index 1e6847a47be..1f8dea41a3e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java @@ -27,7 +27,8 @@ public class FleetControllerClusterTest { new ClusterResourceLimits.Builder(false, featureFlags.resourceLimitDisk(), featureFlags.resourceLimitMemory()) - .build(clusterElement).getClusterControllerLimits()) + .build(clusterElement).getClusterControllerLimits(), + false) .build(root.getDeployState(), root, clusterElement.getXml()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java index 9fe38512fc0..d344be3da9a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java @@ -182,14 +182,12 @@ public class NodeResourcesTuningTest { } @Test public void require_that_concurrent_flush_threads_is_1_with_low_memory() { - assertEquals(2, fromMemAndCpu(17, 9).flush().maxconcurrent()); - assertEquals(2, fromMemAndCpu(17, 64).flush().maxconcurrent()); // still capped by max - assertEquals(2, fromMemAndCpu(65, 8).flush().maxconcurrent()); // still capped by max - assertEquals(2, fromMemAndCpu(33, 8).flush().maxconcurrent()); - assertEquals(1, fromMemAndCpu(31, 8).flush().maxconcurrent()); - assertEquals(1, fromMemAndCpu(15, 8).flush().maxconcurrent()); - assertEquals(1, fromMemAndCpu(17, 8).flush().maxconcurrent()); + assertEquals(1, fromMemAndCpu(1, 8).flush().maxconcurrent()); assertEquals(1, fromMemAndCpu(15, 8).flush().maxconcurrent()); + assertEquals(1, fromMemAndCpu(16, 8).flush().maxconcurrent()); + assertEquals(2, fromMemAndCpu(17, 8).flush().maxconcurrent()); + assertEquals(2, fromMemAndCpu(65, 8).flush().maxconcurrent()); // still capped by max + assertEquals(2, fromMemAndCpu(65, 65).flush().maxconcurrent()); // still capped by max } private static void assertDocumentStoreMaxFileSize(long expFileSizeBytes, int wantedMemoryGb) { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java index 5d5775275c3..17025b10568 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java @@ -23,7 +23,9 @@ import java.util.stream.Collectors; import static java.nio.file.Files.readAttributes; /** - * Deletes file references and url downloads that have not been used for some time + * Deletes file references and url downloads that have not been used for some time. + * See {@link com.yahoo.vespa.config.proxy.filedistribution.RequestTracker} for how we track + * when a file reference or download was last used. * * @author hmusum */ @@ -32,7 +34,7 @@ class FileReferencesAndDownloadsMaintainer implements Runnable { private static final Logger log = Logger.getLogger(FileReferencesAndDownloadsMaintainer.class.getName()); private static final File defaultUrlDownloadDir = UrlDownloadRpcServer.downloadDir; private static final File defaultFileReferencesDownloadDir = FileDownloader.defaultDownloadDirectory; - private static final Duration defaultDurationToKeepFiles = Duration.ofDays(14); + private static final Duration defaultDurationToKeepFiles = Duration.ofDays(21); private static final Duration interval = Duration.ofMinutes(1); private final ScheduledExecutorService executor = diff --git a/configdefinitions/src/vespa/fleetcontroller.def b/configdefinitions/src/vespa/fleetcontroller.def index 10eb408ed69..98b4c3b0216 100644 --- a/configdefinitions/src/vespa/fleetcontroller.def +++ b/configdefinitions/src/vespa/fleetcontroller.def @@ -64,11 +64,6 @@ init_progress_time int default=0 ## we dont change the state too often. min_time_between_new_systemstates int default=10000 -## Sets how many milliseconds to wait between each state poll for old nodes -## requiring state polling. (4.1 or older) -## TODO: Not used, remove in Vespa 9 -state_polling_frequency int default=5000 - ## The maximum amount of premature crashes a node is allowed to have in a row ## before the fleetcontroller disables that node. max_premature_crashes int default=100000 @@ -181,9 +176,6 @@ min_merge_completion_ratio double default=1.0 ## transition logic aims to minimize the window of time where active states diverge. enable_two_phase_cluster_state_transitions bool default=false -## Deprecated - not used -determine_buckets_from_bucket_space_metric bool default=true - # If enabled, the cluster controller observes reported (categorized) resource usage from content nodes (via host info), # and decides whether external feed should be blocked (or unblocked) in the entire cluster. # diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index f6988a6b566..955b1bc8f4f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -534,7 +534,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye NestedTransaction transaction = new NestedTransaction(); Optional<ApplicationTransaction> applicationTransaction = hostProvisioner.map(provisioner -> provisioner.lock(applicationId)) .map(lock -> new ApplicationTransaction(lock, transaction)); - try (var applicationLock = tenantApplications.lock(applicationId)) { + try (@SuppressWarnings("unused") var applicationLock = tenantApplications.lock(applicationId)) { Optional<Long> activeSession = tenantApplications.activeSessionOf(applicationId); CompletionWaiter waiter; if (activeSession.isPresent()) { @@ -796,7 +796,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye NestedTransaction transaction = new NestedTransaction(); Optional<ApplicationTransaction> applicationTransaction = hostProvisioner.map(provisioner -> provisioner.lock(applicationId)) .map(lock -> new ApplicationTransaction(lock, transaction)); - try (var sessionLock = tenant.getApplicationRepo().lock(applicationId)) { + try (@SuppressWarnings("unused") var sessionLock = tenant.getApplicationRepo().lock(applicationId)) { Optional<Session> activeSession = getActiveSession(applicationId); var sessionZooKeeperClient = tenant.getSessionRepository().createSessionZooKeeperClient(session.getSessionId()); CompletionWaiter waiter = sessionZooKeeperClient.createActiveWaiter(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 88e3134ccad..cddcb0f316d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -74,7 +74,7 @@ public class TenantApplications implements RequestHandler, HostValidator { private final MetricUpdater tenantMetricUpdater; private final Clock clock; private final TenantFileSystemDirs tenantFileSystemDirs; - private final ConfigserverConfig configserverConfig; + private final String serverId; private final ListFlag<String> incompatibleVersions; public TenantApplications(TenantName tenant, Curator curator, StripedExecutor<TenantName> zkWatcherExecutor, @@ -95,7 +95,7 @@ public class TenantApplications implements RequestHandler, HostValidator { this.hostRegistry = hostRegistry; this.tenantFileSystemDirs = tenantFileSystemDirs; this.clock = clock; - this.configserverConfig = configserverConfig; + this.serverId = configserverConfig.serverId(); this.incompatibleVersions = PermanentFlags.INCOMPATIBLE_VERSIONS.bindTo(flagSource); } @@ -230,7 +230,7 @@ public class TenantApplications implements RequestHandler, HostValidator { */ public void activateApplication(ApplicationSet applicationSet, long activeSessionId) { ApplicationId id = applicationSet.getId(); - try (Lock lock = lock(id)) { + try (@SuppressWarnings("unused") Lock lock = lock(id)) { if ( ! exists(id)) return; // Application was deleted before activation. if (applicationSet.getApplicationGeneration() != activeSessionId) @@ -269,7 +269,7 @@ public class TenantApplications implements RequestHandler, HostValidator { public void removeApplicationsExcept(Set<ApplicationId> applications) { for (ApplicationId activeApplication : applicationMapper.listApplicationIds()) { if ( ! applications.contains(activeApplication)) { - try (var applicationLock = lock(activeApplication)){ + try (@SuppressWarnings("unused") var applicationLock = lock(activeApplication)){ removeApplication(activeApplication); } } @@ -404,11 +404,11 @@ public class TenantApplications implements RequestHandler, HostValidator { public TenantFileSystemDirs getTenantFileSystemDirs() { return tenantFileSystemDirs; } public CompletionWaiter createRemoveApplicationWaiter(ApplicationId applicationId) { - return RemoveApplicationWaiter.createAndInitialize(curator, applicationId, configserverConfig.serverId()); + return RemoveApplicationWaiter.createAndInitialize(curator, applicationId, serverId); } public CompletionWaiter getRemoveApplicationWaiter(ApplicationId applicationId) { - return RemoveApplicationWaiter.create(curator, applicationId, configserverConfig.serverId()); + return RemoveApplicationWaiter.create(curator, applicationId, serverId); } /** diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 7a2377594a1..62431ce4c06 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -22,10 +22,10 @@ import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; +import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.config.provision.HostName; import com.yahoo.vespa.config.server.tenant.SecretStoreExternalIdRetriever; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; @@ -33,7 +33,6 @@ import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.flags.UnboundFlag; - import java.io.File; import java.net.URI; import java.security.cert.X509Certificate; @@ -41,7 +40,7 @@ import java.util.List; import java.util.Optional; import java.util.Set; import java.util.concurrent.ExecutorService; -import java.util.function.ToIntFunction; +import java.util.function.Predicate; import static com.yahoo.config.provision.NodeResources.Architecture; import static com.yahoo.vespa.config.server.ConfigServerSpec.fromConfig; @@ -174,7 +173,7 @@ public class ModelContextImpl implements ModelContext { private final double feedNiceness; private final List<String> allowedAthenzProxyIdentities; private final int maxActivationInhibitedOutOfSyncGroups; - private final ToIntFunction<ClusterSpec.Type> jvmOmitStackTraceInFastThrow; + private final Predicate<ClusterSpec.Type> jvmOmitStackTraceInFastThrow; private final double resourceLimitDisk; private final double resourceLimitMemory; private final double minNodeRatioPerGroup; @@ -204,6 +203,7 @@ public class ModelContextImpl implements ModelContext { private final int heapPercentage; private final boolean enableGlobalPhase; private final String summaryDecodePolicy; + private final Predicate<ClusterSpec.Id> allowMoreThanOneContentGroupDown; public FeatureFlags(FlagSource source, ApplicationId appId, Version version) { this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -219,7 +219,7 @@ public class ModelContextImpl implements ModelContext { this.mbus_network_threads = flagValue(source, appId, version, Flags.MBUS_NUM_NETWORK_THREADS); this.allowedAthenzProxyIdentities = flagValue(source, appId, version, Flags.ALLOWED_ATHENZ_PROXY_IDENTITIES); this.maxActivationInhibitedOutOfSyncGroups = flagValue(source, appId, version, Flags.MAX_ACTIVATION_INHIBITED_OUT_OF_SYNC_GROUPS); - this.jvmOmitStackTraceInFastThrow = type -> flagValueAsInt(source, appId, version, type, PermanentFlags.JVM_OMIT_STACK_TRACE_IN_FAST_THROW); + this.jvmOmitStackTraceInFastThrow = type -> flagValue(source, appId, version, type, PermanentFlags.JVM_OMIT_STACK_TRACE_IN_FAST_THROW); this.resourceLimitDisk = flagValue(source, appId, version, PermanentFlags.RESOURCE_LIMIT_DISK); this.resourceLimitMemory = flagValue(source, appId, version, PermanentFlags.RESOURCE_LIMIT_MEMORY); this.minNodeRatioPerGroup = flagValue(source, appId, version, Flags.MIN_NODE_RATIO_PER_GROUP); @@ -250,6 +250,7 @@ public class ModelContextImpl implements ModelContext { this.heapPercentage = flagValue(source, appId, version, PermanentFlags.HEAP_SIZE_PERCENTAGE); this.enableGlobalPhase = flagValue(source, appId, version, Flags.ENABLE_GLOBAL_PHASE); this.summaryDecodePolicy = flagValue(source, appId, version, Flags.SUMMARY_DECODE_POLICY); + this.allowMoreThanOneContentGroupDown = clusterId -> flagValue(source, appId, version, clusterId, Flags.ALLOW_MORE_THAN_ONE_CONTENT_GROUP_DOWN); } @Override public int heapSizePercentage() { return heapPercentage; } @@ -270,7 +271,7 @@ public class ModelContextImpl implements ModelContext { @Override public List<String> allowedAthenzProxyIdentities() { return allowedAthenzProxyIdentities; } @Override public int maxActivationInhibitedOutOfSyncGroups() { return maxActivationInhibitedOutOfSyncGroups; } @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { - return translateJvmOmitStackTraceInFastThrowIntToString(jvmOmitStackTraceInFastThrow, type); + return translateJvmOmitStackTraceInFastThrowToString(jvmOmitStackTraceInFastThrow, type); } @Override public double resourceLimitDisk() { return resourceLimitDisk; } @Override public double resourceLimitMemory() { return resourceLimitMemory; } @@ -304,6 +305,7 @@ public class ModelContextImpl implements ModelContext { } @Override public boolean useRestrictedDataPlaneBindings() { return useRestrictedDataPlaneBindings; } @Override public boolean enableGlobalPhase() { return enableGlobalPhase; } + @Override public boolean allowMoreThanOneContentGroupDown(ClusterSpec.Id id) { return allowMoreThanOneContentGroupDown.test(id); } private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) @@ -331,17 +333,21 @@ public class ModelContextImpl implements ModelContext { .boxedValue(); } - static int flagValueAsInt(FlagSource source, - ApplicationId appId, - Version version, - ClusterSpec.Type clusterType, - UnboundFlag<? extends Boolean, ?, ?> flag) { - return flagValue(source, appId, version, clusterType, flag) ? 1 : 0; + private static <V> V flagValue(FlagSource source, + ApplicationId appId, + Version vespaVersion, + ClusterSpec.Id clusterId, + UnboundFlag<? extends V, ?, ?> flag) { + return flag.bindTo(source) + .with(FetchVector.Dimension.APPLICATION_ID, appId.serializedForm()) + .with(FetchVector.Dimension.CLUSTER_ID, clusterId.value()) + .with(FetchVector.Dimension.VESPA_VERSION, vespaVersion.toFullString()) + .boxedValue(); } - private String translateJvmOmitStackTraceInFastThrowIntToString(ToIntFunction<ClusterSpec.Type> function, - ClusterSpec.Type clusterType) { - return function.applyAsInt(clusterType) == 1 ? "" : "-XX:-OmitStackTraceInFastThrow"; + private String translateJvmOmitStackTraceInFastThrowToString(Predicate<ClusterSpec.Type> function, + ClusterSpec.Type clusterType) { + return function.test(clusterType) ? "" : "-XX:-OmitStackTraceInFastThrow"; } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index 86c0c90ca12..62a1704b350 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -97,8 +97,6 @@ public class ApplicationHandler extends HttpHandler { if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/filedistributionstatus")) return filedistributionStatus(applicationId(path), request); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/logs")) return logs(applicationId(path), request); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/metrics/deployment")) return deploymentMetrics(applicationId(path)); - // TODO: Remove when all usage has migrated to .../metrics/searchnode - if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/metrics/proton")) return searchNodeMetrics(applicationId(path)); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/metrics/searchnode")) return searchNodeMetrics(applicationId(path)); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindexing")) return getReindexingStatus(applicationId(path)); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/service/{service}/{hostname}/status/{*}")) return serviceStatusPage(applicationId(path), path.get("service"), path.get("hostname"), path.getRest(), request); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java index 2948b82dd96..22ef6cc2547 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java @@ -88,7 +88,7 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer { createLocalSessionIfMissing(applicationId, sessionId); } } - return asSuccessFactor(attempts, failures); + return asSuccessFactorDeviation(attempts, failures); } private static FileDownloader createFileDownloader(List<String> otherConfigServersInCluster, diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java index dad687aae67..ef48c3bdb98 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java @@ -46,9 +46,9 @@ public abstract class ConfigServerMaintainer extends Maintainer { } @Override - public void completed(String job, double successFactor, long durationMs) { + public void completed(String job, double successFactorDeviation, long durationMs) { var context = metric.createContext(Map.of("job", job)); - metric.set("maintenance.successFactor", successFactor, context); + metric.set("maintenance.successFactorDeviation", successFactorDeviation, context); metric.set("maintenance.duration", durationMs, context); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java index a9672455d09..e81c9b6bcbd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java @@ -77,7 +77,7 @@ public class ReindexingMaintainer extends ConfigServerMaintainer { } }); } - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactorDeviation(attempts.get(), failures.get()); } private Supplier<Long> lazyGeneration(Application application) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java index 728f3e8510f..2ad04fdd572 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java @@ -12,8 +12,8 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.ConfigKey; -import com.yahoo.vespa.config.server.ConfigServerDB; import com.yahoo.vespa.config.server.ConfigActivationListener; +import com.yahoo.vespa.config.server.ConfigServerDB; import com.yahoo.vespa.config.server.ServerCache; import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; import com.yahoo.vespa.config.server.host.HostRegistry; @@ -26,7 +26,6 @@ import com.yahoo.vespa.config.server.tenant.TestTenantRepository; import com.yahoo.vespa.curator.CompletionTimeoutException; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.curator.mock.MockCuratorFramework; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.model.VespaModel; @@ -37,22 +36,17 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.xml.sax.SAXException; - import java.io.File; import java.io.IOException; import java.time.Clock; import java.time.Duration; import java.util.Arrays; -import java.util.Collection; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.IntStream; -import static com.yahoo.vespa.config.server.application.TenantApplications.RemoveApplicationWaiter; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -195,7 +189,6 @@ public class TenantApplicationsTest { public static class MockConfigActivationListener implements ConfigActivationListener { public final AtomicInteger activated = new AtomicInteger(0); final AtomicInteger removed = new AtomicInteger(0); - final Map<String, Collection<String>> tenantHosts = new LinkedHashMap<>(); @Override public void configActivated(ApplicationSet application) { @@ -247,22 +240,22 @@ public class TenantApplicationsTest { @Test public void testRemoveApplication2of3Respond() throws InterruptedException { - Curator curator = new MockCurator3ConfigServers(); - Thread t1 = setupWaiter(curator); - notifyCompletion(curator, 2); + TenantApplications applications = createZKAppRepo(new InMemoryFlagSource()); + Thread t1 = setupWaiter(applications); + notifyCompletion(applications, 2); t1.join(); } @Test public void testRemoveApplicationAllRespond() throws InterruptedException { - Curator curator = new MockCurator3ConfigServers(); - Thread t1 = setupWaiter(curator); - notifyCompletion(curator, 3); + TenantApplications applications = createZKAppRepo(new InMemoryFlagSource()); + Thread t1 = setupWaiter(applications); + notifyCompletion(applications, 3); t1.join(); } - private Thread setupWaiter(Curator curator) { - Curator.CompletionWaiter waiter = RemoveApplicationWaiter.createAndInitialize(curator, createApplicationId(), "cfg1", Duration.ofSeconds(1)); + private Thread setupWaiter(TenantApplications applications) { + Curator.CompletionWaiter waiter = applications.getRemoveApplicationWaiter(createApplicationId()); Thread t1 = new Thread(() -> { try { waiter.awaitCompletion(Duration.ofSeconds(120)); @@ -274,10 +267,10 @@ public class TenantApplicationsTest { return t1; } - private void notifyCompletion(Curator curator, int respondentCount) { + private void notifyCompletion(TenantApplications applications, int respondentCount) { IntStream.range(0, respondentCount) - .forEach(i -> RemoveApplicationWaiter.create(curator, createApplicationId(), "cfg" + i, Duration.ofSeconds(1)) - .notifyCompletion()); + .forEach(i -> applications.createRemoveApplicationWaiter(createApplicationId()) + .notifyCompletion()); } private TenantApplications createZKAppRepo() { @@ -332,12 +325,4 @@ public class TenantApplicationsTest { flagSource); } - private static class MockCurator3ConfigServers extends Curator { - - public MockCurator3ConfigServers() { - super("host1:2181,host2:2181,host3:2181", "host1:2181,host2:2181,host3:2181", (retryPolicy) -> new MockCuratorFramework(true, false)); - } - - } - } diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 36531fbf5e1..84411b31274 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -209,8 +209,11 @@ ], "methods" : [ "public void <init>(byte[])", + "public byte[] value()", "public int compareTo(com.yahoo.prelude.hitfield.RawBase64)", "public java.lang.String toString()", + "public boolean equals(java.lang.Object)", + "public int hashCode()", "public bridge synthetic int compareTo(java.lang.Object)" ], "fields" : [ ] diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java index 88a37ea5a02..92ce6abb319 100644 --- a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java +++ b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java @@ -6,11 +6,11 @@ import com.yahoo.search.Query; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import static com.yahoo.text.Lowercase.toLowerCase; @@ -32,6 +32,16 @@ public class IndexFacts { private Map<String, List<String>> clusterByDocument; + private static class DocumentTypeListOffset { + public final int offset; + public final SearchDefinition searchDefinition; + + public DocumentTypeListOffset(int offset, SearchDefinition searchDefinition) { + this.offset = offset; + this.searchDefinition = searchDefinition; + } + } + /** A Map of all known search definitions indexed by name */ private Map<String, SearchDefinition> searchDefinitions = new LinkedHashMap<>(); @@ -100,32 +110,34 @@ public class IndexFacts { private boolean isIndexFromDocumentTypes(String indexName, List<String> documentTypes) { if ( ! isInitialized()) return true; - if (documentTypes.isEmpty()) return unionSearchDefinition.getIndex(indexName) != null; + if (documentTypes.isEmpty()) { + return unionSearchDefinition.getIndex(indexName) != null; + } - for (String docName : documentTypes) { - SearchDefinition sd = searchDefinitions.get(docName); - if (sd != null) { - Index index = sd.getIndex(indexName); - if (index != null) return true; + DocumentTypeListOffset sd = chooseSearchDefinition(documentTypes, 0); + while (sd != null) { + Index index = sd.searchDefinition.getIndex(indexName); + if (index != null) { + return true; } + sd = chooseSearchDefinition(documentTypes, sd.offset); } + return false; } private String getCanonicNameFromDocumentTypes(String indexName, List<String> documentTypes) { if (!isInitialized()) return indexName; - String lowerCased = toLowerCase(indexName); if (documentTypes.isEmpty()) { - Index index = unionSearchDefinition.getIndexByLowerCase(lowerCased); + Index index = unionSearchDefinition.getIndexByLowerCase(toLowerCase(indexName)); return index == null ? indexName : index.getName(); } - for (String docName : documentTypes) { - SearchDefinition sd = searchDefinitions.get(docName); - if (sd != null) { - Index index = sd.getIndexByLowerCase(lowerCased); - if (index != null) return index.getName(); - } + DocumentTypeListOffset sd = chooseSearchDefinition(documentTypes, 0); + while (sd != null) { + Index index = sd.searchDefinition.getIndexByLowerCase(toLowerCase(indexName)); + if (index != null) return index.getName(); + sd = chooseSearchDefinition(documentTypes, sd.offset); } return indexName; } @@ -146,12 +158,13 @@ public class IndexFacts { return index; } - for (String docName : documentTypes) { - SearchDefinition sd = searchDefinitions.get(docName); - if (sd != null) { - Index index = sd.getIndex(canonicName); - if (index != null) return index; - } + DocumentTypeListOffset sd = chooseSearchDefinition(documentTypes, 0); + + while (sd != null) { + Index index = sd.searchDefinition.getIndex(canonicName); + + if (index != null) return index; + sd = chooseSearchDefinition(documentTypes, sd.offset); } return Index.nullIndex; } @@ -174,7 +187,7 @@ public class IndexFacts { * Given a search list which is a mixture of document types and cluster * names, and a restrict list which is a list of document types, return a * set of all valid document types for this combination. Most use-cases for - * fetching index settings will involve calling this method with the + * fetching index settings will involve calling this method with the the * incoming query's {@link com.yahoo.search.query.Model#getSources()} and * {@link com.yahoo.search.query.Model#getRestrict()} as input parameters * before calling any other method of this class. @@ -183,20 +196,20 @@ public class IndexFacts { * @param restrict the restrict list for a query * @return a (possibly empty) set of valid document types */ - private Set<String> resolveDocumentTypes(Collection<String> sources, Set<String> restrict, + private Set<String> resolveDocumentTypes(Collection<String> sources, Collection<String> restrict, Set<String> candidateDocumentTypes) { sources = emptyCollectionIfNull(sources); - restrict = emptySetIfNull(restrict); + restrict = emptyCollectionIfNull(restrict); if (sources.isEmpty()) { if ( ! restrict.isEmpty()) { - return Set.copyOf(restrict); + return new TreeSet<>(restrict); } else { return candidateDocumentTypes; } } - Set<String> toSearch = new HashSet<>(); + Set<String> toSearch = new TreeSet<>(); for (String source : sources) { // source: a document type or a cluster containing them List<String> clusterDocTypes = clusters.get(source); if (clusterDocTypes == null) { // source was a document type @@ -222,8 +235,21 @@ public class IndexFacts { private Collection<String> emptyCollectionIfNull(Collection<String> collection) { return collection == null ? List.of() : collection; } - private Set<String> emptySetIfNull(Set<String> collection) { - return collection == null ? Set.of() : collection; + + /** + * Chooses the correct search definition, default if in doubt. + * + * @return the search definition to use + */ + private DocumentTypeListOffset chooseSearchDefinition(List<String> documentTypes, int index) { + while (index < documentTypes.size()) { + String docName = documentTypes.get(index++); + SearchDefinition sd = searchDefinitions.get(docName); + if (sd != null) { + return new DocumentTypeListOffset(index, sd); + } + } + return null; } /** @@ -253,6 +279,10 @@ public class IndexFacts { return frozen; } + private void ensureNotFrozen() { + if (frozen) throw new IllegalStateException("Tried to modify frozen IndexFacts instance."); + } + public String getDefaultPosition(String sdName) { SearchDefinition sd; if (sdName == null) { @@ -270,16 +300,12 @@ public class IndexFacts { return new Session(query); } - public Session newSession() { - return new Session(Set.of(), Set.of()); - } - - public Session newSession(Collection<String> sources, Set<String> restrict) { + public Session newSession(Collection<String> sources, Collection<String> restrict) { return new Session(sources, restrict); } public Session newSession(Collection<String> sources, - Set<String> restrict, + Collection<String> restrict, Set<String> candidateDocumentTypes) { return new Session(sources, restrict, candidateDocumentTypes); } @@ -297,12 +323,12 @@ public class IndexFacts { documentTypes = List.copyOf(resolveDocumentTypes(query)); } - private Session(Collection<String> sources, Set<String> restrict) { + private Session(Collection<String> sources, Collection<String> restrict) { // Assumption: Search definition name equals document name. documentTypes = List.copyOf(resolveDocumentTypes(sources, restrict, searchDefinitions.keySet())); } - private Session(Collection<String> sources, Set<String> restrict, Set<String> candidateDocumentTypes) { + private Session(Collection<String> sources, Collection<String> restrict, Set<String> candidateDocumentTypes) { documentTypes = List.copyOf(resolveDocumentTypes(sources, restrict, candidateDocumentTypes)); } diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index f0e3e3f3e44..46332d632fe 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.cluster; -import com.yahoo.component.ComponentId; import com.yahoo.component.annotation.Inject; +import com.yahoo.component.ComponentId; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.QrSearchersConfig; @@ -28,6 +28,10 @@ import com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher; import com.yahoo.yolean.Exceptions; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Set; import java.util.UUID; @@ -53,7 +57,8 @@ public class ClusterSearcher extends Searcher { private final String searchClusterName; - private final SchemaResolver schemaResolver; + // The set of document types contained in this search cluster + private final Set<String> schemas; private final long maxQueryTimeout; // in milliseconds private final long maxQueryCacheTimeout; // in milliseconds @@ -79,7 +84,7 @@ public class ClusterSearcher extends Searcher { searchClusterName = clusterConfig.clusterName(); QrSearchersConfig.Searchcluster searchClusterConfig = getSearchClusterConfigFromClusterName(qrsConfig, searchClusterName); this.globalPhaseRanker = searchClusterConfig.globalphase() ? globalPhaseRanker : null; - this.schemaResolver = new SchemaResolver(documentDbConfig); + schemas = new LinkedHashSet<>(); maxQueryTimeout = ParameterParser.asMilliSeconds(clusterConfig.maxQueryTimeout(), DEFAULT_MAX_QUERY_TIMEOUT); maxQueryCacheTimeout = ParameterParser.asMilliSeconds(clusterConfig.maxQueryCacheTimeout(), DEFAULT_MAX_QUERY_CACHE_TIMEOUT); @@ -88,6 +93,9 @@ public class ClusterSearcher extends Searcher { .com().yahoo().prelude().fastsearch().FastSearcher().docsum() .defaultclass()); + for (DocumentdbInfoConfig.Documentdb docDb : documentDbConfig.documentdb()) + schemas.add(docDb.name()); + String uniqueServerId = UUID.randomUUID().toString(); if (searchClusterConfig.indexingmode() == STREAMING) { server = vdsCluster(uniqueServerId, searchClusterIndex, @@ -149,7 +157,7 @@ public class ClusterSearcher extends Searcher { /** Do not use, for internal testing purposes only. **/ ClusterSearcher(Set<String> schemas, VespaBackEndSearcher searcher, Executor executor) { - this.schemaResolver = new SchemaResolver(schemas); + this.schemas = schemas; searchClusterName = "testScenario"; maxQueryTimeout = DEFAULT_MAX_QUERY_TIMEOUT; maxQueryCacheTimeout = DEFAULT_MAX_QUERY_CACHE_TIMEOUT; @@ -221,9 +229,8 @@ public class ClusterSearcher extends Searcher { } private Result doSearch(Searcher searcher, Query query, Execution execution) { - var schemas = schemaResolver.resolve(query, execution); if (schemas.size() > 1) { - return searchMultipleDocumentTypes(searcher, query, execution, schemas); + return searchMultipleDocumentTypes(searcher, query, execution); } else { String docType = schemas.iterator().next(); query.getModel().setRestrict(docType); @@ -259,7 +266,8 @@ public class ClusterSearcher extends Searcher { } } - private Result searchMultipleDocumentTypes(Searcher searcher, Query query, Execution execution, Set<String> schemas) { + private Result searchMultipleDocumentTypes(Searcher searcher, Query query, Execution execution) { + Set<String> schemas = resolveSchemas(query, execution.context().getIndexFacts()); List<Query> queries = createQueries(query, schemas); if (queries.size() == 1) { return perSchemaSearch(searcher, queries.get(0), execution); @@ -293,7 +301,25 @@ public class ClusterSearcher extends Searcher { } Set<String> resolveSchemas(Query query, IndexFacts indexFacts) { - return schemaResolver.resolve(query, indexFacts); + Set<String> restrict = query.getModel().getRestrict(); + if (restrict == null || restrict.isEmpty()) { + Set<String> sources = query.getModel().getSources(); + return (sources == null || sources.isEmpty()) + ? schemas + : new HashSet<>(indexFacts.newSession(sources, Collections.emptyList(), schemas).documentTypes()); + } else { + return filterValidDocumentTypes(restrict); + } + } + + private Set<String> filterValidDocumentTypes(Collection<String> restrict) { + Set<String> retval = new LinkedHashSet<>(); + for (String docType : restrict) { + if (docType != null && schemas.contains(docType)) { + retval.add(docType); + } + } + return retval; } private List<Query> createQueries(Query query, Set<String> docTypes) { diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/SchemaResolver.java b/container-search/src/main/java/com/yahoo/prelude/cluster/SchemaResolver.java deleted file mode 100644 index 3a2125d1d38..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/SchemaResolver.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.prelude.cluster; - -import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; -import com.yahoo.search.Query; -import com.yahoo.search.searchchain.Execution; - -import java.util.Collection; -import java.util.LinkedHashSet; -import java.util.Set; - -/** - * Resolves schemas from query and execution context - * - * @author bjorncs - */ -class SchemaResolver { - - private final Set<String> schemas; - - SchemaResolver(DocumentdbInfoConfig cfg) { - this(cfg.documentdb().stream().map(DocumentdbInfoConfig.Documentdb::name).toList()); - } - - SchemaResolver(Collection<String> schemas) { - this.schemas = new LinkedHashSet<>(schemas); - } - - Set<String> resolve(Query query, Execution execution) { - return resolve(query, execution.context().getIndexFacts()); - } - - Set<String> resolve(Query query, IndexFacts indexFacts) { - if (schemas.size() == 1) return Set.of(schemas.iterator().next()); - var restrict = query.getModel().getRestrict(); - if (restrict == null || restrict.isEmpty()) { - Set<String> sources = query.getModel().getSources(); - return (sources == null || sources.isEmpty()) - ? schemas - : new LinkedHashSet<>(indexFacts.newSession(sources, Set.of(), schemas).documentTypes()); - } else { - return filterValidDocumentTypes(restrict); - } - } - - private Set<String> filterValidDocumentTypes(Collection<String> restrict) { - Set<String> retval = new LinkedHashSet<>(); - for (String docType : restrict) { - if (docType != null && schemas.contains(docType)) { - retval.add(docType); - } - } - return retval; - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/RawBase64.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/RawBase64.java index 2071e43f54c..485e2c9a8c3 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/RawBase64.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/RawBase64.java @@ -3,16 +3,22 @@ package com.yahoo.prelude.hitfield; import java.util.Arrays; import java.util.Base64; +import java.util.Objects; /** + * Wraps a byte [] and renders it as base64 encoded string * @author baldersheim */ public class RawBase64 implements Comparable<RawBase64> { + private final static Base64.Encoder encoder = Base64.getEncoder().withoutPadding(); private final byte[] content; public RawBase64(byte[] content) { + Objects.requireNonNull(content); this.content = content; } + public byte [] value() { return content; } + @Override public int compareTo(RawBase64 rhs) { return Arrays.compareUnsigned(content, rhs.content); @@ -20,6 +26,19 @@ public class RawBase64 implements Comparable<RawBase64> { @Override public String toString() { - return Base64.getEncoder().encodeToString(content); + return encoder.encodeToString(content); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RawBase64 rawBase64 = (RawBase64) o; + return Arrays.equals(content, rawBase64.content); + } + + @Override + public int hashCode() { + return Arrays.hashCode(content); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/MultiRangeItem.java b/container-search/src/main/java/com/yahoo/prelude/query/MultiRangeItem.java index 7ba7a13936f..3dac4cb92c0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/MultiRangeItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/MultiRangeItem.java @@ -271,7 +271,7 @@ public class MultiRangeItem<Type extends Number> extends MultiTermItem { if (endInclusive) metadata |= 0b00000100; encoder = type.encoderFor(sortedRanges()); - metadata |= encoder.id << 3; + metadata |= (byte)(encoder.id << 3); buffer.put(metadata); putString(startIndex, buffer); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/MultiTermItem.java b/container-search/src/main/java/com/yahoo/prelude/query/MultiTermItem.java index a7ca62d153c..03a661499e0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/MultiTermItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/MultiTermItem.java @@ -67,8 +67,8 @@ abstract class MultiTermItem extends SimpleTaggableItem { super.encodeThis(buffer); byte metadata = 0; - metadata |= (operatorType().code << 5) & 0b11100000; - metadata |= ( termType().code ) & 0b00011111; + metadata |= (byte)((byte)(operatorType().code << 5) & (byte)0b11100000); + metadata |= (byte)(termType().code & (byte)0b00011111); buffer.put(metadata); buffer.putInt(terms()); encodeBlueprint(buffer); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/CustomParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/CustomParser.java index 2bd408220cd..e3b2278475b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/CustomParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/CustomParser.java @@ -6,6 +6,7 @@ import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.query.Item; import com.yahoo.search.query.parser.Parser; +import java.util.Collections; import java.util.Set; /** @@ -22,7 +23,7 @@ public interface CustomParser extends Parser { Set<String> toSearch, IndexFacts indexFacts, String defaultIndexName) { if (indexFacts == null) indexFacts = new IndexFacts(); - return parse(queryToParse, filterToParse, parsingLanguage, indexFacts.newSession(toSearch, Set.of()), defaultIndexName); + return parse(queryToParse, filterToParse, parsingLanguage, indexFacts.newSession(toSearch, Collections.emptySet()), defaultIndexName); } Item parse(String queryToParse, String filterToParse, Language parsingLanguage, diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java index 9952ec64d13..c1d415b8e27 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java @@ -8,6 +8,7 @@ import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.query.Substring; +import java.util.Collections; import java.util.List; import static com.yahoo.prelude.query.parser.Token.Kind.*; @@ -62,7 +63,7 @@ public final class Tokenizer { * @return a read-only list of tokens. This list can only be used by this thread */ public List<Token> tokenize(String string) { - return tokenize(string, new IndexFacts().newSession()); + return tokenize(string, new IndexFacts().newSession(Collections.emptySet(), Collections.emptySet())); } /** @@ -170,10 +171,13 @@ public final class Tokenizer { // this is a heuristic to check whether we probably have reached the end of an URL element for (int i = tokens.size() - 1; i >= 0; --i) { switch (tokens.get(i).kind) { - case COLON -> { if (i == indexLastExplicitlyChangedAt) return false; } - case SPACE -> { return true; } - default -> { } - // do nothing + case COLON: + if (i == indexLastExplicitlyChangedAt) return false; + break; + case SPACE: + return true; + default: + // do nothing } } // really not sure whether we should choose false instead, on cause of the guard at diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/BucketGroupId.java b/container-search/src/main/java/com/yahoo/search/grouping/result/BucketGroupId.java index 05efc134465..0bea390ad63 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/result/BucketGroupId.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/result/BucketGroupId.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.result; -import static com.yahoo.text.Lowercase.toLowerCase; - /** * This abstract class is used in {@link Group} instances where the identifying expression evaluated to a {@link * com.yahoo.search.grouping.request.BucketValue}. The range is inclusive-from and exclusive-to. diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/HitRenderer.java b/container-search/src/main/java/com/yahoo/search/grouping/result/HitRenderer.java index 343fea82b6e..91c46960ab0 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/result/HitRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/result/HitRenderer.java @@ -7,7 +7,6 @@ import com.yahoo.text.Utf8String; import com.yahoo.text.XMLWriter; import java.io.IOException; -import java.util.Arrays; import java.util.Map; /** @@ -63,28 +62,15 @@ public abstract class HitRenderer { private static void renderGroupId(GroupId id, XMLWriter writer) { writer.openTag(TAG_GROUP_ID).attribute(ATR_TYPE, id.getTypeName()); - if (id instanceof ValueGroupId) { - writer.content(getIdValue((ValueGroupId)id), false); - } else if (id instanceof BucketGroupId) { - BucketGroupId bucketId = (BucketGroupId)id; - writer.openTag(TAG_BUCKET_FROM).content(getBucketFrom(bucketId), false).closeTag(); - writer.openTag(TAG_BUCKET_TO).content(getBucketTo(bucketId), false).closeTag(); + if (id instanceof ValueGroupId<?> valueGroupId) { + writer.content(valueGroupId.getValue(), false); + } else if (id instanceof BucketGroupId bucketId) { + writer.openTag(TAG_BUCKET_FROM).content(bucketId.getFrom(), false).closeTag(); + writer.openTag(TAG_BUCKET_TO).content(bucketId.getTo(), false).closeTag(); } writer.closeTag(); } - private static Object getIdValue(ValueGroupId id) { - return id instanceof RawId ? Arrays.toString(((RawId)id).getValue()) : id.getValue(); - } - - private static Object getBucketFrom(BucketGroupId id) { - return id instanceof RawBucketId ? Arrays.toString(((RawBucketId)id).getFrom()) : id.getFrom(); - } - - private static Object getBucketTo(BucketGroupId id) { - return id instanceof RawBucketId ? Arrays.toString(((RawBucketId)id).getTo()) : id.getTo(); - } - private static void renderContinuations(Map<String, Continuation> continuations, XMLWriter writer) { for (Map.Entry<String, Continuation> entry : continuations.entrySet()) { renderContinuation(entry.getKey(), entry.getValue(), writer); diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/RawBucketId.java b/container-search/src/main/java/com/yahoo/search/grouping/result/RawBucketId.java index 9576f548f4a..9b5ad6660b0 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/result/RawBucketId.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/result/RawBucketId.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.result; -import java.util.Arrays; +import com.yahoo.prelude.hitfield.RawBase64; /** * This class is used in {@link Group} instances where the identifying @@ -9,7 +9,7 @@ import java.util.Arrays; * * @author Ulf Lilleengen */ -public class RawBucketId extends BucketGroupId<byte[]> { +public class RawBucketId extends BucketGroupId<RawBase64> { /** * Constructs a new instance of this class. @@ -18,6 +18,6 @@ public class RawBucketId extends BucketGroupId<byte[]> { * @param to The identifying exclusive-to raw buffer. */ public RawBucketId(byte[] from, byte[] to) { - super("raw_bucket", from, Arrays.toString(from), to, Arrays.toString(to)); + super("raw_bucket", new RawBase64(from), new RawBase64(to)); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/RawId.java b/container-search/src/main/java/com/yahoo/search/grouping/result/RawId.java index de711d0c218..fd0d38c37fd 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/result/RawId.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/result/RawId.java @@ -1,14 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.result; -import java.util.Arrays; +import com.yahoo.prelude.hitfield.RawBase64; /** * This class is used in {@link Group} instances where the identifying expression evaluated to a {@link Byte} array. * * @author Simon Thoresen Hult */ -public class RawId extends ValueGroupId<byte[]> { +public class RawId extends ValueGroupId<RawBase64> { /** * Constructs a new instance of this class. @@ -16,6 +16,6 @@ public class RawId extends ValueGroupId<byte[]> { * @param value The identifying byte array. */ public RawId(byte[] value) { - super("raw", value, Arrays.toString(value)); + super("raw", new RawBase64(value)); } } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java index 7f006b098cd..e746706f9c5 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/ResultBuilder.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.vespa; +import com.yahoo.prelude.hitfield.RawBase64; import com.yahoo.search.grouping.Continuation; import com.yahoo.search.grouping.GroupingRequest; import com.yahoo.search.grouping.result.BoolId; @@ -28,6 +29,7 @@ import com.yahoo.searchlib.aggregation.Hit; import com.yahoo.searchlib.aggregation.HitsAggregationResult; import com.yahoo.searchlib.aggregation.MaxAggregationResult; import com.yahoo.searchlib.aggregation.MinAggregationResult; +import com.yahoo.searchlib.aggregation.RawData; import com.yahoo.searchlib.aggregation.StandardDeviationAggregationResult; import com.yahoo.searchlib.aggregation.SumAggregationResult; import com.yahoo.searchlib.aggregation.XorAggregationResult; @@ -169,7 +171,7 @@ class ResultBuilder { } else { String label = transform.getLabel(result.getTag()); if (label != null) { - group.setField(label, newResult(result, tag)); + group.setField(label, convertResult(newResult(result, tag))); } } } @@ -228,24 +230,27 @@ class ResultBuilder { return new RawId(res.getRaw()); } else if (res instanceof StringResultNode) { return new StringId(res.getString()); - } else if (res instanceof FloatBucketResultNode) { - FloatBucketResultNode bucketId = (FloatBucketResultNode)res; + } else if (res instanceof FloatBucketResultNode bucketId) { return new DoubleBucketId(bucketId.getFrom(), bucketId.getTo()); - } else if (res instanceof IntegerBucketResultNode) { - IntegerBucketResultNode bucketId = (IntegerBucketResultNode)res; + } else if (res instanceof IntegerBucketResultNode bucketId) { return new LongBucketId(bucketId.getFrom(), bucketId.getTo()); - } else if (res instanceof StringBucketResultNode) { - StringBucketResultNode bucketId = (StringBucketResultNode)res; + } else if (res instanceof StringBucketResultNode bucketId) { return new StringBucketId(bucketId.getFrom(), bucketId.getTo()); - } else if (res instanceof RawBucketResultNode) { - RawBucketResultNode bucketId = (RawBucketResultNode)res; + } else if (res instanceof RawBucketResultNode bucketId) { return new RawBucketId(bucketId.getFrom(), bucketId.getTo()); } else { throw new UnsupportedOperationException(res.getClass().getName()); } } - Object newResult(ExpressionNode execResult, int tag) { + private Object convertResult(Object value) { + if (value instanceof RawData raw) { + return new RawBase64(raw.getData()); + } + return value; + } + + private Object newResult(ExpressionNode execResult, int tag) { if (execResult instanceof AverageAggregationResult) { return ((AverageAggregationResult)execResult).getAverage().getNumber(); } else if (execResult instanceof CountAggregationResult) { diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java index 99c3477274d..01bbef13129 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/Binding.java @@ -48,7 +48,7 @@ public class Binding implements Comparable<Binding> { for (int i = 0; i <= maxDimensions; i++) { String value = i < dimensionBinding.getDimensions().size() ? dimensionBinding.getValues().get(i) : null; if (value == null) - generality += Math.pow(2, maxDimensions - i-1); + generality += (int)Math.pow(2, maxDimensions - i - 1); else context.put(dimensionBinding.getDimensions().get(i), value); } diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index b36c8788877..90f4e6ae65c 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -32,8 +32,6 @@ import com.yahoo.search.grouping.result.AbstractList; import com.yahoo.search.grouping.result.BucketGroupId; import com.yahoo.search.grouping.result.Group; import com.yahoo.search.grouping.result.GroupId; -import com.yahoo.search.grouping.result.RawBucketId; -import com.yahoo.search.grouping.result.RawId; import com.yahoo.search.grouping.result.RootGroup; import com.yahoo.search.grouping.result.ValueGroupId; import com.yahoo.search.result.Coverage; @@ -57,7 +55,6 @@ import java.math.BigDecimal; import java.math.BigInteger; import java.nio.charset.StandardCharsets; import java.util.ArrayDeque; -import java.util.Arrays; import java.util.Deque; import java.util.Map; import java.util.Optional; @@ -420,31 +417,19 @@ public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { } protected void renderGroupMetadata(GroupId id) throws IOException { - if (!(id instanceof ValueGroupId || id instanceof BucketGroupId)) return; + if (!(id instanceof ValueGroupId<?> || id instanceof BucketGroupId)) return; - if (id instanceof ValueGroupId valueId) { - generator.writeStringField(GROUPING_VALUE, getIdValue(valueId)); + if (id instanceof ValueGroupId<?> valueId) { + generator.writeStringField(GROUPING_VALUE, valueId.getValue().toString()); } else { BucketGroupId<?> bucketId = (BucketGroupId<?>) id; generator.writeObjectFieldStart(BUCKET_LIMITS); - generator.writeStringField(BUCKET_FROM, getBucketFrom(bucketId)); - generator.writeStringField(BUCKET_TO, getBucketTo(bucketId)); + generator.writeStringField(BUCKET_FROM, bucketId.getFrom().toString()); + generator.writeStringField(BUCKET_TO, bucketId.getTo().toString()); generator.writeEndObject(); } } - private static String getIdValue(ValueGroupId<?> id) { - return (id instanceof RawId ? Arrays.toString(((RawId) id).getValue()) : id.getValue()).toString(); - } - - private static String getBucketFrom(BucketGroupId<?> id) { - return (id instanceof RawBucketId ? Arrays.toString(((RawBucketId) id).getFrom()) : id.getFrom()).toString(); - } - - private static String getBucketTo(BucketGroupId<?> id) { - return (id instanceof RawBucketId ? Arrays.toString(((RawBucketId) id).getTo()) : id.getTo()).toString(); - } - protected void renderTotalHitCount(Hit hit) throws IOException { if ( ! (getRecursionLevel() == 1 && hit instanceof HitGroup)) return; diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java index 3a6be1521e2..1ff5574ec03 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java @@ -13,6 +13,7 @@ import com.yahoo.prelude.query.parser.Tokenizer; import org.junit.jupiter.api.Test; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import static com.yahoo.prelude.query.parser.Token.Kind.COLON; @@ -28,9 +29,7 @@ import static com.yahoo.prelude.query.parser.Token.Kind.SPACE; import static com.yahoo.prelude.query.parser.Token.Kind.STAR; import static com.yahoo.prelude.query.parser.Token.Kind.UNDERSCORE; import static com.yahoo.prelude.query.parser.Token.Kind.WORD; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; /** * Tests the tokenizer @@ -284,7 +283,7 @@ public class TokenizerTestCase { sd.addIndex(index2); IndexFacts facts = new IndexFacts(new IndexModel(sd)); - IndexFacts.Session session = facts.newSession(); + IndexFacts.Session session = facts.newSession(Collections.emptySet(), Collections.emptySet()); Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); List<?> tokens = tokenizer.tokenize("normal a:b (normal testexact1:/,%#%&+-+ ) testexact2:ho_/&%&/()/aa*::*& b:c", "default", session); // tokenizer.print(); @@ -329,7 +328,7 @@ public class TokenizerTestCase { IndexFacts facts = new IndexFacts(new IndexModel(sd)); Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); - IndexFacts.Session session = facts.newSession(); + IndexFacts.Session session = facts.newSession(Collections.emptySet(), Collections.emptySet()); List<?> tokens = tokenizer.tokenize("normal a:b (normal testexact1:/,%#%&+-+ ) testexact2:ho_/&%&/()/aa*::*&", session); assertEquals(new Token(WORD, "normal"), tokens.get(0)); assertEquals(new Token(SPACE, " "), tokens.get(1)); @@ -366,7 +365,7 @@ public class TokenizerTestCase { IndexFacts facts = new IndexFacts(new IndexModel(sd)); Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); - IndexFacts.Session session = facts.newSession(); + IndexFacts.Session session = facts.newSession(Collections.emptySet(), Collections.emptySet()); List<?> tokens = tokenizer.tokenize("normal a:b (normal testexact1:/,%#%&+-+ ) testexact2:ho_/&%&/()/aa*::*", session); assertEquals(new Token(WORD, "normal"), tokens.get(0)); assertEquals(new Token(SPACE, " "), tokens.get(1)); @@ -403,7 +402,7 @@ public class TokenizerTestCase { IndexFacts facts = new IndexFacts(new IndexModel(sd)); Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); - IndexFacts.Session session = facts.newSession(); + IndexFacts.Session session = facts.newSession(Collections.emptySet(), Collections.emptySet()); List<?> tokens = tokenizer.tokenize("normal a:b (normal testexact1:!/%#%&+-+ ) testexact2:ho_/&%&/()/aa*::*&b:", session); assertEquals(new Token(WORD, "normal"), tokens.get(0)); assertEquals(new Token(SPACE, " "), tokens.get(1)); @@ -440,7 +439,7 @@ public class TokenizerTestCase { sd.addIndex(index2); IndexFacts indexFacts = new IndexFacts(new IndexModel(sd)); - IndexFacts.Session facts = indexFacts.newSession(); + IndexFacts.Session facts = indexFacts.newSession(Collections.emptySet(), Collections.emptySet()); Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); List<?> tokens = tokenizer.tokenize("normal a:b (normal testexact1:foo) testexact2:bar", facts); diff --git a/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java index dbcb393c922..e6c5a18c9da 100644 --- a/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java @@ -15,12 +15,8 @@ import org.junit.jupiter.api.Test; import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.Set; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; /** * Tests using synthetic index names for IndexFacts class. @@ -184,7 +180,7 @@ public class IndexFactsTestCase { query.getModel().getSources().add("one"); query.getModel().getRestrict().add("two"); - IndexFacts.Session indexFacts = createIndexFacts().newSession(List.of("clusterOne"), Set.of()); + IndexFacts.Session indexFacts = createIndexFacts().newSession(List.of("clusterOne"), List.of()); assertTrue(indexFacts.isIndex("a")); assertFalse(indexFacts.isIndex("b")); assertTrue(indexFacts.isIndex("d")); diff --git a/container-search/src/test/java/com/yahoo/search/grouping/result/GroupIdTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/result/GroupIdTestCase.java index 7b2f0d52742..77ed858b14b 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/result/GroupIdTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/result/GroupIdTestCase.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.grouping.result; +import com.yahoo.prelude.hitfield.RawBase64; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.*; @@ -25,10 +26,10 @@ public class GroupIdTestCase { assertEquals(9L, rangeId.getTo()); valueId = new RawId(new byte[]{6, 9}); - assertArrayEquals(new byte[]{6, 9}, (byte[]) valueId.getValue()); + assertEquals(new RawBase64(new byte[]{6, 9}), valueId.getValue()); rangeId = new RawBucketId(new byte[]{6, 9}, new byte[]{9, 6}); - assertArrayEquals(new byte[]{6, 9}, (byte[]) rangeId.getFrom()); - assertArrayEquals(new byte[]{9, 6}, (byte[]) rangeId.getTo()); + assertEquals(new RawBase64(new byte[]{6, 9}), rangeId.getFrom()); + assertEquals(new RawBase64(new byte[]{9, 6}), rangeId.getTo()); valueId = new StringId("69"); assertEquals("69", valueId.getValue()); @@ -47,8 +48,8 @@ public class GroupIdTestCase { assertEquals("group:long:69", new LongId(69L).toString()); assertEquals("group:long_bucket:6:9", new LongBucketId(6L, 9L).toString()); assertEquals("group:null", new NullId().toString()); - assertEquals("group:raw:[6, 9]", new RawId(new byte[]{6, 9}).toString()); - assertEquals("group:raw_bucket:[6, 9]:[9, 6]", new RawBucketId(new byte[]{6, 9}, new byte[]{9, 6}).toString()); + assertEquals("group:raw:Bgk", new RawId(new byte[]{6, 9}).toString()); + assertEquals("group:raw_bucket:Bgk:CQY", new RawBucketId(new byte[]{6, 9}, new byte[]{9, 6}).toString()); assertTrue(new RootId(0).toString().startsWith("group:root:")); assertEquals("group:string:69", new StringId("69").toString()); assertEquals("group:string_bucket:6:9", new StringBucketId("6", "9").toString()); diff --git a/container-search/src/test/java/com/yahoo/search/grouping/result/HitRendererTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/result/HitRendererTestCase.java index 8e98f49df48..69bd848ebcd 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/result/HitRendererTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/result/HitRendererTestCase.java @@ -57,7 +57,7 @@ public class HitRendererTestCase { "</group>\n"); assertRender(newGroup(new RawId(Utf8.toBytes("foo"))), "<group relevance=\"1.0\">\n" + - "<id type=\"raw\">[102, 111, 111]</id>\n" + + "<id type=\"raw\">Zm9v</id>\n" + "</group>\n"); assertRender(newGroup(new StringId("foo")), "<group relevance=\"1.0\">\n" + @@ -85,7 +85,7 @@ public class HitRendererTestCase { "</group>\n"); assertRender(newGroup(new RawBucketId(Utf8.toBytes("bar"), Utf8.toBytes("baz"))), "<group relevance=\"1.0\">\n" + - "<id type=\"raw_bucket\">\n<from>[98, 97, 114]</from>\n<to>[98, 97, 122]</to>\n</id>\n" + + "<id type=\"raw_bucket\">\n<from>YmFy</from>\n<to>YmF6</to>\n</id>\n" + "</group>\n"); } diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultBuilderTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultBuilderTestCase.java index 019a022b7e6..b0b48bb8731 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultBuilderTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultBuilderTestCase.java @@ -10,12 +10,39 @@ import com.yahoo.search.grouping.result.GroupList; import com.yahoo.search.grouping.result.HitList; import com.yahoo.search.result.HitGroup; import com.yahoo.search.result.Relevance; -import com.yahoo.searchlib.aggregation.*; +import com.yahoo.searchlib.aggregation.AggregationResult; +import com.yahoo.searchlib.aggregation.AverageAggregationResult; +import com.yahoo.searchlib.aggregation.CountAggregationResult; +import com.yahoo.searchlib.aggregation.ExpressionCountAggregationResult; +import com.yahoo.searchlib.aggregation.FS4Hit; +import com.yahoo.searchlib.aggregation.Group; +import com.yahoo.searchlib.aggregation.Grouping; +import com.yahoo.searchlib.aggregation.HitsAggregationResult; +import com.yahoo.searchlib.aggregation.MaxAggregationResult; +import com.yahoo.searchlib.aggregation.MinAggregationResult; +import com.yahoo.searchlib.aggregation.SumAggregationResult; +import com.yahoo.searchlib.aggregation.XorAggregationResult; import com.yahoo.searchlib.aggregation.hll.SparseSketch; -import com.yahoo.searchlib.expression.*; +import com.yahoo.searchlib.expression.FloatBucketResultNode; +import com.yahoo.searchlib.expression.FloatResultNode; +import com.yahoo.searchlib.expression.IntegerBucketResultNode; +import com.yahoo.searchlib.expression.IntegerResultNode; +import com.yahoo.searchlib.expression.NullResultNode; +import com.yahoo.searchlib.expression.RawBucketResultNode; +import com.yahoo.searchlib.expression.RawResultNode; +import com.yahoo.searchlib.expression.ResultNode; +import com.yahoo.searchlib.expression.StringBucketResultNode; +import com.yahoo.searchlib.expression.StringResultNode; import org.junit.jupiter.api.Test; -import java.util.*; + +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.fail; @@ -33,19 +60,19 @@ public class ResultBuilderTestCase { assertGroupId("group:6.9", new FloatResultNode(6.9)); assertGroupId("group:69", new IntegerResultNode(69)); assertGroupId("group:null", new NullResultNode()); - assertGroupId("group:[6, 9]", new RawResultNode(new byte[]{6, 9})); + assertGroupId("group:Bgk", new RawResultNode(new byte[]{6, 9})); assertGroupId("group:a", new StringResultNode("a")); assertGroupId("group:6.9:9.6", new FloatBucketResultNode(6.9, 9.6)); assertGroupId("group:6:9", new IntegerBucketResultNode(6, 9)); assertGroupId("group:a:b", new StringBucketResultNode("a", "b")); - assertGroupId("group:[6, 9]:[9, 6]", new RawBucketResultNode(new RawResultNode(new byte[]{6, 9}), + assertGroupId("group:Bgk:CQY", new RawBucketResultNode(new RawResultNode(new byte[]{6, 9}), new RawResultNode(new byte[]{9, 6}))); } @Test void requireThatUnknownGroupIdThrows() { assertBuildFail("all(group(a) each(output(count())))", - Arrays.asList(newGrouping(new Group().setTag(2).setId(new MyResultNode()))), + List.of(newGrouping(new Group().setTag(2).setId(new MyResultNode()))), "com.yahoo.search.grouping.vespa.ResultBuilderTestCase$MyResultNode"); } @@ -61,9 +88,17 @@ public class ResultBuilderTestCase { } @Test + void requireThatAllBasicResultsCanBeConverted() { + assertResult("69", new MinAggregationResult(new IntegerResultNode(69))); + assertResult("69.3", new MinAggregationResult(new FloatResultNode(69.3))); + assertResult("69.6", new MinAggregationResult(new StringResultNode("69.6"))); + assertResult("Bgk", new MinAggregationResult(new RawResultNode(new byte[]{6,9}))); + } + + @Test void requireThatUnknownExpressionNodeThrows() { assertBuildFail("all(group(a) each(output(count())))", - Arrays.asList(newGrouping(newGroup(2, 2, new MyAggregationResult().setTag(3)))), + List.of(newGrouping(newGroup(2, 2, new MyAggregationResult().setTag(3)))), "com.yahoo.search.grouping.vespa.ResultBuilderTestCase$MyAggregationResult"); } @@ -127,10 +162,10 @@ public class ResultBuilderTestCase { @Test void requireThatParallelResultsAreTransformed() { assertBuild("all(group(foo) each(output(count())) as(bar) each(output(count())) as(baz))", - Arrays.asList(new Grouping().setRoot(newGroup(1, 0)), + List.of(new Grouping().setRoot(newGroup(1, 0)), new Grouping().setRoot(newGroup(1, 0)))); assertBuildFail("all(group(foo) each(output(count())) as(bar) each(output(count())) as(baz))", - Arrays.asList(new Grouping().setRoot(newGroup(2)), + List.of(new Grouping().setRoot(newGroup(2)), new Grouping().setRoot(newGroup(3))), "Expected 1 group, got 2."); } @@ -138,15 +173,15 @@ public class ResultBuilderTestCase { @Test void requireThatTagsAreHandledCorrectly() { assertBuild("all(group(a) each(output(count())))", - Arrays.asList(newGrouping( + List.of(newGrouping( newGroup(7, new CountAggregationResult(0))))); } @Test void requireThatEmptyBranchesArePruned() { - assertBuildFail("all()", Collections.<Grouping>emptyList(), "Expected 1 group, got 0."); - assertBuildFail("all(group(a))", Collections.<Grouping>emptyList(), "Expected 1 group, got 0."); - assertBuildFail("all(group(a) each())", Collections.<Grouping>emptyList(), "Expected 1 group, got 0."); + assertBuildFail("all()", List.of(), "Expected 1 group, got 0."); + assertBuildFail("all(group(a))", List.of(), "Expected 1 group, got 0."); + assertBuildFail("all(group(a) each())", List.of(), "Expected 1 group, got 0."); Grouping grouping = newGrouping(newGroup(2, new CountAggregationResult(69).setTag(3))); String expectedOutput = "RootGroup{id=group:root}[GroupList{label=a}[Group{id=group:2, count()=69}[]]]"; @@ -189,14 +224,14 @@ public class ResultBuilderTestCase { "HitList{label=bar}[Hit{id=hit:1}, Hit{id=hit:2}]]]]"); assertLayout("all(group(foo) each(each(output(summary())) as(bar)" + " each(output(summary())) as(baz)))", - Arrays.asList(newGrouping(newGroup(2, newHitList(3, 2))), + List.of(newGrouping(newGroup(2, newHitList(3, 2))), newGrouping(newGroup(2, newHitList(4, 2)))), "RootGroup{id=group:root}[GroupList{label=foo}[Group{id=group:2}[" + "HitList{label=bar}[Hit{id=hit:1}, Hit{id=hit:2}], " + "HitList{label=baz}[Hit{id=hit:1}, Hit{id=hit:2}]]]]"); assertLayout("all(group(foo) each(each(output(summary())))" + " each(each(output(summary()))) as(bar))", - Arrays.asList(newGrouping(newGroup(2, newHitList(3, 2))), + List.of(newGrouping(newGroup(2, newHitList(3, 2))), newGrouping(newGroup(4, newHitList(5, 2)))), "RootGroup{id=group:root}[" + "GroupList{label=foo}[Group{id=group:2}[HitList{label=hits}[Hit{id=hit:1}, Hit{id=hit:2}]]], " + @@ -273,18 +308,18 @@ public class ResultBuilderTestCase { assertResultCont("all(group(a) max(2) each(output(count())) as(foo)" + " each(output(count())) as(bar))", - Arrays.asList(newGrouping(newGroup(2, 1, new CountAggregationResult(1))), + List.of(newGrouping(newGroup(2, 1, new CountAggregationResult(1))), newGrouping(newGroup(4, 2, new CountAggregationResult(4)))), "[]"); assertResultCont("all(group(a) max(2) each(output(count())) as(foo)" + " each(output(count())) as(bar))", - Arrays.asList(newGrouping(newGroup(2, 1, new CountAggregationResult(1))), + List.of(newGrouping(newGroup(2, 1, new CountAggregationResult(1))), newGrouping(newGroup(4, 2, new CountAggregationResult(4)))), newOffset(newResultId(0), 2, 1), "[0=1]"); assertResultCont("all(group(a) max(2) each(output(count())) as(foo)" + " each(output(count())) as(bar))", - Arrays.asList(newGrouping(newGroup(2, 1, new CountAggregationResult(1))), + List.of(newGrouping(newGroup(2, 1, new CountAggregationResult(1))), newGrouping(newGroup(4, 2, new CountAggregationResult(4)))), newComposite(newOffset(newResultId(0), 2, 2), newOffset(newResultId(1), 4, 1)), @@ -299,18 +334,18 @@ public class ResultBuilderTestCase { assertResultCont("all(group(a) each(max(2) each(output(summary()))) as(foo)" + " each(max(2) each(output(summary()))) as(bar))", - Arrays.asList(newGrouping(newGroup(2, newHitList(3, 4))), + List.of(newGrouping(newGroup(2, newHitList(3, 4))), newGrouping(newGroup(4, newHitList(5, 4)))), "[]"); assertResultCont("all(group(a) each(max(2) each(output(summary()))) as(foo)" + " each(max(2) each(output(summary()))) as(bar))", - Arrays.asList(newGrouping(newGroup(2, newHitList(3, 4))), + List.of(newGrouping(newGroup(2, newHitList(3, 4))), newGrouping(newGroup(4, newHitList(5, 4)))), newOffset(newResultId(0, 0, 0), 3, 1), "[0.0.0=1]"); assertResultCont("all(group(a) each(max(2) each(output(summary()))) as(foo)" + " each(max(2) each(output(summary()))) as(bar))", - Arrays.asList(newGrouping(newGroup(2, newHitList(3, 4))), + List.of(newGrouping(newGroup(2, newHitList(3, 4))), newGrouping(newGroup(4, newHitList(5, 4)))), newComposite(newOffset(newResultId(0, 0, 0), 3, 2), newOffset(newResultId(1, 0, 0), 5, 1)), @@ -404,7 +439,7 @@ public class ResultBuilderTestCase { void requireThatGroupListContinuationsCanBeSetInSiblingGroupLists() { String request = "all(group(a) max(2) each(output(count())) as(foo)" + " each(output(count())) as(bar))"; - List<Grouping> result = Arrays.asList(newGrouping(newGroup(2, 1, new CountAggregationResult(1)), + List<Grouping> result = List.of(newGrouping(newGroup(2, 1, new CountAggregationResult(1)), newGroup(2, 2, new CountAggregationResult(2)), newGroup(2, 3, new CountAggregationResult(3)), newGroup(2, 4, new CountAggregationResult(4))), @@ -646,7 +681,7 @@ public class ResultBuilderTestCase { void requireThatHitListContinuationsCanBeSetInSiblingHitLists() { String request = "all(group(a) each(max(2) each(output(summary()))) as(foo)" + " each(max(2) each(output(summary()))) as(bar))"; - List<Grouping> result = Arrays.asList(newGrouping(newGroup(2, newHitList(3, 4))), + List<Grouping> result = List.of(newGrouping(newGroup(2, newHitList(3, 4))), newGrouping(newGroup(4, newHitList(5, 4)))); assertContinuation(request, result, newComposite(newOffset(newResultId(0, 0, 0), 3, 0), newOffset(newResultId(1, 0, 0), 5, 5)), @@ -839,7 +874,7 @@ public class ResultBuilderTestCase { } private static void assertResultCont(String request, Grouping result, Continuation cont, String expected) { - assertOutput(request, Arrays.asList(result), cont, new ResultContWriter(), expected); + assertOutput(request, List.of(result), cont, new ResultContWriter(), expected); } private static void assertResultCont(String request, List<Grouping> result, String expected) { @@ -851,11 +886,11 @@ public class ResultBuilderTestCase { } private static void assertContinuation(String request, Grouping result, String expected) { - assertOutput(request, Arrays.asList(result), null, new ContinuationWriter(), expected); + assertOutput(request, List.of(result), null, new ContinuationWriter(), expected); } private static void assertContinuation(String request, Grouping result, Continuation cont, String expected) { - assertOutput(request, Arrays.asList(result), cont, new ContinuationWriter(), expected); + assertOutput(request, List.of(result), cont, new ContinuationWriter(), expected); } private static void assertContinuation(String request, List<Grouping> result, Continuation cont, String expected) { @@ -863,7 +898,7 @@ public class ResultBuilderTestCase { } private static void assertLayout(String request, Grouping result, String expected) { - assertOutput(request, Arrays.asList(result), null, new LayoutWriter(), expected); + assertOutput(request, List.of(result), null, new LayoutWriter(), expected); } private static void assertLayout(String request, List<Grouping> result, String expected) { @@ -953,8 +988,7 @@ public class ResultBuilderTestCase { } String toString(Continuation cnt) { - if (cnt instanceof OffsetContinuation) { - OffsetContinuation off = (OffsetContinuation)cnt; + if (cnt instanceof OffsetContinuation off) { String id = off.getResultId().toString().replace(", ", "."); return id.substring(5, id.length() - 1) + "=" + off.getOffset(); } else if (cnt instanceof CompositeContinuation) { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java index 2c496b1e3e5..89226d3309e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java @@ -42,13 +42,17 @@ public class PlanRegistryMock implements PlanRegistry { private final boolean supported; public MockPlan(String planId, boolean billed, boolean supported, double cpuPrice, double memPrice, double dgbPrice, double gpuPrice, int quota, String description) { - this(PlanId.from(planId), billed, supported, new MockCostCalculator(cpuPrice, memPrice, dgbPrice, gpuPrice), () -> Quota.unlimited().withBudget(quota), description); + this(PlanId.from(planId), billed, supported, new MockCostCalculator(cpuPrice, memPrice, dgbPrice, gpuPrice), () -> createQuota(quota), description); } public MockPlan(String planId, boolean billed, boolean supported, String cpuPrice, String memPrice, String dgbPrice, String gpuPrice, int quota, String description) { - this(PlanId.from(planId), billed, supported, new MockCostCalculator(cpuPrice, memPrice, dgbPrice, gpuPrice), () -> Quota.unlimited().withBudget(quota), description); + this(PlanId.from(planId), billed, supported, new MockCostCalculator(cpuPrice, memPrice, dgbPrice, gpuPrice), () -> createQuota(quota), description); } + private static Quota createQuota(int quota) { + return quota == 0 ? Quota.zero() : Quota.unlimited().withBudget(quota); + } + public MockPlan(PlanId planId, boolean billed, boolean supported, MockCostCalculator calculator, QuotaCalculator quota, String description) { this.planId = planId; this.billed = billed; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java index ecea1ce6913..cff61f1a50a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java @@ -69,7 +69,9 @@ public class EndpointCertificateValidatorImpl implements EndpointCertificateVali // Normally because the cert is in the process of being provisioned - this will cause a retry in InternalStepRunner throw new EndpointCertificateException(EndpointCertificateException.Type.CERT_NOT_AVAILABLE, "Certificate not found in secret store"); } catch (EndpointCertificateException e) { - log.log(Level.WARNING, "Certificate validation failure for " + serializedInstanceId, e); + if (!e.type().equals(EndpointCertificateException.Type.CERT_NOT_AVAILABLE)) { // such failures are normal and will be retried, it takes some time to show up in the secret store + log.log(Level.WARNING, "Certificate validation failure for " + serializedInstanceId, e); + } throw e; } catch (Exception e) { log.log(Level.WARNING, "Certificate validation failure for " + serializedInstanceId, e); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java index 04604ae7007..eb2005bf268 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java @@ -1,11 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.deployment; -import ai.vespa.validation.Validation; import com.yahoo.component.Version; import java.time.Instant; -import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; @@ -33,6 +31,7 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { private final Optional<String> sourceUrl; private final Optional<String> commit; private final Optional<String> bundleHash; + private final Optional<Instant> obsoleteAt; private final boolean hasPackage; private final boolean shouldSkip; private final Optional<String> description; @@ -41,7 +40,7 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { public ApplicationVersion(RevisionId id, Optional<SourceRevision> source, Optional<String> authorEmail, Optional<Version> compileVersion, Optional<Integer> allowedMajor, Optional<Instant> buildTime, Optional<String> sourceUrl, Optional<String> commit, Optional<String> bundleHash, - boolean hasPackage, boolean shouldSkip, Optional<String> description, int risk) { + Optional<Instant> obsoleteAt, boolean hasPackage, boolean shouldSkip, Optional<String> description, int risk) { if (commit.isPresent() && commit.get().length() > 128) throw new IllegalArgumentException("Commit may not be longer than 128 characters"); @@ -61,6 +60,7 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { this.sourceUrl = requireNonNull(sourceUrl, "sourceUrl cannot be null"); this.commit = requireNonNull(commit, "commit cannot be null"); this.bundleHash = bundleHash; + this.obsoleteAt = obsoleteAt; this.hasPackage = hasPackage; this.shouldSkip = shouldSkip; this.description = description; @@ -71,19 +71,9 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { return id; } - /** Create an application package version from a completed build, without an author email */ - public static ApplicationVersion from(RevisionId id, SourceRevision source) { - return new ApplicationVersion(id, Optional.of(source), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), true, false, Optional.empty(), 0); - } - - /** Creates a version from a completed build, an author email, and build metadata. */ - public static ApplicationVersion from(RevisionId id, SourceRevision source, String authorEmail, Version compileVersion, Instant buildTime) { - return new ApplicationVersion(id, Optional.of(source), Optional.of(authorEmail), Optional.of(compileVersion), Optional.empty(), Optional.of(buildTime), Optional.empty(), Optional.empty(), Optional.empty(), true, false, Optional.empty(), 0); - } - /** Creates a minimal version for a development build. */ public static ApplicationVersion forDevelopment(RevisionId id, Optional<Version> compileVersion, Optional<Integer> allowedMajor) { - return new ApplicationVersion(id, Optional.empty(), Optional.empty(), compileVersion, allowedMajor, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), true, false, Optional.empty(), 0); + return new ApplicationVersion(id, Optional.empty(), Optional.empty(), compileVersion, allowedMajor, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), true, false, Optional.empty(), 0); } /** Creates a version from a completed build, an author email, and build metadata. */ @@ -91,7 +81,7 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { Optional<Version> compileVersion, Optional<Integer> allowedMajor, Optional<Instant> buildTime, Optional<String> sourceUrl, Optional<String> commit, Optional<String> bundleHash, Optional<String> description, int risk) { return new ApplicationVersion(id, source, authorEmail, compileVersion, allowedMajor, buildTime, - sourceUrl, commit, bundleHash, true, false, description, risk); + sourceUrl, commit, bundleHash, Optional.empty(), true, false, description, risk); } /** Returns a unique identifier for this version or "unknown" if version is not known */ @@ -150,7 +140,17 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { /** Returns a copy of this without a package stored. */ public ApplicationVersion withoutPackage() { - return new ApplicationVersion(id, source, authorEmail, compileVersion, allowedMajor, buildTime, sourceUrl, commit, bundleHash, false, shouldSkip, description, risk); + return new ApplicationVersion(id, source, authorEmail, compileVersion, allowedMajor, buildTime, sourceUrl, commit, bundleHash, obsoleteAt, false, shouldSkip, description, risk); + } + + /** Returns a copy of this which is obsolete now. */ + public ApplicationVersion obsoleteAt(Instant now) { + return new ApplicationVersion(id, source, authorEmail, compileVersion, allowedMajor, buildTime, sourceUrl, commit, bundleHash, Optional.of(now), hasPackage, shouldSkip, description, risk); + } + + /** Returns the instant at which this became obsolete, i.e., no longer relevant for automated deployments. */ + public Optional<Instant> obsoleteAt() { + return obsoleteAt; } /** Whether we still have the package for this revision. */ @@ -160,7 +160,7 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { /** Returns a copy of this which will not be rolled out to production. */ public ApplicationVersion skipped() { - return new ApplicationVersion(id, source, authorEmail, compileVersion, allowedMajor, buildTime, sourceUrl, commit, bundleHash, hasPackage, true, description, risk); + return new ApplicationVersion(id, source, authorEmail, compileVersion, allowedMajor, buildTime, sourceUrl, commit, bundleHash, obsoleteAt, hasPackage, true, description, risk); } /** Whether we still have the package for this revision. */ diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java index 945e0730fe6..34a7c1a6f7c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java @@ -113,6 +113,8 @@ public class NodeRepositoryNode { private String cloudAccount; @JsonProperty("wireguardPubKey") private String wireguardPubKey; + @JsonProperty("archiveUri") + private String archiveUri; public String getUrl() { return url; @@ -460,6 +462,10 @@ public class NodeRepositoryNode { public void setWireguardPubKey(String wireguardPubKey) { this.wireguardPubKey = wireguardPubKey; } + public String getArchiveUri() { return archiveUri; } + + public void setArchiveUri(String archiveUri) { this.archiveUri = archiveUri; } + // --- Helper methods for code that (wrongly) consume this directly public boolean hasType(NodeType type) { @@ -521,6 +527,7 @@ public class NodeRepositoryNode { ", switchHostname='" + switchHostname + '\'' + ", cloudAccount='" + cloudAccount + '\'' + ", wireguardPubKey='" + wireguardPubKey + '\'' + + ", archiveUri='" + archiveUri + '\'' + '}'; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/HostAction.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/HostAction.java index f3bee721343..85c7f78eabc 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/HostAction.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/HostAction.java @@ -67,6 +67,7 @@ public class HostAction { OUT_OF_SYNC, NONE, RETIRING, + READY, RETIRED, COMPLETE } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/VcmrReport.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/VcmrReport.java index 969e6fb1e01..660f3c50556 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/VcmrReport.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/vcmr/VcmrReport.java @@ -11,7 +11,6 @@ import java.time.ZonedDateTime; import java.util.HashMap; import java.util.HashSet; import java.util.Map; -import java.util.Objects; import java.util.Set; import static com.yahoo.yolean.Exceptions.uncheck; @@ -47,18 +46,18 @@ public class VcmrReport { /** * @return true if list of VCMRs is changed */ - public boolean addVcmr(String id, ZonedDateTime plannedStartTime, ZonedDateTime plannedEndtime) { - var vcmr = new Vcmr(id, plannedStartTime, plannedEndtime); + public boolean addVcmr(ChangeRequestSource source) { + var vcmr = new Vcmr(source.getId(), source.getStatus().name(), source.getPlannedStartTime(), source.getPlannedEndTime()); if (vcmrs.contains(vcmr)) return false; // Remove to catch any changes in start/end time - removeVcmr(id); + removeVcmr(source.getId()); return vcmrs.add(vcmr); } public boolean removeVcmr(String id) { - return vcmrs.removeIf(vcmr -> id.equals(vcmr.getId())); + return vcmrs.removeIf(vcmr -> id.equals(vcmr.id())); } public static String getReportId() { @@ -93,55 +92,9 @@ public class VcmrReport { return "VCMRReport{" + vcmrs + "}"; } - public static class Vcmr { - - private String id; - private ZonedDateTime plannedStartTime; - private ZonedDateTime plannedEndTime; - - Vcmr(@JsonProperty("id") String id, - @JsonProperty("plannedStartTime") ZonedDateTime plannedStartTime, - @JsonProperty("plannedEndTime") ZonedDateTime plannedEndTime) { - this.id = id; - this.plannedStartTime = plannedStartTime; - this.plannedEndTime = plannedEndTime; - } - - public String getId() { - return id; - } - - public ZonedDateTime getPlannedStartTime() { - return plannedStartTime; - } - - public ZonedDateTime getPlannedEndTime() { - return plannedEndTime; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Vcmr vcmr = (Vcmr) o; - return Objects.equals(id, vcmr.id) && - Objects.equals(plannedStartTime, vcmr.plannedStartTime) && - Objects.equals(plannedEndTime, vcmr.plannedEndTime); - } - - @Override - public int hashCode() { - return Objects.hash(id, plannedStartTime, plannedEndTime); - } - - @Override - public String toString() { - return "VCMR{" + - "id='" + id + '\'' + - ", plannedStartTime=" + plannedStartTime + - ", plannedEndTime=" + plannedEndTime + - '}'; - } - } + public record Vcmr (@JsonProperty("id") String id, + @JsonProperty("status") String status, + @JsonProperty("plannedStartTime") ZonedDateTime plannedStartTime, + @JsonProperty("plannedEndTime") ZonedDateTime plannedEndTime) {} } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index a1534ebc533..08a8440fbe2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -30,6 +30,7 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; +import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan; import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing; @@ -47,6 +48,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; +import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics.Warning; @@ -60,6 +62,7 @@ import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageValid import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade; import com.yahoo.vespa.hosted.controller.certificate.EndpointCertificates; import com.yahoo.vespa.hosted.controller.concurrent.Once; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.deployment.JobStatus; import com.yahoo.vespa.hosted.controller.deployment.Run; @@ -76,7 +79,6 @@ import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence; import com.yahoo.yolean.Exceptions; - import java.io.ByteArrayInputStream; import java.security.Principal; import java.security.cert.X509Certificate; @@ -586,7 +588,16 @@ public class ApplicationController { } // Validate new deployment spec thoroughly before storing it. - controller.jobController().deploymentStatus(application.get()); + DeploymentStatus status = controller.jobController().deploymentStatus(application.get()); + Change dummyChange = Change.of(RevisionId.forProduction(Long.MAX_VALUE)); // Should always run everywhere. + for (var jobs : status.jobsToRun(applicationPackage.deploymentSpec().instanceNames().stream() + .collect(toMap(name -> name, __ -> dummyChange))) + .entrySet()) { + for (var job : jobs.getValue()) { + decideCloudAccountOf(new DeploymentId(jobs.getKey().application(), job.type().zone()), + applicationPackage.deploymentSpec()); + } + } for (Notification notification : controller.notificationsDb().listNotifications(NotificationSource.from(application.get().id()), true)) { if ( notification.source().instance().isPresent() @@ -696,7 +707,7 @@ public class ApplicationController { throw new IllegalArgumentException("Requested cloud account '" + requestedAccount.get().value() + "' is not valid for tenant '" + tenant + "'"); } - if (!controller.zoneRegistry().hasZone(zoneId, requestedAccount.get())) { + if ( ! controller.zoneRegistry().hasZone(zoneId, requestedAccount.get())) { throw new IllegalArgumentException("Zone " + zoneId + " is not configured in requested cloud account '" + requestedAccount.get().value() + "'"); } @@ -1049,4 +1060,14 @@ public class ApplicationController { collectingAndThen(counting(), Long::intValue))); } + public void verifyPlan(TenantName tenantName) { + var planId = controller.serviceRegistry().billingController().getPlan(tenantName); + Optional<Plan> plan = controller.serviceRegistry().planRegistry().plan(planId); + if (plan.isEmpty()) + throw new IllegalArgumentException("Tenant '" + tenantName.value() + "' has no plan, not allowed to deploy. See https://cloud.vespa.ai/support"); + if (plan.get().quota().calculate().equals(Quota.zero())) + throw new IllegalArgumentException("Tenant '" + tenantName.value() + "' has a plan '" + + plan.get().displayName() + "' with zero quota, not allowed to deploy. See https://cloud.vespa.ai/support"); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java index 64cad599168..5ebb3d53529 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.component.Version; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import java.util.Objects; @@ -23,7 +22,7 @@ import static java.util.Objects.requireNonNull; */ public final class Change { - private static final Change empty = new Change(Optional.empty(), Optional.empty(), false); + private static final Change empty = new Change(Optional.empty(), Optional.empty(), false, false); /** The platform version we are upgrading to, or empty if none */ private final Optional<Version> platform; @@ -32,23 +31,27 @@ public final class Change { private final Optional<RevisionId> revision; /** Whether this change is a pin to its contained Vespa version, or to the application's current. */ - private final boolean pinned; + private final boolean platformPinned; - private Change(Optional<Version> platform, Optional<RevisionId> revision, boolean pinned) { + /** Whether this change is a pin to its contained application revision, or to the application's current. */ + private final boolean revisionPinned; + + private Change(Optional<Version> platform, Optional<RevisionId> revision, boolean platformPinned, boolean revisionPinned) { this.platform = requireNonNull(platform, "platform cannot be null"); this.revision = requireNonNull(revision, "revision cannot be null"); if (revision.isPresent() && ( ! revision.get().isProduction())) { throw new IllegalArgumentException("Application version to deploy must be a known version"); } - this.pinned = pinned; + this.platformPinned = platformPinned; + this.revisionPinned = revisionPinned; } public Change withoutPlatform() { - return new Change(Optional.empty(), revision, pinned); + return new Change(Optional.empty(), revision, platformPinned, revisionPinned); } public Change withoutApplication() { - return new Change(platform, Optional.empty(), pinned); + return new Change(platform, Optional.empty(), platformPinned, revisionPinned); } /** Returns whether a change should currently be deployed */ @@ -58,7 +61,7 @@ public final class Change { /** Returns whether this is the empty change. */ public boolean isEmpty() { - return ! hasTargets() && ! pinned; + return ! hasTargets() && ! platformPinned && ! revisionPinned; } /** Returns the platform version carried by this. */ @@ -67,42 +70,55 @@ public final class Change { /** Returns the application version carried by this. */ public Optional<RevisionId> revision() { return revision; } - public boolean isPinned() { return pinned; } + public boolean isPlatformPinned() { return platformPinned; } + + public boolean isRevisionPinned() { return revisionPinned; } /** Returns an instance representing no change */ public static Change empty() { return empty; } /** Returns a version of this change which replaces or adds this platform change */ public Change with(Version platformVersion) { - if (pinned) + if (platformPinned) throw new IllegalArgumentException("Not allowed to set a platform version when pinned."); - return new Change(Optional.of(platformVersion), revision, pinned); + return new Change(Optional.of(platformVersion), revision, platformPinned, revisionPinned); } /** Returns a version of this change which replaces or adds this revision change */ public Change with(RevisionId revision) { - return new Change(platform, Optional.of(revision), pinned); + if (revisionPinned) + throw new IllegalArgumentException("Not allowed to set a revision when pinned."); + + return new Change(platform, Optional.of(revision), platformPinned, revisionPinned); + } + + /** Returns a change with the versions of this, and with the platform version pinned. */ + public Change withPlatformPin() { + return new Change(platform, revision, true, revisionPinned); + } + + /** Returns a change with the versions of this, and with the platform version unpinned. */ + public Change withoutPlatformPin() { + return new Change(platform, revision, false, revisionPinned); } /** Returns a change with the versions of this, and with the platform version pinned. */ - public Change withPin() { - return new Change(platform, revision, true); + public Change withRevisionPin() { + return new Change(platform, revision, platformPinned, true); } /** Returns a change with the versions of this, and with the platform version unpinned. */ - public Change withoutPin() { - return new Change(platform, revision, false); + public Change withoutRevisionPin() { + return new Change(platform, revision, platformPinned, false); } /** Returns the change obtained when overwriting elements of the given change with any present in this */ public Change onTopOf(Change other) { - if (platform.isPresent()) - other = other.with(platform.get()); - if (revision.isPresent()) - other = other.with(revision.get()); - if (pinned) - other = other.withPin(); + if (platform.isPresent()) other = other.with(platform.get()); + if (revision.isPresent()) other = other.with(revision.get()); + if (platformPinned) other = other.withPlatformPin(); + if (revisionPinned) other = other.withRevisionPin(); return other; } @@ -111,34 +127,38 @@ public final class Change { if (this == o) return true; if (!(o instanceof Change)) return false; Change change = (Change) o; - return pinned == change.pinned && + return platformPinned == change.platformPinned && + revisionPinned == change.revisionPinned && Objects.equals(platform, change.platform) && Objects.equals(revision, change.revision); } @Override public int hashCode() { - return Objects.hash(platform, revision, pinned); + return Objects.hash(platform, revision, platformPinned, revisionPinned); } @Override public String toString() { StringJoiner changes = new StringJoiner(" and "); - if (pinned) + if (platformPinned) changes.add("pin to " + platform.map(Version::toString).orElse("current platform")); else platform.ifPresent(version -> changes.add("upgrade to " + version)); - revision.ifPresent(revision -> changes.add("revision change to " + revision)); + if (revisionPinned) + changes.add("pin to " + revision.map(RevisionId::toString).orElse("current revision")); + else + revision.ifPresent(revision -> changes.add("revision change to " + revision)); changes.setEmptyValue("no change"); return changes.toString(); } public static Change of(RevisionId revision) { - return new Change(Optional.empty(), Optional.of(revision), false); + return new Change(Optional.empty(), Optional.of(revision), false, false); } public static Change of(Version platformChange) { - return new Change(Optional.of(platformChange), Optional.empty(), false); + return new Change(Optional.of(platformChange), Optional.empty(), false, false); } /** Returns whether this change carries a revision downgrade relative to the given revision. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index 1eb68c14353..80b52e0c7a4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -31,7 +31,8 @@ import static java.util.Comparator.comparing; */ public class Endpoint { - private static final String OATH_DNS_SUFFIX = ".vespa.oath.cloud"; + private static final String MAIN_OATH_DNS_SUFFIX = ".vespa.oath.cloud"; + private static final String CD_OATH_DNS_SUFFIX = ".cd.vespa.oath.cloud"; private static final String PUBLIC_DNS_SUFFIX = ".vespa-app.cloud"; private static final String PUBLIC_CD_DNS_SUFFIX = ".cd.vespa-app.cloud"; @@ -243,18 +244,13 @@ public class Endpoint { /** Returns the DNS suffix used for endpoints in given system */ private static String dnsSuffix(SystemName system) { - switch (system) { - case cd, main -> { - return OATH_DNS_SUFFIX; - } - case Public -> { - return PUBLIC_DNS_SUFFIX; - } - case PublicCd -> { - return PUBLIC_CD_DNS_SUFFIX; - } + return switch (system) { + case cd -> CD_OATH_DNS_SUFFIX; + case main -> MAIN_OATH_DNS_SUFFIX; + case Public -> PUBLIC_DNS_SUFFIX; + case PublicCd -> PUBLIC_CD_DNS_SUFFIX; default -> throw new IllegalArgumentException("No DNS suffix declared for system " + system); - } + }; } /** Returns the DNS suffix used for internal names (i.e. names not exposed to tenants) in given system */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java index 2441da19b90..b94779994e4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java @@ -21,7 +21,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.OptionalInt; import java.util.function.Function; import static java.util.Comparator.comparing; @@ -81,8 +80,7 @@ public class InstanceList extends AbstractFilteringList<ApplicationId, InstanceL /** Returns the subset of instances that are allowed to upgrade to the given version at the given time */ public InstanceList canUpgradeAt(Version version, Instant instant) { return matching(id -> instances.get(id).instanceSteps().get(id.instance()) - .readyAt(Change.of(version)) - .map(readyAt -> ! readyAt.isAfter(instant)).orElse(false)); + .readiness(Change.of(version)).okAt(instant)); } /** Returns the subset of instances which have at least one production deployment */ @@ -127,7 +125,7 @@ public class InstanceList extends AbstractFilteringList<ApplicationId, InstanceL /** Returns the subset of instances which are not pinned to a certain Vespa version. */ public InstanceList unpinned() { - return matching(id -> ! instance(id).change().isPinned()); + return matching(id -> ! instance(id).change().isPlatformPinned()); } /** Returns the subset of instances which are currently failing a job. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageStream.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageStream.java index f86073cfb25..3880b028eb0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageStream.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageStream.java @@ -5,6 +5,7 @@ import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; +import java.nio.file.attribute.FileTime; import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; @@ -32,6 +33,7 @@ public class ApplicationPackageStream { private final Supplier<Predicate<String>> filter; private final Supplier<InputStream> in; private final AtomicReference<ApplicationPackage> truncatedPackage = new AtomicReference<>(); + private final FileTime createdAt = FileTime.fromMillis(System.currentTimeMillis()); /** Stream that effectively copies the input stream to its {@link #truncatedPackage()} when exhausted. */ public ApplicationPackageStream(Supplier<InputStream> in) { @@ -60,7 +62,7 @@ public class ApplicationPackageStream { * and the first to be exhausted will populate the truncated application package. */ public InputStream zipStream() { - return new Stream(in.get(), replacer.get(), filter.get(), truncatedPackage); + return new Stream(in.get(), replacer.get(), filter.get(), createdAt, truncatedPackage); } /** @@ -85,6 +87,7 @@ public class ApplicationPackageStream { private final ZipInputStream inZip; private final Replacer replacer; private final Predicate<String> filter; + private final FileTime createdAt; private byte[] currentOut = new byte[0]; private InputStream currentIn = InputStream.nullInputStream(); private boolean includeCurrent = false; @@ -92,11 +95,12 @@ public class ApplicationPackageStream { private boolean closed = false; private boolean done = false; - private Stream(InputStream in, Replacer replacer, Predicate<String> filter, AtomicReference<ApplicationPackage> truncatedPackage) { + private Stream(InputStream in, Replacer replacer, Predicate<String> filter, FileTime createdAt, AtomicReference<ApplicationPackage> truncatedPackage) { this.in = in; this.inZip = new ZipInputStream(in); this.replacer = replacer; this.filter = filter; + this.createdAt = createdAt; this.truncatedPackage = truncatedPackage; } @@ -129,10 +133,12 @@ public class ApplicationPackageStream { ZipEntry next = inZip.getNextEntry(); String name; + FileTime modifiedAt; InputStream content = null; if (next == null) { // We may still have replacements to fill in, but if we don't, we're done filling, forever! name = replacer.next(); + modifiedAt = createdAt; if (name == null) { outZip.close(); // This typically makes new output available, so must check for that after this. teeZip.close(); @@ -144,6 +150,7 @@ public class ApplicationPackageStream { } else { name = next.getName(); + modifiedAt = next.getLastModifiedTime(); content = new FilterInputStream(inZip) { @Override public void close() { } }; // Protect inZip from replacements closing it. } @@ -153,8 +160,8 @@ public class ApplicationPackageStream { currentIn = InputStream.nullInputStream(); } else { - if (includeCurrent) teeZip.putNextEntry(new ZipEntry(name)); - outZip.putNextEntry(new ZipEntry(name)); + if (includeCurrent) teeZip.putNextEntry(new ZipEntry(name) {{ setLastModifiedTime(modifiedAt); }}); + outZip.putNextEntry(new ZipEntry(name) {{ setLastModifiedTime(modifiedAt); }}); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java index aa6e3b0c44d..cbd0f685d80 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java @@ -49,7 +49,7 @@ public record AuditLog(List<Entry> entries) { public record Entry(Instant at, String principal, Method method, String resource, Optional<String> data, Client client) implements Comparable<Entry> { - private final static int maxDataLength = 1024; + final static int maxDataLength = 1024; private final static Comparator<Entry> comparator = Comparator.comparing(Entry::at).reversed(); public Entry(Instant at, Client client, String principal, Method method, String resource, byte[] data) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java index 033cd0a52c9..13b3d9d170f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java @@ -4,11 +4,12 @@ package com.yahoo.vespa.hosted.controller.auditlog; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.jdisc.http.HttpHeaders; import com.yahoo.transaction.Mutex; +import com.yahoo.vespa.hosted.controller.auditlog.AuditLog.Entry; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.UncheckedIOException; +import java.io.InputStream; +import java.io.SequenceInputStream; import java.net.URI; import java.security.Principal; import java.time.Clock; @@ -17,6 +18,9 @@ import java.time.Instant; import java.util.Objects; import java.util.Optional; +import static com.yahoo.yolean.Exceptions.uncheck; +import static java.util.Objects.requireNonNullElse; + /** * This provides read and write operations for the audit log. * @@ -58,14 +62,8 @@ public class AuditLogger { "misconfiguration and should not happen"); } - byte[] data = new byte[0]; - try { - if (request.getData() != null) { - data = request.getData().readAllBytes(); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } + InputStream requestData = requireNonNullElse(request.getData(), InputStream.nullInputStream()); + byte[] data = uncheck(() -> requestData.readNBytes(Entry.maxDataLength)); AuditLog.Entry.Client client = parseClient(request); Instant now = clock.instant(); @@ -80,7 +78,9 @@ public class AuditLogger { } // Create a new input stream to allow callers to consume request body - return new HttpRequest(request.getJDiscRequest(), new ByteArrayInputStream(data), request.propertyMap()); + return new HttpRequest(request.getJDiscRequest(), + new SequenceInputStream(new ByteArrayInputStream(data), requestData), + request.propertyMap()); } private static AuditLog.Entry.Client parseClient(HttpRequest request) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index a76e76611c2..0f1bbfeb25e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -39,11 +39,11 @@ import java.util.Collections; import java.util.Deque; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -229,7 +229,7 @@ public class DeploymentStatus { .anyMatch(deployment -> ! compatibleWithCompileVersion.test(deployment.version()))) { for (Version platform : targetsForPolicy(versionStatus, systemVersion, application.deploymentSpec().requireInstance(instance).upgradePolicy())) if (compatibleWithCompileVersion.test(platform)) - return change.withoutPin().with(platform); + return change.withoutPlatformPin().with(platform); } return change; } @@ -265,7 +265,7 @@ public class DeploymentStatus { for (InstanceName instance : application.deploymentSpec().instanceNames()) { Change outstanding = outstandingChange(instance); if (outstanding.hasTargets()) - outstandingChanges.put(instance, outstanding.onTopOf(application.require(instance).change())); + outstandingChanges.put(instance, outstanding.onTopOf(application.require(instance).change().withoutRevisionPin())); } var testJobs = jobsToRun(outstandingChanges, true).entrySet().stream() .filter(entry -> ! entry.getKey().type().isProduction()); @@ -303,7 +303,11 @@ public class DeploymentStatus { fallbackPlatform(change, job)); if (step.completedAt(change, firstProductionJobWithDeploymentInCloud).isEmpty()) { JobType typeWithZone = job.type().isSystemTest() ? JobType.systemTest(zones, cloud) : JobType.stagingTest(zones, cloud); - jobs.merge(job, List.of(new Job(typeWithZone, versions, step.readyAt(change), change)), DeploymentStatus::union); + Readiness readiness = step.readiness(change, firstProductionJobWithDeploymentInCloud); + jobs.merge(job, List.of(new Job(typeWithZone, + versions, + readiness.okAt(now) && jobs().get(job).get().isRunning() ? readiness.running() : readiness, + change)), DeploymentStatus::union); } }); }); @@ -390,7 +394,7 @@ public class DeploymentStatus { } /** The set of jobs that need to run for the given changes to be considered complete. */ - private Map<JobId, List<Job>> jobsToRun(Map<InstanceName, Change> changes) { + public Map<JobId, List<Job>> jobsToRun(Map<InstanceName, Change> changes) { return jobsToRun(changes, false); } @@ -498,21 +502,23 @@ public class DeploymentStatus { } /** Earliest instant when job was triggered with given versions, or both system and staging tests were successful. */ - public Optional<Instant> verifiedAt(JobId job, Versions versions) { - Optional<Instant> triggeredAt = allJobs.get(job) - .flatMap(status -> status.runs().values().stream() - .filter(run -> run.versions().equals(versions)) - .findFirst()) - .map(Run::start); - Optional<Instant> systemTestedAt = testedAt(job, systemTest(job.type()), versions); - Optional<Instant> stagingTestedAt = testedAt(job, stagingTest(job.type()), versions); - if (systemTestedAt.isEmpty() || stagingTestedAt.isEmpty()) return triggeredAt; - Optional<Instant> testedAt = systemTestedAt.get().isAfter(stagingTestedAt.get()) ? systemTestedAt : stagingTestedAt; - return triggeredAt.isPresent() && triggeredAt.get().isBefore(testedAt.get()) ? triggeredAt : testedAt; + public Readiness verifiedAt(JobId job, Versions versions) { + Readiness triggered = allJobs.get(job) + .flatMap(status -> status.runs().values().stream() + .filter(run -> run.versions().equals(versions)) + .findFirst()) + .map(Run::start) + .map(Readiness::new) + .orElse(Readiness.unverified); + Readiness systemTested = testedAt(job, systemTest(job.type()), versions); + Readiness stagingTested = testedAt(job, stagingTest(job.type()), versions); + if (! systemTested.ok() || ! stagingTested.ok()) return triggered; + Readiness tested = min(systemTested, stagingTested); + return triggered.ok() && triggered.at().isBefore(tested.at) ? triggered : tested; } /** Earliest instant when versions were tested for the given instance. */ - private Optional<Instant> testedAt(JobId job, JobType type, Versions versions) { + private Readiness testedAt(JobId job, JobType type, Versions versions) { return prerequisiteTests(job, type).stream() .map(test -> allJobs.get(test).stream() .flatMap(status -> RunList.from(status) @@ -522,19 +528,21 @@ public class DeploymentStatus { .asList().stream() .map(run -> run.end().get())) .min(naturalOrder())) - .reduce((o, n) -> o.isEmpty() || n.isEmpty() ? Optional.empty() : o.get().isBefore(n.get()) ? n : o) - .orElse(Optional.empty()); + .map(testedAt -> testedAt.map(Readiness::new).orElse(Readiness.unverified)) + .reduce(Readiness.empty, DeploymentStatus::max); } private Map<JobId, List<Job>> productionJobs(InstanceName instance, Change change, boolean assumeUpgradesSucceed) { Map<JobId, List<Job>> jobs = new LinkedHashMap<>(); - jobSteps.forEach((job, step) -> { + for (Entry<JobId, StepStatus> entry : reversed(List.copyOf(jobSteps.entrySet()))) { + JobId job = entry.getKey(); + StepStatus step = entry.getValue(); if ( ! job.application().instance().equals(instance) || ! job.type().isProduction()) - return; + continue; // Signal strict completion criterion by depending on job itself. if (step.completedAt(change, Optional.of(job)).isPresent()) - return; + continue; // When computing eager test jobs for outstanding changes, assume current change completes successfully. Optional<Deployment> deployment = deploymentFor(job); @@ -544,7 +552,7 @@ public class DeploymentStatus { || areIncompatible(change.platform(), existingRevision, job); if (assumeUpgradesSucceed) { if (deployingCompatibilityChange) // No eager tests for this. - return; + continue; Change currentChange = application.require(instance).change(); Versions target = Versions.from(currentChange, application, deployment, fallbackPlatform(currentChange, job)); @@ -554,21 +562,28 @@ public class DeploymentStatus { List<Job> toRun = new ArrayList<>(); List<Change> changes = deployingCompatibilityChange || allJobs.get(job).flatMap(status -> status.lastCompleted()).isEmpty() - ? List.of(change) - : changes(job, step, change); + ? List.of(change) + : changes(job, step, change); for (Change partial : changes) { - Job jobToRun = new Job(job.type(), - Versions.from(partial, application, existingPlatform, existingRevision, fallbackPlatform(partial, job)), - step.readyAt(partial, Optional.of(job)), - partial); - toRun.add(jobToRun); + Versions versions = Versions.from(partial, application, existingPlatform, existingRevision, fallbackPlatform(partial, job)); + Readiness readiness = step.readiness(partial, Optional.of(job)); + // This job is blocked if it is already running ... + readiness = jobs().get(job).get().isRunning() && readiness.okAt(now) ? readiness.running() : readiness; + // ... or if it is a deployment, and a test job for the current state is not yet complete, + // which is the case when the next versions to run that test with is not the same as we want to deploy here. + List<Job> tests = job.type().isTest() ? null : jobs.get(new JobId(job.application(), JobType.productionTestOf(job.type().zone()))); + readiness = tests != null && ! versions.targetsMatch(tests.get(0).versions) && readiness.okAt(now) ? readiness.blocked() : readiness; + toRun.add(new Job(job.type(), versions, readiness, partial)); // Assume first partial change is applied before the second. - existingPlatform = Optional.of(jobToRun.versions.targetPlatform()); - existingRevision = Optional.of(jobToRun.versions.targetRevision()); + existingPlatform = Optional.of(versions.targetPlatform()); + existingRevision = Optional.of(versions.targetRevision()); } jobs.put(job, toRun); - }); - return jobs; + } + Map<JobId, List<Job>> jobsInOrder = new LinkedHashMap<>(); + for (Entry<JobId, List<Job>> entry : reversed(List.copyOf(jobs.entrySet()))) + jobsInOrder.put(entry.getKey(), entry.getValue()); + return jobsInOrder; } private boolean areIncompatible(Optional<Version> platform, Optional<RevisionId> revision, JobId job) { @@ -581,7 +596,8 @@ public class DeploymentStatus { /** Changes to deploy with the given job, possibly split in two steps. */ private List<Change> changes(JobId job, StepStatus step, Change change) { - if (change.platform().isEmpty() || change.revision().isEmpty() || change.isPinned()) + if ( change.platform().isEmpty() || change.revision().isEmpty() + || change.isPlatformPinned() || change.isRevisionPinned()) return List.of(change); if ( step.completedAt(change.withoutApplication(), Optional.of(job)).isPresent() @@ -606,8 +622,7 @@ public class DeploymentStatus { // the revision is now blocked by waiting for the production test to verify the upgrade. // In this case we must abandon the production test on the pure upgrade, so the revision can be deployed. if (platformDeployedAt.isPresent() && revisionDeployedAt.isEmpty()) { - if (jobSteps.get(deployment).readyAt(change, Optional.of(deployment)) - .map(ready -> ! now.isBefore(ready)).orElse(false)) { + if (jobSteps.get(deployment).readiness(change, Optional.of(deployment)).okAt(now)) { return switch (rollout) { // If separate rollout, this test should keep blocking the revision, unless there are failures. case separate -> hasFailures(jobSteps.get(deployment), jobSteps.get(job)) ? List.of(change) : List.of(change.withoutApplication(), change); @@ -676,12 +691,15 @@ public class DeploymentStatus { for (Job productionJob : versionsList) if (allJobs.successOn(testType, productionJob.versions()) .instance(testJob.application().instance()) - .asList().isEmpty()) + .asList().isEmpty()) { + Readiness readiness = jobSteps().get(testJob).readiness(productionJob.change, Optional.of(job)); testJobs.merge(testJob, List.of(new Job(testJob.type(), productionJob.versions(), - jobSteps().get(testJob).readyAt(productionJob.change), + readiness.okAt(now) && jobs().get(testJob).get().isRunning() ? readiness.running() : readiness, productionJob.change)), DeploymentStatus::union); + + } }); } } @@ -894,16 +912,17 @@ public class DeploymentStatus { abstract Optional<Instant> completedAt(Change change, Optional<JobId> dependent); /** The time at which this step is ready to run the specified change and / or versions. */ - public Optional<Instant> readyAt(Change change) { return readyAt(change, Optional.empty()); } + public Readiness readiness(Change change) { return readiness(change, Optional.empty()); } /** The time at which this step is ready to run the specified change and / or versions. */ - Optional<Instant> readyAt(Change change, Optional<JobId> dependent) { + Readiness readiness(Change change, Optional<JobId> dependent) { return dependenciesCompletedAt(change, dependent) + .map(Readiness::new) .map(ready -> Stream.of(blockedUntil(change), pausedUntil(), coolingDownUntil(change, dependent)) - .flatMap(Optional::stream) - .reduce(ready, maxBy(naturalOrder()))); + .reduce(ready, maxBy(naturalOrder()))) + .orElse(Readiness.notReady); } /** The time at which all dependencies completed on the given change and / or versions. */ @@ -918,13 +937,13 @@ public class DeploymentStatus { } /** The time until which this step is blocked by a change blocker. */ - public Optional<Instant> blockedUntil(Change change) { return Optional.empty(); } + public Readiness blockedUntil(Change change) { return Readiness.empty; } /** The time until which this step is paused by user intervention. */ - public Optional<Instant> pausedUntil() { return Optional.empty(); } + public Readiness pausedUntil() { return Readiness.empty; } /** The time until which this step is cooling down, due to consecutive failures. */ - public Optional<Instant> coolingDownUntil(Change change, Optional<JobId> dependent) { return Optional.empty(); } + public Readiness coolingDownUntil(Change change, Optional<JobId> dependent) { return Readiness.empty; } /** Whether this step is declared in the deployment spec, or is an implicit step. */ public boolean isDeclared() { return true; } @@ -940,7 +959,8 @@ public class DeploymentStatus { @Override Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { - return readyAt(change, dependent).map(completion -> completion.plus(step().delay())); + return Optional.ofNullable(readiness(change, dependent).at()) + .map(completion -> completion.plus(step().delay())); } } @@ -964,12 +984,12 @@ public class DeploymentStatus { /** The time at which this step is ready to run the specified change and / or versions. */ @Override - public Optional<Instant> readyAt(Change change) { + public Readiness readiness(Change change) { return status.jobSteps.keySet().stream() .filter(job -> job.type().isProduction() && job.application().instance().equals(instance.name())) - .map(job -> super.readyAt(change, Optional.of(job))) - .reduce((o, n) -> o.isEmpty() || n.isEmpty() ? Optional.empty() : n.get().isBefore(o.get()) ? n : o) - .orElseGet(() -> super.readyAt(change, Optional.empty())); + .map(job -> super.readiness(change, Optional.of(job))) + .reduce((a, b) -> ! a.ok() ? a : ! b.ok() ? b : min(a, b)) + .orElseGet(() -> super.readiness(change, Optional.empty())); } /** @@ -986,7 +1006,7 @@ public class DeploymentStatus { } @Override - public Optional<Instant> blockedUntil(Change change) { + public Readiness blockedUntil(Change change) { for (Instant current = now; now.plus(Duration.ofDays(7)).isAfter(current); ) { boolean blocked = false; for (DeploymentSpec.ChangeBlocker blocker : spec.changeBlocker()) { @@ -999,9 +1019,9 @@ public class DeploymentStatus { } } if ( ! blocked) - return current == now ? Optional.empty() : Optional.of(current); + return current == now ? Readiness.empty : new Readiness(current, DelayCause.changeBlocked); } - return Optional.of(now.plusSeconds(1 << 30)); // Some time in the future that doesn't look like anything you'd expect. + return new Readiness(now.plusSeconds(1 << 30), DelayCause.changeBlocked); // Some time in the future that doesn't look like anything you'd expect. } } @@ -1023,31 +1043,34 @@ public class DeploymentStatus { public Optional<JobId> job() { return Optional.of(job.id()); } @Override - public Optional<Instant> pausedUntil() { - return status.application().require(job.id().application().instance()).jobPause(job.id().type()); + public Readiness pausedUntil() { + return status.application().require(job.id().application().instance()).jobPause(job.id().type()) + .map(pause -> new Readiness(pause, DelayCause.paused)) + .orElse(Readiness.empty); } @Override - public Optional<Instant> coolingDownUntil(Change change, Optional<JobId> dependent) { - if (job.lastTriggered().isEmpty()) return Optional.empty(); - if (job.lastCompleted().isEmpty()) return Optional.empty(); - if (job.firstFailing().isEmpty() || ! job.firstFailing().get().hasEnded()) return Optional.empty(); + public Readiness coolingDownUntil(Change change, Optional<JobId> dependent) { + if (job.lastTriggered().isEmpty()) return Readiness.empty; + if (job.lastCompleted().isEmpty()) return Readiness.empty; + if (job.firstFailing().isEmpty() || ! job.firstFailing().get().hasEnded()) return Readiness.empty; Versions lastVersions = job.lastCompleted().get().versions(); Versions toRun = Versions.from(change, status.application, dependent.flatMap(status::deploymentFor), status.fallbackPlatform(change, job.id())); - if ( ! toRun.targetsMatch(lastVersions)) return Optional.empty(); + if ( ! toRun.targetsMatch(lastVersions)) return Readiness.empty; if ( job.id().type().environment().isTest() && ! dependent.map(JobId::type).map(status::findCloud).map(List.of(CloudName.AWS, CloudName.GCP)::contains).orElse(true) - && job.isNodeAllocationFailure()) return Optional.empty(); + && job.isNodeAllocationFailure()) return Readiness.empty; - if (job.lastStatus().get() == invalidApplication) return Optional.of(status.now.plus(Duration.ofDays(36524))); // 100 years + if (job.lastStatus().get() == invalidApplication) return new Readiness(status.now.plus(Duration.ofSeconds(1 << 30)), DelayCause.invalidPackage); Instant firstFailing = job.firstFailing().get().end().get(); Instant lastCompleted = job.lastCompleted().get().end().get(); - return firstFailing.equals(lastCompleted) ? Optional.of(lastCompleted) - : Optional.of(lastCompleted.plus(Duration.ofMinutes(10)) - .plus(Duration.between(firstFailing, lastCompleted) - .dividedBy(2))) - .filter(status.now::isBefore); + Duration penalty = firstFailing.equals(lastCompleted) ? Duration.ZERO + : Duration.ofMinutes(10) + .plus(Duration.between(firstFailing, lastCompleted) + .dividedBy(2)); + return lastCompleted.plus(penalty).isAfter(status.now) ? new Readiness(lastCompleted.plus(penalty), DelayCause.coolingDown) + : Readiness.empty; } private static JobStepStatus ofProductionDeployment(DeclaredZone step, List<StepStatus> dependencies, @@ -1059,24 +1082,23 @@ public class DeploymentStatus { return new JobStepStatus(StepType.deployment, step, dependencies, job, status) { @Override - public Optional<Instant> readyAt(Change change, Optional<JobId> dependent) { - Optional<Instant> readyAt = super.readyAt(change, dependent); - Optional<Instant> testedAt = status.verifiedAt(job.id(), Versions.from(change, status.application, existingDeployment, status.fallbackPlatform(change, job.id()))); - if (readyAt.isEmpty() || testedAt.isEmpty()) return Optional.empty(); - return readyAt.get().isAfter(testedAt.get()) ? readyAt : testedAt; + public Readiness readiness(Change change, Optional<JobId> dependent) { + Readiness readyAt = super.readiness(change, dependent); + Readiness testedAt = status.verifiedAt(job.id(), Versions.from(change, status.application, existingDeployment, status.fallbackPlatform(change, job.id()))); + return max(readyAt, testedAt); } /** Complete if deployment is on pinned version, and last successful deployment, or if given versions is strictly a downgrade, and this isn't forced by a pin. */ @Override Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { - if ( change.isPinned() + if ( change.isPlatformPinned() && change.platform().isPresent() && ! existingDeployment.map(Deployment::version).equals(change.platform())) return Optional.empty(); if ( change.revision().isPresent() - && ! existingDeployment.map(Deployment::revision).equals(change.revision()) - && dependent.equals(job())) // Job should (re-)run in this case, but other dependents need not wait. + && change.isRevisionPinned() + && ! existingDeployment.map(Deployment::revision).equals(change.revision())) return Optional.empty(); Change fullChange = status.application().require(job.id().application().instance()).change(); @@ -1103,11 +1125,11 @@ public class DeploymentStatus { JobId prodId = new JobId(job.id().application(), JobType.deploymentTo(job.id().type().zone())); return new JobStepStatus(StepType.test, step, dependencies, job, status) { @Override - Optional<Instant> readyAt(Change change, Optional<JobId> dependent) { - Optional<Instant> readyAt = super.readyAt(change, dependent); - Optional<Instant> deployedAt = status.jobSteps().get(prodId).completedAt(change, Optional.of(prodId)); - if (readyAt.isEmpty() || deployedAt.isEmpty()) return Optional.empty(); - return readyAt.get().isAfter(deployedAt.get()) ? readyAt : deployedAt; + Readiness readiness(Change change, Optional<JobId> dependent) { + Readiness readyAt = super.readiness(change, dependent); + Readiness deployedAt = status.jobSteps().get(prodId).completedAt(change, Optional.of(prodId)) + .map(Readiness::new).orElse(Readiness.notReady); + return max(readyAt, deployedAt); } @Override @@ -1163,13 +1185,13 @@ public class DeploymentStatus { private final JobType type; private final Versions versions; - private final Optional<Instant> readyAt; + private final Readiness readiness; private final Change change; - public Job(JobType type, Versions versions, Optional<Instant> readyAt, Change change) { + public Job(JobType type, Versions versions, Readiness readiness, Change change) { this.type = type; this.versions = type.isSystemTest() ? versions.withoutSources() : versions; - this.readyAt = readyAt; + this.readiness = readiness; this.change = change; } @@ -1181,8 +1203,8 @@ public class DeploymentStatus { return versions; } - public Optional<Instant> readyAt() { - return readyAt; + public Readiness readiness() { + return readiness; } @Override @@ -1190,19 +1212,60 @@ public class DeploymentStatus { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Job job = (Job) o; - return type.zone().equals(job.type.zone()) && versions.equals(job.versions) && readyAt.equals(job.readyAt) && change.equals(job.change); + return type.zone().equals(job.type.zone()) && versions.equals(job.versions) && readiness.equals(job.readiness) && change.equals(job.change); } @Override public int hashCode() { - return Objects.hash(type.zone(), versions, readyAt, change); + return Objects.hash(type.zone(), versions, readiness, change); } @Override public String toString() { - return change + " with versions " + versions + ", ready at " + readyAt; + return change + " with versions " + versions + ", " + readiness; } } + public enum DelayCause { none, unverified, notReady, blocked, running, coolingDown, invalidPackage, changeBlocked, paused } + public record Readiness(Instant at, DelayCause cause) implements Comparable<Readiness> { + public static final Readiness unverified = new Readiness(null, DelayCause.unverified); + public static final Readiness notReady = new Readiness(null, DelayCause.notReady); + public static final Readiness empty = new Readiness(Instant.EPOCH, DelayCause.none); + public Readiness(Instant at) { this(at, DelayCause.none); } + public Readiness blocked() { return new Readiness(at, DelayCause.blocked); } + public Readiness running() { return new Readiness(at, DelayCause.running); } + public boolean ok() { return at != null; } + public boolean okAt(Instant at) { return ok() && cause != DelayCause.running && cause != DelayCause.blocked && ! at.isBefore(this.at); } + @Override public int compareTo(Readiness o) { + return at == null ? o.at == null ? 0 : 1 + : o.at == null ? -1 : at.compareTo(o.at); + } + @Override public String toString() { + return ok() ? "ready at " + at + switch (cause) { + case none -> ""; + case coolingDown -> ": cooling down after repeated failures"; + case blocked -> ": waiting for verification test to complete"; + case running -> ": waiting for current run to complete"; + case invalidPackage -> ": invalid application package, must resubmit"; + case changeBlocked -> ": deployment configuration blocks changes"; + case paused -> ": manually paused"; + default -> throw new IllegalStateException(cause + " should not have an instant at which it is ready"); + } + : "not ready" + switch (cause) { + case unverified -> ": waiting for verification test to complete"; + case notReady -> ": waiting for dependencies to complete"; + default -> throw new IllegalStateException(cause + " should have an instant at which it is ready"); + }; + } + } + + static <T extends Comparable<T>> T min(T a, T b) { + return a.compareTo(b) > 0 ? b : a; + } + + static <T extends Comparable<T>> T max(T a, T b) { + return a.compareTo(b) < 0 ? b : a; + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index a5cb839e9c9..4e699f2c28f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -20,6 +20,8 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.DelayCause; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.Readiness; import java.math.BigDecimal; import java.time.Clock; @@ -80,8 +82,7 @@ public class DeploymentTrigger { Change outstanding = status.outstandingChange(instanceName); boolean deployOutstanding = outstanding.hasTargets() && status.instanceSteps().get(instanceName) - .readyAt(outstanding) - .map(readyAt -> ! readyAt.isAfter(clock.instant())).orElse(false) + .readiness(outstanding).okAt(clock.instant()) && acceptNewRevision(status, instanceName, outstanding.revision().get()); application = application.with(instanceName, instance -> withRemainingChange(instance, @@ -235,7 +236,7 @@ public class DeploymentTrigger { if ( ! upgradeRevision && change.revision().isPresent()) change = change.withoutApplication(); if ( ! upgradePlatform && change.platform().isPresent()) change = change.withoutPlatform(); Versions versions = Versions.from(change, application, status.deploymentFor(job), status.fallbackPlatform(change, job)); - DeploymentStatus.Job toTrigger = new DeploymentStatus.Job(job.type(), versions, Optional.of(controller.clock().instant()), instance.change()); + DeploymentStatus.Job toTrigger = new DeploymentStatus.Job(job.type(), versions, new Readiness(controller.clock().instant()), instance.change()); Map<JobId, List<DeploymentStatus.Job>> testJobs = status.testJobs(Map.of(job, List.of(toTrigger))); Map<JobId, List<DeploymentStatus.Job>> jobs = testJobs.isEmpty() || ! requireTests @@ -329,15 +330,14 @@ public class DeploymentTrigger { /** Cancels the indicated part of the given application's change. */ public void cancelChange(ApplicationId instanceId, ChangesToCancel cancellation) { applications().lockApplicationOrThrow(TenantAndApplicationId.from(instanceId), application -> { - Change change; - switch (cancellation) { - case ALL: change = Change.empty(); break; - case VERSIONS: change = Change.empty().withPin(); break; - case PLATFORM: change = application.get().require(instanceId.instance()).change().withoutPlatform(); break; - case APPLICATION: change = application.get().require(instanceId.instance()).change().withoutApplication(); break; - case PIN: change = application.get().require(instanceId.instance()).change().withoutPin(); break; - default: throw new IllegalArgumentException("Unknown cancellation choice '" + cancellation + "'!"); - } + Change change = switch (cancellation) { + case ALL -> Change.empty(); + case PLATFORM -> application.get().require(instanceId.instance()).change().withoutPlatform(); + case APPLICATION -> application.get().require(instanceId.instance()).change().withoutApplication(); + case PIN -> application.get().require(instanceId.instance()).change().withoutPlatformPin(); + case PLATFORM_PIN -> application.get().require(instanceId.instance()).change().withoutPlatformPin(); + case APPLICATION_PIN -> application.get().require(instanceId.instance()).change().withoutRevisionPin(); + }; applications().store(application.with(instanceId.instance(), instance -> withRemainingChange(instance, change, @@ -346,7 +346,7 @@ public class DeploymentTrigger { }); } - public enum ChangesToCancel { ALL, PLATFORM, APPLICATION, VERSIONS, PIN } + public enum ChangesToCancel { ALL, PLATFORM, APPLICATION, PIN, PLATFORM_PIN, APPLICATION_PIN } // ---------- Conveniences ---------- @@ -374,17 +374,17 @@ public class DeploymentTrigger { List<Job> jobs = new ArrayList<>(); Map<JobId, List<DeploymentStatus.Job>> jobsToRun = status.jobsToRun(); jobsToRun.forEach((jobId, jobsList) -> { + abortIfOutdated(status, jobsToRun, jobId); DeploymentStatus.Job job = jobsList.get(0); - if ( job.readyAt().isPresent() - && ! clock.instant().isBefore(job.readyAt().get()) + if ( job.readiness().okAt(clock.instant()) && ! controller.jobController().isDisabled(new JobId(jobId.application(), job.type())) - && ! (jobId.type().isProduction() && isUnhealthyInAnotherZone(status.application(), jobId)) - && abortIfRunning(status, jobsToRun, jobId)) // Abort and trigger this later if running with outdated parameters. + && ! (jobId.type().isProduction() && isUnhealthyInAnotherZone(status.application(), jobId))) { jobs.add(deploymentJob(status.application().require(jobId.application().instance()), job.versions(), job.type(), status.instanceJobs(jobId.application().instance()).get(jobId.type()).isNodeAllocationFailure(), - job.readyAt().get())); + job.readiness().at())); + } }); return Collections.unmodifiableList(jobs); } @@ -403,41 +403,29 @@ public class DeploymentTrigger { return false; } - private void abortIfOutdated(DeploymentStatus status, Map<JobId, List<DeploymentStatus.Job>> jobs, JobId job) { - status.jobs().get(job) - .flatMap(JobStatus::lastTriggered) - .filter(last -> ! last.hasEnded() && last.reason().isEmpty()) - .ifPresent(last -> { - if (jobs.get(job).stream().noneMatch(versions -> versions.versions().targetsMatch(last.versions()) - && versions.versions().sourcesMatchIfPresent(last.versions()))) { - String blocked = jobs.get(job).stream() - .map(scheduled -> scheduled.versions().toString()) - .collect(Collectors.joining(", ")); - log.log(Level.INFO, "Aborting outdated run " + last + ", which is blocking runs: " + blocked); - controller.jobController().abort(last.id(), "run no longer scheduled, and is blocking scheduled runs: " + blocked); - } - }); + private void abortIfOutdated(JobStatus job, List<DeploymentStatus.Job> jobs) { + job.lastTriggered() + .filter(last -> ! last.hasEnded() && last.reason().isEmpty()) + .ifPresent(last -> { + if (jobs.stream().noneMatch(versions -> versions.versions().targetsMatch(last.versions()) + && versions.versions().sourcesMatchIfPresent(last.versions()))) { + String blocked = jobs.stream() + .map(scheduled -> scheduled.versions().toString()) + .collect(Collectors.joining(", ")); + log.log(Level.INFO, "Aborting outdated run " + last + ", which is blocking runs: " + blocked); + controller.jobController().abort(last.id(), "run no longer scheduled, and is blocking scheduled runs: " + blocked); + } + }); } /** Returns whether the job is free to start, and also aborts it if it's running with outdated versions. */ - private boolean abortIfRunning(DeploymentStatus status, Map<JobId, List<DeploymentStatus.Job>> jobs, JobId job) { - abortIfOutdated(status, jobs, job); - boolean blocked = status.jobs().get(job).get().isRunning(); - - if ( ! job.type().isTest()) { - Optional<JobStatus> productionTest = status.jobs().get(new JobId(job.application(), JobType.productionTestOf(job.type().zone()))); - if (productionTest.isPresent()) { - abortIfOutdated(status, jobs, productionTest.get().id()); - // Production deployments are also blocked by their declared tests, if the next versions to run - // for those are not the same as the versions we're considering running in the deployment job now. - if (productionTest.map(JobStatus::id).map(jobs::get) - .map(versions -> ! versions.get(0).versions().targetsMatch(jobs.get(job).get(0).versions())) - .orElse(false)) - blocked = true; - } - } - - return ! blocked; + private void abortIfOutdated(DeploymentStatus status, Map<JobId, List<DeploymentStatus.Job>> jobs, JobId job) { + Readiness readiness = jobs.get(job).get(0).readiness(); + if (readiness.cause() == DelayCause.running) + abortIfOutdated(status.jobs().get(job).get(), jobs.get(job)); + if (readiness.cause() == DelayCause.blocked && ! job.type().isTest()) + status.jobs().get(new JobId(job.application(), JobType.productionTestOf(job.type().zone()))) + .ifPresent(jobStatus -> abortIfOutdated(jobStatus, jobs.get(jobStatus.id()))); } // ---------- Change management o_O ---------- diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 451f5555eb2..52ddcfd5171 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -279,7 +279,7 @@ public class InternalStepRunner implements StepRunner { switch (e.type()) { case CERT_NOT_AVAILABLE: // Same as CERTIFICATE_NOT_READY above, only from the controller - logger.log("Creating a CA signed certificate for the application. " + + logger.log("Retrieving CA signed certificate for the application. " + "This may take up to " + timeouts.endpointCertificate() + " on first deployment."); if (startTime.plus(timeouts.endpointCertificate()).isBefore(controller.clock().instant())) { logger.log(WARNING, "CA signed certificate for app not available within " + @@ -437,7 +437,7 @@ public class InternalStepRunner implements StepRunner { Version targetPlatform = controller.jobController().run(id).versions().targetPlatform(); Version systemVersion = controller.readSystemVersion(); boolean incompatible = controller.applications().versionCompatibility(id.application()).refuse(targetPlatform, systemVersion); - return incompatible || application(id.application()).change().isPinned() ? targetPlatform : systemVersion; + return incompatible || application(id.application()).change().isPlatformPinned() ? targetPlatform : systemVersion; } private Optional<RunStatus> installTester(RunId id, DualLogger logger) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 73c64be3e47..318a6ffe820 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -111,6 +111,7 @@ import static java.util.logging.Level.WARNING; public class JobController { public static final Duration maxHistoryAge = Duration.ofDays(60); + public static final Duration obsoletePackageExpiry = Duration.ofDays(7); private static final Logger log = Logger.getLogger(JobController.class.getName()); @@ -165,8 +166,8 @@ public class JobController { return Optional.empty(); return active(id).isPresent() - ? Optional.of(logs.readActive(id.application(), id.type(), after)) - : logs.readFinished(id, after); + ? Optional.of(logs.readActive(id.application(), id.type(), after)) + : logs.readFinished(id, after); } } @@ -284,10 +285,10 @@ public class JobController { private Optional<InputStream> getVespaLogsFromLogserver(Run run, long fromMillis, boolean tester) { return deploymentCompletedAt(run, tester).map(at -> - controller.serviceRegistry().configServer().getLogs(new DeploymentId(tester ? run.id().tester().id() : run.id().application(), - run.id().type().zone()), - Map.of("from", Long.toString(Math.max(fromMillis, at.toEpochMilli())), - "to", Long.toString(run.end().orElse(controller.clock().instant()).toEpochMilli())))); + controller.serviceRegistry().configServer().getLogs(new DeploymentId(tester ? run.id().tester().id() : run.id().application(), + run.id().type().zone()), + Map.of("from", Long.toString(Math.max(fromMillis, at.toEpochMilli())), + "to", Long.toString(run.end().orElse(controller.clock().instant()).toEpochMilli())))); } /** Fetches any new test log entries, and records the id of the last of these, for continuation. */ @@ -509,14 +510,14 @@ public class JobController { long successes = runs.values().stream().filter(Run::hasSucceeded).count(); var oldEntries = runs.entrySet().iterator(); for (var old = oldEntries.next(); - old.getKey().number() <= last - historyLength + old.getKey().number() <= last - historyLength || old.getValue().start().isBefore(controller.clock().instant().minus(maxHistoryAge)); old = oldEntries.next()) { // Make sure we keep the last success and the first failing if ( successes == 1 - && old.getValue().hasSucceeded() - && ! old.getValue().start().isBefore(controller.clock().instant().minus(maxHistoryAge))) { + && old.getValue().hasSucceeded() + && ! old.getValue().start().isBefore(controller.clock().instant().minus(maxHistoryAge))) { oldEntries.next(); continue; } @@ -624,7 +625,7 @@ public class JobController { }); } - private LockedApplication withPrunedPackages(LockedApplication application, RevisionId latest){ + private LockedApplication withPrunedPackages(LockedApplication application, RevisionId latest) { TenantAndApplicationId id = application.get().id(); Application wrapped = application.get(); RevisionId oldestDeployed = application.get().oldestDeployedRevision() @@ -632,11 +633,28 @@ public class JobController { .flatMap(instance -> instance.change().revision().stream()) .min(naturalOrder())) .orElse(latest); - controller.applications().applicationStore().prune(id.tenant(), id.application(), oldestDeployed); + RevisionId oldestToKeep = null; + Instant now = controller.clock().instant(); + for (ApplicationVersion version : application.get().revisions().withPackage()) { + if (version.id().compareTo(oldestDeployed) < 0) { + if (version.obsoleteAt().isEmpty()) { + application = application.withRevisions(revisions -> revisions.with(version.obsoleteAt(now))); + if (oldestToKeep == null) + oldestToKeep = version.id(); + } + else { + if (oldestToKeep == null && !version.obsoleteAt().get().isBefore(now.minus(obsoletePackageExpiry))) + oldestToKeep = version.id(); + } + } + } - for (ApplicationVersion version : application.get().revisions().withPackage()) - if (version.id().compareTo(oldestDeployed) < 0) - application = application.withRevisions(revisions -> revisions.with(version.withoutPackage())); + if (oldestToKeep != null) { + controller.applications().applicationStore().prune(id.tenant(), id.application(), oldestToKeep); + for (ApplicationVersion version : application.get().revisions().withPackage()) + if (version.id().compareTo(oldestToKeep) < 0) + application = application.withRevisions(revisions -> revisions.with(version.withoutPackage())); + } return application; } @@ -703,8 +721,8 @@ public class JobController { VersionStatus versionStatus = controller.readVersionStatus(); if ( ! controller.system().isCd() - && platform.isPresent() - && versionStatus.deployableVersions().stream().map(VespaVersion::versionNumber).noneMatch(platform.get()::equals)) + && platform.isPresent() + && versionStatus.deployableVersions().stream().map(VespaVersion::versionNumber).noneMatch(platform.get()::equals)) throw new IllegalArgumentException("platform version " + platform.get() + " is not present in this system"); controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { @@ -713,6 +731,7 @@ public class JobController { // TODO(mpolden): Enable for public CD once all tests have been updated if (controller.system() != SystemName.PublicCd) { controller.applications().validatePackage(applicationPackage, application.get()); + controller.applications().decideCloudAccountOf(new DeploymentId(id, type.zone()), applicationPackage.deploymentSpec()); } controller.applications().store(application); }); @@ -730,8 +749,8 @@ public class JobController { controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> { Version targetPlatform = platform.orElseGet(() -> findTargetPlatform(applicationPackage, deploymentId, application.get().get(id.instance()), versionStatus)); if ( ! allowOutdatedPlatform - && ! controller.readVersionStatus().isOnCurrentMajor(targetPlatform) - && runs(id, type).values().stream().noneMatch(run -> run.versions().targetPlatform().getMajor() == targetPlatform.getMajor())) + && ! controller.readVersionStatus().isOnCurrentMajor(targetPlatform) + && runs(id, type).values().stream().noneMatch(run -> run.versions().targetPlatform().getMajor() == targetPlatform.getMajor())) throw new IllegalArgumentException("platform version " + targetPlatform + " is not on a current major version in this system"); controller.applications().applicationStore().putDev(deploymentId, version.id(), applicationPackage.zippedContent(), diff); @@ -871,7 +890,7 @@ public class JobController { /** Locks all runs and modifies the list of historic runs for the given application and job type. */ private void locked(ApplicationId id, JobType type, Consumer<SortedMap<RunId, Run>> modifications) { - try (Mutex __ = curator.lock(id, type)) { + try (Mutex __ = curator.lock(id, type)) { SortedMap<RunId, Run> runs = new TreeMap<>(curator.readHistoricRuns(id, type)); modifications.accept(runs); curator.writeHistoricRuns(id, type, runs.values()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RevisionHistory.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RevisionHistory.java index bbab9487ea2..272417ba0ac 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RevisionHistory.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RevisionHistory.java @@ -93,7 +93,7 @@ public class RevisionHistory { // Fallback for when an application version isn't known for the given key. private static ApplicationVersion revisionOf(RevisionId id) { - return new ApplicationVersion(id, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), false, false, Optional.empty(), 0); + return new ApplicationVersion(id, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), false, false, Optional.empty(), 0); } /** Returns the production {@link ApplicationVersion} with this revision ID. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java index e7371561636..f752e396c09 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java @@ -126,7 +126,7 @@ public class Versions { private static Version targetPlatform(Application application, Change change, Optional<Version> existing, Supplier<Version> defaultVersion) { - if (change.isPinned() && change.platform().isPresent()) + if (change.isPlatformPinned() && change.platform().isPresent()) return change.platform().get(); return max(change.platform(), existing) @@ -135,6 +135,9 @@ public class Versions { private static RevisionId targetRevision(Application application, Change change, Optional<RevisionId> existing) { + if (change.isRevisionPinned() && change.revision().isPresent()) + return change.revision().get(); + return change.revision() .or(() -> existing) .orElseGet(() -> defaultRevision(application)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java index 02e1818932e..cbdfcf70123 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java @@ -79,7 +79,7 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer { log.log(Level.INFO, "Exception caught when attempting to file an issue for '" + application.id() + "': " + Exceptions.toMessageString(e)); } }); - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactorDeviation(attempts.get(), failures.get()); } private boolean isInCurrentShard(TenantAndApplicationId id) { @@ -122,7 +122,7 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer { log.log(Level.INFO, "Exception caught when attempting to escalate issue with id '" + issueId + "': " + Exceptions.toMessageString(e)); } }); - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactorDeviation(attempts.get(), failures.get()); } private double updateConfirmedApplicationOwners() { @@ -149,7 +149,7 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer { log.log(Level.INFO, "Exception caught when attempting to find confirmed owner of issue with id '" + issueId + "': " + Exceptions.toMessageString(e)); } }); - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactorDeviation(attempts.get(), failures.get()); } private ApplicationList applications() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java index 518027f8099..c4f3c611cc5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java @@ -98,7 +98,7 @@ public class ArchiveUriUpdater extends ControllerMaintainer { } } - return asSuccessFactor(tenantsByZone.size(), failures); + return asSuccessFactorDeviation(tenantsByZone.size(), failures); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java index 25c4121c271..32d06286820 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.artifact.Artifact; import com.yahoo.vespa.hosted.controller.api.integration.artifact.ArtifactRegistry; @@ -13,12 +12,9 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import java.time.Duration; import java.time.Instant; import java.util.Comparator; -import java.util.EnumSet; import java.util.List; -import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; /** * Periodically expire unused artifacts, e.g. container images and RPMs. @@ -32,7 +28,7 @@ public class ArtifactExpirer extends ControllerMaintainer { private static final Duration MIN_AGE = Duration.ofDays(14); public ArtifactExpirer(Controller controller, Duration interval) { - super(controller, interval, null, expiringSystems()); + super(controller, interval); } @Override @@ -56,10 +52,10 @@ public class ArtifactExpirer extends ControllerMaintainer { log.log(Level.INFO, "Expiring " + artifactsToExpire.size() + " artifacts in " + cloudName + ": " + artifactsToExpire); artifactRegistry.deleteAll(artifactsToExpire); } - return 1; + return 0; } catch (RuntimeException e) { log.log(Level.WARNING, "Failed to expire artifacts in " + cloudName + ". Will retry in " + interval(), e); - return 0; + return 1; } } @@ -77,11 +73,4 @@ public class ArtifactExpirer extends ControllerMaintainer { return true; } - /** Returns systems where artifacts can be expired */ - private static Set<SystemName> expiringSystems() { - // Run only in public and main. Public systems have distinct container registries, while main and CD have - // shared registries. - return EnumSet.of(SystemName.Public, SystemName.PublicCd, SystemName.main); - } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java index 32e6ad0d557..fe1930a19ea 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java @@ -42,11 +42,13 @@ public class BcpGroupUpdater extends ControllerMaintainer { private final ApplicationController applications; private final NodeRepository nodeRepository; + private final Double successFactorBaseline; - public BcpGroupUpdater(Controller controller, Duration duration) { - super(controller, duration); + public BcpGroupUpdater(Controller controller, Duration duration, Double successFactorBaseline) { + super(controller, duration, successFactorBaseline); this.applications = controller.applications(); this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); + this.successFactorBaseline = successFactorBaseline; } @Override @@ -58,7 +60,7 @@ public class BcpGroupUpdater extends ControllerMaintainer { for (var application : applications.asList()) { for (var instance : application.instances().values()) { for (var deployment : instance.productionDeployments().values()) { - if (shuttingDown()) return 1.0; + if (shuttingDown()) return 0.0; try { attempts++; var bcpGroups = BcpGroup.groupsFrom(instance, application.deploymentSpec()); @@ -75,12 +77,12 @@ public class BcpGroupUpdater extends ControllerMaintainer { } } } - double successFactor = asSuccessFactor(attempts, failures); - if ( successFactor == 0 ) + double successFactorDeviation = asSuccessFactorDeviation(attempts, failures); + if ( successFactorDeviation == -successFactorBaseline ) log.log(Level.WARNING, "Could not update traffic share on any applications", lastException); - else if ( successFactor < 0.9 ) + else if ( successFactorDeviation < -0.1 ) log.log(Level.FINE, "Could not update traffic share on all applications", lastException); - return successFactor; + return successFactorDeviation; } /** Adds deployment traffic share to the given patch. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java index a7ebaec7c09..b40078eef51 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java @@ -19,6 +19,6 @@ public class BillingDatabaseMaintainer extends ControllerMaintainer { @Override protected double maintain() { controller().serviceRegistry().billingDatabase().maintain(); - return 1; + return 0.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudDatabaseMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudDatabaseMaintainer.java index 914707aa318..68fd5c8bafe 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudDatabaseMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudDatabaseMaintainer.java @@ -19,8 +19,8 @@ public class CloudDatabaseMaintainer extends ControllerMaintainer { controller().serviceRegistry().billingController().updateCache(tenants); } catch (Exception e) { log.warning("Could not update cloud database cache: " + Exceptions.toMessageString(e)); - return 0.0; + return 1.0; } - return 1.0; + return 0.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java index 6ecad482cd2..f9c93a87c44 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java @@ -64,7 +64,7 @@ public class ContactInformationMaintainer extends ControllerMaintainer { interval()); } } - return asSuccessFactor(attempts, failures); + return asSuccessFactorDeviation(attempts, failures); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java index c861d522818..f21803283eb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java @@ -12,7 +12,6 @@ import java.util.EnumSet; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.logging.Logger; /** * A maintainer is some job which runs at a fixed interval to perform some maintenance task in the controller. @@ -26,13 +25,22 @@ public abstract class ControllerMaintainer extends Maintainer { /** The systems in which this maintainer should run */ private final Set<SystemName> activeSystems; + public ControllerMaintainer(Controller controller, Duration interval) { - this(controller, interval, null, EnumSet.allOf(SystemName.class)); + this(controller, interval, null, EnumSet.allOf(SystemName.class), 1.0); + } + + public ControllerMaintainer(Controller controller, Duration interval, Double successFactorBaseline) { + this(controller, interval, null, EnumSet.allOf(SystemName.class), successFactorBaseline); } public ControllerMaintainer(Controller controller, Duration interval, String name, Set<SystemName> activeSystems) { + this(controller, interval, name, activeSystems, 1.0); + } + + public ControllerMaintainer(Controller controller, Duration interval, String name, Set<SystemName> activeSystems, Double successFactorBaseline) { super(name, interval, controller.clock(), controller.jobControl(), - new ControllerJobMetrics(controller.metric()), controller.curator().cluster(), true); + new ControllerJobMetrics(controller.metric()), controller.curator().cluster(), true, successFactorBaseline); this.controller = controller; this.activeSystems = Set.copyOf(Objects.requireNonNull(activeSystems)); } @@ -54,8 +62,8 @@ public abstract class ControllerMaintainer extends Maintainer { } @Override - public void completed(String job, double successFactor, long durationMs) { - metric.set("maintenance.successFactor", successFactor, metric.createContext(Map.of("job", job))); + public void completed(String job, double successFactorDeviation, long durationMs) { + metric.set("maintenance.successFactorDeviation", successFactorDeviation, metric.createContext(Map.of("job", job))); metric.set("maintenance.duration", durationMs, metric.createContext(Map.of("job", job))); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 05159b38ec6..7f48a1115c5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -40,6 +40,7 @@ public class ControllerMaintenance extends AbstractComponent { @SuppressWarnings("unused") // instantiated by Dependency Injection public ControllerMaintenance(Controller controller, Metric metric, UserManagement userManagement, AthenzClientFactory athenzClientFactory) { Intervals intervals = new Intervals(controller.system()); + SuccessFactorBaseline successFactorBaseline = new SuccessFactorBaseline(controller.system()); upgrader = new Upgrader(controller, intervals.defaultInterval); osUpgradeScheduler = new OsUpgradeScheduler(controller, intervals.osUpgradeScheduler); maintainers.add(upgrader); @@ -68,7 +69,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new HostInfoUpdater(controller, intervals.hostInfoUpdater)); maintainers.add(new ReindexingTriggerer(controller, intervals.reindexingTriggerer)); maintainers.add(new EndpointCertificateMaintainer(controller, intervals.endpointCertificateMaintainer)); - maintainers.add(new BcpGroupUpdater(controller, intervals.trafficFractionUpdater)); + maintainers.add(new BcpGroupUpdater(controller, intervals.trafficFractionUpdater, successFactorBaseline.trafficFractionUpdater)); maintainers.add(new ArchiveUriUpdater(controller, intervals.archiveUriUpdater)); maintainers.add(new ArchiveAccessMaintainer(controller, metric, intervals.archiveAccessMaintainer)); maintainers.add(new TenantRoleMaintainer(controller, intervals.tenantRoleMaintainer)); @@ -189,4 +190,15 @@ public class ControllerMaintenance extends AbstractComponent { } + private static class SuccessFactorBaseline { + + private final Double defaultSuccessFactorBaseline; + private final Double trafficFractionUpdater; + + public SuccessFactorBaseline(SystemName system) { + Objects.requireNonNull(system); + this.defaultSuccessFactorBaseline = 1.0; + this.trafficFractionUpdater = system.isCd() ? 0.5 : 0.7; + } + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java index 403b5aed1ce..668893d5a7e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java @@ -34,7 +34,7 @@ public class CostReportMaintainer extends ControllerMaintainer { protected double maintain() { var csv = CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller(), clock, consumer.fixedAllocations()); consumer.consume(csv); - return 1.0; + return 0.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java index 97f3f955a20..c22cb1efdb3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java @@ -5,8 +5,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.yolean.Exceptions; @@ -47,7 +45,7 @@ public class DeploymentExpirer extends ControllerMaintainer { } } } - return asSuccessFactor(attempts, failures); + return asSuccessFactorDeviation(attempts, failures); } /** Returns whether given deployment has expired according to its TTL */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java index f6029eade37..d8d89177a9e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java @@ -38,7 +38,7 @@ public class DeploymentInfoMaintainer extends ControllerMaintainer { } } } - return asSuccessFactor(attempts, failures); + return asSuccessFactorDeviation(attempts, failures); } private Collection<DeploymentId> instanceDeployments(Instance instance) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java index 6b058537c2d..c352fb053dc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java @@ -77,7 +77,7 @@ public class DeploymentIssueReporter extends ControllerMaintainer { fileDeploymentIssueFor(application); else store(application.id(), null); - return 1.0; + return 0.0; } /** @@ -87,24 +87,24 @@ public class DeploymentIssueReporter extends ControllerMaintainer { */ private double maintainPlatformIssue(List<Application> applications) { if (controller().system() == SystemName.cd) - return 1.0; + return 0.0; VersionStatus versionStatus = controller().readVersionStatus(); Version systemVersion = controller().systemVersion(versionStatus); if (versionStatus.version(systemVersion).confidence() != broken) - return 1.0; + return 0.0; DeploymentStatusList statuses = controller().jobController().deploymentStatuses(ApplicationList.from(applications)); if (statuses.failingUpgradeToVersionSince(systemVersion, controller().clock().instant().minus(upgradeGracePeriod)).isEmpty()) - return 1.0; + return 0.0; List<ApplicationId> failingApplications = statuses.failingUpgradeToVersionSince(systemVersion, controller().clock().instant()) .mapToList(status -> status.application().id().defaultInstance()); // TODO jonmv: Send only tenant and application, here and elsewhere in this. deploymentIssues.fileUnlessOpen(failingApplications, systemVersion); - return 1.0; + return 0.0; } private Tenant ownerOf(TenantAndApplicationId applicationId) { @@ -145,7 +145,7 @@ public class DeploymentIssueReporter extends ControllerMaintainer { log.log(Level.INFO, "Exception caught when attempting to escalate issue with id '" + issueId + "': " + Exceptions.toMessageString(e)); } })); - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactorDeviation(attempts.get(), failures.get()); } private void store(TenantAndApplicationId id, IssueId issueId) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java index fa917d2eb4e..427ee7c4b06 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java @@ -96,7 +96,7 @@ public class DeploymentMetricsMaintainer extends ControllerMaintainer { } catch (InterruptedException e) { throw new RuntimeException(e); } - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactorDeviation(attempts.get(), failures.get()); } static DeploymentMetrics updateDeploymentMetrics(DeploymentMetrics current, List<ClusterMetrics> metrics) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java index 934a1b4fa2f..8a3a2a11e09 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java @@ -78,7 +78,7 @@ public class DeploymentUpgrader extends ControllerMaintainer { ": " + Exceptions.toMessageString(e) + ". Retrying in " + interval()); } - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactorDeviation(attempts.get(), failures.get()); } /** Returns whether query and feed metrics are ~zero, or currently platform has been deployed for a week. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java index 7af96d10f2f..5218da91c46 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java @@ -26,7 +26,7 @@ public class EnclaveAccessMaintainer extends ControllerMaintainer { return controller().serviceRegistry().enclaveAccessService().allowAccessFor(externalAccounts()); } catch (RuntimeException e) { logger.log(WARNING, "Failed sharing resources with enclave", e); - return 0; + return 1.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index 0b96d8adc1a..713782eb7b9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -75,10 +75,10 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { deleteOrReportUnmanagedCertificates(); } catch (Exception e) { log.log(Level.SEVERE, "Exception caught while maintaining endpoint certificates", e); - return 0.0; + return 1.0; } - return 1.0; + return 0.0; } private void updateRefreshedCertificates() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java index 1f21c688540..31236f4fcda 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java @@ -73,7 +73,7 @@ public class HostInfoUpdater extends ControllerMaintainer { LOG.info("Updated information for " + hostsUpdated + " hosts(s)"); } } - return 1.0; + return 0.0; } private static Optional<String> modelNameOf(NodeEntity nodeEntity) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java index b051590ac5a..68c79fdca7e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java @@ -50,7 +50,7 @@ public abstract class InfrastructureUpgrader<TARGET extends VersionTarget> exten @Override protected double maintain() { return target().map(target -> upgradeAll(target, managedApplications)) - .orElse(1.0); + .orElse(0.0); } /** Deploy a list of system applications until they converge on the given version */ @@ -81,7 +81,7 @@ public abstract class InfrastructureUpgrader<TARGET extends VersionTarget> exten break; } } - return asSuccessFactor(attempts, failures); + return asSuccessFactorDeviation(attempts, failures); } /** Returns whether all applications have converged to the target version in zone */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 294d5bad42d..67188eb5e3a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -90,7 +90,7 @@ public class MetricsReporter extends ControllerMaintainer { reportBrokenSystemVersion(versionStatus); reportTenantMetrics(); reportZmsQuotaMetrics(); - return 1.0; + return 0.0; } private void reportBrokenSystemVersion(VersionStatus versionStatus) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java index f930e64fc5a..f0d218ae6cf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java @@ -40,7 +40,7 @@ public class OsUpgradeScheduler extends ControllerMaintainer { if (!change.get().scheduleAt(now)) continue; controller().upgradeOsIn(cloud, change.get().version(), false); } - return 1.0; + return 0.0; } /** Returns the wanted change for cloud at given instant, if any */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java index 119540eaa68..c643df6af68 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java @@ -22,12 +22,12 @@ public class OsVersionStatusUpdater extends ControllerMaintainer { try { OsVersionStatus newStatus = OsVersionStatus.compute(controller()); controller().updateOsVersionStatus(newStatus); - return 1.0; + return 0.0; } catch (Exception e) { log.log(Level.WARNING, "Failed to compute OS version status: " + Exceptions.toMessageString(e) + ". Retrying in " + interval()); } - return 0.0; + return 1.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java index 400673bfd0c..945b6d32a30 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java @@ -53,11 +53,11 @@ public class ReindexingTriggerer extends ControllerMaintainer { controller().applications().reindex(id, deployment.zone(), List.of(), List.of(), true, speed, "bakground reindexing, to account for changes in built-in linguistics components"); }); - return 1.0; + return 0.0; } catch (RuntimeException e) { log.log(Level.WARNING, "Failed to trigger reindexing: " + Exceptions.toMessageString(e)); - return 0.0; + return 1.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index 3f20c2eac8f..52206d41c00 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -98,13 +98,13 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { } catch (Exception e) { log.log(Level.WARNING, "Failed to collect resource snapshots. Retrying in " + interval() + ". Error: " + Exceptions.toMessageString(e)); - return 0.0; + return 1.0; } if (systemName.isPublic()) reportResourceSnapshots(resourceSnapshots); if (systemName.isPublic()) reportAllScalingEvents(); updateDeploymentCost(resourceSnapshots); - return 1.0; + return 0.0; } void updateDeploymentCost(Collection<ResourceSnapshot> resourceSnapshots) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java index 18ed154fcf1..59871f716e0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java @@ -43,7 +43,7 @@ public class ResourceTagMaintainer extends ControllerMaintainer { if (taggedResources > 0) log.log(Level.INFO, "Tagged " + taggedResources + " resources in " + zone.getId()); }); - return 1.0; + return 0.0; } private Map<HostName, ApplicationId> getTenantOfParentHosts(ZoneId zoneId) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java index 74bb89e4105..aaf730cc158 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java @@ -47,9 +47,9 @@ public class RetriggerMaintainer extends ControllerMaintainer { controller().curator().writeRetriggerEntries(remaining); } catch (Exception e) { logger.log(Level.WARNING, "Exception while triggering jobs", e); - return 0.0; + return 1.0; } - return 1.0; + return 0.0; } /** Returns true if a job is ready to run, i.e. is currently not running */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleCleanupMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleCleanupMaintainer.java index f29df1bc0d5..e3a3415e170 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleCleanupMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleCleanupMaintainer.java @@ -26,6 +26,6 @@ public class TenantRoleCleanupMaintainer extends ControllerMaintainer { controller().serviceRegistry().tenantSecretService().cleanupSecretStores(deletedTenants); } - return 1.0; + return 0.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java index bd121871c7c..c7b236880fd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java @@ -37,7 +37,7 @@ public class TenantRoleMaintainer extends ControllerMaintainer { controller().tenants().updateLastTenantRolesMaintained(t.name(), updated); }); - return 1.0; + return 0.0; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index 82b3141e503..edcfcc317a7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -64,7 +64,7 @@ public class Upgrader extends ControllerMaintainer { for (UpgradePolicy policy : UpgradePolicy.values()) updateTargets(versionStatus, deploymentStatuses, policy); - return 1.0; + return 0.0; } private DeploymentStatusList deploymentStatuses(VersionStatus versionStatus) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java index 03987efab8b..7c4645a6e48 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java @@ -11,6 +11,7 @@ import java.util.Optional; import java.util.logging.Logger; import java.util.stream.Collectors; + /** * Maintains user management resources. * For now, ensures there's no discrepnacy between expected tenant/application roles and auth0/athenz roles @@ -46,7 +47,7 @@ public class UserManagementMaintainer extends ControllerMaintainer { }); } - return 1.0; + return 0.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java index da0fa890960..d4c4b4efda7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java @@ -90,7 +90,7 @@ public class VcmrMaintainer extends ControllerMaintainer { } }); updateMetrics(); - return 1.0; + return 0.0; } /** @@ -139,7 +139,10 @@ public class VcmrMaintainer extends ControllerMaintainer { return Stream.empty(); } var spareCapacity = hasSpareCapacity(zone, nodes); - return nodes.stream().map(node -> nextAction(zone, node, changeRequest, spareCapacity)); + var impactedProxyCount = nodes.stream() + .filter(node -> node.type() == NodeType.proxy) + .count(); + return nodes.stream().map(node -> nextAction(zone, node, changeRequest, spareCapacity, impactedProxyCount)); }).toList(); } @@ -162,7 +165,7 @@ public class VcmrMaintainer extends ControllerMaintainer { .findFirst(); } - private HostAction nextAction(ZoneId zoneId, Node node, VespaChangeRequest changeRequest, boolean spareCapacity) { + private HostAction nextAction(ZoneId zoneId, Node node, VespaChangeRequest changeRequest, boolean spareCapacity, long impactedProxyCount) { var hostAction = getPreviousAction(node, changeRequest) .orElse(new HostAction(node.hostname().value(), State.NONE, Instant.now())); @@ -176,7 +179,8 @@ public class VcmrMaintainer extends ControllerMaintainer { if (isLowImpact(changeRequest)) return hostAction; - addReport(zoneId, changeRequest, node); + if (shouldAddReport(node, changeRequest.getChangeRequestSource().getId(), hostAction)) + addReport(zoneId, changeRequest, node); if (isOutOfSync(node, hostAction)) return hostAction.withState(State.OUT_OF_SYNC); @@ -187,7 +191,13 @@ public class VcmrMaintainer extends ControllerMaintainer { return hostAction.withState(State.PENDING_RETIREMENT); } - if (node.type() != NodeType.host || !spareCapacity) { + if (!spareCapacity) { + return hostAction.withState(State.REQUIRES_OPERATOR_ACTION); + } + + if (node.type() != NodeType.host) { + if (node.type() == NodeType.proxy && impactedProxyCount == 1) + return hostAction.withState(State.READY); return hostAction.withState(State.REQUIRES_OPERATOR_ACTION); } @@ -267,6 +277,16 @@ public class VcmrMaintainer extends ControllerMaintainer { && node.state() == Node.State.active; } + private boolean shouldAddReport(Node node, String vcmrId, HostAction previousAction) { + var vcmrReport = VcmrReport.fromReports(node.reports()); + var hasReport = vcmrReport.getVcmrs().stream().map(VcmrReport.Vcmr::id).anyMatch(id -> id.equals(vcmrId)); + // Don't add report if none exists and this is not initial assessment + // Presumably removed manually by operator. + if (!hasReport && previousAction.getState() != State.NONE) + return false; + return true; + } + // Determines if node state is unexpected based on previous action taken private boolean isOutOfSync(Node node, HostAction action) { return action.getState() == State.RETIRED && node.state() != Node.State.parked || @@ -343,8 +363,7 @@ public class VcmrMaintainer extends ControllerMaintainer { private void addReport(ZoneId zoneId, VespaChangeRequest changeRequest, Node node) { var report = VcmrReport.fromReports(node.reports()); - var source = changeRequest.getChangeRequestSource(); - if (report.addVcmr(source.getId(), source.getPlannedStartTime(), source.getPlannedEndTime())) { + if (report.addVcmr(changeRequest.getChangeRequestSource())) { updateReport(zoneId, node, report); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java index 154455c5198..1c4d13aa16d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java @@ -39,12 +39,12 @@ public class VersionStatusUpdater extends ControllerMaintainer { controller().serviceRegistry().systemMonitor().reportSystemVersion(version.versionNumber(), convert(version.confidence())); }); - return 1.0; + return 0.0; } catch (Exception e) { log.log(Level.WARNING, "Failed to compute version status: " + Exceptions.toMessageString(e) + ". Retrying in " + interval()); } - return 0.0; + return 1.0; } static SystemMonitor.Confidence convert(VespaVersion.Confidence confidence) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index ee12c9957b1..e5006ab9785 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -82,7 +82,8 @@ public class ApplicationSerializer { private static final String versionsField = "versions"; private static final String prodVersionsField = "prodVersions"; private static final String devVersionsField = "devVersions"; - private static final String pinnedField = "pinned"; + private static final String platformPinnedField = "pinned"; + private static final String revisionPinnedField = "revisionPinned"; private static final String deploymentIssueField = "deploymentIssueId"; private static final String ownershipIssueIdField = "ownershipIssueId"; private static final String ownerField = "confirmedOwner"; @@ -118,6 +119,7 @@ public class ApplicationSerializer { private static final String riskField = "risk"; private static final String authorEmailField = "authorEmailField"; private static final String deployedDirectlyField = "deployedDirectly"; + private static final String obsoleteAtField = "obsoleteAt"; private static final String hasPackageField = "hasPackage"; private static final String shouldSkipField = "shouldSkip"; private static final String compileVersionField = "compileVersion"; @@ -265,6 +267,7 @@ public class ApplicationSerializer { applicationVersion.sourceUrl().ifPresent(url -> object.setString(sourceUrlField, url)); applicationVersion.commit().ifPresent(commit -> object.setString(commitField, commit)); object.setBool(deployedDirectlyField, applicationVersion.isDeployedDirectly()); + applicationVersion.obsoleteAt().ifPresent(at -> object.setLong(obsoleteAtField, at.toEpochMilli())); object.setBool(hasPackageField, applicationVersion.hasPackage()); object.setBool(shouldSkipField, applicationVersion.shouldSkip()); applicationVersion.description().ifPresent(description -> object.setString(descriptionField, description)); @@ -295,8 +298,10 @@ public class ApplicationSerializer { object.setString(versionField, deploying.platform().get().toString()); if (deploying.revision().isPresent()) toSlime(deploying.revision().get(), object); - if (deploying.isPinned()) - object.setBool(pinnedField, true); + if (deploying.isPlatformPinned()) + object.setBool(platformPinnedField, true); + if (deploying.isRevisionPinned()) + object.setBool(revisionPinnedField, true); } private void toSlime(RotationStatus status, Cursor array) { @@ -487,6 +492,7 @@ public class ApplicationSerializer { Optional<Instant> buildTime = SlimeUtils.optionalInstant(object.field(buildTimeField)); Optional<String> sourceUrl = SlimeUtils.optionalString(object.field(sourceUrlField)); Optional<String> commit = SlimeUtils.optionalString(object.field(commitField)); + Optional<Instant> obsoleteAt = SlimeUtils.optionalInstant(object.field(obsoleteAtField)); boolean hasPackage = object.field(hasPackageField).asBool(); boolean shouldSkip = object.field(shouldSkipField).asBool(); Optional<String> description = SlimeUtils.optionalString(object.field(descriptionField)); @@ -494,7 +500,7 @@ public class ApplicationSerializer { Optional<String> bundleHash = SlimeUtils.optionalString(object.field(bundleHashField)); return new ApplicationVersion(id, sourceRevision, authorEmail, compileVersion, allowedMajor, buildTime, - sourceUrl, commit, bundleHash, hasPackage, shouldSkip, description, risk); + sourceUrl, commit, bundleHash, obsoleteAt, hasPackage, shouldSkip, description, risk); } private Optional<SourceRevision> sourceRevisionFromSlime(Inspector object) { @@ -520,8 +526,10 @@ public class ApplicationSerializer { change = Change.of(Version.fromString(versionFieldValue.asString())); if (object.field(applicationBuildNumberField).valid()) change = change.with(revisionFromSlime(object, null)); - if (object.field(pinnedField).asBool()) - change = change.withPin(); + if (object.field(platformPinnedField).asBool()) + change = change.withPlatformPin(); + if (object.field(revisionPinnedField).asBool()) + change = change.withRevisionPin(); return change; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 8a0e2d01d8c..ded27ee1060 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -131,7 +131,6 @@ import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; - import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -261,11 +260,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/package")) return applicationPackage(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/diff/{number}")) return applicationPackageDiff(path.get("tenant"), path.get("application"), path.get("number")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return deploying(path.get("tenant"), path.get("application"), "default", request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), "default", request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance")) return applications(path.get("tenant"), Optional.of(path.get("application")), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return instance(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(controller, appIdFromPath(path), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.applications().requireApplication(TenantAndApplicationId.from(path.get("tenant"), path.get("application"))), controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)).descendingMap(), Optional.ofNullable(request.getProperty("limit")), request.getUri()); // (((＼（✘෴✘）／))) if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/package")) return devApplicationPackage(appIdFromPath(path), jobTypeFromPath(path)); @@ -328,14 +325,18 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return createApplication(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/platform")) return deployPlatform(path.get("tenant"), path.get("application"), "default", false, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deployPlatform(path.get("tenant"), path.get("application"), "default", true, request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/application")) return deployApplication(path.get("tenant"), path.get("application"), "default", request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/platform-pin")) return deployPlatform(path.get("tenant"), path.get("application"), "default", true, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/application-pin")) return deployApplication(path.get("tenant"), path.get("application"), "default", true, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/application")) return deployApplication(path.get("tenant"), path.get("application"), "default", false, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/key")) return addDeployKey(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/submit")) return submit(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return createInstance(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploy/{jobtype}")) return jobDeploy(appIdFromPath(path), jobTypeFromPath(path), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/platform")) return deployPlatform(path.get("tenant"), path.get("application"), path.get("instance"), false, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/pin")) return deployPlatform(path.get("tenant"), path.get("application"), path.get("instance"), true, request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/application")) return deployApplication(path.get("tenant"), path.get("application"), path.get("instance"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/platform-pin")) return deployPlatform(path.get("tenant"), path.get("application"), path.get("instance"), true, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/application-pin")) return deployApplication(path.get("tenant"), path.get("application"), path.get("instance"), true, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/application")) return deployApplication(path.get("tenant"), path.get("application"), path.get("instance"), false, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/submit")) return submit(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return trigger(appIdFromPath(path), jobTypeFromPath(path), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/pause")) return pause(appIdFromPath(path), jobTypeFromPath(path)); @@ -1897,7 +1898,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { JobControllerApiHandlerHelper.toSlime(response.setObject("applicationVersion"), application.revisions().get(deployment.revision())); if ( ! status.jobsToRun().containsKey(stepStatus.job().get())) response.setString("status", "complete"); - else if (stepStatus.readyAt(instance.change()).map(controller.clock().instant()::isBefore).orElse(true)) + else if ( ! stepStatus.readiness(instance.change()).okAt(controller.clock().instant())) response.setString("status", "pending"); else response.setString("status", "running"); @@ -2060,7 +2061,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if ( ! instance.change().isEmpty()) { instance.change().platform().ifPresent(version -> root.setString("platform", version.toString())); instance.change().revision().ifPresent(revision -> root.setString("application", revision.toString())); - root.setBool("pinned", instance.change().isPinned()); + root.setBool("pinned", instance.change().isPlatformPinned()); + root.setBool("platform-pinned", instance.change().isPlatformPinned()); + root.setBool("application-pinned", instance.change().isRevisionPinned()); } return new SlimeJsonResponse(slime); } @@ -2173,7 +2176,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { .collect(joining(", "))); Change change = Change.of(version); if (pin) - change = change.withPin(); + change = change.withPlatformPin(); controller.applications().deploymentTrigger().forceChange(id, change, isOperator(request)); response.append("Triggered ").append(change).append(" for ").append(id); @@ -2182,7 +2185,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { } /** Trigger deployment to the last known application package for the given application. */ - private HttpResponse deployApplication(String tenantName, String applicationName, String instanceName, HttpRequest request) { + private HttpResponse deployApplication(String tenantName, String applicationName, String instanceName, boolean pin, HttpRequest request) { ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); Inspector buildField = toSlime(request.getData()).get().field("build"); long build = buildField.valid() ? buildField.asLong() : -1; @@ -2192,6 +2195,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { RevisionId revision = build == -1 ? application.get().revisions().last().get().id() : getRevision(application.get(), build); Change change = Change.of(revision); + if (pin) + change = change.withRevisionPin(); controller.applications().deploymentTrigger().forceChange(id, change, isOperator(request)); response.append("Triggered ").append(change).append(" for ").append(id); }); @@ -2232,7 +2237,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return; } - ChangesToCancel cancel = ChangesToCancel.valueOf(choice.toUpperCase()); + ChangesToCancel cancel = ChangesToCancel.valueOf(choice.replaceAll("-", "_").toUpperCase()); controller.applications().deploymentTrigger().cancelChange(id, cancel); response.append("Changed deployment from '").append(change).append("' to '").append(controller.applications().requireInstance(id).change()).append("' for ").append(id); }); @@ -3004,7 +3009,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { byte[] testPackage = dataParts.getOrDefault(EnvironmentResource.APPLICATION_TEST_ZIP, new byte[0]); Submission submission = new Submission(applicationPackage, testPackage, sourceUrl, sourceRevision, authorEmail, description, risk); - controller.applications().verifyApplicationIdentityConfiguration(TenantName.from(tenant), + TenantName tenantName = TenantName.from(tenant); + controller.applications().verifyPlan(tenantName); + controller.applications().verifyApplicationIdentityConfiguration(tenantName, Optional.empty(), Optional.empty(), applicationPackage, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index ce60e0054c4..9ff8c7df18b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -15,7 +15,6 @@ import com.yahoo.text.Text; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.NotExistsException; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; @@ -26,6 +25,8 @@ import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.deployment.ConvergenceSummary; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.DelayCause; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.Readiness; import com.yahoo.vespa.hosted.controller.deployment.JobController; import com.yahoo.vespa.hosted.controller.deployment.JobStatus; import com.yahoo.vespa.hosted.controller.deployment.Run; @@ -269,17 +270,39 @@ class JobControllerApiHandlerHelper { stepObject.setString("instance", stepStatus.instance().value()); // TODO: recursively search dependents for what is the relevant partial change when this is a delay step ... - Optional<Instant> readyAt = stepStatus.job().map(jobsToRun::get).map(jobs -> jobs.get(0).readyAt()) - .orElse(stepStatus.readyAt(change)); - readyAt.ifPresent(ready -> stepObject.setLong("readyAt", ready.toEpochMilli())); - readyAt.filter(controller.clock().instant()::isBefore) - .ifPresent(until -> stepObject.setLong("delayedUntil", until.toEpochMilli())); - stepStatus.pausedUntil().ifPresent(until -> stepObject.setLong("pausedUntil", until.toEpochMilli())); - stepStatus.coolingDownUntil(change, Optional.empty()).ifPresent(until -> stepObject.setLong("coolingDownUntil", until.toEpochMilli())); - stepStatus.blockedUntil(Change.of(controller.systemVersion(versionStatus))) // Dummy version — just anything with a platform. - .ifPresent(until -> stepObject.setLong("platformBlockedUntil", until.toEpochMilli())); - stepStatus.blockedUntil(Change.of(RevisionId.forProduction(1))) // Dummy version — just anything with an application. - .ifPresent(until -> stepObject.setLong("applicationBlockedUntil", until.toEpochMilli())); + Readiness readiness = stepStatus.job().map(jobsToRun::get).map(job -> job.get(0).readiness()) + .orElse(stepStatus.readiness(change)); + if (readiness.ok()) { + stepObject.setLong("readyAt", readiness.at().toEpochMilli()); + if ( ! readiness.okAt(controller.clock().instant())) { + Instant until = readiness.at(); + stepObject.setLong("delayedUntil", readiness.at().toEpochMilli()); + switch (readiness.cause()) { + case paused -> stepObject.setLong("pausedUntil", until.toEpochMilli()); + case coolingDown -> stepObject.setLong("coolingDownUntil", until.toEpochMilli()); + case changeBlocked -> { + Readiness platformReadiness = stepStatus.readiness(Change.of(controller.systemVersion(versionStatus))); // Dummy version — just anything with a platform. + if (platformReadiness.cause() == DelayCause.changeBlocked) + stepObject.setLong("platformBlockedUntil", platformReadiness.at().toEpochMilli()); + Readiness applicationReadiness = stepStatus.readiness(Change.of(RevisionId.forProduction(1))); // Dummy version — just anything with an application. + if (applicationReadiness.cause() == DelayCause.changeBlocked) + stepObject.setLong("applicationBlockedUntil", applicationReadiness.at().toEpochMilli()); + } + } + } + } + stepObject.setString("delayCause", + switch (readiness.cause()) { + case none -> null; + case invalidPackage -> "invalidPackage"; + case paused -> "paused"; + case coolingDown -> "coolingDown"; + case changeBlocked -> "changeBlocked"; + case blocked -> "blocked"; + case running -> "running"; + case notReady -> "notReady"; + case unverified -> "unverified"; + }); if (stepStatus.type() == DeploymentStatus.StepType.delay) stepStatus.completedAt(change).ifPresent(completed -> stepObject.setLong("completedAt", completed.toEpochMilli())); @@ -289,7 +312,9 @@ class JobControllerApiHandlerHelper { if ( ! change.isEmpty()) { change.platform().ifPresent(version -> deployingObject.setString("platform", version.toFullString())); change.revision().ifPresent(revision -> toSlime(deployingObject.setObject("application"), application.revisions().get(revision))); - if (change.isPinned()) deployingObject.setBool("pinned", true); + if (change.isPlatformPinned()) deployingObject.setBool("pinned", true); + if (change.isPlatformPinned()) deployingObject.setBool("platformPinned", true); + if (change.isRevisionPinned()) deployingObject.setBool("revisionPinned", true); } Cursor latestVersionsObject = stepObject.setObject("latestVersions"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index 759b2366229..6e5635e8c8c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -22,6 +22,8 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.DelayCause; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.Readiness; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Versions; @@ -169,11 +171,14 @@ public class DeploymentApiHandler extends ThreadedHttpRequestHandler { instanceObject.setString("application", instance.application().value()); instanceObject.setString("instance", instance.instance().value()); instanceObject.setBool("upgrading", status.application().require(instance.instance()).change().platform().equals(Optional.of(statistics.version()))); - instanceObject.setBool("pinned", status.application().require(instance.instance()).change().isPinned()); + instanceObject.setBool("pinned", status.application().require(instance.instance()).change().isPlatformPinned()); + instanceObject.setBool("platformPinned", status.application().require(instance.instance()).change().isPlatformPinned()); + instanceObject.setBool("revisionPinned", status.application().require(instance.instance()).change().isRevisionPinned()); DeploymentStatus.StepStatus stepStatus = status.instanceSteps().get(instance.instance()); if (stepStatus != null) { // Instance may not have any steps, i.e. an empty deployment spec has been submitted - stepStatus.blockedUntil(Change.of(statistics.version())) - .ifPresent(until -> instanceObject.setLong("blockedUntil", until.toEpochMilli())); + Readiness platformReadiness = stepStatus.blockedUntil(Change.of(statistics.version())); + if (platformReadiness.cause() == DelayCause.changeBlocked) + instanceObject.setLong("blockedUntil", platformReadiness.at().toEpochMilli()); } instanceObject.setString("upgradePolicy", toString(status.application().deploymentSpec().instance(instance.instance()) .map(DeploymentInstanceSpec::upgradePolicy) @@ -185,10 +190,12 @@ public class DeploymentApiHandler extends ThreadedHttpRequestHandler { if ( ! job.application().equals(instance)) return; Cursor jobObject = jobsArray.addObject(); jobObject.setString("name", job.type().jobName()); - jobStatus.pausedUntil().ifPresent(until -> jobObject.setLong("pausedUntil", until.toEpochMilli())); - jobStatus.coolingDownUntil(status.application().require(instance.instance()).change(), Optional.empty()) - .ifPresent(until -> jobObject.setLong("coolingDownUntil", until.toEpochMilli())); if (jobsToRun.containsKey(job)) { + Readiness readiness = jobsToRun.get(job).get(0).readiness(); + switch (readiness.cause()) { + case paused -> jobObject.setLong("pausedUntil", readiness.at().toEpochMilli()); + case coolingDown -> jobObject.setLong("coolingDownUntil", readiness.at().toEpochMilli()); + } List<Versions> versionsOnThisPlatform = jobsToRun.get(job).stream() .map(DeploymentStatus.Job::versions) .filter(versions -> versions.targetPlatform().equals(statistics.version())) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java index e9947f3d565..45c00848407 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java @@ -41,8 +41,8 @@ public record VespaVersion(Version version, .not().upgradingTo(statistics.version()); InstanceList failingOnThis = all.matching(instance -> statistics.failingUpgrades().stream().anyMatch(run -> run.id().application().equals(instance))); - // 'broken' if any canary fails - if ( ! failingOnThis.with(UpgradePolicy.canary).isEmpty()) + // 'broken' if any canary fails, and no non-canary is upgraded + if ( ! failingOnThis.with(UpgradePolicy.canary).isEmpty() && productionOnThis.not().with(UpgradePolicy.canary).isEmpty()) return Confidence.broken; // 'broken' if 6 non-canary was broken by this, and that is at least 5% of all diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 8f0536480f5..04c8c46e1ef 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -22,6 +22,7 @@ import com.yahoo.path.Path; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistryMock; import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; @@ -40,6 +41,7 @@ import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.deployment.JobController; import com.yahoo.vespa.hosted.controller.deployment.Submission; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.notification.Notification; @@ -54,7 +56,6 @@ import com.yahoo.vespa.hosted.controller.routing.rotation.RotationLock; import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import org.junit.jupiter.api.Test; - import java.io.InputStream; import java.time.Duration; import java.time.Instant; @@ -106,9 +107,11 @@ public class ControllerTest { Version version1 = tester.configServer().initialVersion(); var context = tester.newDeploymentContext(); context.submit(applicationPackage); - assertEquals(ApplicationVersion.from(RevisionId.forProduction(1), DeploymentContext.defaultSourceRevision, "a@b", new Version("6.1"), Instant.ofEpochSecond(1)), - context.application().revisions().get(context.instance().change().revision().get()), - "Application version is known from completion of initial job"); + RevisionId id = RevisionId.forProduction(1); + Version compileVersion = new Version("6.1"); + assertEquals(new ApplicationVersion(id, Optional.of(DeploymentContext.defaultSourceRevision), Optional.of("a@b"), Optional.of(compileVersion), Optional.empty(), Optional.of(Instant.ofEpochSecond(1)), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), true, false, Optional.empty(), 0), + context.application().revisions().get(context.instance().change().revision().get()), + "Application version is known from completion of initial job"); context.runJob(systemTest); context.runJob(stagingTest); @@ -220,6 +223,59 @@ public class ControllerTest { } @Test + void testPackagePruning() { + DeploymentContext app = tester.newDeploymentContext().submit().deploy(); + RevisionId revision1 = app.lastSubmission().get(); + assertTrue(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision1.number())); + + app.submit().deploy(); + RevisionId revision2 = app.lastSubmission().get(); + assertTrue(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision1.number())); + assertTrue(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision2.number())); + + // Revision 1 is marked as obsolete now + app.submit().deploy(); + RevisionId revision3 = app.lastSubmission().get(); + assertTrue(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision1.number())); + assertTrue(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision2.number())); + assertTrue(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision3.number())); + + // Time advances, and revision 2 is marked as obsolete now + tester.clock().advance(JobController.obsoletePackageExpiry); + app.submit().deploy(); + RevisionId revision4 = app.lastSubmission().get(); + assertTrue(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision1.number())); + assertTrue(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision2.number())); + assertTrue(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision3.number())); + assertTrue(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision4.number())); + + // Time advances, and revision is now old enough to be pruned + tester.clock().advance(Duration.ofMillis(1)); + app.submit().deploy(); + RevisionId revision5 = app.lastSubmission().get(); + assertFalse(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision1.number())); + assertTrue(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision2.number())); + assertTrue(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision3.number())); + assertTrue(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision4.number())); + assertTrue(tester.controllerTester().serviceRegistry().applicationStore() + .hasBuild(app.instanceId().tenant(), app.instanceId().application(), revision5.number())); + } + + @Test void testGlobalRotationStatus() { var context = tester.newDeploymentContext(); var zone1 = ZoneId.from("prod", "us-west-1"); @@ -1429,9 +1485,14 @@ public class ControllerTest { // Deployment fails because zone is not configured in requested cloud account tester.controllerTester().flagSource().withListFlag(PermanentFlags.CLOUD_ACCOUNTS.id(), List.of(cloudAccount), String.class); - context.submit(applicationPackage) - .runJobExpectingFailure(systemTest, "Zone test.us-east-1 is not configured in requested cloud account '012345678912'") - .abortJob(stagingTest); + assertEquals("Zone test.us-east-1 is not configured in requested cloud account '012345678912'", + assertThrows(IllegalArgumentException.class, + () -> context.submit(applicationPackage)) + .getMessage()); + assertEquals("Zone dev.us-east-1 is not configured in requested cloud account '012345678912'", + assertThrows(IllegalArgumentException.class, + () -> context.runJob(devUsEast1, applicationPackage)) + .getMessage()); // Deployment to prod succeeds once all zones are configured in requested account tester.controllerTester().zoneRegistry().configureCloudAccount(CloudAccount.from(cloudAccount), @@ -1508,4 +1569,19 @@ public class ControllerTest { assertFalse(tester.configServer().application(deployment.applicationId(), deployment.zoneId()).isPresent()); } + @Test + void testVerifyPlan() { + DeploymentId deployment = tester.newDeploymentContext().deploymentIdIn(ZoneId.from("prod", "us-west-1")); + TenantName tenant = deployment.applicationId().tenant(); + + tester.controller().serviceRegistry().billingController().setPlan(tenant, PlanRegistryMock.nonePlan.id(), false, false); + try { + tester.controller().applications().verifyPlan(tenant); + fail("should have thrown an exception"); + } catch (IllegalArgumentException e) { + assertEquals("Tenant 'tenant' has a plan 'None Plan - for testing purposes' with zero quota, not allowed to deploy. " + + "See https://cloud.vespa.ai/support", e.getMessage()); + } + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java index a76d2eca521..ca31ceebc17 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java @@ -126,7 +126,7 @@ public class EndpointTest { Endpoint.of(instance1).target(cluster, prodZone).on(Port.tls()).in(SystemName.main), // Prod endpoint in CD - "https://cd.a1.t1.us-north-1.vespa.oath.cloud/", + "https://cd.a1.t1.us-north-1.cd.vespa.oath.cloud/", Endpoint.of(instance1).target(cluster, prodZone).on(Port.tls()).in(SystemName.cd), // Test endpoint in main @@ -300,7 +300,7 @@ public class EndpointTest { .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) .in(SystemName.main), - "cd.a2.t2.us-east-3-r.vespa.oath.cloud", + "cd.a2.t2.us-east-3-r.cd.vespa.oath.cloud", Endpoint.of(app2) .targetApplication(EndpointId.defaultId(), ClusterSpec.Id.from("qrs"), Map.of(new DeploymentId(app2.instance("i1"), ZoneId.from("prod", "us-east-3")), 1)) @@ -335,7 +335,7 @@ public class EndpointTest { .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) .in(SystemName.main), - "https://cd.a2.t2.a.vespa.oath.cloud/", + "https://cd.a2.t2.a.cd.vespa.oath.cloud/", Endpoint.of(app2) .targetApplication(EndpointId.defaultId(), ClusterSpec.Id.from("qrs"), Map.of(new DeploymentId(app2.instance("i1"), ZoneId.from("prod", "us-east-3")), 1)) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 226fb785bf6..6e5c2458c92 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -653,15 +653,21 @@ public class DeploymentTriggerTest { assertEquals(appVersion1, latestDeployed(app.instance())); // Downgrading application version. - tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(appVersion0)); - assertEquals(Change.of(appVersion0), app.instance().change()); + tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(appVersion0).withRevisionPin()); + assertEquals(Change.of(appVersion0).withRevisionPin(), app.instance().change()); app.runJob(stagingTest) - .runJob(productionUsCentral1) - .runJob(productionUsEast3) - .runJob(productionUsWest1); - assertEquals(Change.empty(), app.instance().change()); + .runJob(productionUsCentral1) + .runJob(productionUsEast3) + .runJob(productionUsWest1); + assertEquals(Change.empty().withRevisionPin(), app.instance().change()); assertEquals(appVersion0, app.instance().deployments().get(productionUsEast3.zone()).revision()); assertEquals(appVersion0, latestDeployed(app.instance())); + + tester.outstandingChangeDeployer().run(); + assertEquals(Change.empty().withRevisionPin(), app.instance().change()); + tester.deploymentTrigger().cancelChange(app.instanceId(), ALL); + tester.outstandingChangeDeployer().run(); + assertEquals(Change.of(appVersion1), app.instance().change()); } @Test @@ -756,8 +762,8 @@ public class DeploymentTriggerTest { // Last job has a different deployment target, so tests need to run again. app1.runJob(productionEuWest1) // Upgrade completes, and revision is the only change. - .runJob(productionUsCentral1) // With only revision change, central should run to cover a previous failure. - .runJob(productionEuWest1); // Finally, west changes revision. + .runJob(productionUsCentral1) // With only revision change, central should run to cover a previous failure. + .runJob(productionEuWest1); // Finally, west changes revision. assertEquals(Change.empty(), app1.instance().change()); assertEquals(Optional.of(RunStatus.success), app1.instanceJobs().get(productionUsCentral1).lastStatus()); } @@ -1239,13 +1245,13 @@ public class DeploymentTriggerTest { assertEquals(Change.empty(), app.instance().change()); // Application is pinned to previous version, and downgrades to that. Tests are re-run. - tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(version0).withPin()); + tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(version0).withPlatformPin()); app.runJob(stagingTest).runJob(productionUsEast3); tester.clock().advance(Duration.ofMinutes(1)); app.failDeployment(testUsEast3); tester.clock().advance(Duration.ofMinutes(11)); // Job is cooling down after consecutive failures. app.runJob(testUsEast3); - assertEquals(Change.empty().withPin(), app.instance().change()); + assertEquals(Change.empty().withPlatformPin(), app.instance().change()); // A new upgrade is attempted, and production tests wait for redeployment. tester.controllerTester().upgradeSystem(version2); @@ -2234,7 +2240,7 @@ public class DeploymentTriggerTest { .majorVersion(7) .compileVersion(version1) .build()); - tester.deploymentTrigger().forceChange(app.instanceId(), app.instance().change().withPin()); + tester.deploymentTrigger().forceChange(app.instanceId(), app.instance().change().withPlatformPin()); app.deploy(); assertEquals(version1, tester.jobs().last(app.instanceId(), productionUsEast3).get().versions().targetPlatform()); assertEquals(version1, app.application().revisions().get(tester.jobs().last(app.instanceId(), productionUsEast3).get().versions().targetRevision()).compileVersion().get()); @@ -2251,7 +2257,7 @@ public class DeploymentTriggerTest { // The new app enters a platform block window, and is pinned to the old platform; // the new submission overrides both those settings, as the new revision should roll out regardless. tester.atMondayMorning(); - tester.deploymentTrigger().forceChange(newApp.instanceId(), Change.empty().withPin()); + tester.deploymentTrigger().forceChange(newApp.instanceId(), Change.empty().withPlatformPin()); newApp.submit(new ApplicationPackageBuilder().compileVersion(version2) .systemTest() .blockChange(false, true, "mon", "0-23", "UTC") @@ -2280,11 +2286,11 @@ public class DeploymentTriggerTest { tester.upgrader().run(); assertEquals(Change.of(newRevision).with(version1), newApp.instance().change()); - tester.deploymentTrigger().forceChange(newApp.instanceId(), newApp.instance().change().withPin()); + tester.deploymentTrigger().forceChange(newApp.instanceId(), newApp.instance().change().withPlatformPin()); tester.outstandingChangeDeployer().run(); - assertEquals(Change.of(newRevision).with(version1).withPin(), newApp.instance().change()); + assertEquals(Change.of(newRevision).with(version1).withPlatformPin(), newApp.instance().change()); tester.upgrader().run(); - assertEquals(Change.of(newRevision).with(version1).withPin(), newApp.instance().change()); + assertEquals(Change.of(newRevision).with(version1).withPlatformPin(), newApp.instance().change()); newApp.deploy(); assertEquals(version1, tester.jobs().last(newApp.instanceId(), productionUsEast3).get().versions().targetPlatform()); @@ -2381,7 +2387,7 @@ public class DeploymentTriggerTest { .build())) .getMessage()); - tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(oldVersion).with(app.application().revisions().last().get().id()).withPin()); + tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(oldVersion).with(app.application().revisions().last().get().id()).withPlatformPin()); app.deploy(); assertEquals(oldVersion, app.deployment(ZoneId.from("prod", "us-east-3")).version()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdaterTest.java index 5deba19c5ea..b8b05bfcfc1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdaterTest.java @@ -45,7 +45,7 @@ public class BcpGroupUpdaterTest { tester.controllerTester().upgradeSystem(Version.fromString("7.1")); var context = tester.newDeploymentContext(); var deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(tester.controller(), Duration.ofDays(1)); - var updater = new BcpGroupUpdater(tester.controller(), Duration.ofDays(1)); + var updater = new BcpGroupUpdater(tester.controller(), Duration.ofDays(1), 1.0); ZoneId prod1 = ZoneId.from("prod", "ap-northeast-1"); ZoneId prod2 = ZoneId.from("prod", "us-east-3"); ZoneId prod3 = ZoneId.from("prod", "us-west-1"); @@ -57,7 +57,7 @@ public class BcpGroupUpdaterTest { setQpsMetric(50.0, context.application().id().defaultInstance(), prod1, tester); setBcpMetrics(1.5, 0.1, 0.45, context.instanceId(), prod1, "cluster1", tester); deploymentMetricsMaintainer.maintain(); - assertEquals(1.0, updater.maintain(), 0.0000001); + assertEquals(0.0, updater.maintain(), 0.0000001); assertTrafficFraction(1.0, 1.0, context.instanceId(), prod1, tester); assertNoBcpGroupInfo(context.instanceId(), prod1, "cluster1", tester, "No other regions in group"); @@ -67,7 +67,7 @@ public class BcpGroupUpdaterTest { setQpsMetric(20.0, context.application().id().defaultInstance(), prod2, tester); setBcpMetrics(100.0, 0.1, 0.45, context.instanceId(), prod1, "cluster1", tester); deploymentMetricsMaintainer.maintain(); - assertEquals(1.0, updater.maintain(), 0.0000001); + assertEquals(0.0, updater.maintain(), 0.0000001); assertTrafficFraction(0.75, 1.0, context.instanceId(), prod1, tester); assertTrafficFraction(0.25, 1.0, context.instanceId(), prod2, tester); assertNoBcpGroupInfo(context.instanceId(), prod1, "cluster1", tester, @@ -75,7 +75,7 @@ public class BcpGroupUpdaterTest { assertBcpGroupInfo(100.0, 0.1, 0.45, context.instanceId(), prod2, "cluster1", tester); setBcpMetrics(50.0, 0.2, 0.5, context.instanceId(), prod2, "cluster1", tester); - assertEquals(1.0, updater.maintain(), 0.0000001); + assertEquals(0.0, updater.maintain(), 0.0000001); assertBcpGroupInfo(50.0, 0.2, 0.5, context.instanceId(), prod1, "cluster1", tester); @@ -85,7 +85,7 @@ public class BcpGroupUpdaterTest { setQpsMetric(45.0, context.application().id().defaultInstance(), prod2, tester); setQpsMetric(02.0, context.application().id().defaultInstance(), prod3, tester); deploymentMetricsMaintainer.maintain(); - assertEquals(1.0, updater.maintain(), 0.0000001); + assertEquals(0.0, updater.maintain(), 0.0000001); assertTrafficFraction(0.53, 0.53 + (double)45/2 / 100, context.instanceId(), prod1, tester); assertTrafficFraction(0.45, 0.45 + (double)53/2 / 100, context.instanceId(), prod2, tester); assertTrafficFraction(0.02, 0.02 + (double)53/2 / 100, context.instanceId(), prod3, tester); @@ -129,7 +129,7 @@ public class BcpGroupUpdaterTest { locked -> tester.controller().applications().store(locked.with(deploymentSpec))); var deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(tester.controller(), Duration.ofDays(1)); - var updater = new BcpGroupUpdater(tester.controller(), Duration.ofDays(1)); + var updater = new BcpGroupUpdater(tester.controller(), Duration.ofDays(1), 1.0); ZoneId ap1 = ZoneId.from("prod", "ap-northeast-1"); ZoneId ap2 = ZoneId.from("prod", "ap-southeast-1"); @@ -150,7 +150,7 @@ public class BcpGroupUpdaterTest { setQpsMetric(40.0, context.application().id().defaultInstance(), eu1, tester); deploymentMetricsMaintainer.maintain(); - assertEquals(1.0, updater.maintain(), 0.0000001); + assertEquals(0.0, updater.maintain(), 0.0000001); assertTrafficFraction(0.5, 0.5, context.instanceId(), ap1, tester); assertTrafficFraction(0.0, 0.5, context.instanceId(), ap2, tester); assertTrafficFraction(0.1, 0.1, context.instanceId(), us1, tester); @@ -197,7 +197,7 @@ public class BcpGroupUpdaterTest { locked -> tester.controller().applications().store(locked.with(deploymentSpec))); var deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(tester.controller(), Duration.ofDays(1)); - var updater = new BcpGroupUpdater(tester.controller(), Duration.ofDays(1)); + var updater = new BcpGroupUpdater(tester.controller(), Duration.ofDays(1), 1.0); ZoneId ap1 = ZoneId.from("prod", "ap-northeast-1"); ZoneId ap2 = ZoneId.from("prod", "ap-southeast-1"); @@ -221,7 +221,7 @@ public class BcpGroupUpdaterTest { setQpsMetric(60.0, context.application().id().defaultInstance(), eu1, tester); deploymentMetricsMaintainer.maintain(); - assertEquals(1.0, updater.maintain(), 0.0000001); + assertEquals(0.0, updater.maintain(), 0.0000001); assertTrafficFraction(0.10, 0.10 + 50 / 200.0 / 1.5, context.instanceId(), ap1, tester); assertTrafficFraction(0.25, 0.25 + 30 / 200.0 / 1.5, context.instanceId(), ap2, tester); assertTrafficFraction(0.00, 0.00 + 40 / 200.0 / 2.5, context.instanceId(), us1, tester); @@ -242,7 +242,7 @@ public class BcpGroupUpdaterTest { setBcpMetrics(300, 0.3, 0.3, context.instanceId(), us3, "cluster2", tester); setBcpMetrics(100, 0.1, 0.1, context.instanceId(), eu1, "cluster2", tester); - assertEquals(1.0, updater.maintain(), 0.0000001); + assertEquals(0.0, updater.maintain(), 0.0000001); assertNoBcpGroupInfo(context.instanceId(), ap1, "cluster1", tester, "No info in ap"); assertNoBcpGroupInfo(context.instanceId(), ap2, "cluster1", tester, "No info in ap"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java index 63e2c99cb6e..6452edc9e61 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java @@ -38,14 +38,14 @@ public class ControllerMaintainerTest { void records_metric() { TestControllerMaintainer maintainer = new TestControllerMaintainer(tester.controller(), SystemName.main, new AtomicInteger()); maintainer.run(); - assertEquals(1.0, successFactorMetric(), 0.0000001); + assertEquals(0.0, successFactorDeviationMetric(), 0.0000001); maintainer.success = false; maintainer.run(); maintainer.run(); - assertEquals(0.0, successFactorMetric(), 0.0000001); + assertEquals(1.0, successFactorDeviationMetric(), 0.0000001); maintainer.success = true; maintainer.run(); - assertEquals(1.0, successFactorMetric(), 0.0000001); + assertEquals(0.0, successFactorDeviationMetric(), 0.0000001); } private long consecutiveFailuresMetric() { @@ -54,10 +54,10 @@ public class ControllerMaintainerTest { "maintenance.consecutiveFailures").get().longValue(); } - private long successFactorMetric() { + private long successFactorDeviationMetric() { MetricsMock metrics = (MetricsMock) tester.controller().metric(); return metrics.getMetric((context) -> "TestControllerMaintainer".equals(context.get("job")), - "maintenance.successFactor").get().longValue(); + "maintenance.successFactorDeviation").get().longValue(); } private static class TestControllerMaintainer extends ControllerMaintainer { @@ -73,7 +73,7 @@ public class ControllerMaintainerTest { @Override protected double maintain() { executions.incrementAndGet(); - return success ? 1.0 : 0.0; + return success ? 0.0 : 1.0; } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java index 934e15ad623..49cf8c634ba 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java @@ -49,7 +49,7 @@ public class EndpointCertificateMaintainerTest { @Test void old_and_unused_cert_is_deleted() { tester.curator().writeEndpointCertificateMetadata(ApplicationId.defaultId(), exampleMetadata); - assertEquals(1.0, maintainer.maintain(), 0.0000001); + assertEquals(0.0, maintainer.maintain(), 0.0000001); assertTrue(tester.curator().readEndpointCertificateMetadata(ApplicationId.defaultId()).isEmpty()); } @@ -57,7 +57,7 @@ public class EndpointCertificateMaintainerTest { void unused_but_recently_used_cert_is_not_deleted() { EndpointCertificateMetadata recentlyRequestedCert = exampleMetadata.withLastRequested(tester.clock().instant().minusSeconds(3600).getEpochSecond()); tester.curator().writeEndpointCertificateMetadata(ApplicationId.defaultId(), recentlyRequestedCert); - assertEquals(1.0, maintainer.maintain(), 0.0000001); + assertEquals(0.0, maintainer.maintain(), 0.0000001); assertEquals(Optional.of(recentlyRequestedCert), tester.curator().readEndpointCertificateMetadata(ApplicationId.defaultId())); } @@ -69,7 +69,7 @@ public class EndpointCertificateMaintainerTest { secretStore.setSecret(exampleMetadata.keyName(), "foo", 1); secretStore.setSecret(exampleMetadata.certName(), "bar", 1); - assertEquals(1.0, maintainer.maintain(), 0.0000001); + assertEquals(0.0, maintainer.maintain(), 0.0000001); var updatedCert = Optional.of(recentlyRequestedCert.withLastRefreshed(tester.clock().instant().getEpochSecond()).withVersion(1)); @@ -90,7 +90,7 @@ public class EndpointCertificateMaintainerTest { deploymentContext.submit(applicationPackage).runJob(systemTest).runJob(stagingTest).runJob(productionUsWest1); - assertEquals(1.0, maintainer.maintain(), 0.0000001); + assertEquals(0.0, maintainer.maintain(), 0.0000001); var metadata = tester.curator().readEndpointCertificateMetadata(appId).orElseThrow(); tester.controller().serviceRegistry().endpointCertificateProvider().certificateDetails(metadata.rootRequestId()); // cert should not be deleted, the app is deployed! } @@ -110,7 +110,7 @@ public class EndpointCertificateMaintainerTest { var originalMetadata = tester.curator().readEndpointCertificateMetadata(appId).orElseThrow(); // cert should not be deleted, the app is deployed! - assertEquals(1.0, maintainer.maintain(), 0.0000001); + assertEquals(0.0, maintainer.maintain(), 0.0000001); assertEquals(tester.curator().readEndpointCertificateMetadata(appId), Optional.of(originalMetadata)); tester.controller().serviceRegistry().endpointCertificateProvider().certificateDetails(originalMetadata.rootRequestId()); @@ -121,7 +121,7 @@ public class EndpointCertificateMaintainerTest { tester.controller().serviceRegistry().endpointCertificateProvider().requestCaSignedCertificate(appId, originalMetadata.requestedDnsSans(), Optional.of(originalMetadata)); // We should now pick up the new key and cert version + uuid, but not force trigger deployment yet - assertEquals(1.0, maintainer.maintain(), 0.0000001); + assertEquals(0.0, maintainer.maintain(), 0.0000001); deploymentContext.assertNotRunning(productionUsWest1); var updatedMetadata = tester.curator().readEndpointCertificateMetadata(appId).orElseThrow(); assertNotEquals(originalMetadata.leafRequestId().orElseThrow(), updatedMetadata.leafRequestId().orElseThrow()); @@ -130,7 +130,7 @@ public class EndpointCertificateMaintainerTest { // after another 4 days, we should force trigger deployment if it hasn't already happened tester.clock().advance(Duration.ofDays(4).plusSeconds(1)); deploymentContext.assertNotRunning(productionUsWest1); - assertEquals(1.0, maintainer.maintain(), 0.0000001); + assertEquals(0.0, maintainer.maintain(), 0.0000001); deploymentContext.assertRunning(productionUsWest1); } @@ -156,7 +156,7 @@ public class EndpointCertificateMaintainerTest { ApplicationId unknown = ApplicationId.fromSerializedForm("applicationid:is:unknown"); endpointCertificateProvider.requestCaSignedCertificate(unknown, List.of("a", "b", "c"), Optional.empty()); // Unknown to controller! - assertEquals(1.0, maintainer.maintain(), 0.0000001); + assertEquals(0.0, maintainer.maintain(), 0.0000001); assertTrue(endpointCertificateProvider.dnsNamesOf(unknown).isEmpty()); assertTrue(endpointCertificateProvider.listCertificates().isEmpty()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index 11110d6edaa..96c1d7c545d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -5,7 +5,6 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.test.ManualClock; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.application.Change; @@ -27,7 +26,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.OptionalInt; import java.util.Set; import java.util.stream.Collectors; @@ -856,10 +854,10 @@ public class UpgraderTest { // Create an application with pinned platform version. var context = tester.newDeploymentContext().submit().deploy(); - tester.deploymentTrigger().forceChange(context.instanceId(), Change.empty().withPin()); + tester.deploymentTrigger().forceChange(context.instanceId(), Change.empty().withPlatformPin()); assertFalse(context.instance().change().hasTargets()); - assertTrue(context.instance().change().isPinned()); + assertTrue(context.instance().change().isPlatformPinned()); assertEquals(3, context.instance().deployments().size()); // Application does not upgrade. @@ -867,21 +865,21 @@ public class UpgraderTest { tester.controllerTester().upgradeSystem(version1); tester.upgrader().maintain(); assertFalse(context.instance().change().hasTargets()); - assertTrue(context.instance().change().isPinned()); + assertTrue(context.instance().change().isPlatformPinned()); // New application package is deployed. context.submit().deploy(); assertFalse(context.instance().change().hasTargets()); - assertTrue(context.instance().change().isPinned()); + assertTrue(context.instance().change().isPlatformPinned()); // Application upgrades to new version when pin is removed. tester.deploymentTrigger().cancelChange(context.instanceId(), PIN); tester.upgrader().maintain(); assertTrue(context.instance().change().hasTargets()); - assertFalse(context.instance().change().isPinned()); + assertFalse(context.instance().change().isPlatformPinned()); // Application is pinned to new version, and upgrade is therefore not cancelled, even though confidence is broken. - tester.deploymentTrigger().forceChange(context.instanceId(), Change.empty().withPin()); + tester.deploymentTrigger().forceChange(context.instanceId(), Change.empty().withPlatformPin()); tester.upgrader().maintain(); tester.triggerJobs(); assertEquals(version1, context.instance().change().platform().get()); @@ -890,7 +888,7 @@ public class UpgraderTest { context.runJob(systemTest).runJob(stagingTest).runJob(productionUsCentral1) .timeOutUpgrade(productionUsWest1); tester.deploymentTrigger().cancelChange(context.instanceId(), ALL); - tester.deploymentTrigger().forceChange(context.instanceId(), Change.of(version0).withPin()); + tester.deploymentTrigger().forceChange(context.instanceId(), Change.of(version0).withPlatformPin()); assertEquals(version0, context.instance().change().platform().get()); // Application downgrades to pinned version. @@ -913,7 +911,7 @@ public class UpgraderTest { // Keep app 1 on current version tester.controller().applications().lockApplicationIfPresent(app1.application().id(), app -> tester.controller().applications().store(app.with(app1.instance().name(), - instance -> instance.withChange(instance.change().withPin())))); + instance -> instance.withChange(instance.change().withPlatformPin())))); // New version is released Version version1 = Version.fromString("6.2"); @@ -935,7 +933,7 @@ public class UpgraderTest { // App 1 is unpinned and upgrades to latest 6 tester.controller().applications().lockApplicationIfPresent(app1.application().id(), app -> tester.controller().applications().store(app.with(app1.instance().name(), - instance -> instance.withChange(instance.change().withoutPin())))); + instance -> instance.withChange(instance.change().withoutPlatformPin())))); tester.upgrader().maintain(); assertEquals(version1, app1.instance().change().platform().orElseThrow(), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainerTest.java index 52bd8e9c618..39bf61df9ed 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainerTest.java @@ -58,7 +58,7 @@ public class VcmrMaintainerTest { @Test void recycle_hosts_after_completion() { var vcmrReport = new VcmrReport(); - vcmrReport.addVcmr("id123", ZonedDateTime.now(), ZonedDateTime.now()); + vcmrReport.addVcmr(new ChangeRequestSource("aws", "id123", "url", ChangeRequestSource.Status.WAITING_FOR_APPROVAL , ZonedDateTime.now(), ZonedDateTime.now())); var parkedNode = createNode(host1, NodeType.host, Node.State.parked, true); var failedNode = createNode(host2, NodeType.host, Node.State.failed, false); var reports = vcmrReport.toNodeReports(); @@ -181,7 +181,7 @@ public class VcmrMaintainerTest { activeNode = nodeRepo.list(zoneId, NodeFilter.all().hostnames(host2)).get(0); var report = VcmrReport.fromReports(activeNode.reports()); var reportAdded = report.getVcmrs().stream() - .filter(vcmr -> vcmr.getId().equals(changeRequestId)) + .filter(vcmr -> vcmr.id().equals(changeRequestId)) .count() == 1; assertTrue(reportAdded); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 589fc25700f..b71d3cf838b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -101,16 +101,17 @@ public class ApplicationSerializerTest { Optional.empty(), Optional.of("best commit"), Optional.of("hash1"), + Optional.of(Instant.ofEpochMilli(777)), true, false, Optional.of("~(˘▾˘)~"), 3); assertEquals("https://github/org/repo/tree/commit1", applicationVersion1.sourceUrl().get()); - ApplicationVersion applicationVersion2 = ApplicationVersion.from(RevisionId.forDevelopment(31, new JobId(id1, DeploymentContext.productionUsEast3)), - new SourceRevision("repo1", "branch1", "commit1"), "a@b", - Version.fromString("6.3.1"), - Instant.ofEpochMilli(496)); + RevisionId id = RevisionId.forDevelopment(31, new JobId(id1, DeploymentContext.productionUsEast3)); + SourceRevision source = new SourceRevision("repo1", "branch1", "commit1"); + Version compileVersion = Version.fromString("6.3.1"); + ApplicationVersion applicationVersion2 = new ApplicationVersion(id, Optional.of(source), Optional.of("a@b"), Optional.of(compileVersion), Optional.empty(), Optional.of(Instant.ofEpochMilli(496)), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), true, false, Optional.empty(), 0); Instant activityAt = Instant.parse("2018-06-01T10:15:30.00Z"); deployments.add(new Deployment(zone1, CloudAccount.empty, applicationVersion1.id(), Version.fromString("1.2.3"), Instant.ofEpochMilli(3), DeploymentMetrics.none, DeploymentActivity.none, QuotaUsage.none, OptionalDouble.empty())); @@ -143,7 +144,7 @@ public class ApplicationSerializerTest { Map.of(), List.of(), RotationStatus.EMPTY, - Change.of(Version.fromString("6.7")).withPin())); + Change.of(Version.fromString("6.7")).withPlatformPin().withRevisionPin())); Application original = new Application(TenantAndApplicationId.from(id1), Instant.now().truncatedTo(ChronoUnit.MILLIS), @@ -174,6 +175,7 @@ public class ApplicationSerializerTest { assertEquals(original.revisions().last().get().sourceUrl(), serialized.revisions().last().get().sourceUrl()); assertEquals(original.revisions().last().get().commit(), serialized.revisions().last().get().commit()); assertEquals(original.revisions().last().get().bundleHash(), serialized.revisions().last().get().bundleHash()); + assertEquals(original.revisions().last().get().obsoleteAt(), serialized.revisions().last().get().obsoleteAt()); assertEquals(original.revisions().last().get().hasPackage(), serialized.revisions().last().get().hasPackage()); assertEquals(original.revisions().last().get().shouldSkip(), serialized.revisions().last().get().shouldSkip()); assertEquals(original.revisions().last().get().description(), serialized.revisions().last().get().description()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 9a34989aeff..76bcbe078ff 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -541,24 +541,22 @@ public class ApplicationApiTest extends ControllerContainerTest { "{\"message\":\"No deployment in progress for tenant1.application1.instance1 at this time\"}"); // POST pinning to a given version to an application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/platform-pin", POST) .userIdentity(USER_ID) .data("6.1.0"), "{\"message\":\"Triggered pin to 6.1 for tenant1.application1.instance1\"}"); assertTrue(tester.controller().auditLogger().readLog().entries().stream() - .anyMatch(entry -> entry.resource().equals("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin?")), + .anyMatch(entry -> entry.resource().equals("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/platform-pin?")), "Action is logged to audit log"); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) - .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":true}"); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin", GET) - .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":true}"); + .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":true,\"platform-pinned\":true,\"application-pinned\":false}"); // DELETE only the pin to a given version - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/platform-pin", DELETE) .userIdentity(USER_ID), "{\"message\":\"Changed deployment from 'pin to 6.1' to 'upgrade to 6.1' for tenant1.application1.instance1\"}"); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) - .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":false}"); + .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":false,\"platform-pinned\":false,\"application-pinned\":false}"); // POST pinning again tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin", POST) @@ -566,14 +564,14 @@ public class ApplicationApiTest extends ControllerContainerTest { .data("6.1"), "{\"message\":\"Triggered pin to 6.1 for tenant1.application1.instance1\"}"); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) - .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":true}"); + .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":true,\"platform-pinned\":true,\"application-pinned\":false}"); // DELETE only the version, but leave the pin tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/platform", DELETE) .userIdentity(USER_ID), "{\"message\":\"Changed deployment from 'pin to 6.1' to 'pin to current platform' for tenant1.application1.instance1\"}"); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) - .userIdentity(USER_ID), "{\"pinned\":true}"); + .userIdentity(USER_ID), "{\"pinned\":true,\"platform-pinned\":true,\"application-pinned\":false}"); // DELETE also the pin to a given version tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin", DELETE) @@ -582,6 +580,32 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) .userIdentity(USER_ID), "{}"); + // POST pinning to a given revision to an application + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/application-pin", POST) + .userIdentity(USER_ID) + .data(""), + "{\"message\":\"Triggered pin to build 1 for tenant1.application1.instance1\"}"); + assertTrue(tester.controller().auditLogger().readLog().entries().stream() + .anyMatch(entry -> entry.resource().equals("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/application-pin?")), + "Action is logged to audit log"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) + .userIdentity(USER_ID), "{\"application\":\"build 1\",\"pinned\":false,\"platform-pinned\":false,\"application-pinned\":true}"); + + // DELETE only the pin to a given revision + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/application-pin", DELETE) + .userIdentity(USER_ID), + "{\"message\":\"Changed deployment from 'pin to build 1' to 'revision change to build 1' for tenant1.application1.instance1\"}"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) + .userIdentity(USER_ID), "{\"application\":\"build 1\",\"pinned\":false,\"platform-pinned\":false,\"application-pinned\":false}"); + + // DELETE deploying to a given revision + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/application", DELETE) + .userIdentity(USER_ID), + "{\"message\":\"Changed deployment from 'revision change to build 1' to 'no change' for tenant1.application1.instance1\"}"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) + .userIdentity(USER_ID), "{}"); + + // POST a pause to a production job tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-west-1/pause", POST) .userIdentity(USER_ID), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json index a02fb1fb375..6793553faca 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json @@ -9,6 +9,7 @@ "declared": true, "instance": "default", "readyAt": 0, + "delayCause": null, "deploying": { "application": { "build": 3, @@ -155,6 +156,7 @@ "declared": false, "instance": "default", "readyAt": 0, + "delayCause": null, "jobName": "system-test", "url": "https://some.url:43/instance/default/job/system-test", "environment": "test", @@ -344,6 +346,7 @@ "readyAt": 15153000, "delayedUntil": 15153000, "coolingDownUntil": 15153000, + "delayCause": "coolingDown", "jobName": "staging-test", "url": "https://some.url:43/instance/default/job/staging-test", "environment": "staging", @@ -777,6 +780,8 @@ "declared": true, "instance": "default", "readyAt": 14403000, + "delayedUntil": 14403000, + "delayCause": "running", "jobName": "production-us-central-1", "url": "https://some.url:43/instance/default/job/production-us-central-1", "environment": "prod", @@ -902,6 +907,7 @@ ], "declared": true, "instance": "default", + "delayCause": "notReady", "jobName": "test-us-central-1", "url": "https://some.url:43/instance/default/job/test-us-central-1", "environment": "prod", @@ -1042,6 +1048,7 @@ ], "declared": true, "instance": "default", + "delayCause": "notReady", "jobName": "production-us-west-1", "url": "https://some.url:43/instance/default/job/production-us-west-1", "environment": "prod", @@ -1150,6 +1157,7 @@ ], "declared": true, "instance": "default", + "delayCause": "notReady", "jobName": "production-us-east-3", "url": "https://some.url:43/instance/default/job/production-us-east-3", "environment": "prod", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json index 35dd6fc5398..0b7c64c72a5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json @@ -9,6 +9,7 @@ "declared": true, "instance": "instance1", "readyAt": 0, + "delayCause": null, "deploying": { "application": { "build": 4, @@ -47,6 +48,14 @@ "sourceUrl": "repository1/tree/commit1", "commit": "commit1" } + }, + { + "application": { + "build": 1, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + } } ], "blockers": [ ] @@ -59,6 +68,8 @@ "declared": false, "instance": "instance1", "readyAt": 0, + "delayedUntil": 0, + "delayCause": "running", "jobName": "system-test", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test", "environment": "test", @@ -187,6 +198,8 @@ "declared": false, "instance": "instance1", "readyAt": 0, + "delayedUntil": 0, + "delayCause": "running", "jobName": "staging-test", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test", "environment": "staging", @@ -348,6 +361,7 @@ ], "declared": true, "instance": "instance1", + "delayCause": "unverified", "jobName": "production-us-central-1", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-central-1", "environment": "prod", @@ -405,6 +419,7 @@ ], "declared": true, "instance": "instance1", + "delayCause": "notReady", "jobName": "production-us-west-1", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-west-1", "environment": "prod", @@ -462,6 +477,7 @@ ], "declared": true, "instance": "instance1", + "delayCause": "notReady", "jobName": "production-us-east-3", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3", "environment": "prod", @@ -547,6 +563,7 @@ ], "declared": true, "instance": "instance2", + "delayCause": "notReady", "deploying": { "application": { "build": 4, @@ -585,6 +602,14 @@ "sourceUrl": "repository1/tree/commit1", "commit": "commit1" } + }, + { + "application": { + "build": 1, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1" + } } ], "blockers": [ ] @@ -598,6 +623,7 @@ ], "declared": true, "instance": "instance2", + "delayCause": "unverified", "jobName": "production-us-central-1", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance2/job/production-us-central-1", "environment": "prod", @@ -624,6 +650,7 @@ ], "declared": true, "instance": "instance2", + "delayCause": "notReady", "jobName": "production-us-west-1", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance2/job/production-us-west-1", "environment": "prod", @@ -650,6 +677,7 @@ ], "declared": true, "instance": "instance2", + "delayCause": "notReady", "jobName": "production-us-east-3", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance2/job/production-us-east-3", "environment": "prod", @@ -697,6 +725,15 @@ "description": "my best commit yet", "risk": 9001, "deployable": false + }, + { + "build": 1, + "compileVersion": "6.1.0", + "sourceUrl": "repository1/tree/commit1", + "commit": "commit1", + "description": "my best commit yet", + "risk": 9001, + "deployable": true } ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index 7ee5f6db9b9..c942a7ad63d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -18,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import org.junit.jupiter.api.Test; import java.io.File; +import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -76,7 +77,7 @@ public class DeploymentApiTest extends ControllerContainerTest { deploymentTester.upgrader().maintain(); deploymentTester.triggerJobs(); productionApp.runJob(DeploymentContext.systemTest).runJob(DeploymentContext.stagingTest).runJob(DeploymentContext.productionUsWest1); - failingApp.failDeployment(DeploymentContext.systemTest).failDeployment(DeploymentContext.stagingTest); + failingApp.failDeployment(DeploymentContext.systemTest).failDeployment(DeploymentContext.stagingTest).timeOutConvergence(DeploymentContext.stagingTest); deploymentTester.upgrader().maintain(); deploymentTester.triggerJobs(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json index 51398daa1d4..ac43fbf2a80 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json @@ -37,16 +37,17 @@ "instance": "default", "upgrading": false, "pinned": false, + "platformPinned": false, + "revisionPinned": false, "upgradePolicy": "default", "compileVersion": "6.1.0", "jobs": [ { - "name": "system-test", - "coolingDownUntil": 1600000000000 + "name": "system-test" }, { "name": "staging-test", - "coolingDownUntil": 1600000000000 + "coolingDownUntil": 1600022201500 }, { "name": "production-us-west-1" @@ -79,6 +80,8 @@ "instance": "i2", "upgrading": false, "pinned": false, + "platformPinned": false, + "revisionPinned": false, "upgradePolicy": "default", "compileVersion": "6.1.0", "jobs": [ @@ -141,7 +144,7 @@ "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1", "upgradePolicy": "default", "failing": "staging-test", - "status": "error" + "status": "installationFailed" } ], "productionApplications": [ @@ -165,14 +168,6 @@ "running": "system-test" }, { - "tenant": "tenant1", - "application": "application1", - "instance": "default", - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1", - "upgradePolicy": "default", - "running": "staging-test" - }, - { "tenant": "tenant2", "application": "application2", "instance": "i2", @@ -188,17 +183,18 @@ "instance": "default", "upgrading": true, "pinned": false, + "platformPinned": false, + "revisionPinned": false, "upgradePolicy": "default", "compileVersion": "6.1.0", "jobs": [ { "name": "system-test", - "coolingDownUntil": 1600000000000, "pending": "application" }, { "name": "staging-test", - "coolingDownUntil": 1600000000000, + "coolingDownUntil": 1600022201500, "pending": "platform" }, { @@ -222,15 +218,10 @@ }, "staging-test": { "failing": { - "number": 2, - "start": 1600000000000, - "end": 1600000000000, - "status": "error" - }, - "running": { "number": 3, "start": 1600000000000, - "status": "running" + "end": 1600014401000, + "status": "installationFailed" } } }, @@ -250,15 +241,10 @@ }, "staging-test": { "failing": { - "number": 2, - "start": 1600000000000, - "end": 1600000000000, - "status": "error" - }, - "running": { "number": 3, "start": 1600000000000, - "status": "running" + "end": 1600014401000, + "status": "installationFailed" } } } @@ -269,6 +255,8 @@ "instance": "i1", "upgrading": false, "pinned": false, + "platformPinned": false, + "revisionPinned": false, "upgradePolicy": "default", "compileVersion": "6.1.0", "jobs": [ @@ -289,15 +277,15 @@ "system-test": { "failing": { "number": 3, - "start": 1600000000000, - "end": 1600000000000, + "start": 1600014401000, + "end": 1600014401000, "status": "error" } }, "staging-test": { "running": { "number": 3, - "start": 1600000000000, + "start": 1600014401000, "status": "running" } }, @@ -329,6 +317,8 @@ "instance": "i2", "upgrading": true, "pinned": false, + "platformPinned": false, + "revisionPinned": false, "upgradePolicy": "default", "compileVersion": "6.1.0", "jobs": [ @@ -341,7 +331,7 @@ "production-us-west-1": { "running": { "number": 2, - "start": 1600000000000, + "start": 1600014401000, "status": "running" } } @@ -350,7 +340,7 @@ "production-us-west-1": { "running": { "number": 2, - "start": 1600000000000, + "start": 1600014401000, "status": "running" } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java index 43ad01fc5c2..168a1345c39 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java @@ -18,7 +18,6 @@ import org.junit.jupiter.api.Test; import java.net.URI; import java.util.List; import java.util.Set; -import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -155,7 +154,7 @@ public class RotationRepositoryTest { var application2 = tester.newDeploymentContext("tenant2", "app2", "default"); application2.submit(applicationPackage).deploy(); assertEquals(List.of(new RotationId("foo-1")), rotationIds(application2.instance().rotations())); - assertEquals("https://cd.app2.tenant2.global.vespa.oath.cloud/", + assertEquals("https://cd.app2.tenant2.global.cd.vespa.oath.cloud/", tester.controller().routing().readDeclaredEndpointsOf(application2.instanceId()).primary().get().url().toString()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index f08e92a515d..7afa5c7f44a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -349,6 +349,13 @@ public class VersionStatusTest { assertEquals(Confidence.high, confidence(tester.controller(), version0), "Confidence remains unchanged for version0: High"); assertEquals(VespaVersion.Confidence.high, confidence(tester.controller(), version2), "90% of defaults deployed successfully: High"); + // Canary failing a new revision does not affect confidence + canary0.submit(canaryPolicy).failDeployment(systemTest); + tester.controllerTester().computeVersionStatus(); + assertEquals(Confidence.high, confidence(tester.controller(), version0), "Confidence remains unchanged for version0: High"); + assertEquals(VespaVersion.Confidence.high, confidence(tester.controller(), version2), "90% of defaults deployed successfully: High"); + canary0.deploy(); + // A new version is released, all canaries upgrade successfully, but enough "default" apps fail to mark version // as broken Version version3 = new Version("6.5"); diff --git a/default_build_settings.cmake b/default_build_settings.cmake index c82bb91accc..2785a98a396 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -4,11 +4,11 @@ include(VespaExtendedDefaultBuildSettings OPTIONAL) function(setup_vespa_default_build_settings_darwin) message("-- Setting up default build settings for darwin") - set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS_PREFIX}/lib" "/usr/local/opt/bison/lib" "/usr/local/opt/flex/lib" "/usr/local/opt/icu4c/lib" "/usr/local/opt/openssl@1.1/lib" "/usr/local/opt/openblas/lib") - list(APPEND DEFAULT_EXTRA_LINK_DIRECTORY "/usr/local/lib") + set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS_PREFIX}/lib" "${VESPA_HOMEBREW_PREFIX}/opt/bison/lib" "${VESPA_HOMEBREW_PREFIX}/opt/flex/lib" "${VESPA_HOMEBREW_PREFIX}/opt/icu4c/lib" "${VESPA_HOMEBREW_PREFIX}/opt/openssl@1.1/lib" "${VESPA_HOMEBREW_PREFIX}/opt/openblas/lib") + list(APPEND DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_HOMEBREW_PREFIX}/lib") set(DEFAULT_EXTRA_LINK_DIRECTORY "${DEFAULT_EXTRA_LINK_DIRECTORY}" PARENT_SCOPE) - set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS_PREFIX}/include" "/usr/local/opt/flex/include" "/usr/local/opt/icu4c/include" "/usr/local/opt/openssl@1.1/include" "/usr/local/opt/openblas/include") - list(APPEND DEFAULT_EXTRA_INCLUDE_DIRECTORY "/usr/local/include") + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS_PREFIX}/include" "${VESPA_HOMEBREW_PREFIX}/opt/flex/include" "${VESPA_HOMEBREW_PREFIX}/opt/icu4c/include" "${VESPA_HOMEBREW_PREFIX}/opt/openssl@1.1/include" "${VESPA_HOMEBREW_PREFIX}/opt/openblas/include") + list(APPEND DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_HOMEBREW_PREFIX}/include") set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${DEFAULT_EXTRA_INCLUDE_DIRECTORY}" PARENT_SCOPE) endfunction() @@ -84,7 +84,7 @@ endfunction() function(vespa_use_default_cmake_prefix_path) set(DEFAULT_CMAKE_PREFIX_PATH ${VESPA_DEPS_PREFIX}) if (APPLE) - list(APPEND DEFAULT_CMAKE_PREFIX_PATH "/usr/local/opt/bison" "/usr/local/opt/flex" "/usr/local/opt/openssl@1.1" "/usr/local/opt/openblas" "/usr/local/opt/icu4c") + list(APPEND DEFAULT_CMAKE_PREFIX_PATH "${VESPA_HOMEBREW_PREFIX}/opt/bison" "${VESPA_HOMEBREW_PREFIX}/opt/flex" "${VESPA_HOMEBREW_PREFIX}/opt/openssl@1.1" "${VESPA_HOMEBREW_PREFIX}/opt/openblas" "${VESPA_HOMEBREW_PREFIX}/opt/icu4c") endif() message("-- DEFAULT_CMAKE_PREFIX_PATH is ${DEFAULT_CMAKE_PREFIX_PATH}") if(NOT DEFINED CMAKE_PREFIX_PATH) @@ -203,8 +203,8 @@ function(vespa_use_default_cxx_compiler) unset(DEFAULT_CMAKE_CXX_COMPILER) if(NOT DEFINED VESPA_COMPILER_VARIANT OR VESPA_COMPILER_VARIANT STREQUAL "gcc") if(APPLE) - set(DEFAULT_CMAKE_C_COMPILER "/usr/local/bin/gcc-12") - set(DEFAULT_CMAKE_CXX_COMPILER "/usr/local/bin/g++-12") + set(DEFAULT_CMAKE_C_COMPILER "${VESPA_HOMEBREW_PREFIX}/bin/gcc-12") + set(DEFAULT_CMAKE_CXX_COMPILER "${VESPA_HOMEBREW_PREFIX}/bin/g++-12") elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "amzn 2") set(DEFAULT_CMAKE_C_COMPILER "/usr/bin/gcc10-gcc") set(DEFAULT_CMAKE_CXX_COMPILER "/usr/bin/gcc10-g++") @@ -215,8 +215,8 @@ function(vespa_use_default_cxx_compiler) endif() elseif(VESPA_COMPILER_VARIANT STREQUAL "clang") if(APPLE) - set(DEFAULT_CMAKE_C_COMPILER, "/usr/local/opt/llvm/bin/clang") - set(DEFAULT_CMAKE_CXX_COMPILER "/usr/local/opt/llvm/bin/clang++") + set(DEFAULT_CMAKE_C_COMPILER, "${VESPA_HOMEBREW_PREFIX}/opt/llvm/bin/clang") + set(DEFAULT_CMAKE_CXX_COMPILER "${VESPA_HOMEBREW_PREFIX}/opt/llvm/bin/clang++") elseif(EXISTS "/usr/bin/clang" AND EXISTS "/usr/bin/clang++") set(DEFAULT_CMAKE_C_COMPILER "/usr/bin/clang") set(DEFAULT_CMAKE_CXX_COMPILER "/usr/bin/clang++") diff --git a/docprocs/src/test/cfg/ilscripts.cfg b/docprocs/src/test/cfg/ilscripts.cfg index c58028f1056..6e4c75f46a7 100644 --- a/docprocs/src/test/cfg/ilscripts.cfg +++ b/docprocs/src/test/cfg/ilscripts.cfg @@ -11,3 +11,4 @@ ilscript[0].content[2] "input song | attribute song" ilscript[0].content[4] "input artist . " ". input title | index combined" ilscript[0].content[5] "(input artist || "") . " " . (input title || "") | index combinedWithFallback" + diff --git a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java index 110564bea46..795f8e93187 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java +++ b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java @@ -5,7 +5,6 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.yahoo.document.DataType; import com.yahoo.document.DocumentId; import com.yahoo.document.Field; -import com.yahoo.document.PositionDataType; import com.yahoo.document.PrimitiveDataType; import com.yahoo.document.datatypes.Array; import com.yahoo.document.datatypes.BoolFieldValue; @@ -41,7 +40,6 @@ import java.nio.charset.StandardCharsets; import java.util.Base64; import java.util.Iterator; import java.util.Map; -import java.util.Set; /** * @author Steinar Knutsen @@ -49,7 +47,7 @@ import java.util.Set; */ public class JsonSerializationHelper { - private final static Base64.Encoder base64Encoder = Base64.getEncoder(); // Important: _basic_ format + private final static Base64.Encoder base64Encoder = Base64.getEncoder().withoutPadding(); // Important: _basic_ format static class JsonSerializationException extends RuntimeException { public JsonSerializationException(Exception base) { @@ -166,8 +164,7 @@ public class JsonSerializationHelper { public static void serializeStructField(FieldWriter fieldWriter, JsonGenerator generator, FieldBase field, Struct value) { DataType dt = value.getDataType(); - if (dt instanceof GeoPosType) { - var gpt = (GeoPosType)dt; + if (dt instanceof GeoPosType gpt) { if (gpt.renderJsonAsVespa8()) { serializeGeoPos(generator, field, value, gpt); return; diff --git a/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateFlags.java b/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateFlags.java index e3510676148..11ded80ed2a 100644 --- a/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateFlags.java +++ b/document/src/main/java/com/yahoo/document/serialization/DocumentUpdateFlags.java @@ -23,7 +23,7 @@ public class DocumentUpdateFlags { } public void setCreateIfNonExistent(boolean value) { flags &= ~1; // clear flag - flags |= value ? 1 : 0; // set flag + flags |= value ? (byte)1 : (byte)0; // set flag } public int injectInto(int value) { return extractValue(value) | (flags << 28); diff --git a/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java b/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java index 9c1df0cd6c7..d35693f785f 100644 --- a/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java +++ b/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java @@ -34,6 +34,8 @@ import java.util.Map; @SuppressWarnings("removal") public class XmlSerializationHelper { + private final static Base64.Encoder base64Encoder = Base64.getEncoder().withoutPadding(); + public static void printArrayXml(Array array, XmlStream xml) { List<FieldValue> lst = array.getValues(); for (FieldValue value : lst) { @@ -98,7 +100,7 @@ public class XmlSerializationHelper { public static void printRawXml(Raw r, XmlStream xml) { xml.addAttribute("binaryencoding", "base64"); - xml.addContent(Base64.getEncoder().encodeToString(r.getByteBuffer().array())); + xml.addContent(base64Encoder.encodeToString(r.getByteBuffer().array())); } public static void printStringXml(StringFieldValue s, XmlStream xml) { @@ -106,7 +108,7 @@ public class XmlSerializationHelper { if (containsNonPrintableCharactersString(content)) { byte[] bytecontent = Utf8.toBytes(content); xml.addAttribute("binaryencoding", "base64"); - xml.addContent(Base64.getEncoder().encodeToString(bytecontent)); + xml.addContent(base64Encoder.encodeToString(bytecontent)); } else { xml.addContent(content); } diff --git a/document/src/test/java/com/yahoo/document/DocumentTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTestCase.java index 33b77cb1878..4470865b636 100644 --- a/document/src/test/java/com/yahoo/document/DocumentTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentTestCase.java @@ -52,7 +52,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { " <mailid>emailfromalicetobob&someone</mailid>\n" + " <date>-2013512400</date>\n" + " <attachmentcount>2</attachmentcount>\n" + - " <rawfield binaryencoding=\"base64\">AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiYw==</rawfield>\n"; + " <rawfield binaryencoding=\"base64\">AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiYw</rawfield>\n"; private static final String SERTEST_DOC_AS_XML_WEIGHT1 = " <weightedfield>\n" + diff --git a/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java b/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java index 08a5c9a124c..af7469de31b 100644 --- a/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java +++ b/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java @@ -504,7 +504,7 @@ public class DocumentUpdateJsonSerializerTest { " 'update': 'DOCUMENT_ID',", " 'fields': {", " 'raw_field': {", - " 'assign': 'RG9uJ3QgYmVsaWV2ZSBoaXMgbGllcw=='", + " 'assign': 'RG9uJ3QgYmVsaWV2ZSBoaXMgbGllcw'", " }", " }", "}" diff --git a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java index 0c130ab9a42..a761a9adfb6 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonReaderTestCase.java @@ -668,7 +668,7 @@ public class JsonReaderTestCase { @Test public void testRaw() throws IOException { String base64 = new String(new JsonStringEncoder().quoteAsString( - Base64.getEncoder().encodeToString(Utf8.toBytes("smoketest")))); + Base64.getEncoder().withoutPadding().encodeToString(Utf8.toBytes("smoketest")))); String s = fieldStringFromBase64RawContent(base64); assertEquals("smoketest", s); } diff --git a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java index eab33afc3e4..4f15a2fe368 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java @@ -291,7 +291,7 @@ public class JsonWriterTestCase { String payload = new String( new JsonStringEncoder().quoteAsString( "c3RyaW5nIGxvbmcgZW5vdWdoIHRvIGVtaXQgbW9yZSB0aGFuIDc2IGJhc2U2NCBjaGFyYWN0ZXJzIGFuZC" + - "B3aGljaCBzaG91bGQgY2VydGFpbmx5IG5vdCBiZSBuZXdsaW5lLWRlbGltaXRlZCE=")); + "B3aGljaCBzaG91bGQgY2VydGFpbmx5IG5vdCBiZSBuZXdsaW5lLWRlbGltaXRlZCE")); String docId = "id:unittest:testraw::whee"; diff --git a/document/src/vespa/document/select/parse_utils.cpp b/document/src/vespa/document/select/parse_utils.cpp index 95461442349..4c116d5bff4 100644 --- a/document/src/vespa/document/select/parse_utils.cpp +++ b/document/src/vespa/document/select/parse_utils.cpp @@ -24,7 +24,7 @@ parse_i64(const char* str, size_t len, int64_t& out) { } bool parse_double(const char* str, size_t len, double& out) { -#if defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 160000 +#if defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 170000 // Temporary workaround that also handles underflow (cf. issue 3081) // until libc++ supports std::from_chars for double char *str_end = const_cast<char*>(str) + len; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 4ba5911dd30..ea17d9967ae 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -57,7 +57,7 @@ public class Flags { public static final UnboundBooleanFlag DROP_CACHES = defineFeatureFlag( "drop-caches", false, - List.of("hakonhall", "baldersheim"), "2023-03-06", "2023-04-05", + List.of("hakonhall", "baldersheim"), "2023-03-06", "2023-06-05", "Drop caches on tenant hosts", "Takes effect on next tick", ZONE_ID, @@ -309,7 +309,7 @@ public class Flags { public static final UnboundBooleanFlag ENABLE_PROXY_PROTOCOL_MIXED_MODE = defineFeatureFlag( "enable-proxy-protocol-mixed-mode", true, - List.of("tokle"), "2022-05-09", "2023-03-31", + List.of("tokle"), "2022-05-09", "2023-04-30", "Enable or disable proxy protocol mixed mode", "Takes effect on redeployment", APPLICATION_ID); @@ -385,6 +385,25 @@ public class Flags { "Takes effect on next config server container start", ZONE_ID); + public static final UnboundBooleanFlag NODE_ADMIN_TENANT_SERVICE_REGISTRY = defineFeatureFlag( + "node-admin-tenant-service-registry", false, + List.of("olaa"), "2023-04-12", "2023-06-12", + "Whether AthenzCredentialsMaintainer in node-admin should create tenant service identity certificate", + "Takes effect on next tick", + ZONE_ID, HOSTNAME + ); + + public static final UnboundBooleanFlag ENABLE_CROWDSTRIKE = defineFeatureFlag( + "enable-crowdstrike", true, List.of("andreer"), "2023-04-13", "2023-06-13", + "Whether to enable CrowdStrike.", "Takes effect on next host admin tick", + HOSTNAME); + + public static final UnboundBooleanFlag ALLOW_MORE_THAN_ONE_CONTENT_GROUP_DOWN = defineFeatureFlag( + "allow-more-than-one-content-group-down", false, List.of("hmusum"), "2023-04-14", "2023-06-14", + "Whether to enable possible configuration of letting more than one content group down", + "Takes effect at redeployment", + HOSTNAME); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index 336edf74e8f..dea0dbd3623 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -347,6 +347,12 @@ public class PermanentFlags { "Takes effect immediately", TENANT_ID); + public static final UnboundIntFlag KEEP_FILE_REFERENCES_ON_TENANT_NODES = defineIntFlag( + "keep-file-references-on-tenant-nodes", 14, + "How many days to keep file references on tenant nodes (based on last modification time)", + "Takes effect on restart of Docker container", + ZONE_ID, APPLICATION_ID + ); private PermanentFlags() {} diff --git a/functions.cmake b/functions.cmake index 7fa0b0db954..7f217867314 100644 --- a/functions.cmake +++ b/functions.cmake @@ -746,6 +746,14 @@ function(vespa_detect_build_platform) elseif(APPLE) set(OS_DISTRO "darwin") set(OS_DISTRO_VERSION ${CMAKE_SYSTEM_VERSION}) + if(EXISTS "/opt/homebrew/bin/brew") + set(VESPA_HOMEBREW_PREFIX "/opt/homebrew") + elseif(EXISTS "/usr/local/bin/brew") + set(VESPA_HOMEBREW_PREFIX "/usr/local") + else() + message(FATAL_ERROR "-- Cannot determine homebrew prefix") + endif() + set(VESPA_HOMEBREW_PREFIX ${VESPA_HOMEBREW_PREFIX} PARENT_SCOPE) endif() if(OS_DISTRO) set(VESPA_OS_DISTRO ${OS_DISTRO} PARENT_SCOPE) diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionConverter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionConverter.java index 45d8637aa3e..231f6fb7598 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionConverter.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionConverter.java @@ -1,122 +1,22 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.indexinglanguage; -import com.yahoo.collections.Pair; -import com.yahoo.vespa.indexinglanguage.expressions.*; - -import java.lang.reflect.InvocationTargetException; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; /** * @author Simon Thoresen Hult */ -@SuppressWarnings({ "UnusedDeclaration" }) public abstract class ExpressionConverter implements Cloneable { - public final Expression convert(Expression exp) { - if (exp == null) { - return null; - } - if (shouldConvert(exp)) { - return doConvert(exp); - } - if (!(exp instanceof CompositeExpression)) { - return exp; - } - try { - // The class.getMethod here takes 8% of the cpu time in reading the SSBE application package - // TODO: Implement double dispatch through visitor instead? - return (Expression)ExpressionConverter.class.getMethod("innerConvert", exp.getClass()).invoke(this, exp); - } catch (IllegalAccessException | NoSuchMethodException e) { - throw new UnsupportedOperationException(exp.getClass().getName(), e); - } catch (InvocationTargetException e) { - Throwable t = e.getTargetException(); - throw t instanceof RuntimeException ? (RuntimeException)t : new RuntimeException(t); - } - } - - public Expression innerConvert(ArithmeticExpression exp) { - return new ArithmeticExpression(convert(exp.getLeftHandSide()), - exp.getOperator(), - convert(exp.getRightHandSide())); - } - - public Expression innerConvert(CatExpression exp) { - List<Expression> lst = new LinkedList<>(); - for (Expression innerExp : exp) { - Expression next = convert(innerExp); - if (next != null) { - lst.add(next); - } - } - return new CatExpression(lst); - } - - public Expression innerConvert(ForEachExpression exp) { - return new ForEachExpression(convert(exp.getInnerExpression())); - } - - public Expression innerConvert(GuardExpression exp) { - return new GuardExpression(convert(exp.getInnerExpression())); - } - - public Expression innerConvert(IfThenExpression exp) { - return new IfThenExpression(branch().convert(exp.getLeftHandSide()), - exp.getComparator(), - branch().convert(exp.getRightHandSide()), - branch().convert(exp.getIfTrueExpression()), - branch().convert(exp.getIfFalseExpression())); - } - - public Expression innerConvert(ParenthesisExpression exp) { - return new ParenthesisExpression(convert(exp.getInnerExpression())); - } - - public Expression innerConvert(ScriptExpression exp) { - List<StatementExpression> lst = new LinkedList<>(); - for (Expression innerExp : exp) { - StatementExpression next = (StatementExpression)branch().convert(innerExp); - if (next != null) { - lst.add(next); - } - } - return new ScriptExpression(lst); - } - - public Expression innerConvert(SelectInputExpression exp) { - List<Pair<String, Expression>> cases = new LinkedList<>(); - for (Pair<String, Expression> pair : exp.getCases()) { - cases.add(new Pair<>(pair.getFirst(), branch().convert(pair.getSecond()))); - } - return new SelectInputExpression(cases); - } - - public Expression innerConvert(StatementExpression exp) { - List<Expression> lst = new LinkedList<>(); - for (Expression innerExp : exp) { - Expression next = convert(innerExp); - if (next != null) { - lst.add(next); - } - } - return new StatementExpression(lst); - } - - public Expression innerConvert(SwitchExpression exp) { - Map<String, Expression> cases = new HashMap<>(); - for (Map.Entry<String, Expression> entry : exp.getCases().entrySet()) { - Expression next = branch().convert(entry.getValue()); - if (next != null) { - cases.put(entry.getKey(), next); - } - } - return new SwitchExpression(cases, branch().convert(exp.getDefaultExpression())); + public final Expression convert(Expression expression) { + if (expression == null) return null; + if (shouldConvert(expression)) + return doConvert(expression); + else + return expression.convertChildren(this); } - protected ExpressionConverter branch() { + public ExpressionConverter branch() { return this; } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ValueTransformProvider.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ValueTransformProvider.java index bbd8c5ebcb8..623f940b06b 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ValueTransformProvider.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ValueTransformProvider.java @@ -20,7 +20,7 @@ public abstract class ValueTransformProvider extends ExpressionConverter { } @Override - protected final ExpressionConverter branch() { + public final ExpressionConverter branch() { return clone(); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java index e4bc2dae965..b7ee444975f 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; import com.yahoo.document.NumericDataType; import com.yahoo.document.datatypes.*; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; @@ -55,6 +56,12 @@ public final class ArithmeticExpression extends CompositeExpression { this.rhs = rhs; } + @Override + public ArithmeticExpression convertChildren(ExpressionConverter converter) { + // TODO: branch()? + return new ArithmeticExpression(converter.convert(lhs), op, converter.convert(rhs)); + } + public Expression getLeftHandSide() { return lhs; } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java index 4c14c633fbf..564ab015e10 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java @@ -9,6 +9,7 @@ import com.yahoo.document.datatypes.Array; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import java.util.*; @@ -26,6 +27,11 @@ public final class CatExpression extends ExpressionList<Expression> { } @Override + public CatExpression convertChildren(ExpressionConverter converter) { + return new CatExpression(convertChildList(converter)); + } + + @Override protected void doExecute(ExecutionContext context) { FieldValue input = context.getValue(); DataType inputType = input != null ? input.getDataType() : null; diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java index 4f83cbfdd8c..5dbb9292a9d 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import java.util.Arrays; import java.util.Collection; @@ -32,6 +33,11 @@ public class ChoiceExpression extends ExpressionList<Expression> { } @Override + public ChoiceExpression convertChildren(ExpressionConverter converter) { + return new ChoiceExpression(asList().stream().map(choice -> converter.branch().convert(choice)).toList()); + } + + @Override protected void doExecute(ExecutionContext context) { FieldValue input = context.getValue(); for (Expression expression : this) { @@ -44,9 +50,9 @@ public class ChoiceExpression extends ExpressionList<Expression> { @Override protected void doVerify(VerificationContext context) { DataType input = context.getValueType(); + context.setValueType(input); for (Expression exp : this) context.setValueType(input).execute(exp); - context.setValueType(input); } private static DataType resolveInputType(Collection<? extends Expression> list) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpression.java index 27e5524f4ad..8c00aad6bb0 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpression.java @@ -2,12 +2,16 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; /** * @author Simon Thoresen Hult */ public abstract class CompositeExpression extends Expression { + @Override + public abstract CompositeExpression convertChildren(ExpressionConverter converter); + protected CompositeExpression(DataType inputType) { super(inputType); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java index bf8201ee7ee..f498b871096 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java @@ -35,6 +35,12 @@ public abstract class Expression extends Selectable { this.inputType = inputType; } + /** + * Returns an expression where the children of this has been converted using the given converter. + * This default implementation returns this as it has no children. + */ + public Expression convertChildren(ExpressionConverter converter) { return this; } + /** Sets the document type and field the statement this expression is part of will write to */ public void setStatementOutput(DocumentType documentType, Field field) {} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java index e2ff1de7126..57de66f80a0 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; import com.yahoo.document.Field; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; @@ -11,6 +12,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Objects; /** * @author Simon Thoresen Hult @@ -26,6 +28,10 @@ public abstract class ExpressionList<T extends Expression> extends CompositeExpr } } + protected List<Expression> convertChildList(ExpressionConverter converter) { + return asList().stream().map(converter::convert).filter(Objects::nonNull).toList(); + } + @Override public void setStatementOutput(DocumentType documentType, Field field) { for (Expression expression : expressions) diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java index 0f3a445bcb9..3053a391823 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java @@ -6,6 +6,7 @@ import com.yahoo.document.datatypes.Array; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.Struct; import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.indexinglanguage.FieldValueConverter; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; @@ -27,6 +28,11 @@ public final class ForEachExpression extends CompositeExpression { } @Override + public ForEachExpression convertChildren(ExpressionConverter converter) { + return new ForEachExpression(converter.convert(exp)); + } + + @Override public void setStatementOutput(DocumentType documentType, Field field) { exp.setStatementOutput(documentType, field); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java index da7cfcdcaee..38a05c3056c 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; import com.yahoo.document.Field; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; import com.yahoo.vespa.indexinglanguage.UpdateAdapter; import com.yahoo.vespa.objects.ObjectOperation; @@ -28,6 +29,11 @@ public final class GuardExpression extends CompositeExpression { } @Override + public GuardExpression convertChildren(ExpressionConverter converter) { + return new GuardExpression(converter.convert(exp)); + } + + @Override public void setStatementOutput(DocumentType documentType, Field field) { exp.setStatementOutput(documentType, field); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java index 8a29c8e8645..f05795aa234 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java @@ -6,6 +6,7 @@ import com.yahoo.document.DocumentType; import com.yahoo.document.Field; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.NumericFieldValue; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; @@ -57,6 +58,15 @@ public final class IfThenExpression extends CompositeExpression { } @Override + public IfThenExpression convertChildren(ExpressionConverter converter) { + return new IfThenExpression(converter.branch().convert(lhs), + cmp, + converter.branch().convert(rhs), + converter.branch().convert(ifTrue), + converter.branch().convert(ifFalse)); + } + + @Override public void setStatementOutput(DocumentType documentType, Field field) { lhs.setStatementOutput(documentType, field); rhs.setStatementOutput(documentType, field); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java index 30c824d410d..bba1b09cda2 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java @@ -21,7 +21,9 @@ public final class InputExpression extends Expression { public InputExpression(String fieldName) { super(null); - this.fieldName = Objects.requireNonNull(fieldName); + if (fieldName == null) + throw new IllegalArgumentException("'input' must be given a field name as argument"); + this.fieldName = fieldName; } public String getFieldName() { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java index 60b059f3ef1..6e476f5f7e4 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; import com.yahoo.document.Field; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; @@ -24,6 +25,11 @@ public class ParenthesisExpression extends CompositeExpression { } @Override + public ParenthesisExpression convertChildren(ExpressionConverter converter) { + return new ParenthesisExpression(converter.convert(innerExp)); + } + + @Override public void setStatementOutput(DocumentType documentType, Field field) { innerExp.setStatementOutput(documentType, field); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java index f0c37960a99..1a640c9924e 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java @@ -6,6 +6,7 @@ import com.yahoo.document.datatypes.FieldValue; import com.yahoo.language.Linguistics; import com.yahoo.language.process.Embedder; import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.indexinglanguage.ScriptParser; import com.yahoo.vespa.indexinglanguage.ScriptParserContext; import com.yahoo.vespa.indexinglanguage.parser.IndexingInput; @@ -17,6 +18,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; /** * @author Simon Thoresen Hult @@ -36,6 +38,14 @@ public final class ScriptExpression extends ExpressionList<StatementExpression> } @Override + public ScriptExpression convertChildren(ExpressionConverter converter) { + return new ScriptExpression(asList().stream() + .map(child -> (StatementExpression)converter.branch().convert(child)) + .filter(Objects::nonNull) + .toList()); + } + + @Override protected void doExecute(ExecutionContext context) { FieldValue input = context.getValue(); for (StatementExpression statement : this) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java index 212b60525f9..bb8111f358e 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java @@ -6,10 +6,12 @@ import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; import com.yahoo.document.Field; import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; @@ -32,6 +34,13 @@ public final class SelectInputExpression extends CompositeExpression { } @Override + public SelectInputExpression convertChildren(ExpressionConverter converter) { + return new SelectInputExpression(cases.stream() + .map(c -> new Pair<>(c.getFirst(), converter.branch().convert(c.getSecond()))) + .toList()); + } + + @Override public void setStatementOutput(DocumentType documentType, Field field) { for (var casePair : cases) casePair.getSecond().setStatementOutput(documentType, field); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java index 8516ddb883d..75f206ef47d 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java @@ -5,6 +5,7 @@ import com.yahoo.document.DataType; import com.yahoo.language.Linguistics; import com.yahoo.language.process.Embedder; import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.indexinglanguage.ScriptParser; import com.yahoo.vespa.indexinglanguage.ScriptParserContext; import com.yahoo.vespa.indexinglanguage.parser.IndexingInput; @@ -15,6 +16,7 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; /** @@ -45,6 +47,14 @@ public final class StatementExpression extends ExpressionList<Expression> { public List<String> getInputFields() { return inputFields; } @Override + public StatementExpression convertChildren(ExpressionConverter converter) { + return new StatementExpression(asList().stream() + .map(child -> converter.convert(child)) + .filter(Objects::nonNull) + .toList()); + } + + @Override protected void doExecute(ExecutionContext context) { for (Expression expression : this) { context.execute(expression); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java index 86913d8c1ba..c7cf7066483 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java @@ -7,6 +7,7 @@ import com.yahoo.document.Field; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.text.StringUtilities; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; @@ -32,6 +33,17 @@ public final class SwitchExpression extends CompositeExpression { this.cases.putAll(cases); } + @Override + public SwitchExpression convertChildren(ExpressionConverter converter) { + var convertedCases = new LinkedHashMap<String, Expression>(); + for (var entry : cases.entrySet()) { + var converted = converter.branch().convert(entry.getValue()); + if (converted != null) + convertedCases.put(entry.getKey(), converted); + } + return new SwitchExpression(convertedCases, converter.branch().convert(defaultExp)); + } + public boolean isEmpty() { return defaultExp == null && cases.isEmpty(); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationContext.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationContext.java index c667a0019c2..fb1338b8b65 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationContext.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/VerificationContext.java @@ -11,7 +11,7 @@ import java.util.Map; */ public class VerificationContext implements FieldTypeAdapter, Cloneable { - private final Map<String, DataType> variables = new HashMap<String, DataType>(); + private final Map<String, DataType> variables = new HashMap<>(); private final FieldTypeAdapter adapter; private DataType value; private String outputField; diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java index f1e1be0ae41..8aeaa084e1b 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.indexinglanguage; import com.yahoo.collections.Pair; -import com.yahoo.document.DataType; import com.yahoo.document.datatypes.IntegerFieldValue; import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.vespa.indexinglanguage.expressions.ArithmeticExpression; @@ -11,9 +10,7 @@ import com.yahoo.vespa.indexinglanguage.expressions.Base64DecodeExpression; import com.yahoo.vespa.indexinglanguage.expressions.Base64EncodeExpression; import com.yahoo.vespa.indexinglanguage.expressions.CatExpression; import com.yahoo.vespa.indexinglanguage.expressions.ClearStateExpression; -import com.yahoo.vespa.indexinglanguage.expressions.CompositeExpression; import com.yahoo.vespa.indexinglanguage.expressions.EchoExpression; -import com.yahoo.vespa.indexinglanguage.expressions.ExecutionContext; import com.yahoo.vespa.indexinglanguage.expressions.Expression; import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression; import com.yahoo.vespa.indexinglanguage.expressions.GetFieldExpression; @@ -54,7 +51,6 @@ import com.yahoo.vespa.indexinglanguage.expressions.ToStringExpression; import com.yahoo.vespa.indexinglanguage.expressions.ToWsetExpression; import com.yahoo.vespa.indexinglanguage.expressions.TokenizeExpression; import com.yahoo.vespa.indexinglanguage.expressions.TrimExpression; -import com.yahoo.vespa.indexinglanguage.expressions.VerificationContext; import com.yahoo.vespa.indexinglanguage.expressions.ZCurveExpression; import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig; import org.junit.Test; @@ -73,7 +69,6 @@ import static org.junit.Assert.fail; */ public class ExpressionConverterTestCase { - @SuppressWarnings("unchecked") @Test public void requireThatAllExpressionTypesCanBeTraversed() { assertConvertable(new ArithmeticExpression(new InputExpression("foo"), ArithmeticExpression.Operator.ADD, @@ -167,16 +162,6 @@ public class ExpressionConverterTestCase { } @Test - public void requireThatUnknownCompositeThrows() { - try { - new MyTraverser().convert(new MyComposite()); - fail(); - } catch (UnsupportedOperationException e) { - assertEquals(NoSuchMethodException.class, e.getCause().getClass()); - } - } - - @Test public void requireThatConversionExceptionCanBeThrown() { final RuntimeException expectedCause = new RuntimeException(); try { @@ -254,24 +239,4 @@ public class ExpressionConverterTestCase { } } - private static class MyComposite extends CompositeExpression { - - MyComposite() { - super(null); - } - @Override - protected void doExecute(ExecutionContext context) { - - } - - @Override - protected void doVerify(VerificationContext context) { - - } - - @Override - public DataType createdOutputType() { - return null; - } - } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceTestCase.java index 351c925ed56..e6d5c550e93 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceTestCase.java @@ -2,9 +2,17 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; +import com.yahoo.document.Document; import com.yahoo.document.Field; +import com.yahoo.document.datatypes.LongFieldValue; import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.language.Linguistics; +import com.yahoo.language.process.Embedder; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.vespa.indexinglanguage.ExpressionSearcher; import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import com.yahoo.vespa.indexinglanguage.parser.ParseException; +import com.yahoo.yolean.Exceptions; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -38,10 +46,53 @@ public class ChoiceTestCase { var adapter = new SimpleTestAdapter(new Field("foo", DataType.STRING), new Field("bar", DataType.STRING)); adapter.setValue("foo", new StringFieldValue("foo1")); adapter.setValue("bar", new StringFieldValue("bar1")); + choice.verify(adapter); ExecutionContext context = new ExecutionContext(adapter); choice.execute(context); assertEquals("foo1", context.getValue().getWrappedValue()); } } + @Test + public void testChoiceWithConstant() throws ParseException { + var choice = parse("input timestamp || 99999999L | attribute timestamp"); + + { // value is set + var adapter = new SimpleTestAdapter(new Field("timestamp", DataType.LONG)); + choice.verify(adapter); + adapter.setValue("timestamp", new LongFieldValue(34)); + ExecutionContext context = new ExecutionContext(adapter); + choice.execute(context); + assertEquals(34L, context.getValue().getWrappedValue()); + } + + { // fallback to default + var adapter = new SimpleTestAdapter(new Field("timestamp", DataType.LONG)); + choice.verify(adapter); + ExecutionContext context = new ExecutionContext(adapter); + choice.execute(context); + assertEquals(99999999L, context.getValue().getWrappedValue()); + } + } + + @Test + public void testIllegalChoiceExpression() throws ParseException { + try { + parse("input (foo || 99999999) | attribute"); + } + catch (IllegalArgumentException e) { + assertEquals("'input' must be given a field name as argument", Exceptions.toMessageString(e)); + } + } + + @Test + public void testInnerConvert() throws ParseException { + var expression = parse("(input foo || 99999999) | attribute"); + new ExpressionSearcher<>(AttributeExpression.class).searchIn(expression); // trigger innerConvert + } + + private static Expression parse(String s) throws ParseException { + return Expression.fromString(s, new SimpleLinguistics(), Embedder.throwsOnUse.asMap()); + } + } diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/core/ExportPackagesIT.java b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ExportPackagesIT.java index e9aba0893f9..7ec3406be1f 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/core/ExportPackagesIT.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ExportPackagesIT.java @@ -62,8 +62,8 @@ public class ExportPackagesIT { String expectedValue = expectedProperties.getProperty(ExportPackages.EXPORT_PACKAGES); assertNotNull(expectedValue, "Missing exportPackages property in file."); - Set<String> actualPackages = getPackages(actualValue); - Set<String> expectedPackages = getPackages(expectedValue); + Set<String> actualPackages = removeNewPackageOnJava20(removeJavaVersion(getPackages(actualValue))); + Set<String> expectedPackages = removeNewPackageOnJava20(removeJavaVersion(getPackages(expectedValue))); if (!actualPackages.equals(expectedPackages)) { StringBuilder message = getDiff(actualPackages, expectedPackages); message.append("\n\nIf this test fails due to an intentional change in exported packages, run the following command:\n") @@ -73,6 +73,14 @@ public class ExportPackagesIT { } } + private static Set<String> removeJavaVersion(Set<String> packages) { + return packages.stream().map(p -> p.replaceAll(".JavaSE_\\d+", "")).collect(Collectors.toSet()); + } + + private static Set<String> removeNewPackageOnJava20(Set<String> packages) { + return packages.stream().filter(p -> ! p.contains("java.lang.foreign")).collect(Collectors.toSet()); + } + private static StringBuilder getDiff(Set<String> actual, Set<String> expected) { StringBuilder sb = new StringBuilder(); Set<String> onlyInActual = onlyInSet1(actual, expected); diff --git a/model-integration/pom.xml b/model-integration/pom.xml index 9bb60827a68..c27ed9d2c31 100644 --- a/model-integration/pom.xml +++ b/model-integration/pom.xml @@ -106,6 +106,11 @@ </dependency> <dependency> + <groupId>org.lz4</groupId> + <artifactId>lz4-java</artifactId> + </dependency> + + <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <scope>test</scope> diff --git a/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java b/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java index b40e2b5be72..bf56d233f89 100644 --- a/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java +++ b/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java @@ -32,10 +32,9 @@ import java.util.Map; */ public class BertBaseEmbedder extends AbstractComponent implements Embedder { - private final static int TOKEN_CLS = 101; // [CLS] - private final static int TOKEN_SEP = 102; // [SEP] - private final int maxTokens; + private final int startSequenceToken; + private final int endSequenceToken; private final String inputIdsName; private final String attentionMaskName; private final String tokenTypeIdsName; @@ -48,6 +47,8 @@ public class BertBaseEmbedder extends AbstractComponent implements Embedder { @Inject public BertBaseEmbedder(OnnxRuntime onnx, BertBaseEmbedderConfig config) { maxTokens = config.transformerMaxTokens(); + startSequenceToken = config.transformerStartSequenceToken(); + endSequenceToken = config.transformerEndSequenceToken(); inputIdsName = config.transformerInputIds(); attentionMaskName = config.transformerAttentionMask(); tokenTypeIdsName = config.transformerTokenTypeIds(); @@ -98,7 +99,7 @@ public class BertBaseEmbedder extends AbstractComponent implements Embedder { if (!type.dimensions().get(0).isIndexed()) { throw new IllegalArgumentException("Error in embedding to type '" + type + "': dimension should be indexed."); } - List<Integer> tokens = embedWithSeperatorTokens(text, context, maxTokens); + List<Integer> tokens = embedWithSeparatorTokens(text, context, maxTokens); return embedTokens(tokens, type); } @@ -109,6 +110,7 @@ public class BertBaseEmbedder extends AbstractComponent implements Embedder { Tensor attentionMask = createAttentionMask(inputSequence); Tensor tokenTypeIds = createTokenTypeIds(inputSequence); + Map<String, Tensor> inputs; if (!"".equals(tokenTypeIdsName)) { inputs = Map.of(inputIdsName, inputSequence.expand("d0"), @@ -138,14 +140,14 @@ public class BertBaseEmbedder extends AbstractComponent implements Embedder { return builder.build(); } - private List<Integer> embedWithSeperatorTokens(String text, Context context, int maxLength) { + private List<Integer> embedWithSeparatorTokens(String text, Context context, int maxLength) { List<Integer> tokens = new ArrayList<>(); - tokens.add(TOKEN_CLS); + tokens.add(startSequenceToken); tokens.addAll(embed(text, context)); - tokens.add(TOKEN_SEP); + tokens.add(endSequenceToken); if (tokens.size() > maxLength) { tokens = tokens.subList(0, maxLength-1); - tokens.add(TOKEN_SEP); + tokens.add(endSequenceToken); } return tokens; } diff --git a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java index 7cdc27b6d63..02fa7b68dc4 100644 --- a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java +++ b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java @@ -7,6 +7,7 @@ import ai.onnxruntime.OnnxTensor; import ai.onnxruntime.OnnxValue; import ai.onnxruntime.OrtException; import ai.onnxruntime.OrtSession; +import ai.vespa.modelintegration.evaluator.OnnxRuntime.ModelPathOrData; import ai.vespa.modelintegration.evaluator.OnnxRuntime.ReferencedOrtSession; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; @@ -28,7 +29,11 @@ public class OnnxEvaluator implements AutoCloseable { private final ReferencedOrtSession session; OnnxEvaluator(String modelPath, OnnxEvaluatorOptions options, OnnxRuntime runtime) { - session = createSession(modelPath, runtime, options, true); + session = createSession(ModelPathOrData.of(modelPath), runtime, options, true); + } + + OnnxEvaluator(byte[] data, OnnxEvaluatorOptions options, OnnxRuntime runtime) { + session = createSession(ModelPathOrData.of(data), runtime, options, true); } public Tensor evaluate(Map<String, Tensor> inputs, String output) { @@ -125,19 +130,20 @@ public class OnnxEvaluator implements AutoCloseable { } } - private static ReferencedOrtSession createSession(String modelPath, OnnxRuntime runtime, OnnxEvaluatorOptions options, boolean tryCuda) { + private static ReferencedOrtSession createSession( + ModelPathOrData model, OnnxRuntime runtime, OnnxEvaluatorOptions options, boolean tryCuda) { if (options == null) { options = new OnnxEvaluatorOptions(); } try { - return runtime.acquireSession(modelPath, options, tryCuda && options.requestingGpu()); + return runtime.acquireSession(model, options, tryCuda && options.requestingGpu()); } catch (OrtException e) { if (e.getCode() == OrtException.OrtErrorCode.ORT_NO_SUCHFILE) { - throw new IllegalArgumentException("No such file: " + modelPath); + throw new IllegalArgumentException("No such file: " + model.path().get()); } if (tryCuda && isCudaError(e) && !options.gpuDeviceRequired()) { // Failed in CUDA native code, but GPU device is optional, so we can proceed without it - return createSession(modelPath, runtime, options, false); + return createSession(model, runtime, options, false); } if (isCudaError(e)) { throw new IllegalArgumentException("GPU device is required, but CUDA initialization failed", e); @@ -146,6 +152,9 @@ public class OnnxEvaluator implements AutoCloseable { } } + // For unit testing + OrtSession ortSession() { return session.instance(); } + private String mapToInternalName(String outputName) throws OrtException { var info = session.instance().getOutputInfo(); var internalNames = info.keySet(); diff --git a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxRuntime.java b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxRuntime.java index 42830041c02..ece1db55c1e 100644 --- a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxRuntime.java +++ b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxRuntime.java @@ -10,9 +10,15 @@ import com.yahoo.component.annotation.Inject; import com.yahoo.jdisc.ResourceReference; import com.yahoo.jdisc.refcount.DebugReferencesWithStack; import com.yahoo.jdisc.refcount.References; +import net.jpountz.xxhash.XXHashFactory; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; @@ -26,14 +32,22 @@ import static com.yahoo.yolean.Exceptions.throwUnchecked; public class OnnxRuntime extends AbstractComponent { // For unit testing - @FunctionalInterface interface OrtSessionFactory { + interface OrtSessionFactory { OrtSession create(String path, OrtSession.SessionOptions opts) throws OrtException; + OrtSession create(byte[] data, OrtSession.SessionOptions opts) throws OrtException; } private static final Logger log = Logger.getLogger(OnnxRuntime.class.getName()); private static final OrtEnvironmentResult ortEnvironment = getOrtEnvironment(); - private static final OrtSessionFactory defaultFactory = (path, opts) -> ortEnvironment().createSession(path, opts); + private static final OrtSessionFactory defaultFactory = new OrtSessionFactory() { + @Override public OrtSession create(String path, OrtSession.SessionOptions opts) throws OrtException { + return ortEnvironment().createSession(path, opts); + } + @Override public OrtSession create(byte[] data, OrtSession.SessionOptions opts) throws OrtException { + return ortEnvironment().createSession(data, opts); + } + }; private final Object monitor = new Object(); private final Map<OrtSessionId, SharedOrtSession> sessions = new HashMap<>(); @@ -43,6 +57,14 @@ public class OnnxRuntime extends AbstractComponent { OnnxRuntime(OrtSessionFactory factory) { this.factory = factory; } + public OnnxEvaluator evaluatorOf(byte[] model) { + return new OnnxEvaluator(model, null, this); + } + + public OnnxEvaluator evaluatorOf(byte[] model, OnnxEvaluatorOptions options) { + return new OnnxEvaluator(model, options, this); + } + public OnnxEvaluator evaluatorOf(String modelPath) { return new OnnxEvaluator(modelPath, null, this); } @@ -105,8 +127,8 @@ public class OnnxRuntime extends AbstractComponent { }; } - ReferencedOrtSession acquireSession(String modelPath, OnnxEvaluatorOptions options, boolean loadCuda) throws OrtException { - var sessionId = new OrtSessionId(modelPath, options, loadCuda); + ReferencedOrtSession acquireSession(ModelPathOrData model, OnnxEvaluatorOptions options, boolean loadCuda) throws OrtException { + var sessionId = new OrtSessionId(calculateModelHash(model), options, loadCuda); synchronized (monitor) { var sharedSession = sessions.get(sessionId); if (sharedSession != null) { @@ -114,8 +136,9 @@ public class OnnxRuntime extends AbstractComponent { } } + var opts = options.getOptions(loadCuda); // Note: identical models loaded simultaneously will result in duplicate session instances - var session = factory.create(modelPath, options.getOptions(loadCuda)); + var session = model.path().isPresent() ? factory.create(model.path().get(), opts) : factory.create(model.data().get(), opts); log.fine(() -> "Created new session (%s)".formatted(System.identityHashCode(session))); var sharedSession = new SharedOrtSession(sessionId, session); @@ -125,25 +148,52 @@ public class OnnxRuntime extends AbstractComponent { return referencedSession; } + private static long calculateModelHash(ModelPathOrData model) { + if (model.path().isPresent()) { + try (var hasher = XXHashFactory.fastestInstance().newStreamingHash64(0); + var in = Files.newInputStream(Paths.get(model.path().get()))) { + byte[] buffer = new byte[8192]; + int bytesRead; + while ((bytesRead = in.read(buffer)) != -1) { + hasher.update(buffer, 0, bytesRead); + } + return hasher.getValue(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } else { + var data = model.data().get(); + return XXHashFactory.fastestInstance().hash64().hash(data, 0, data.length, 0); + } + } + int sessionsCached() { synchronized(monitor) { return sessions.size(); } } - public static class ReferencedOrtSession implements AutoCloseable { + static class ReferencedOrtSession implements AutoCloseable { private final OrtSession instance; private final ResourceReference ref; - public ReferencedOrtSession(OrtSession instance, ResourceReference ref) { + ReferencedOrtSession(OrtSession instance, ResourceReference ref) { this.instance = instance; this.ref = ref; } - public OrtSession instance() { return instance; } + OrtSession instance() { return instance; } @Override public void close() { ref.close(); } } + record ModelPathOrData(Optional<String> path, Optional<byte[]> data) { + static ModelPathOrData of(String path) { return new ModelPathOrData(Optional.of(path), Optional.empty()); } + static ModelPathOrData of(byte[] data) { return new ModelPathOrData(Optional.empty(), Optional.of(data)); } + ModelPathOrData { + if (path.isEmpty() == data.isEmpty()) throw new IllegalArgumentException("Either path or data must be non-empty"); + } + } + // Assumes options are never modified after being stored in `onnxSessions` - record OrtSessionId(String modelPath, OnnxEvaluatorOptions options, boolean loadCuda) {} + private record OrtSessionId(long modelHash, OnnxEvaluatorOptions options, boolean loadCuda) {} - record OrtEnvironmentResult(OrtEnvironment env, Throwable failure) {} + private record OrtEnvironmentResult(OrtEnvironment env, Throwable failure) {} private class SharedOrtSession { private final OrtSessionId id; diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/OrderedTensorType.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/OrderedTensorType.java index ceb9a27924d..eee60d56c55 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/OrderedTensorType.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/OrderedTensorType.java @@ -96,14 +96,14 @@ public class OrderedTensorType { * so that they are correctly laid out in memory for Vespa. * Used when importing tensors. */ - public int toDirectIndex(int index) { + public long toDirectIndex(int index) { if (dimensions.size() == 0) { return 0; } if (dimensionMap == null) { throw new IllegalArgumentException("Dimension map is not available"); } - int directIndex = 0; + long directIndex = 0; long rest = index; for (int i = 0; i < dimensions.size(); ++i) { long address = rest / innerSizesOriginal[i]; diff --git a/model-integration/src/main/resources/configdefinitions/embedding.bert-base-embedder.def b/model-integration/src/main/resources/configdefinitions/embedding.bert-base-embedder.def index 14d953eeef9..ef42d81e1fe 100644 --- a/model-integration/src/main/resources/configdefinitions/embedding.bert-base-embedder.def +++ b/model-integration/src/main/resources/configdefinitions/embedding.bert-base-embedder.def @@ -17,6 +17,10 @@ transformerInputIds string default=input_ids transformerAttentionMask string default=attention_mask transformerTokenTypeIds string default=token_type_ids +# special token ids +transformerStartSequenceToken int default=101 +transformerEndSequenceToken int default=102 + # Output name transformerOutput string default=output_0 diff --git a/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java b/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java index 5aba54de11b..5a367ef83e4 100644 --- a/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java +++ b/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java @@ -5,30 +5,26 @@ package ai.vespa.modelintegration.evaluator; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import org.junit.Test; -import org.junit.jupiter.api.BeforeAll; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.junit.Assume.assumeNotNull; +import static org.junit.Assume.assumeTrue; /** * @author lesters */ public class OnnxEvaluatorTest { - private static OnnxRuntime runtime; - - @BeforeAll - public static void beforeAll() { - if (OnnxRuntime.isRuntimeAvailable()) runtime = new OnnxRuntime(); - } - @Test public void testSimpleModel() { - assumeNotNull(runtime); + assumeTrue(OnnxRuntime.isRuntimeAvailable()); + var runtime = new OnnxRuntime(); OnnxEvaluator evaluator = runtime.evaluatorOf("src/test/models/onnx/simple/simple.onnx"); // Input types @@ -53,7 +49,8 @@ public class OnnxEvaluatorTest { @Test public void testBatchDimension() { - assumeNotNull(runtime); + assumeTrue(OnnxRuntime.isRuntimeAvailable()); + var runtime = new OnnxRuntime(); OnnxEvaluator evaluator = runtime.evaluatorOf("src/test/models/onnx/pytorch/one_layer.onnx"); // Input types @@ -72,21 +69,23 @@ public class OnnxEvaluatorTest { @Test public void testMatMul() { - assumeNotNull(runtime); + assumeTrue(OnnxRuntime.isRuntimeAvailable()); + var runtime = new OnnxRuntime(); String expected = "tensor<float>(d0[2],d1[4]):[38,44,50,56,83,98,113,128]"; String input1 = "tensor<float>(d0[2],d1[3]):[1,2,3,4,5,6]"; String input2 = "tensor<float>(d0[3],d1[4]):[1,2,3,4,5,6,7,8,9,10,11,12]"; - assertEvaluate("simple/matmul.onnx", expected, input1, input2); + assertEvaluate(runtime, "simple/matmul.onnx", expected, input1, input2); } @Test public void testTypes() { - assumeNotNull(runtime); - assertEvaluate("add_double.onnx", "tensor(d0[1]):[3]", "tensor(d0[1]):[1]", "tensor(d0[1]):[2]"); - assertEvaluate("add_float.onnx", "tensor<float>(d0[1]):[3]", "tensor<float>(d0[1]):[1]", "tensor<float>(d0[1]):[2]"); - assertEvaluate("add_int64.onnx", "tensor<double>(d0[1]):[3]", "tensor<double>(d0[1]):[1]", "tensor<double>(d0[1]):[2]"); - assertEvaluate("cast_int8_float.onnx", "tensor<float>(d0[1]):[-128]", "tensor<int8>(d0[1]):[128]"); - assertEvaluate("cast_float_int8.onnx", "tensor<int8>(d0[1]):[-1]", "tensor<float>(d0[1]):[255]"); + assumeTrue(OnnxRuntime.isRuntimeAvailable()); + var runtime = new OnnxRuntime(); + assertEvaluate(runtime, "add_double.onnx", "tensor(d0[1]):[3]", "tensor(d0[1]):[1]", "tensor(d0[1]):[2]"); + assertEvaluate(runtime, "add_float.onnx", "tensor<float>(d0[1]):[3]", "tensor<float>(d0[1]):[1]", "tensor<float>(d0[1]):[2]"); + assertEvaluate(runtime, "add_int64.onnx", "tensor<double>(d0[1]):[3]", "tensor<double>(d0[1]):[1]", "tensor<double>(d0[1]):[2]"); + assertEvaluate(runtime, "cast_int8_float.onnx", "tensor<float>(d0[1]):[-128]", "tensor<int8>(d0[1]):[128]"); + assertEvaluate(runtime, "cast_float_int8.onnx", "tensor<int8>(d0[1]):[-1]", "tensor<float>(d0[1]):[255]"); // ONNX Runtime 1.8.0 does not support much of bfloat16 yet // assertEvaluate("cast_bfloat16_float.onnx", "tensor<float>(d0[1]):[1]", "tensor<bfloat16>(d0[1]):[1]"); @@ -94,7 +93,8 @@ public class OnnxEvaluatorTest { @Test public void testNotIdentifiers() { - assumeNotNull(runtime); + assumeTrue(OnnxRuntime.isRuntimeAvailable()); + var runtime = new OnnxRuntime(); OnnxEvaluator evaluator = runtime.evaluatorOf("src/test/models/onnx/badnames.onnx"); var inputInfo = evaluator.getInputInfo(); var outputInfo = evaluator.getOutputInfo(); @@ -159,7 +159,18 @@ public class OnnxEvaluatorTest { assertEquals(3, allResults.size()); } - private void assertEvaluate(String model, String output, String... input) { + @Test + public void testLoadModelFromBytes() throws IOException { + assumeTrue(OnnxRuntime.isRuntimeAvailable()); + var runtime = new OnnxRuntime(); + var model = Files.readAllBytes(Paths.get("src/test/models/onnx/simple/simple.onnx")); + var evaluator = runtime.evaluatorOf(model); + assertEquals(3, evaluator.getInputs().size()); + assertEquals(1, evaluator.getOutputs().size()); + evaluator.close(); + } + + private void assertEvaluate(OnnxRuntime runtime, String model, String output, String... input) { OnnxEvaluator evaluator = runtime.evaluatorOf("src/test/models/onnx/" + model); Map<String, Tensor> inputs = new HashMap<>(); for (int i = 0; i < input.length; ++i) { diff --git a/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxRuntimeTest.java b/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxRuntimeTest.java index 81b1237e770..fdbd4fa4e5c 100644 --- a/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxRuntimeTest.java +++ b/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxRuntimeTest.java @@ -2,16 +2,18 @@ package ai.vespa.modelintegration.evaluator; -import ai.onnxruntime.OrtException; -import ai.onnxruntime.OrtSession; import org.junit.jupiter.api.Test; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; + import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertSame; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; /** * @author bjorncs @@ -19,30 +21,81 @@ import static org.mockito.Mockito.verify; class OnnxRuntimeTest { @Test - void reuses_sessions_while_active() throws OrtException { - var runtime = new OnnxRuntime((__, ___) -> mock(OrtSession.class)); - var session1 = runtime.acquireSession("model1", new OnnxEvaluatorOptions(), false); - var session2 = runtime.acquireSession("model1", new OnnxEvaluatorOptions(), false); - var session3 = runtime.acquireSession("model2", new OnnxEvaluatorOptions(), false); - assertSame(session1.instance(), session2.instance()); - assertNotSame(session1.instance(), session3.instance()); + void reuses_sessions_while_active() { + assumeTrue(OnnxRuntime.isRuntimeAvailable()); + OnnxRuntime runtime = new OnnxRuntime(); + String model1 = "src/test/models/onnx/simple/simple.onnx"; + var evaluator1 = runtime.evaluatorOf(model1); + var evaluator2 = runtime.evaluatorOf(model1); + String model2 = "src/test/models/onnx/simple/matmul.onnx"; + var evaluator3 = runtime.evaluatorOf(model2); + assertSameSession(evaluator1, evaluator2); + assertNotSameSession(evaluator1, evaluator3); assertEquals(2, runtime.sessionsCached()); - session1.close(); - session2.close(); + evaluator1.close(); + evaluator2.close(); assertEquals(1, runtime.sessionsCached()); - verify(session1.instance()).close(); - verify(session3.instance(), never()).close(); + assertClosed(evaluator1); + assertNotClosed(evaluator3); - session3.close(); + evaluator3.close(); assertEquals(0, runtime.sessionsCached()); - verify(session3.instance()).close(); + assertClosed(evaluator3); - var session4 = runtime.acquireSession("model1", new OnnxEvaluatorOptions(), false); - assertNotSame(session1.instance(), session4.instance()); + var session4 = runtime.evaluatorOf(model1); + assertNotSameSession(evaluator1, session4); assertEquals(1, runtime.sessionsCached()); session4.close(); assertEquals(0, runtime.sessionsCached()); - verify(session4.instance()).close(); + assertClosed(session4); + } + + @Test + void loads_model_from_byte_array() throws IOException { + assumeTrue(OnnxRuntime.isRuntimeAvailable()); + var runtime = new OnnxRuntime(); + byte[] bytes = Files.readAllBytes(Paths.get("src/test/models/onnx/simple/simple.onnx")); + var evaluator1 = runtime.evaluatorOf(bytes); + var evaluator2 = runtime.evaluatorOf(bytes); + assertEquals(3, evaluator1.getInputs().size()); + assertEquals(1, runtime.sessionsCached()); + assertSameSession(evaluator1, evaluator2); + evaluator2.close(); + evaluator1.close(); + assertEquals(0, runtime.sessionsCached()); + assertClosed(evaluator1); + } + + @Test + void loading_same_model_from_bytes_and_file_resolve_to_same_instance() throws IOException { + assumeTrue(OnnxRuntime.isRuntimeAvailable()); + var runtime = new OnnxRuntime(); + String modelPath = "src/test/models/onnx/simple/simple.onnx"; + byte[] bytes = Files.readAllBytes(Paths.get(modelPath)); + try (var evaluator1 = runtime.evaluatorOf(bytes); + var evaluator2 = runtime.evaluatorOf(modelPath)) { + assertSameSession(evaluator1, evaluator2); + assertEquals(1, runtime.sessionsCached()); + } + } + + private static void assertClosed(OnnxEvaluator evaluator) { assertTrue(isClosed(evaluator), "Session is not closed"); } + private static void assertNotClosed(OnnxEvaluator evaluator) { assertFalse(isClosed(evaluator), "Session is closed"); } + private static void assertSameSession(OnnxEvaluator evaluator1, OnnxEvaluator evaluator2) { + assertSame(evaluator1.ortSession(), evaluator2.ortSession()); + } + private static void assertNotSameSession(OnnxEvaluator evaluator1, OnnxEvaluator evaluator2) { + assertNotSame(evaluator1.ortSession(), evaluator2.ortSession()); + } + + private static boolean isClosed(OnnxEvaluator evaluator) { + try { + evaluator.getInputs(); + return false; + } catch (IllegalStateException e) { + assertEquals("Asking for inputs from a closed OrtSession.", e.getMessage()); + return true; + } } }
\ No newline at end of file diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java index 7367a254b4a..92cccf86ecb 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.node.admin.configserver.cores; import com.yahoo.config.provision.DockerImage; import java.nio.file.Path; +import java.time.Instant; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -12,7 +13,11 @@ import java.util.Optional; * @author hakonhall */ public class CoreDumpMetadata { + public enum Type { CORE_DUMP, JVM_HEAP, OOM } + + private Type type; private String binPath; + private Instant created; private List<String> backtrace; private List<String> backtraceAllThreads; private Path coreDumpPath; @@ -24,30 +29,36 @@ public class CoreDumpMetadata { public CoreDumpMetadata() {} - public Optional<String> binPath() { return Optional.ofNullable(binPath); }; - public Optional<List<String>> backtrace() { return Optional.ofNullable(backtrace); }; - public Optional<List<String>> backtraceAllThreads() { return Optional.ofNullable(backtraceAllThreads); }; - public Optional<Path> coredumpPath() { return Optional.ofNullable(coreDumpPath); }; + public Optional<Type> type() { return Optional.ofNullable(type); } + public Optional<String> binPath() { return Optional.ofNullable(binPath); } + public Optional<Instant> created() { return Optional.ofNullable(created); } + public Optional<List<String>> backtrace() { return Optional.ofNullable(backtrace); } + public Optional<List<String>> backtraceAllThreads() { return Optional.ofNullable(backtraceAllThreads); } + public Optional<Path> coredumpPath() { return Optional.ofNullable(coreDumpPath); } public Optional<String> decryptionToken() { return Optional.ofNullable(decryptionToken); } - public Optional<String> kernelVersion() { return Optional.ofNullable(kernelVersion); }; - public Optional<String> cpuMicrocodeVersion() { return Optional.ofNullable(cpuMicrocodeVersion); }; - public Optional<DockerImage> dockerImage() { return Optional.ofNullable(dockerImage); }; - public Optional<String> vespaVersion() { return Optional.ofNullable(vespaVersion); }; + public Optional<String> kernelVersion() { return Optional.ofNullable(kernelVersion); } + public Optional<String> cpuMicrocodeVersion() { return Optional.ofNullable(cpuMicrocodeVersion); } + public Optional<DockerImage> dockerImage() { return Optional.ofNullable(dockerImage); } + public Optional<String> vespaVersion() { return Optional.ofNullable(vespaVersion); } - public CoreDumpMetadata setBinPath(String binPath) { this.binPath = binPath; return this; }; - public CoreDumpMetadata setBacktrace(List<String> backtrace) { this.backtrace = backtrace; return this; }; - public CoreDumpMetadata setBacktraceAllThreads(List<String> backtraceAllThreads) { this.backtraceAllThreads = backtraceAllThreads; return this; }; - public CoreDumpMetadata setCoreDumpPath(Path coreDumpPath) { this.coreDumpPath = coreDumpPath; return this; }; + public CoreDumpMetadata setType(Type type) { this.type = type; return this; } + public CoreDumpMetadata setBinPath(String binPath) { this.binPath = binPath; return this; } + public CoreDumpMetadata setCreated(Instant created) { this.created = created; return this; } + public CoreDumpMetadata setBacktrace(List<String> backtrace) { this.backtrace = backtrace; return this; } + public CoreDumpMetadata setBacktraceAllThreads(List<String> backtraceAllThreads) { this.backtraceAllThreads = backtraceAllThreads; return this; } + public CoreDumpMetadata setCoreDumpPath(Path coreDumpPath) { this.coreDumpPath = coreDumpPath; return this; } public CoreDumpMetadata setDecryptionToken(String decryptionToken) { this.decryptionToken = decryptionToken; return this; } - public CoreDumpMetadata setKernelVersion(String kernelVersion) { this.kernelVersion = kernelVersion; return this; }; - public CoreDumpMetadata setCpuMicrocodeVersion(String cpuMicrocodeVersion) { this.cpuMicrocodeVersion = cpuMicrocodeVersion; return this; }; - public CoreDumpMetadata setDockerImage(DockerImage dockerImage) { this.dockerImage = dockerImage; return this; }; - public CoreDumpMetadata setVespaVersion(String vespaVersion) { this.vespaVersion = vespaVersion; return this; }; + public CoreDumpMetadata setKernelVersion(String kernelVersion) { this.kernelVersion = kernelVersion; return this; } + public CoreDumpMetadata setCpuMicrocodeVersion(String cpuMicrocodeVersion) { this.cpuMicrocodeVersion = cpuMicrocodeVersion; return this; } + public CoreDumpMetadata setDockerImage(DockerImage dockerImage) { this.dockerImage = dockerImage; return this; } + public CoreDumpMetadata setVespaVersion(String vespaVersion) { this.vespaVersion = vespaVersion; return this; } @Override public String toString() { return "CoreDumpMetadata{" + - "binPath=" + binPath + + "type=" + type + + ", binPath=" + binPath + + ", created=" + created + ", backtrace=" + backtrace + ", backtraceAllThreads=" + backtraceAllThreads + ", coreDumpPath=" + coreDumpPath + @@ -64,7 +75,9 @@ public class CoreDumpMetadata { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; CoreDumpMetadata metadata = (CoreDumpMetadata) o; - return Objects.equals(binPath, metadata.binPath) && + return type == metadata.type && + Objects.equals(binPath, metadata.binPath) && + Objects.equals(created, metadata.created) && Objects.equals(backtrace, metadata.backtrace) && Objects.equals(backtraceAllThreads, metadata.backtraceAllThreads) && Objects.equals(coreDumpPath, metadata.coreDumpPath) && @@ -77,7 +90,7 @@ public class CoreDumpMetadata { @Override public int hashCode() { - return Objects.hash(binPath, backtrace, backtraceAllThreads, coreDumpPath, decryptionToken, kernelVersion, + return Objects.hash(type, binPath, created, backtrace, backtraceAllThreads, coreDumpPath, decryptionToken, kernelVersion, cpuMicrocodeVersion, dockerImage, vespaVersion); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java index 27cf28b8e1e..a9620ebabc2 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java @@ -14,6 +14,7 @@ import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; +import java.time.Instant; import java.util.List; import java.util.Optional; @@ -31,6 +32,8 @@ public class ReportCoreDumpRequest { public List<String> backtrace; public List<String> backtrace_all_threads; + public Long created; + public String type; public String bin_path; public String coredump_path; public String cpu_microcode_version; @@ -44,7 +47,9 @@ public class ReportCoreDumpRequest { /** Fill this from metadata and return this. */ @JsonIgnore public ReportCoreDumpRequest fillFrom(CoreDumpMetadata metadata) { + metadata.type().ifPresent(type -> this.type = type.name()); metadata.binPath().ifPresent(binPath -> this.bin_path = binPath); + metadata.created().ifPresent(created -> this.created = created.toEpochMilli()); metadata.backtrace().ifPresent(backtrace -> this.backtrace = List.copyOf(backtrace)); metadata.backtraceAllThreads().ifPresent(backtraceAllThreads -> this.backtrace_all_threads = List.copyOf(backtraceAllThreads)); metadata.coredumpPath().ifPresent(coredumpPath -> this.coredump_path = coredumpPath.toString()); @@ -58,7 +63,9 @@ public class ReportCoreDumpRequest { @JsonIgnore public void populateMetadata(CoreDumpMetadata metadata, FileSystem fileSystem) { + if (type != null) metadata.setType(CoreDumpMetadata.Type.valueOf(type)); if (bin_path != null) metadata.setBinPath(bin_path); + if (created != null) metadata.setCreated(Instant.ofEpochMilli(created)); if (backtrace != null) metadata.setBacktrace(backtrace); if (backtrace_all_threads != null) metadata.setBacktraceAllThreads(backtrace_all_threads); if (coredump_path != null) metadata.setCoreDumpPath(fileSystem.getPath(coredump_path)); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/DropDocumentsReport.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/DropDocumentsReport.java new file mode 100644 index 00000000000..0d88f10ebf9 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/reports/DropDocumentsReport.java @@ -0,0 +1,55 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports; + +import com.fasterxml.jackson.annotation.JsonGetter; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author freva + */ +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class DropDocumentsReport extends BaseReport { + private static final String REPORT_ID = "dropDocuments"; + private static final String DROPPED_AT_FIELD = "droppedAt"; + private static final String READIED_AT_FIELD = "readiedAt"; + private static final String STARTED_AT_FIELD = "startedAt"; + + private final Long droppedAt; + private final Long readiedAt; + private final Long startedAt; + + public DropDocumentsReport(@JsonProperty(CREATED_FIELD) Long createdMillisOrNull, + @JsonProperty(DROPPED_AT_FIELD) Long droppedAtOrNull, + @JsonProperty(READIED_AT_FIELD) Long readiedAtOrNull, + @JsonProperty(STARTED_AT_FIELD) Long startedAtOrNull) { + super(createdMillisOrNull, null); + this.droppedAt = droppedAtOrNull; + this.readiedAt = readiedAtOrNull; + this.startedAt = startedAtOrNull; + } + + @JsonGetter(DROPPED_AT_FIELD) + public Long droppedAt() { return droppedAt; } + + @JsonGetter(READIED_AT_FIELD) + public Long readiedAt() { return readiedAt; } + + @JsonGetter(STARTED_AT_FIELD) + public Long startedAt() { return startedAt; } + + public DropDocumentsReport withDroppedAt(long droppedAt) { + return new DropDocumentsReport(getCreatedMillisOrNull(), droppedAt, readiedAt, startedAt); + } + + public DropDocumentsReport withStartedAt(long startedAt) { + return new DropDocumentsReport(getCreatedMillisOrNull(), droppedAt, readiedAt, startedAt); + } + + public static String reportId() { + return REPORT_ID; + } + +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java index 28773767d24..5d4628b41b6 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java @@ -5,6 +5,7 @@ import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoreDumpMetadata; import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; @@ -96,17 +97,19 @@ public class CoreCollector { } CoreDumpMetadata collect(NodeAgentContext context, ContainerPath coredumpPath) { - var metadata = new CoreDumpMetadata(); + var metadata = new CoreDumpMetadata() + .setCreated(new UnixPath(coredumpPath).getLastModifiedTime()); if (JAVA_HEAP_DUMP_PATTERN.matcher(coredumpPath.getFileName().toString()).find()) { - metadata.setBinPath("java") + metadata.setType(CoreDumpMetadata.Type.JVM_HEAP) + .setBinPath("java") .setBacktrace(List.of("Heap dump, no backtrace available")); return metadata; } try { String binPath = readBinPath(context, coredumpPath); - metadata.setBinPath(binPath); + metadata.setType(CoreDumpMetadata.Type.CORE_DUMP).setBinPath(binPath); if (Path.of(binPath).getFileName().toString().equals("java")) { metadata.setBacktraceAllThreads(readJstack(context, coredumpPath, binPath)); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index 15be7accb7d..3fb9c73367d 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -19,6 +19,10 @@ import com.yahoo.vespa.athenz.identityprovider.client.CsrGenerator; import com.yahoo.vespa.athenz.identityprovider.client.DefaultIdentityDocumentClient; import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; import com.yahoo.vespa.athenz.utils.SiaUtils; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.node.admin.component.ConfigServerInfo; import com.yahoo.vespa.hosted.node.admin.container.ContainerName; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; @@ -47,6 +51,9 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; import java.util.logging.Logger; +import static com.yahoo.vespa.hosted.node.admin.maintenance.identity.AthenzCredentialsMaintainer.IdentityType.NODE; +import static com.yahoo.vespa.hosted.node.admin.maintenance.identity.AthenzCredentialsMaintainer.IdentityType.TENANT; + /** * A maintainer that is responsible for providing and refreshing Athenz credentials for a container. * @@ -68,7 +75,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { private final String certificateDnsSuffix; private final ServiceIdentityProvider hostIdentityProvider; private final IdentityDocumentClient identityDocumentClient; - private final boolean useInternalZts; + private final BooleanFlag tenantServiceIdentityFlag; // Used as an optimization to ensure ZTS is not DDoS'ed on continuously failing refresh attempts private final Map<ContainerName, Instant> lastRefreshAttempt = new ConcurrentHashMap<>(); @@ -78,7 +85,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { ConfigServerInfo configServerInfo, String certificateDnsSuffix, ServiceIdentityProvider hostIdentityProvider, - boolean useInternalZts, + FlagSource flagSource, Clock clock) { this.ztsEndpoint = ztsEndpoint; this.ztsTrustStorePath = ztsTrustStorePath; @@ -89,24 +96,33 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { hostIdentityProvider, new AthenzIdentityVerifier(Set.of(configServerInfo.getConfigServerIdentity()))); this.clock = clock; - this.useInternalZts = useInternalZts; + this.tenantServiceIdentityFlag = Flags.NODE_ADMIN_TENANT_SERVICE_REGISTRY.bindTo(flagSource); } public boolean converge(NodeAgentContext context) { + var modified = false; + modified |= maintain(context, NODE); + if (shouldWriteTenantServiceIdentity(context)) + modified |= maintain(context, TENANT); + return modified; + } + + private boolean maintain(NodeAgentContext context, IdentityType identityType) { if (context.isDisabled(NodeAgentTask.CredentialsMaintainer)) return false; try { context.log(logger, Level.FINE, "Checking certificate"); - ContainerPath containerSiaDirectory = context.paths().of(CONTAINER_SIA_DIRECTORY, context.users().vespa()); - ContainerPath privateKeyFile = (ContainerPath) SiaUtils.getPrivateKeyFile(containerSiaDirectory, context.identity()); - ContainerPath certificateFile = (ContainerPath) SiaUtils.getCertificateFile(containerSiaDirectory, context.identity()); - ContainerPath identityDocumentFile = containerSiaDirectory.resolve("vespa-node-identity-document.json"); + ContainerPath siaDirectory = context.paths().of(CONTAINER_SIA_DIRECTORY, context.users().vespa()); + ContainerPath identityDocumentFile = siaDirectory.resolve(identityType.getIdentityDocument()); + AthenzIdentity athenzIdentity = getAthenzIdentity(context, identityType, identityDocumentFile); + ContainerPath privateKeyFile = (ContainerPath) SiaUtils.getPrivateKeyFile(siaDirectory, athenzIdentity); + ContainerPath certificateFile = (ContainerPath) SiaUtils.getCertificateFile(siaDirectory, athenzIdentity); if (!Files.exists(privateKeyFile) || !Files.exists(certificateFile) || !Files.exists(identityDocumentFile)) { context.log(logger, "Certificate/private key/identity document file does not exist"); Files.createDirectories(privateKeyFile.getParent()); Files.createDirectories(certificateFile.getParent()); Files.createDirectories(identityDocumentFile.getParent()); - registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile); + registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, identityType, athenzIdentity); return true; } @@ -116,11 +132,11 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { var doc = EntityBindingsMapper.readSignedIdentityDocumentFromFile(identityDocumentFile); if (doc.outdated()) { context.log(logger, "Identity document is outdated (version=%d)", doc.documentVersion()); - registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile); + registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, identityType, athenzIdentity); return true; } else if (isCertificateExpired(expiry, now)) { context.log(logger, "Certificate has expired (expiry=%s)", expiry.toString()); - registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile); + registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, identityType, athenzIdentity); return true; } @@ -134,7 +150,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { return false; } else { lastRefreshAttempt.put(context.containerName(), now); - refreshIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, doc); + refreshIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, doc, identityType, athenzIdentity); return true; } } @@ -182,12 +198,12 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { now)) > 0; } - private void registerIdentity(NodeAgentContext context, ContainerPath privateKeyFile, ContainerPath certificateFile, ContainerPath identityDocumentFile) { + private void registerIdentity(NodeAgentContext context, ContainerPath privateKeyFile, ContainerPath certificateFile, ContainerPath identityDocumentFile, IdentityType identityType, AthenzIdentity identity) { KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); - SignedIdentityDocument doc = identityDocumentClient.getNodeIdentityDocument(context.hostname().value()); + SignedIdentityDocument doc = signedIdentityDocument(context, identityType); CsrGenerator csrGenerator = new CsrGenerator(certificateDnsSuffix, doc.providerService().getFullName()); Pkcs10Csr csr = csrGenerator.generateInstanceCsr( - context.identity(), doc.providerUniqueId(), doc.ipAddresses(), doc.clusterType(), keyPair); + identity, doc.providerUniqueId(), doc.ipAddresses(), doc.clusterType(), keyPair); // Allow all zts hosts while removing SIS HostnameVerifier ztsHostNameVerifier = (hostname, sslSession) -> true; @@ -195,7 +211,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { InstanceIdentity instanceIdentity = ztsClient.registerInstance( doc.providerService(), - context.identity(), + identity, EntityBindingsMapper.toAttestationData(doc), csr); EntityBindingsMapper.writeSignedIdentityDocumentToFile(identityDocumentFile, doc); @@ -214,11 +230,11 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { .orElse(ztsEndpoint); } private void refreshIdentity(NodeAgentContext context, ContainerPath privateKeyFile, ContainerPath certificateFile, - ContainerPath identityDocumentFile, SignedIdentityDocument doc) { + ContainerPath identityDocumentFile, SignedIdentityDocument doc, IdentityType identityType, AthenzIdentity identity) { KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); CsrGenerator csrGenerator = new CsrGenerator(certificateDnsSuffix, doc.providerService().getFullName()); Pkcs10Csr csr = csrGenerator.generateInstanceCsr( - context.identity(), doc.providerUniqueId(), doc.ipAddresses(), doc.clusterType(), keyPair); + identity, doc.providerUniqueId(), doc.ipAddresses(), doc.clusterType(), keyPair); SSLContext containerIdentitySslContext = new SslContextBuilder().withKeyStore(privateKeyFile, certificateFile) .withTrustStore(ztsTrustStorePath) @@ -231,7 +247,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { InstanceIdentity instanceIdentity = ztsClient.refreshInstance( doc.providerService(), - context.identity(), + identity, doc.providerUniqueId().asDottedString(), csr); writePrivateKeyAndCertificate(privateKeyFile, keyPair.getPrivate(), certificateFile, instanceIdentity.certificate()); @@ -239,7 +255,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { } catch (ZtsClientException e) { if (e.getErrorCode() == 403 && e.getDescription().startsWith("Certificate revoked")) { context.log(logger, Level.SEVERE, "Certificate cannot be refreshed as it is revoked by ZTS - re-registering the instance now", e); - registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile); + registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, identityType, identity); } else { throw e; } @@ -272,4 +288,46 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { private static boolean isCertificateExpired(Instant expiry, Instant now) { return now.isAfter(expiry.minus(EXPIRY_MARGIN)); } + + private SignedIdentityDocument signedIdentityDocument(NodeAgentContext context, IdentityType identityType) { + return switch (identityType) { + case NODE -> identityDocumentClient.getNodeIdentityDocument(context.hostname().value()); + case TENANT -> identityDocumentClient.getTenantIdentityDocument(context.hostname().value()); + }; + } + + private AthenzIdentity getAthenzIdentity(NodeAgentContext context, IdentityType identityType, ContainerPath identityDocumentFile) { + return switch (identityType) { + case NODE -> context.identity(); + case TENANT -> getTenantIdentity(context, identityDocumentFile); + }; + } + + private AthenzIdentity getTenantIdentity(NodeAgentContext context, ContainerPath identityDocumentFile) { + if (Files.exists(identityDocumentFile)) { + return EntityBindingsMapper.readSignedIdentityDocumentFromFile(identityDocumentFile).serviceIdentity(); + } else { + return identityDocumentClient.getTenantIdentityDocument(context.hostname().value()).serviceIdentity(); + } + } + + private boolean shouldWriteTenantServiceIdentity(NodeAgentContext context) { + return tenantServiceIdentityFlag + .with(FetchVector.Dimension.HOSTNAME, context.hostname().value()) + .value(); + } + + enum IdentityType { + NODE("vespa-node-identity-document.json"), + TENANT("vespa-tenant-identity-document.json"); + + private String identityDocument; + IdentityType(String identityDocument) { + this.identityDocument = identityDocument; + } + + public String getIdentityDocument() { + return identityDocument; + } + } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java index 3a12191a0de..c743f1c8c85 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java @@ -71,11 +71,14 @@ public class SyncFileInfo { if (filename.startsWith("vespa.log")) { dir = "logs/vespa/"; compression = Compression.ZSTD; - minDurationBetweenSync = filename.length() == 9 ? rotatedOnly ? Duration.ofHours(1) : Duration.ZERO : null; + if (filename.length() == 9) { + if (!rotatedOnly) remoteFilename = "vespa.log-" + DATE_TIME_FORMATTER.format(new UnixPath(logFile).getLastModifiedTime()); + minDurationBetweenSync = rotatedOnly ? Duration.ofHours(1) : Duration.ZERO; + } } else if (filename.startsWith("zookeeper.") && filename.endsWith(".log")) { compression = Compression.ZSTD; dir = "logs/zookeeper/"; - remoteFilename = filename.endsWith(".0.log") ? "zookeeper.log" : + remoteFilename = rotatedOnly && filename.endsWith(".0.log") ? "zookeeper.log" : "zookeeper.log-" + DATE_TIME_FORMATTER.format(new UnixPath(logFile).getLastModifiedTime()); minDurationBetweenSync = filename.endsWith(".0.log") ? rotatedOnly ? Duration.ofHours(1) : Duration.ZERO : null; } else { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 20359410321..f2f690106fa 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeMembers import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.DropDocumentsReport; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException; import com.yahoo.vespa.hosted.node.admin.container.Container; @@ -29,6 +30,7 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.identity.CredentialsMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.VespaServiceDumper; import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; +import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; import java.time.Clock; import java.time.Duration; @@ -228,6 +230,12 @@ public class NodeAgentImpl implements NodeAgent { changed = true; } + Optional<DropDocumentsReport> report = context.node().reports().getReport(DropDocumentsReport.reportId(), DropDocumentsReport.class); + if (report.isPresent() && report.get().startedAt() == null && report.get().readiedAt() != null) { + newNodeAttributes.withReport(DropDocumentsReport.reportId(), report.get().withStartedAt(clock.millis()).toJsonNode()); + changed = true; + } + if (changed) { context.log(logger, "Publishing new set of attributes to node repo: %s -> %s", currentNodeAttributes, newNodeAttributes); @@ -433,6 +441,21 @@ public class NodeAgentImpl implements NodeAgent { .orElse(false); } + private void dropDocsIfNeeded(NodeAgentContext context, Optional<Container> container) { + Optional<DropDocumentsReport> report = context.node().reports() + .getReport(DropDocumentsReport.reportId(), DropDocumentsReport.class); + if (report.isEmpty() || report.get().readiedAt() != null) return; + + if (report.get().droppedAt() == null) { + container.ifPresent(c -> removeContainer(context, c, List.of("Dropping documents"), true)); + FileFinder.from(context.paths().underVespaHome("var/db/vespa/search")).deleteRecursively(context); + nodeRepository.updateNodeAttributes(context.node().hostname(), + new NodeAttributes().withReport(DropDocumentsReport.reportId(), report.get().withDroppedAt(clock.millis()).toJsonNode())); + } + + throw ConvergenceException.ofTransient("Documents already dropped, waiting for signal to start the container"); + } + public void converge(NodeAgentContext context) { try { doConverge(context); @@ -494,6 +517,7 @@ public class NodeAgentImpl implements NodeAgent { context.log(logger, "Waiting for image to download " + context.node().wantedDockerImage().get().asString()); return; } + dropDocsIfNeeded(context, container); container = removeContainerIfNeededUpdateContainerState(context, container); credentialsMaintainers.forEach(maintainer -> maintainer.converge(context)); if (container.isEmpty()) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesImpl.java index 1b5e6c57f5c..b7a1cb8a16e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/IPAddressesImpl.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.task.util.network; +import java.io.UncheckedIOException; import java.net.InetAddress; import java.net.UnknownHostException; @@ -14,7 +15,7 @@ public class IPAddressesImpl implements IPAddresses { try { return InetAddress.getAllByName(hostname); } catch (UnknownHostException e) { - throw new RuntimeException(e); + throw new UncheckedIOException(e); } } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java index c7d34a12f43..9662d4184df 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java @@ -91,7 +91,7 @@ public class SystemCtl { public String getServiceProperty(TaskContext context, String unit, String property) { return newCommandLine(context) .add("systemctl", "show", "--property", property, "--value", unit + ".service") - .execute() + .executeSilently() .getOutput(); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java index f49dd2e705b..b35f4d6c790 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java @@ -16,6 +16,7 @@ import org.mockito.ArgumentCaptor; import java.nio.file.FileSystem; import java.nio.file.Path; +import java.time.Instant; import java.util.List; import java.util.Optional; @@ -40,6 +41,8 @@ class CoresTest { private final HostName hostname = HostName.of("foo.com"); private final String id = "5c987afb-347a-49ee-a0c5-bef56bbddeb0"; private final CoreDumpMetadata metadata = new CoreDumpMetadata() + .setType(CoreDumpMetadata.Type.OOM) + .setCreated(Instant.ofEpochMilli(12345678)) .setKernelVersion("4.18.0-372.26.1.el8_6.x86_64") .setCpuMicrocodeVersion("0x1000065") .setCoreDumpPath(fileSystem.getPath("/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813")) @@ -83,9 +86,11 @@ class CoresTest { "bin_path": "/usr/bin/java", "coredump_path": "/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813", "cpu_microcode_version": "0x1000065", + "created": 12345678, "decryption_token": "987def", "docker_image": "us-central1-docker.pkg.dev/vespa-external-cd/vespa-cloud/vespa/cloud-tenant-rhel8:8.68.8", "kernel_version": "4.18.0-372.26.1.el8_6.x86_64", + "type": "OOM", "vespa_version": "8.68.8" }""", JsonTestHelper.normalize(uncheck(() -> mapper.writeValueAsString(bodyJsonPojoCaptor.getValue())))); @@ -128,9 +133,11 @@ class CoresTest { "bin_path": "/usr/bin/java", "coredump_path": "/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813", "cpu_microcode_version": "0x1000065", + "created": 12345678, "decryption_token": "987def", "docker_image": "us-central1-docker.pkg.dev/vespa-external-cd/vespa-cloud/vespa/cloud-tenant-rhel8:8.68.8", "kernel_version": "4.18.0-372.26.1.el8_6.x86_64", + "type": "OOM", "vespa_version": "8.68.8" }""", JsonTestHelper.normalize(new UnixPath(path).readUtf8File())); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java index 4fa18c71da0..b4a35d6012c 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java @@ -5,11 +5,13 @@ import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoreDumpMetadata; import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.jupiter.api.Test; +import java.time.Instant; import java.util.List; import static com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoreCollector.GDB_PATH_RHEL8; @@ -22,12 +24,18 @@ import static org.mockito.Mockito.when; * @author freva */ public class CoreCollectorTest { + private static final Instant CORE_CREATED = Instant.ofEpochMilli(2233445566L); + private final ContainerOperations docker = mock(ContainerOperations.class); private final CoreCollector coreCollector = new CoreCollector(docker); private final NodeAgentContext context = NodeAgentContextImpl.builder("container-123.domain.tld") .fileSystem(TestFileSystem.create()).build(); - private final ContainerPath TEST_CORE_PATH = context.paths().of("/tmp/core.1234"); + private final ContainerPath TEST_CORE_PATH = (ContainerPath) new UnixPath(context.paths().of("/tmp/core.1234")) + .createParents() + .createNewFile() + .setLastModifiedTime(CORE_CREATED) + .toPath(); private final String TEST_BIN_PATH = "/usr/bin/program"; private final List<String> GDB_BACKTRACE = List.of("[New Thread 2703]", "Core was generated by `/usr/bin/program\'.", "Program terminated with signal 11, Segmentation fault.", @@ -143,6 +151,8 @@ public class CoreCollectorTest { String.join("\n", GDB_BACKTRACE)); var expected = new CoreDumpMetadata().setBinPath(TEST_BIN_PATH) + .setCreated(CORE_CREATED) + .setType(CoreDumpMetadata.Type.CORE_DUMP) .setBacktrace(GDB_BACKTRACE) .setBacktraceAllThreads(GDB_BACKTRACE); assertEquals(expected, coreCollector.collect(context, TEST_CORE_PATH)); @@ -156,7 +166,7 @@ public class CoreCollectorTest { mockExec(new String[]{GDB_PATH_RHEL8 + " -n -ex set print frame-arguments none -ex bt -batch /usr/bin/program /tmp/core.1234"}, "", "Failure"); - var expected = new CoreDumpMetadata().setBinPath(TEST_BIN_PATH); + var expected = new CoreDumpMetadata().setBinPath(TEST_BIN_PATH).setCreated(CORE_CREATED).setType(CoreDumpMetadata.Type.CORE_DUMP); assertEquals(expected, coreCollector.collect(context, TEST_CORE_PATH)); } @@ -174,6 +184,8 @@ public class CoreCollectorTest { jstack); var expected = new CoreDumpMetadata().setBinPath(jdkPath) + .setCreated(CORE_CREATED) + .setType(CoreDumpMetadata.Type.CORE_DUMP) .setBacktraceAllThreads(List.of(jstack)); assertEquals(expected, coreCollector.collect(context, TEST_CORE_PATH)); } @@ -181,9 +193,14 @@ public class CoreCollectorTest { @Test void metadata_for_java_heap_dump() { var expected = new CoreDumpMetadata().setBinPath("java") + .setType(CoreDumpMetadata.Type.JVM_HEAP) + .setCreated(CORE_CREATED) .setBacktrace(List.of("Heap dump, no backtrace available")); - assertEquals(expected, coreCollector.collect(context, context.paths().of("/dump_java_pid123.hprof"))); + assertEquals(expected, coreCollector.collect(context, (ContainerPath) new UnixPath(context.paths().of("/dump_java_pid123.hprof")) + .createNewFile() + .setLastModifiedTime(CORE_CREATED) + .toPath())); } private void mockExec(String[] cmd, String output) { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java index 3c91a9f32d1..b7aee6706b1 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java @@ -74,8 +74,9 @@ public class SyncFileInfoTest { @Test void vespa_logs() { + new UnixPath(vespaLogPath1).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z")); assertForLogFile(vespaLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log.zst", ZSTD, Duration.ofHours(1), true); - assertForLogFile(vespaLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log.zst", ZSTD, Duration.ZERO, false); + assertForLogFile(vespaLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2022-05-09.14-22-11.zst", ZSTD, Duration.ZERO, false); assertForLogFile(vespaLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2021-02-12.zst", ZSTD, true); assertForLogFile(vespaLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2021-02-12.zst", ZSTD, false); @@ -83,8 +84,9 @@ public class SyncFileInfoTest { @Test void zookeeper_logs() { + new UnixPath(zkLogPath0).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-13T13:13:45Z")); assertForLogFile(zkLogPath0, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log.zst", ZSTD, Duration.ofHours(1), true); - assertForLogFile(zkLogPath0, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log.zst", ZSTD, Duration.ZERO, false); + assertForLogFile(zkLogPath0, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log-2022-05-13.13-13-45.zst", ZSTD, Duration.ZERO, false); new UnixPath(zkLogPath1).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z")); assertForLogFile(zkLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log-2022-05-09.14-22-11.zst", ZSTD, true); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index b8b72308bdd..2db5314dbf2 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -14,6 +14,7 @@ import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeReposit import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.OrchestratorStatus; +import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.reports.DropDocumentsReport; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException; import com.yahoo.vespa.hosted.node.admin.container.Container; @@ -27,6 +28,7 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.acl.AclMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.identity.CredentialsMaintainer; import com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.VespaServiceDumper; import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -38,8 +40,11 @@ import java.time.Instant; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.BiFunction; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -739,6 +744,56 @@ public class NodeAgentImplTest { inOrder.verify(orchestrator, times(1)).resume(eq(hostName)); } + @Test + void drop_all_documents() { + InOrder inOrder = inOrder(orchestrator, nodeRepository); + BiFunction<NodeState, DropDocumentsReport, NodeSpec> specBuilder = (state, report) -> (report == null ? + nodeBuilder(state) : nodeBuilder(state).report(DropDocumentsReport.reportId(), report.toJsonNode())) + .wantedDockerImage(dockerImage).currentDockerImage(dockerImage) + .build(); + NodeAgentImpl nodeAgent = makeNodeAgent(dockerImage, true, Duration.ofSeconds(30)); + + NodeAgentContext context = createContext(specBuilder.apply(NodeState.active, null)); + UnixPath indexPath = new UnixPath(context.paths().underVespaHome("var/db/vespa/search/cluster.foo/0/doc")).createParents().createNewFile(); + mockGetContainer(dockerImage, ContainerResources.from(2, 2, 16), true); + assertTrue(indexPath.exists()); + + // Initially no changes, index is not dropped + nodeAgent.converge(context); + assertTrue(indexPath.exists()); + inOrder.verifyNoMoreInteractions(); + + context = createContext(specBuilder.apply(NodeState.active, new DropDocumentsReport(1L, null, null, null))); + nodeAgent.converge(context); + verify(containerOperations).removeContainer(eq(context), any()); + assertFalse(indexPath.exists()); + inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes().withReport(DropDocumentsReport.reportId(), new DropDocumentsReport(1L, clock.millis(), null, null).toJsonNode()))); + inOrder.verifyNoMoreInteractions(); + + // After droppedAt and before readiedAt are set, we cannot proceed + mockGetContainer(null, false); + context = createContext(specBuilder.apply(NodeState.active, new DropDocumentsReport(1L, 2L, null, null))); + nodeAgent.converge(context); + verify(containerOperations, never()).removeContainer(eq(context), any()); + verify(containerOperations, never()).startContainer(eq(context)); + inOrder.verifyNoMoreInteractions(); + + context = createContext(specBuilder.apply(NodeState.active, new DropDocumentsReport(1L, 2L, 3L, null))); + nodeAgent.converge(context); + verify(containerOperations).startContainer(eq(context)); + inOrder.verifyNoMoreInteractions(); + + mockGetContainer(dockerImage, ContainerResources.from(0, 2, 16), true); + clock.advance(Duration.ofSeconds(31)); + nodeAgent.converge(context); + verify(containerOperations, times(1)).startContainer(eq(context)); + verify(containerOperations, never()).removeContainer(eq(context), any()); + inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes() + .withRebootGeneration(0) + .withReport(DropDocumentsReport.reportId(), new DropDocumentsReport(1L, 2L, 3L, clock.millis()).toJsonNode()))); + inOrder.verifyNoMoreInteractions(); + } + private void verifyThatContainerIsStopped(NodeState nodeState, Optional<ApplicationId> owner) { NodeSpec.Builder nodeBuilder = nodeBuilder(nodeState) .type(NodeType.tenant) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index 4c9fab748d1..856d6e07156 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -59,7 +59,7 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { failures++; } } - return asSuccessFactor(attempts, failures); + return asSuccessFactorDeviation(attempts, failures); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java index c8b736cb25b..f3ea326a3c0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java @@ -130,7 +130,7 @@ public class CapacityChecker { Set<String> ipPool = host.ipConfig().pool().asSet(); for (var child : nodeChildren.get(host)) { hostResources = hostResources.subtract(child.resources().justNumbers()); - occupiedIps += child.ipConfig().primary().stream().filter(ipPool::contains).count(); + occupiedIps += (int)child.ipConfig().primary().stream().filter(ipPool::contains).count(); } availableResources.put(host, new AllocationResources(hostResources, host.ipConfig().pool().asSet().size() - occupiedIps)); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DiskReplacer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DiskReplacer.java index acd5cb61d81..6f2eb726e91 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DiskReplacer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DiskReplacer.java @@ -51,6 +51,6 @@ public class DiskReplacer extends NodeRepositoryMaintainer { log.log(Level.WARNING, "Failed to rebuild " + host.hostname() + ", will retry in " + interval(), e); } } - return this.asSuccessFactor(nodes.size(), failures); + return this.asSuccessFactorDeviation(nodes.size(), failures); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java index 83dadddf76c..a4bc3a1aea5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java @@ -113,7 +113,7 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { } success++; } - return asSuccessFactor(attempts, attempts - success); + return asSuccessFactorDeviation(attempts, attempts - success); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisioner.java index 7ecfc8f7926..0fa16f22061 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisioner.java @@ -53,7 +53,7 @@ public class HostDeprovisioner extends NodeRepositoryMaintainer { log.log(Level.WARNING, "Failed to deprovision " + host.hostname() + ", will retry in " + interval(), e); } } - return asSuccessFactor(hosts.size(), failures); + return asSuccessFactorDeviation(hosts.size(), failures); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java index 86c5a926900..3c77725298d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java @@ -63,7 +63,7 @@ public class HostResumeProvisioner extends NodeRepositoryMaintainer { } } } - return asSuccessFactor(hosts.size(), failures); + return asSuccessFactorDeviation(hosts.size(), failures); } private void setIpConfig(Node host, NodeList children, HostIpConfig hostIpConfig) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index f864ab18920..baa2e596b36 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -94,7 +94,7 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { .collect(Collectors.joining(", ")), interval())); } - return asSuccessFactor(attempts.get(), failed.size()); + return asSuccessFactorDeviation(attempts.get(), failed.size()); } /** Remove reals from inactive load balancers */ @@ -131,7 +131,7 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { interval()), lastException.get()); } - return asSuccessFactor(attempts.get(), failed.size()); + return asSuccessFactorDeviation(attempts.get(), failed.size()); } /** Patch load balancers matching given filter, while holding lock */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index 5af74214648..e6cfe8ca6b5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -176,6 +176,9 @@ public class MetricsReporter extends NodeRepositoryMaintainer { boolean converged = currentVersion.isPresent() && currentVersion.get().equals(wantedVersion); metric.set("wantToChangeVespaVersion", converged ? 0 : 1, context); + if (node.cloudAccount().isEnclave(nodeRepository().zone())) { + metric.set("hasWireguardKey", node.wireguardPubKey().isPresent() ? 1 : 0, context); + } } else { context = getContext(Map.of("state", node.state().name(), "host", node.hostname())); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index afea08711fa..766bc688c62 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.Deployment; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TransientException; import com.yahoo.jdisc.Metric; +import com.yahoo.slime.SlimeUtils; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; @@ -97,7 +98,7 @@ public class NodeFailer extends NodeRepositoryMaintainer { metric.set(throttlingActiveMetric, throttlingActive, null); metric.set(throttledHostFailuresMetric, throttledHostFailures, null); metric.set(throttledNodeFailuresMetric, throttledNodeFailures, null); - return asSuccessFactor(attempts, failures); + return asSuccessFactorDeviation(attempts, failures); } private Collection<FailingNode> findActiveFailingNodes() { @@ -109,7 +110,7 @@ public class NodeFailer extends NodeRepositoryMaintainer { for (Node node : activeNodes) { Instant graceTimeStart = clock().instant().minus(nodeRepository().nodes().suspended(node) ? suspendedDownTimeLimit : downTimeLimit); - if (node.isDown() && node.history().hasEventBefore(History.Event.Type.down, graceTimeStart) && !applicationSuspended(node)) { + if (node.isDown() && node.history().hasEventBefore(History.Event.Type.down, graceTimeStart) && !applicationSuspended(node) && !undergoingCmr(node)) { // Allow a grace period after node re-activation if (!node.history().hasEventAfter(History.Event.Type.activated, graceTimeStart)) failingNodes.add(new FailingNode(node, "Node has been down longer than " + downTimeLimit)); @@ -157,6 +158,19 @@ public class NodeFailer extends NodeRepositoryMaintainer { } } + private boolean undergoingCmr(Node node) { + return node.reports().getReport("vcmr") + .map(report -> + SlimeUtils.entriesStream(report.getInspector().field("upcoming")) + .anyMatch(cmr -> { + var startTime = cmr.field("plannedStartTime").asLong(); + var endTime = cmr.field("plannedEndTime").asLong(); + var now = clock().instant().getEpochSecond(); + return now > startTime && now < endTime; + }) + ).orElse(false); + } + /** Is the node and all active children suspended? */ private boolean allSuspended(Node node, NodeList activeNodes) { if (!nodeRepository().nodes().suspended(node)) return false; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java index 781debe26a0..979a2771082 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java @@ -77,7 +77,7 @@ public class NodeHealthTracker extends NodeRepositoryMaintainer { failures.add(1); } }); - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactorDeviation(attempts.get(), failures.get()); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java index b299369db1a..e28dac1c915 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java @@ -55,10 +55,10 @@ public class NodeMetricsDbMaintainer extends NodeRepositoryMaintainer { nodeRepository().metricsDb().gc(); - return asSuccessFactor(attempts, failures.get()); + return asSuccessFactorDeviation(attempts, failures.get()); } catch (InterruptedException e) { - return asSuccessFactor(attempts, failures.get()); + return asSuccessFactorDeviation(attempts, failures.get()); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java index 3c00e3b708d..b8d37c7eb5c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java @@ -59,7 +59,7 @@ public abstract class NodeRepositoryMaintainer extends Maintainer { @Override public void completed(String job, double successFactor, long duration) { var context = metric.createContext(Map.of("job", job)); - metric.set("maintenance.successFactor", successFactor, context); + metric.set("maintenance.successFactorDeviation", successFactor, context); metric.set("maintenance.duration", duration, context); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java index af368934188..4071559d841 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java @@ -48,7 +48,7 @@ public class ScalingSuggestionsMaintainer extends NodeRepositoryMaintainer { failures++; } } - return asSuccessFactor(attempts, failures); + return asSuccessFactorDeviation(attempts, failures); } private Applications applications() { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index 691f88a9be3..06c1916dd4f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -21,7 +21,6 @@ import java.util.function.Consumer; import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; /** * Performs preparation of node activation changes for a single host group in an application. @@ -107,7 +106,7 @@ public class GroupPreparer { // Offer the nodes on the newly provisioned hosts, this should be enough to cover the deficit List<NodeCandidate> candidates = provisionedHosts.stream() .map(host -> NodeCandidate.createNewExclusiveChild(host.generateNode(), - host.generateHost())) + host.generateHost())) .toList(); allocation.offer(candidates); }; @@ -124,6 +123,14 @@ public class GroupPreparer { hosts.forEach(host -> nodeRepository.nodes().deprovision(host.hostname(), Agent.system, nodeRepository.clock().instant())); throw e; } + } else if (allocation.hostDeficit().isPresent() && requestedNodes.canFail() && + allocation.hasRetiredJustNow() && requestedNodes instanceof NodeSpec.CountNodeSpec cns) { + // Non-dynamically provisioned zone with a deficit because we just now retired some nodes. + // Try again, but without retiring + indices.resetProbe(); + List<Node> accepted = prepareWithLocks(application, cluster, cns.withoutRetiring(), surplusActiveNodes, indices, wantedGroups); + log.warning("Prepared " + application + " " + cluster.id() + " without retirement due to lack of capacity"); + return accepted; } if (! allocation.fulfilled() && requestedNodes.canFail()) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java index 178ea6ed514..35b2fef2c78 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java @@ -1,14 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; -import com.yahoo.component.annotation.Inject; +import ai.vespa.http.DomainName; import com.yahoo.component.Version; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.ActivationContext; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.Deployment; import com.yahoo.config.provision.HostFilter; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InfraDeployer; import com.yahoo.config.provision.NodeType; @@ -116,7 +116,7 @@ public class InfraDeployerImpl implements InfraDeployer { duperModel.infraApplicationActivated( application.getApplicationId(), - hostSpecs.stream().map(HostSpec::hostname).map(HostName::of).toList()); + hostSpecs.stream().map(HostSpec::hostname).map(DomainName::of).toList()); logger.log(Level.FINE, () -> generateActivationLogMessage(hostSpecs, application.getApplicationId())); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index 04f64b070b3..b3198a72d1b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -109,7 +109,7 @@ public class LoadBalancerProvisioner { public void activate(Set<ClusterSpec> clusters, NodeList newActive, ApplicationTransaction transaction) { Map<ClusterSpec.Id, ZoneEndpoint> activatingClusters = clusters.stream() // .collect(Collectors.toMap(ClusterSpec::id, ClusterSpec::zoneEndpoint)); - // TODO: this dies with combined clusters Ü + // TODO: this dies with combined clusters .collect(groupingBy(LoadBalancerProvisioner::effectiveId, reducing(ZoneEndpoint.defaultEndpoint, ClusterSpec::zoneEndpoint, @@ -193,14 +193,13 @@ public class LoadBalancerProvisioner { Optional<LoadBalancer> loadBalancer = db.readLoadBalancer(id); LoadBalancer newLoadBalancer; LoadBalancer.State fromState = loadBalancer.map(LoadBalancer::state).orElse(null); - if ( loadBalancer.isPresent() - && ( ! inAccount(cloudAccount, loadBalancer.get()) - || ! hasCorrectVisibility(loadBalancer.get(), zoneEndpoint))) { - // We have a load balancer, but with the wrong account or visibility. - // Load balancer must be removed before we can provision a new one with the wanted visibility - newLoadBalancer = loadBalancer.get().with(LoadBalancer.State.removable, now); - } - else { + boolean recreateLoadBalancer = loadBalancer.isPresent() && ( ! inAccount(cloudAccount, loadBalancer.get()) + || ! hasCorrectVisibility(loadBalancer.get(), zoneEndpoint)); + if (recreateLoadBalancer) { + // We have a load balancer, but with the wrong account or visibility. + // Load balancer must be removed before we can provision a new one with the wanted visibility + newLoadBalancer = loadBalancer.get().with(LoadBalancer.State.removable, now); + } else { Optional<LoadBalancerInstance> instance = provisionInstance(id, loadBalancer, zoneEndpoint, cloudAccount); newLoadBalancer = loadBalancer.isEmpty() ? new LoadBalancer(id, instance, LoadBalancer.State.reserved, now) : loadBalancer.get().with(instance); @@ -211,8 +210,8 @@ public class LoadBalancerProvisioner { } private static boolean hasCorrectVisibility(LoadBalancer newLoadBalancer, ZoneEndpoint zoneEndpoint) { - return newLoadBalancer.instance().isEmpty() - || newLoadBalancer.instance().get().settings().isPublicEndpoint() == zoneEndpoint.isPublicEndpoint(); + return newLoadBalancer.instance().isEmpty() || + newLoadBalancer.instance().get().settings().isPublicEndpoint() == zoneEndpoint.isPublicEndpoint(); } private void activate(ApplicationTransaction transaction, ClusterSpec.Id cluster, ZoneEndpoint settings, NodeList nodes) { @@ -320,14 +319,6 @@ public class LoadBalancerProvisioner { return loadBalancer.instance().isEmpty() || loadBalancer.instance().get().cloudAccount().equals(cloudAccount); } - /** Returns whether load balancer has given reals, and settings if specified */ - private static boolean isUpToDate(LoadBalancer loadBalancer, Set<Real> reals, ZoneEndpoint zoneEndpoint) { - if (loadBalancer.instance().isEmpty()) - throw new IllegalStateException("Expected a load balancer instance to be present, for " + loadBalancer.id()); - return loadBalancer.instance().get().reals().equals(reals) - && loadBalancer.instance().get().settings().equals(zoneEndpoint); - } - /** Find IP addresses reachable by the load balancer service */ private Set<String> reachableIpAddresses(Node node) { Set<String> reachable = new LinkedHashSet<>(node.ipConfig().primary()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index c6971f0fe02..3af63125474 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -307,10 +307,15 @@ class NodeAllocation { } /** Returns true if this allocation was already fulfilled and resulted in no new changes */ - public boolean fulfilledAndNoChanges() { + boolean fulfilledAndNoChanges() { return fulfilled() && reservableNodes().isEmpty() && newNodes().isEmpty(); } + /** Returns true if this allocation has retired nodes */ + boolean hasRetiredJustNow() { + return wasRetiredJustNow > 0; + } + /** * Returns {@link HostDeficit} describing the host deficit for the given {@link NodeSpec}. * @@ -451,7 +456,7 @@ class NodeAllocation { .toList(); } - public String allocationFailureDetails() { + String allocationFailureDetails() { List<String> reasons = new ArrayList<>(); if (rejectedDueToExclusivity > 0) reasons.add("host exclusivity constraints"); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java index 59c089943ab..28d1e7c1c68 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java @@ -3,8 +3,6 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; @@ -79,7 +77,7 @@ public interface NodeSpec { } static NodeSpec from(int nodeCount, NodeResources resources, boolean exclusive, boolean canFail, CloudAccount cloudAccount) { - return new CountNodeSpec(nodeCount, resources, exclusive, canFail, cloudAccount); + return new CountNodeSpec(nodeCount, resources, exclusive, canFail, canFail, cloudAccount); } static NodeSpec from(NodeType type, CloudAccount cloudAccount) { @@ -93,14 +91,19 @@ public interface NodeSpec { private final NodeResources requestedNodeResources; private final boolean exclusive; private final boolean canFail; + private final boolean considerRetiring; private final CloudAccount cloudAccount; - private CountNodeSpec(int count, NodeResources resources, boolean exclusive, boolean canFail, CloudAccount cloudAccount) { + private CountNodeSpec(int count, NodeResources resources, boolean exclusive, boolean canFail, boolean considerRetiring, CloudAccount cloudAccount) { this.count = count; this.requestedNodeResources = Objects.requireNonNull(resources, "Resources must be specified"); this.exclusive = exclusive; this.canFail = canFail; + this.considerRetiring = considerRetiring; this.cloudAccount = Objects.requireNonNull(cloudAccount); + + if (!canFail && considerRetiring) + throw new IllegalArgumentException("Cannot consider retiring nodes if we cannot fail"); } @Override @@ -127,8 +130,7 @@ public interface NodeSpec { @Override public boolean considerRetiring() { - // If we cannot fail we cannot retire as we may end up without sufficient replacement capacity - return canFail(); + return considerRetiring; } @Override @@ -143,7 +145,11 @@ public interface NodeSpec { @Override public NodeSpec fraction(int divisor) { - return new CountNodeSpec(count/divisor, requestedNodeResources, exclusive, canFail, cloudAccount); + return new CountNodeSpec(count/divisor, requestedNodeResources, exclusive, canFail, considerRetiring, cloudAccount); + } + + public NodeSpec withoutRetiring() { + return new CountNodeSpec(count, requestedNodeResources, exclusive, canFail, false, cloudAccount); } @Override @@ -185,7 +191,7 @@ public interface NodeSpec { } - /** A node spec specifying a node type. This will accept all nodes of this type. */ + /** A node spec specifying a node type. */ class TypeNodeSpec implements NodeSpec { private static final Map<NodeType, Integer> WANTED_NODE_COUNT = Map.of(NodeType.config, 3, @@ -256,7 +262,7 @@ public interface NodeSpec { } @Override - public String toString() { return "request for all nodes of type '" + type + "'"; } + public String toString() { return "request for nodes of type '" + type + "'"; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java index d083d81c196..98afc6e7482 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java @@ -75,8 +75,8 @@ public class ProvisionedHost { /** Generate {@link Node} instance representing the node running on this physical host */ public Node generateNode() { return Node.reserve(Set.of(), nodeHostname(), hostHostname, nodeResources, hostType.childNodeType()) - .cloudAccount(cloudAccount) - .build(); + .cloudAccount(cloudAccount) + .build(); } public String getId() { return id; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java index dfe01f5f1c3..bbe287fc034 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java @@ -11,8 +11,10 @@ import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.WireguardKey; +import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.ObjectTraverser; +import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import com.yahoo.slime.Type; import com.yahoo.vespa.hosted.provision.LockedNodeList; @@ -40,6 +42,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.TreeSet; +import java.util.stream.Stream; import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast; import static com.yahoo.config.provision.NodeResources.DiskSpeed.slow; @@ -54,9 +57,13 @@ import static com.yahoo.config.provision.NodeResources.StorageType.remote; */ public class NodePatcher { + // Same as in DropDocumentsReport.java + private static final String DROP_DOCUMENTS_REPORT = "dropDocuments"; + private static final String WANT_TO_RETIRE = "wantToRetire"; private static final String WANT_TO_DEPROVISION = "wantToDeprovision"; private static final String WANT_TO_REBUILD = "wantToRebuild"; + private static final String REPORTS = "reports"; private static final Set<String> RECURSIVE_FIELDS = Set.of(WANT_TO_RETIRE, WANT_TO_DEPROVISION); private static final Set<String> IP_CONFIG_FIELDS = Set.of("ipAddresses", "additionalIpAddresses", @@ -133,7 +140,29 @@ public class NodePatcher { throw new IllegalArgumentException("Could not set field '" + name + "'", e); } } - nodeRepository.nodes().write(node, lock); + List<Node> nodes = List.of(node); + if (node.state() == Node.State.active && isInDocumentsDroppedState(root.field(REPORTS).field(DROP_DOCUMENTS_REPORT))) { + NodeList clusterNodes = nodeRepository.nodes() + .list(Node.State.active) + .except(node) + .owner(node.allocation().get().owner()) + .cluster(node.allocation().get().membership().cluster().id()); + boolean allNodesDroppedDocuments = clusterNodes.stream().allMatch(cNode -> + cNode.reports().getReport(DROP_DOCUMENTS_REPORT).map(report -> isInDocumentsDroppedState(report.getInspector())).orElse(false)); + if (allNodesDroppedDocuments) { + nodes = Stream.concat(nodes.stream(), clusterNodes.stream()) + .map(cNode -> { + Cursor reportRoot = new Slime().setObject(); + Report report = cNode.reports().getReport(DROP_DOCUMENTS_REPORT).get(); + report.toSlime(reportRoot); + reportRoot.setLong("readiedAt", clock.millis()); + + return cNode.with(cNode.reports().withReport(Report.fromSlime(DROP_DOCUMENTS_REPORT, reportRoot))); + }) + .toList(); + } + } + nodeRepository.nodes().write(nodes, lock); } } @@ -202,18 +231,15 @@ public class NodePatcher { .orElseGet(node.status()::wantToRebuild), Agent.operator, clock.instant()); - case "reports" : + case REPORTS: return nodeWithPatchedReports(node, value); - case "id" : + case "id": return node.withId(asString(value)); case "diskGb": - case "minDiskAvailableGb": return node.with(node.flavor().with(node.flavor().resources().withDiskGb(value.asDouble())), Agent.operator, clock.instant()); case "memoryGb": - case "minMainMemoryAvailableGb": return node.with(node.flavor().with(node.flavor().resources().withMemoryGb(value.asDouble())), Agent.operator, clock.instant()); case "vcpu": - case "minCpuCores": return node.with(node.flavor().with(node.flavor().resources().withVcpu(value.asDouble())), Agent.operator, clock.instant()); case "fastDisk": return node.with(node.flavor().with(node.flavor().resources().with(value.asBool() ? fast : slow)), Agent.operator, clock.instant()); @@ -244,18 +270,12 @@ public class NodePatcher { } private Node applyIpconfigField(Node node, String name, Inspector value, LockedNodeList nodes) { - switch (name) { - case "ipAddresses" -> { - return IP.Config.verify(node.with(node.ipConfig().withPrimary(asStringSet(value))), nodes); - } - case "additionalIpAddresses" -> { - return IP.Config.verify(node.with(node.ipConfig().withPool(node.ipConfig().pool().withIpAddresses(asStringSet(value)))), nodes); - } - case "additionalHostnames" -> { - return IP.Config.verify(node.with(node.ipConfig().withPool(node.ipConfig().pool().withHostnames(asHostnames(value)))), nodes); - } - } - throw new IllegalArgumentException("Could not apply field '" + name + "' on a node: No such modifiable field"); + return switch (name) { + case "ipAddresses" -> IP.Config.verify(node.with(node.ipConfig().withPrimary(asStringSet(value))), nodes); + case "additionalIpAddresses" -> IP.Config.verify(node.with(node.ipConfig().withPool(node.ipConfig().pool().withIpAddresses(asStringSet(value)))), nodes); + case "additionalHostnames" -> IP.Config.verify(node.with(node.ipConfig().withPool(node.ipConfig().pool().withHostnames(asHostnames(value)))), nodes); + default -> throw new IllegalArgumentException("Could not apply field '" + name + "' on a node: No such modifiable field"); + }; } private Node nodeWithPatchedReports(Node node, Inspector reportsInspector) { @@ -374,4 +394,9 @@ public class NodePatcher { return Optional.of(field).filter(Inspector::valid).map(this::asBoolean); } + private static boolean isInDocumentsDroppedState(Inspector report) { + if (!report.valid()) return false; + return report.field("droppedAt").valid() && !report.field("readiedAt").valid(); + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java index 15126ed5845..06ce08df8c3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; +import ai.vespa.http.DomainName; import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.HostName; import com.yahoo.vespa.service.monitor.DuperModelInfraApi; import com.yahoo.vespa.service.monitor.InfraApplicationApi; @@ -20,7 +20,7 @@ import java.util.concurrent.ConcurrentHashMap; public class MockDuperModel implements DuperModelInfraApi { private final Map<ApplicationId, InfraApplicationApi> supportedInfraApps = new HashMap<>(); - private final ConcurrentHashMap<ApplicationId, List<HostName>> activeApps = new ConcurrentHashMap<>(); + private final ConcurrentHashMap<ApplicationId, List<DomainName>> activeApps = new ConcurrentHashMap<>(); @Inject public MockDuperModel() { @@ -46,12 +46,12 @@ public class MockDuperModel implements DuperModelInfraApi { return activeApps.containsKey(applicationId); } - public List<HostName> hostnamesOf(ApplicationId applicationId) { + public List<DomainName> hostnamesOf(ApplicationId applicationId) { return activeApps.getOrDefault(applicationId, List.of()); } @Override - public void infraApplicationActivated(ApplicationId applicationId, List<HostName> hostnames) { + public void infraApplicationActivated(ApplicationId applicationId, List<DomainName> hostnames) { activeApps.put(applicationId, hostnames); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java index 491485b78fc..c63be6d5dc5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.applicationmodel.ServiceInstance; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.hosted.provision.Node; @@ -627,6 +628,49 @@ public class NodeFailerTest { assertFalse(badNode(1, 3, 1, 2)); } + @Test + public void nodes_undergoing_cmr_are_not_failed() { + var tester = NodeFailTester.withTwoApplications(6); + var clock = tester.clock; + var slime = SlimeUtils.jsonToSlime( + String.format(""" + { + "upcoming":[{ + "id": "id-42", + "status": "some-status", + "plannedStartTime": %d, + "plannedEndTime": %d + }] + } + """, clock.instant().getEpochSecond(), clock.instant().plus(Duration.ofMinutes(90)).getEpochSecond()) + ); + var cmrReport = Report.fromSlime("vcmr", slime.get()); + var downHost = tester.nodeRepository.nodes().list(Node.State.active).owner(NodeFailTester.app1).asList().get(1).hostname(); + + var node = tester.nodeRepository.nodes().node(downHost).get(); + tester.nodeRepository.nodes().write(node.with(node.reports().withReport(cmrReport)), () -> {}); + + tester.serviceMonitor.setHostDown(downHost); + tester.runMaintainers(); + node = tester.nodeRepository.nodes().node(downHost).get(); + assertTrue(node.isDown()); + assertEquals(Node.State.active, node.state()); + + // CMR still ongoing, don't fail yet + clock.advance(Duration.ofHours(1)); + tester.runMaintainers(); + node = tester.nodeRepository.nodes().node(downHost).get(); + assertTrue(node.isDown()); + assertEquals(Node.State.active, node.state()); + + // No ongoing CMR anymore, host should be failed + clock.advance(Duration.ofHours(1)); + tester.runMaintainers(); + node = tester.nodeRepository.nodes().node(downHost).get(); + assertTrue(node.isDown()); + assertEquals(Node.State.failed, node.state()); + } + private void addServiceInstances(List<ServiceInstance> list, ServiceStatus status, int num) { for (int i = 0; i < num; ++i) { ServiceInstance service = mock(ServiceInstance.class); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java index c7c6e770fe3..611594cc72e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java @@ -44,7 +44,7 @@ public class NodeMetricsDbMaintainerTest { fetcher, Duration.ofHours(1), new TestMetric()); - assertEquals(maintainer.maintain(), 1.0, 0.0000001); + assertEquals(maintainer.maintain(), 0.0, 0.0000001); List<NodeTimeseries> timeseriesList = tester.nodeRepository().metricsDb().getNodeTimeseries(Duration.ofDays(1), Set.of("host-1.yahoo.com", "host-2.yahoo.com")); assertEquals(2, timeseriesList.size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java index 58f4be18992..aaa4bffc000 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import ai.vespa.http.DomainName; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; @@ -268,7 +269,7 @@ public class RetiredExpirerTest { private Set<String> configServerHostnames(MockDuperModel duperModel) { return duperModel.hostnamesOf(new ConfigServerApplication().getApplicationId()).stream() - .map(com.yahoo.config.provision.HostName::value) + .map(DomainName::value) .collect(Collectors.toSet()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java index 9cd5adef5f4..7763459dd92 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; +import ai.vespa.http.DomainName; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Provisioner; @@ -133,7 +133,7 @@ public class InfraDeployerImplTest { @SuppressWarnings("unchecked") private void verifyActivated(String... hostnames) { verify(duperModelInfraApi).infraApplicationActivated( - eq(application.getApplicationId()), eq(Stream.of(hostnames).map(HostName::of).toList())); + eq(application.getApplicationId()), eq(Stream.of(hostnames).map(DomainName::of).toList())); ArgumentMatcher<ApplicationTransaction> transactionMatcher = t -> { assertEquals(application.getApplicationId(), t.application()); return true; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 978edf3f7e4..2cd0e84c356 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -757,6 +757,23 @@ public class ProvisioningTest { } @Test + public void ignore_retirement_if_no_capacity() { + ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); + tester.makeReadyHosts(3, defaultResources).activateTenantHosts(); + + ApplicationId application = ProvisioningTester.applicationId(); + ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("music")).vespaVersion("4.5.6").build(); + tester.activate(application, tester.prepare(application, cluster, 3, 1, defaultResources)); + + // Mark the nodes as want to retire + NodeList nodes = tester.getNodes(application); + tester.patchNodes(nodes.asList(), node -> node.withWantToRetire(true, Agent.system, tester.clock().instant())); + + tester.activate(application, tester.prepare(application, cluster, 3, 1, defaultResources)); + assertEquals(3, tester.getNodes(application).state(Node.State.active).not().retired().size()); + } + + @Test public void highest_node_indexes_are_retired_first() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).build(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java index fb773f19b8a..0744d82c85b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java @@ -34,7 +34,6 @@ import java.util.stream.Stream; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -171,7 +170,7 @@ public class VirtualNodeProvisioningTest { @Test public void indistinct_distribution_with_known_ready_nodes() { ProvisioningTester tester = new ProvisioningTester.Builder().build(); - tester.makeReadyChildren(3, resources1); + tester.makeReadyChildren(4, resources1); int contentNodeCount = 3; int groups = 1; @@ -191,13 +190,10 @@ public class VirtualNodeProvisioningTest { tester.makeReadyChildren(1, resources1, "parentHost1"); tester.makeReadyChildren(2, resources1, "parentHost2"); - NodeAllocationException expectedException = null; - try { - tester.prepare(applicationId, contentClusterSpec, contentNodeCount, groups, resources1); - } catch (NodeAllocationException e) { - expectedException = e; - } - assertNotNull(expectedException); + tester.activate(applicationId, tester.prepare(applicationId, contentClusterSpec, contentNodeCount, groups, resources1)); + nodes = tester.getNodes(applicationId, Node.State.active); + assertEquals(4, nodes.size()); + assertEquals(1, nodes.retired().size()); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index c9e57c22d11..7affcfebdb3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -647,6 +647,22 @@ public class NodesV2ApiTest { Request.Method.PATCH), "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-reports-4.json"); + + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com", + Utf8.toBytes("{\"reports\": {\"dropDocuments\":{\"createdMillis\":25,\"droppedAt\":36}}}"), + Request.Method.PATCH), + "{\"message\":\"Updated host1.yahoo.com\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com"), + "{\"dropDocuments\":{\"createdMillis\":25,\"droppedAt\":36}}"); + + assertResponse(new Request("http://localhost:8080/nodes/v2/node/host10.yahoo.com", + Utf8.toBytes("{\"reports\": {\"dropDocuments\":{\"createdMillis\":49,\"droppedAt\":456}}}"), + Request.Method.PATCH), + "{\"message\":\"Updated host10.yahoo.com\"}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host10.yahoo.com"), + "{\"dropDocuments\":{\"createdMillis\":49,\"droppedAt\":456,\"readiedAt\":123}}"); + tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com"), + "{\"dropDocuments\":{\"createdMillis\":25,\"droppedAt\":36,\"readiedAt\":123}}"); } @Test @@ -906,13 +922,13 @@ public class NodesV2ApiTest { // Test patching with overrides tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node/" + host, - "{\"minDiskAvailableGb\":5432,\"minMainMemoryAvailableGb\":2345}".getBytes(StandardCharsets.UTF_8), + "{\"diskGb\":5432,\"memoryGb\":2345}".getBytes(StandardCharsets.UTF_8), Request.Method.PATCH), 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'minMainMemoryAvailableGb': Can only override disk GB for configured flavor\"}"); + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not set field 'memoryGb': Can only override disk GB for configured flavor\"}"); assertResponse(new Request("http://localhost:8080/nodes/v2/node/" + host, - "{\"minDiskAvailableGb\":5432}".getBytes(StandardCharsets.UTF_8), + "{\"diskGb\":5432}".getBytes(StandardCharsets.UTF_8), Request.Method.PATCH), "{\"message\":\"Updated " + host + "\"}"); tester.assertResponseContains(new Request("http://localhost:8080/nodes/v2/node/" + host), diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java index 323ae678b0b..73774321ffb 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator; +import ai.vespa.http.DomainName; import com.yahoo.config.model.api.SuperModel; import com.yahoo.config.model.api.SuperModelListener; import com.yahoo.config.model.api.SuperModelProvider; @@ -87,9 +88,9 @@ public class OrchestratorTest { // There is one config server application with 3 nodes ApplicationId applicationId = new ConfigServerApplication().getApplicationId(); - var cfg1 = com.yahoo.config.provision.HostName.of("cfg1"); - var cfg2 = com.yahoo.config.provision.HostName.of("cfg2"); - var cfg3 = com.yahoo.config.provision.HostName.of("cfg3"); + var cfg1 = DomainName.of("cfg1"); + var cfg2 = DomainName.of("cfg2"); + var cfg3 = DomainName.of("cfg3"); duperModelManager.infraApplicationActivated(applicationId, List.of(cfg1, cfg2, cfg3)); duperModelManager.infraApplicationsIsNowComplete(); @@ -135,7 +136,7 @@ public class OrchestratorTest { // etc (should be the same as for cfg1) } - private HostName toApplicationModelHostName(com.yahoo.config.provision.HostName hostname) { + private HostName toApplicationModelHostName(DomainName hostname) { return new HostName(hostname.value()); } diff --git a/parent/pom.xml b/parent/pom.xml index ace6a47d6bc..8d2f802e34b 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -1166,7 +1166,7 @@ <netty.version>4.1.86.Final</netty.version> <netty-tcnative.version>2.0.54.Final</netty-tcnative.version> <onnxruntime.version>1.13.1</onnxruntime.version> <!-- WARNING: sync cloud-tenant-base-dependencies-enforcer/pom.xml --> - <org.json.version>20220320</org.json.version> + <org.json.version>20230227</org.json.version> <org.lz4.version>1.8.0</org.lz4.version> <prometheus.client.version>0.6.0</prometheus.client.version> <protobuf.version>3.21.7</protobuf.version> diff --git a/predicate-search/src/main/java/com/yahoo/search/predicate/benchmarks/ResultMetrics.java b/predicate-search/src/main/java/com/yahoo/search/predicate/benchmarks/ResultMetrics.java index 11103a2a66a..ef65d9e2efa 100644 --- a/predicate-search/src/main/java/com/yahoo/search/predicate/benchmarks/ResultMetrics.java +++ b/predicate-search/src/main/java/com/yahoo/search/predicate/benchmarks/ResultMetrics.java @@ -62,8 +62,8 @@ public class ResultMetrics { } private double percentile(double percentile) { - int targetCount = (int) Math.round(totalQueries * percentile); - int currentCount = 0; + long targetCount = Math.round(totalQueries * percentile); + long currentCount = 0; int index = 0; while (currentCount < targetCount && index < SLOTS) { currentCount += latencyHistogram[index]; diff --git a/predicate-search/src/main/java/com/yahoo/search/predicate/index/CachedPostingListCounter.java b/predicate-search/src/main/java/com/yahoo/search/predicate/index/CachedPostingListCounter.java index e6db1dec7c3..eb8b0b9927b 100644 --- a/predicate-search/src/main/java/com/yahoo/search/predicate/index/CachedPostingListCounter.java +++ b/predicate-search/src/main/java/com/yahoo/search/predicate/index/CachedPostingListCounter.java @@ -75,7 +75,7 @@ public class CachedPostingListCounter { private void countUsingBitVector(byte[] nPostingListsForDocument, int postingListBitmap) { for (int docId = 0; docId < nDocuments; docId++) { - nPostingListsForDocument[docId] += Integer.bitCount(bitVector[docId] & postingListBitmap); + nPostingListsForDocument[docId] += (byte)Integer.bitCount(bitVector[docId] & postingListBitmap); } } @@ -88,8 +88,7 @@ public class CachedPostingListCounter { } public CachedPostingListCounter rebuildCache() { - MinMaxPriorityQueue<Entry> mostExpensive = MinMaxPriorityQueue - .maximumSize(32).expectedSize(32).create(); + MinMaxPriorityQueue<Entry> mostExpensive = MinMaxPriorityQueue.maximumSize(32).expectedSize(32).create(); synchronized (this) { for (ObjectLongPair<int[]> p : frequency.keyValuesView()) { mostExpensive.add(new Entry(p.getOne(), p.getTwo())); diff --git a/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp b/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp index 52b7a7ce60a..d2798c16065 100644 --- a/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_initializer/attribute_initializer_test.cpp @@ -7,8 +7,11 @@ #include <vespa/searchcore/proton/test/attribute_utils.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchlib/attribute/integerbase.h> +#include <vespa/searchlib/attribute/stringbase.h> #include <vespa/searchlib/test/directory_handler.h> #include <vespa/searchcommon/attribute/config.h> +#include <vespa/searchcommon/attribute/i_multi_value_attribute.h> +#include <vespa/vespalib/util/stash.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/testkit/testapp.h> @@ -18,8 +21,10 @@ LOG_SETUP("attribute_initializer_test"); using search::attribute::Config; using search::attribute::BasicType; using search::attribute::CollectionType; +using search::attribute::IMultiValueAttribute; using search::SerialNum; using search::test::DirectoryHandler; +using vespalib::Stash; const vespalib::string test_dir = "test_output"; @@ -60,7 +65,7 @@ Config get_int32_wset_fs() } void -saveAttr(const vespalib::string &name, const Config &cfg, SerialNum serialNum, SerialNum createSerialNum) +saveAttr(const vespalib::string &name, const Config &cfg, SerialNum serialNum, SerialNum createSerialNum, bool mutate_reserved_doc = false) { auto diskLayout = AttributeDiskLayout::create(test_dir); auto dir = diskLayout->createAttributeDir(name); @@ -81,6 +86,15 @@ saveAttr(const vespalib::string &name, const Config &cfg, SerialNum serialNum, S iav.append(docId, 10, 1); iav.append(docId, 11, 1); } + if (mutate_reserved_doc) { + av->clearDoc(0u); + if (cfg.basicType().type() == BasicType::Type::STRING && + cfg.collectionType().type() == CollectionType::Type::WSET) { + auto &sav = dynamic_cast<search::StringAttribute &>(*av); + sav.append(0u, "badly", 15); + sav.append(0u, "broken", 20); + } + } av->save(); writer->markValidSnapshot(serialNum); } @@ -250,6 +264,22 @@ TEST("require that saved attribute is ignored when serial num is not set") EXPECT_EQUAL(1u, av->getNumDocs()); } +TEST("require that reserved document is reinitialized during load") +{ + saveAttr("a", string_wset, 10, 2, true); + Fixture f; + auto av = f.createInitializer({"a", string_wset}, 5)->init().getAttribute(); + EXPECT_EQUAL(2u, av->getCreateSerialNum()); + EXPECT_EQUAL(2u, av->getNumDocs()); + auto mvav = av->as_multi_value_attribute(); + ASSERT_TRUE(mvav != nullptr); + Stash stash; + auto read_view = mvav->make_read_view(IMultiValueAttribute::WeightedSetTag<const char*>(), stash); + ASSERT_TRUE(read_view != nullptr); + auto reserved_values = read_view->get_values(0u); + EXPECT_EQUAL(0u, reserved_values.size()); +} + } TEST_MAIN() diff --git a/searchcore/src/tests/proton/common/attribute_updater/CMakeLists.txt b/searchcore/src/tests/proton/common/attribute_updater/CMakeLists.txt index 5955ee52756..6e285423931 100644 --- a/searchcore/src/tests/proton/common/attribute_updater/CMakeLists.txt +++ b/searchcore/src/tests/proton/common/attribute_updater/CMakeLists.txt @@ -4,5 +4,6 @@ vespa_add_executable(searchcore_attribute_updater_test_app TEST attribute_updater_test.cpp DEPENDS searchcore_pcommon + searchlib_test ) vespa_add_test(NAME searchcore_attribute_updater_test_app COMMAND searchcore_attribute_updater_test_app) diff --git a/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp b/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp index 037bf884e6b..892be2c874f 100644 --- a/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp +++ b/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp @@ -6,6 +6,7 @@ #include <vespa/searchlib/attribute/single_raw_attribute.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> #include <vespa/searchlib/tensor/serialized_fast_value_attribute.h> +#include <vespa/searchlib/test/attribute_builder.h> #include <vespa/searchlib/test/weighted_type_test_utils.h> #include <vespa/searchcommon/attribute/config.h> #include <vespa/document/base/testdocrepo.h> @@ -53,6 +54,7 @@ using search::attribute::Config; using search::attribute::Reference; using search::attribute::ReferenceAttribute; using search::attribute::SingleRawAttribute; +using search::attribute::test::AttributeBuilder; using search::tensor::ITensorAttribute; using search::tensor::DenseTensorAttribute; using search::tensor::SerializedFastValueAttribute; @@ -124,19 +126,19 @@ struct Fixture { void applyArrayUpdates(AttributeVector & vec, std::unique_ptr<FieldValue> assign, std::unique_ptr<FieldValue> first, std::unique_ptr<FieldValue> second) { - applyValueUpdate(vec, 0, std::make_unique<AssignValueUpdate>(std::move(assign))); - applyValueUpdate(vec, 1, std::make_unique<AddValueUpdate>(std::move(second))); - applyValueUpdate(vec, 2, std::make_unique<RemoveValueUpdate>(std::move(first))); - applyValueUpdate(vec, 3, std::make_unique<ClearValueUpdate>()); + applyValueUpdate(vec, 1, std::make_unique<AssignValueUpdate>(std::move(assign))); + applyValueUpdate(vec, 2, std::make_unique<AddValueUpdate>(std::move(second))); + applyValueUpdate(vec, 3, std::make_unique<RemoveValueUpdate>(std::move(first))); + applyValueUpdate(vec, 4, std::make_unique<ClearValueUpdate>()); } void applyWeightedSetUpdates(AttributeVector & vec, std::unique_ptr<FieldValue> assign, std::unique_ptr<FieldValue> first, std::unique_ptr<FieldValue> copyOfFirst, std::unique_ptr<FieldValue> second) { - applyValueUpdate(vec, 0, std::make_unique<AssignValueUpdate>(std::move(assign))); - applyValueUpdate(vec, 1, std::make_unique<AddValueUpdate>(std::move(second), 20)); - applyValueUpdate(vec, 2, std::make_unique<RemoveValueUpdate>(std::move(first))); - applyValueUpdate(vec, 3, std::make_unique<ClearValueUpdate>()); - applyValueUpdate(vec, 4, std::make_unique<MapValueUpdate>(std::move(copyOfFirst), std::make_unique<ArithmeticValueUpdate>(ArithmeticValueUpdate::Add, 10))); + applyValueUpdate(vec, 1, std::make_unique<AssignValueUpdate>(std::move(assign))); + applyValueUpdate(vec, 2, std::make_unique<AddValueUpdate>(std::move(second), 20)); + applyValueUpdate(vec, 3, std::make_unique<RemoveValueUpdate>(std::move(first))); + applyValueUpdate(vec, 4, std::make_unique<ClearValueUpdate>()); + applyValueUpdate(vec, 5, std::make_unique<MapValueUpdate>(std::move(copyOfFirst), std::make_unique<ArithmeticValueUpdate>(ArithmeticValueUpdate::Add, 10))); } void applyValue(AttributeVector& vec, uint32_t docid, std::unique_ptr<FieldValue> value) { @@ -144,35 +146,6 @@ struct Fixture { } }; -template <typename T, typename VectorType> -AttributePtr -create(uint32_t numDocs, T val, int32_t weight, - const std::string & baseName, - const Config &info) -{ - LOG(info, "create attribute vector: %s", baseName.c_str()); - AttributePtr vec = AttributeFactory::createAttribute(baseName, info); - VectorType * api = static_cast<VectorType *>(vec.get()); - for (uint32_t i = 0; i < numDocs; ++i) { - if (!api->addDoc(i)) { - LOG(info, "failed adding doc: %u", i); - return AttributePtr(); - } - if (api->hasMultiValue()) { - if (!api->append(i, val, weight)) { - LOG(info, "failed append to doc: %u", i); - } - } else { - if (!api->update(i, val)) { - LOG(info, "failed update doc: %u", i); - return AttributePtr(); - } - } - } - api->commit(); - return vec; -} - template <typename T> bool check(const AttributePtr &vec, uint32_t docId, const std::vector<T> &values) @@ -223,70 +196,67 @@ TEST_F("require that single attributes are updated", Fixture) using search::attribute::getUndefined; CollectionType ct(CollectionType::SINGLE); { - BasicType bt(BasicType::INT32); - AttributePtr vec = create<int32_t, IntegerAttribute>(3, 32, 0, "in1/int", Config(bt, ct)); - f.applyValueUpdate(*vec, 0, std::make_unique<AssignValueUpdate>(std::make_unique<IntFieldValue>(64))); - f.applyValueUpdate(*vec, 1, std::make_unique<ArithmeticValueUpdate>(ArithmeticValueUpdate::Add, 10)); - f.applyValueUpdate(*vec, 2, std::make_unique<ClearValueUpdate>()); - EXPECT_EQUAL(3u, vec->getNumDocs()); - EXPECT_TRUE(check(vec, 0, std::vector<WeightedInt>{WeightedInt(64)})); - EXPECT_TRUE(check(vec, 1, std::vector<WeightedInt>{WeightedInt(42)})); - EXPECT_TRUE(check(vec, 2, std::vector<WeightedInt>{WeightedInt(getUndefined<int32_t>())})); + auto vec = AttributeBuilder("in1/int", Config(BasicType::INT32)).fill({ 32, 32, 32}).get(); + f.applyValueUpdate(*vec, 1, std::make_unique<AssignValueUpdate>(std::make_unique<IntFieldValue>(64))); + f.applyValueUpdate(*vec, 2, std::make_unique<ArithmeticValueUpdate>(ArithmeticValueUpdate::Add, 10)); + f.applyValueUpdate(*vec, 3, std::make_unique<ClearValueUpdate>()); + EXPECT_EQUAL(4u, vec->getNumDocs()); + EXPECT_TRUE(check(vec, 1, std::vector<WeightedInt>{WeightedInt(64)})); + EXPECT_TRUE(check(vec, 2, std::vector<WeightedInt>{WeightedInt(42)})); + EXPECT_TRUE(check(vec, 3, std::vector<WeightedInt>{WeightedInt(getUndefined<int32_t>())})); } { - BasicType bt(BasicType::FLOAT); - AttributePtr vec = create<float, FloatingPointAttribute>(3, 55.5f, 0, "in1/float",Config(bt, ct)); - f.applyValueUpdate(*vec, 0, std::make_unique<AssignValueUpdate>(std::make_unique<FloatFieldValue>(77.7f))); - f.applyValueUpdate(*vec, 1, std::make_unique<ArithmeticValueUpdate>(ArithmeticValueUpdate::Add, 10)); - f.applyValueUpdate(*vec, 2, std::make_unique<ClearValueUpdate>()); - EXPECT_EQUAL(3u, vec->getNumDocs()); - EXPECT_TRUE(check(vec, 0, std::vector<WeightedFloat>{WeightedFloat(77.7f)})); - EXPECT_TRUE(check(vec, 1, std::vector<WeightedFloat>{WeightedFloat(65.5f)})); - EXPECT_TRUE(std::isnan(vec->getFloat(2))); + auto vec = AttributeBuilder("in1/float", Config(BasicType::FLOAT)).fill({55.5, 55.5, 55.5}).get(); + f.applyValueUpdate(*vec, 1, std::make_unique<AssignValueUpdate>(std::make_unique<FloatFieldValue>(77.7f))); + f.applyValueUpdate(*vec, 2, std::make_unique<ArithmeticValueUpdate>(ArithmeticValueUpdate::Add, 10)); + f.applyValueUpdate(*vec, 3, std::make_unique<ClearValueUpdate>()); + EXPECT_EQUAL(4u, vec->getNumDocs()); + EXPECT_TRUE(check(vec, 1, std::vector<WeightedFloat>{WeightedFloat(77.7f)})); + EXPECT_TRUE(check(vec, 2, std::vector<WeightedFloat>{WeightedFloat(65.5f)})); + EXPECT_TRUE(std::isnan(vec->getFloat(3))); } { - BasicType bt(BasicType::STRING); - AttributePtr vec = create<std::string, StringAttribute>(3, "first", 0, "in1/string",Config(bt, ct)); - f.applyValueUpdate(*vec, 0, std::make_unique<AssignValueUpdate>(StringFieldValue::make("second"))); - f.applyValueUpdate(*vec, 2, std::make_unique<ClearValueUpdate>()); - EXPECT_EQUAL(3u, vec->getNumDocs()); - EXPECT_TRUE(check(vec, 0, std::vector<WeightedString>{WeightedString("second")})); - EXPECT_TRUE(check(vec, 1, std::vector<WeightedString>{WeightedString("first")})); - EXPECT_TRUE(check(vec, 2, std::vector<WeightedString>{WeightedString("")})); + auto vec = AttributeBuilder("in1/string", Config(BasicType::STRING)).fill({"first", "first", "first"}).get(); + f.applyValueUpdate(*vec, 1, std::make_unique<AssignValueUpdate>(StringFieldValue::make("second"))); + f.applyValueUpdate(*vec, 3, std::make_unique<ClearValueUpdate>()); + EXPECT_EQUAL(4u, vec->getNumDocs()); + EXPECT_TRUE(check(vec, 1, std::vector<WeightedString>{WeightedString("second")})); + EXPECT_TRUE(check(vec, 2, std::vector<WeightedString>{WeightedString("first")})); + EXPECT_TRUE(check(vec, 3, std::vector<WeightedString>{WeightedString("")})); } { BasicType bt(BasicType::REFERENCE); Config cfg(bt, ct); AttributePtr vec = AttributeFactory::createAttribute("in1/ref", cfg); + vec->addReservedDoc(); uint32_t startDoc = 0; uint32_t endDoc = 0; EXPECT_TRUE(vec->addDocs(startDoc, endDoc, 3)); - EXPECT_EQUAL(0u, startDoc); - EXPECT_EQUAL(2u, endDoc); - for (uint32_t docId = 0; docId < 3; ++docId) { + EXPECT_EQUAL(1u, startDoc); + EXPECT_EQUAL(3u, endDoc); + for (uint32_t docId = 1; docId < 4; ++docId) { asReferenceAttribute(*vec).update(docId, toGid(doc1)); } vec->commit(); - f.applyValueUpdate(*vec, 0, std::make_unique<AssignValueUpdate>(std::make_unique<ReferenceFieldValue>(dynamic_cast<const ReferenceDataType &>(f.docType->getField("ref").getDataType()), DocumentId(doc2)))); - f.applyValueUpdate(*vec, 2, std::make_unique<ClearValueUpdate>()); - EXPECT_EQUAL(3u, vec->getNumDocs()); - TEST_DO(assertRef(*vec, doc2, 0)); - TEST_DO(assertRef(*vec, doc1, 1)); - TEST_DO(assertNoRef(*vec, 2)); + f.applyValueUpdate(*vec, 1, std::make_unique<AssignValueUpdate>(std::make_unique<ReferenceFieldValue>(dynamic_cast<const ReferenceDataType &>(f.docType->getField("ref").getDataType()), DocumentId(doc2)))); + f.applyValueUpdate(*vec, 3, std::make_unique<ClearValueUpdate>()); + EXPECT_EQUAL(4u, vec->getNumDocs()); + TEST_DO(assertRef(*vec, doc2, 1)); + TEST_DO(assertRef(*vec, doc1, 2)); + TEST_DO(assertNoRef(*vec, 3)); } { - BasicType bt(BasicType::RAW); vespalib::string first_backing("first"); vespalib::ConstArrayRef<char> first(first_backing.data(), first_backing.size()); - AttributePtr vec = create<vespalib::ConstArrayRef<char>, SingleRawAttribute>(4, first, 0, "in1/raw", Config(bt, ct)); - f.applyValueUpdate(*vec, 0, std::make_unique<AssignValueUpdate>(std::make_unique<RawFieldValue>("second"))); - f.applyValueUpdate(*vec, 2, std::make_unique<ClearValueUpdate>()); - f.applyValue(*vec, 3, std::make_unique<RawFieldValue>("third")); - EXPECT_EQUAL(4u, vec->getNumDocs()); - EXPECT_EQUAL(as_vector("second"), as_vector(vec->get_raw(0))); - EXPECT_EQUAL(as_vector("first"), as_vector(vec->get_raw(1))); - EXPECT_EQUAL(as_vector(""), as_vector(vec->get_raw(2))); - EXPECT_EQUAL(as_vector("third"), as_vector(vec->get_raw(3))); + auto vec = AttributeBuilder("in1/raw", Config(BasicType::RAW)).fill({first, first, first, first}).get(); + f.applyValueUpdate(*vec, 1, std::make_unique<AssignValueUpdate>(std::make_unique<RawFieldValue>("second"))); + f.applyValueUpdate(*vec, 3, std::make_unique<ClearValueUpdate>()); + f.applyValue(*vec, 4, std::make_unique<RawFieldValue>("third")); + EXPECT_EQUAL(5u, vec->getNumDocs()); + EXPECT_EQUAL(as_vector("second"), as_vector(vec->get_raw(1))); + EXPECT_EQUAL(as_vector("first"), as_vector(vec->get_raw(2))); + EXPECT_EQUAL(as_vector(""), as_vector(vec->get_raw(3))); + EXPECT_EQUAL(as_vector("third"), as_vector(vec->get_raw(4))); } } @@ -294,52 +264,51 @@ TEST_F("require that array attributes are updated", Fixture) { CollectionType ct(CollectionType::ARRAY); { - BasicType bt(BasicType::INT32); - AttributePtr vec = create<int32_t, IntegerAttribute>(5, 32, 1, "in1/aint", Config(bt, ct)); + using IL = AttributeBuilder::IntList; + auto vec = AttributeBuilder("in1/aint", Config(BasicType::INT32, ct)).fill_array({IL{32}, {32}, {32}, {32}, {32}}).get(); auto first = std::make_unique<IntFieldValue>(32); auto second = std::make_unique<IntFieldValue>(64); auto assign = std::make_unique<ArrayFieldValue>(f.docType->getField("aint").getDataType()); assign->add(*second); f.applyArrayUpdates(*vec, std::move(assign), std::move(first), std::move(second)); - EXPECT_EQUAL(5u, vec->getNumDocs()); - EXPECT_TRUE(check(vec, 0, std::vector<WeightedInt>{WeightedInt(64)})); - EXPECT_TRUE(check(vec, 1, std::vector<WeightedInt>{WeightedInt(32), WeightedInt(64)})); - EXPECT_TRUE(check(vec, 2, std::vector<WeightedInt>{})); + EXPECT_EQUAL(6u, vec->getNumDocs()); + EXPECT_TRUE(check(vec, 1, std::vector<WeightedInt>{WeightedInt(64)})); + EXPECT_TRUE(check(vec, 2, std::vector<WeightedInt>{WeightedInt(32), WeightedInt(64)})); EXPECT_TRUE(check(vec, 3, std::vector<WeightedInt>{})); - EXPECT_TRUE(check(vec, 4, std::vector<WeightedInt>{WeightedInt(32)})); + EXPECT_TRUE(check(vec, 4, std::vector<WeightedInt>{})); + EXPECT_TRUE(check(vec, 5, std::vector<WeightedInt>{WeightedInt(32)})); } { - BasicType bt(BasicType::FLOAT); - AttributePtr vec = create<float, FloatingPointAttribute>(5, 55.5f, 1, "in1/afloat", Config(bt, ct)); + using DL = AttributeBuilder::DoubleList; + auto vec = AttributeBuilder("in1/afloat", Config(BasicType::FLOAT, ct)).fill_array({DL{55.5}, {55.5}, {55.5}, {55.5}, {55.5}}).get(); auto first = std::make_unique<FloatFieldValue>(55.5f); auto second = std::make_unique<FloatFieldValue>(77.7f); auto assign = std::make_unique<ArrayFieldValue>(f.docType->getField("afloat").getDataType()); assign->add(*second); f.applyArrayUpdates(*vec, std::move(assign), std::move(first), std::move(second)); - EXPECT_EQUAL(5u, vec->getNumDocs()); - EXPECT_TRUE(check(vec, 0, std::vector<WeightedFloat>{WeightedFloat(77.7f)})); - EXPECT_TRUE(check(vec, 1, std::vector<WeightedFloat>{WeightedFloat(55.5f), WeightedFloat(77.7f)})); - EXPECT_TRUE(check(vec, 2, std::vector<WeightedFloat>{})); + EXPECT_EQUAL(6u, vec->getNumDocs()); + EXPECT_TRUE(check(vec, 1, std::vector<WeightedFloat>{WeightedFloat(77.7f)})); + EXPECT_TRUE(check(vec, 2, std::vector<WeightedFloat>{WeightedFloat(55.5f), WeightedFloat(77.7f)})); EXPECT_TRUE(check(vec, 3, std::vector<WeightedFloat>{})); - EXPECT_TRUE(check(vec, 4, std::vector<WeightedFloat>{WeightedFloat(55.5f)})); + EXPECT_TRUE(check(vec, 4, std::vector<WeightedFloat>{})); + EXPECT_TRUE(check(vec, 5, std::vector<WeightedFloat>{WeightedFloat(55.5f)})); } { - BasicType bt(BasicType::STRING); - AttributePtr vec = create<std::string, StringAttribute>(5, "first", 1, "in1/astring", Config(bt, ct)); + auto vec = AttributeBuilder("in1/astring", Config(BasicType::STRING, ct)).fill_array({{"first"}, {"first"}, {"first"}, {"first"}, {"first"}}).get(); auto first = StringFieldValue::make("first"); auto second = StringFieldValue::make("second"); auto assign = std::make_unique<ArrayFieldValue>(f.docType->getField("astring").getDataType()); assign->add(*second); f.applyArrayUpdates(*vec, std::move(assign), std::move(first), std::move(second)); - EXPECT_EQUAL(5u, vec->getNumDocs()); - EXPECT_TRUE(check(vec, 0, std::vector<WeightedString>{WeightedString("second")})); - EXPECT_TRUE(check(vec, 1, std::vector<WeightedString>{WeightedString("first"), WeightedString("second")})); - EXPECT_TRUE(check(vec, 2, std::vector<WeightedString>{})); + EXPECT_EQUAL(6u, vec->getNumDocs()); + EXPECT_TRUE(check(vec, 1, std::vector<WeightedString>{WeightedString("second")})); + EXPECT_TRUE(check(vec, 2, std::vector<WeightedString>{WeightedString("first"), WeightedString("second")})); EXPECT_TRUE(check(vec, 3, std::vector<WeightedString>{})); - EXPECT_TRUE(check(vec, 4, std::vector<WeightedString>{WeightedString("first")})); + EXPECT_TRUE(check(vec, 4, std::vector<WeightedString>{})); + EXPECT_TRUE(check(vec, 5, std::vector<WeightedString>{WeightedString("first")})); } } @@ -347,8 +316,8 @@ TEST_F("require that weighted set attributes are updated", Fixture) { CollectionType ct(CollectionType::WSET); { - BasicType bt(BasicType::INT32); - AttributePtr vec = create<int32_t, IntegerAttribute>(5, 32, 100, "in1/wsint", Config(bt, ct)); + using WIL = AttributeBuilder::WeightedIntList; + auto vec = AttributeBuilder("in1/wsint", Config(BasicType::INT32, ct)).fill_wset({WIL{{32, 100}}, {{32, 100}}, {{32, 100}}, {{32, 100}}, {{32, 100}}}).get(); auto first = std::make_unique<IntFieldValue>(32); auto copyOfFirst = std::make_unique<IntFieldValue>(32); auto second = std::make_unique<IntFieldValue>(64); @@ -356,16 +325,16 @@ TEST_F("require that weighted set attributes are updated", Fixture) assign->add(*second, 20); f.applyWeightedSetUpdates(*vec, std::move(assign), std::move(first), std::move(copyOfFirst), std::move(second)); - EXPECT_EQUAL(5u, vec->getNumDocs()); - EXPECT_TRUE(check(vec, 0, std::vector<WeightedInt>{WeightedInt(64, 20)})); - EXPECT_TRUE(check(vec, 1, std::vector<WeightedInt>{WeightedInt(32, 100), WeightedInt(64, 20)})); - EXPECT_TRUE(check(vec, 2, std::vector<WeightedInt>{})); + EXPECT_EQUAL(6u, vec->getNumDocs()); + EXPECT_TRUE(check(vec, 1, std::vector<WeightedInt>{WeightedInt(64, 20)})); + EXPECT_TRUE(check(vec, 2, std::vector<WeightedInt>{WeightedInt(32, 100), WeightedInt(64, 20)})); EXPECT_TRUE(check(vec, 3, std::vector<WeightedInt>{})); - EXPECT_TRUE(check(vec, 4, std::vector<WeightedInt>{WeightedInt(32, 110)})); + EXPECT_TRUE(check(vec, 4, std::vector<WeightedInt>{})); + EXPECT_TRUE(check(vec, 5, std::vector<WeightedInt>{WeightedInt(32, 110)})); } { - BasicType bt(BasicType::FLOAT); - AttributePtr vec = create<float, FloatingPointAttribute>(5, 55.5f, 100, "in1/wsfloat", Config(bt, ct)); + using WDL = AttributeBuilder::WeightedDoubleList; + auto vec = AttributeBuilder("in1/wsfloat", Config(BasicType::FLOAT, ct)).fill_wset({WDL{{55.5, 100}}, {{55.5, 100}}, {{55.5, 100}}, {{55.5, 100}}, {{55.5, 100}}}).get(); auto first = std::make_unique<FloatFieldValue>(55.5f); auto copyOfFirst = std::make_unique<FloatFieldValue>(55.5f); auto second = std::make_unique<FloatFieldValue>(77.7f); @@ -373,16 +342,15 @@ TEST_F("require that weighted set attributes are updated", Fixture) assign->add(*second, 20); f.applyWeightedSetUpdates(*vec, std::move(assign), std::move(first), std::move(copyOfFirst), std::move(second)); - EXPECT_EQUAL(5u, vec->getNumDocs()); - EXPECT_TRUE(check(vec, 0, std::vector<WeightedFloat>{WeightedFloat(77.7f, 20)})); - EXPECT_TRUE(check(vec, 1, std::vector<WeightedFloat>{WeightedFloat(55.5f, 100), WeightedFloat(77.7f, 20)})); - EXPECT_TRUE(check(vec, 2, std::vector<WeightedFloat>{})); + EXPECT_EQUAL(6u, vec->getNumDocs()); + EXPECT_TRUE(check(vec, 1, std::vector<WeightedFloat>{WeightedFloat(77.7f, 20)})); + EXPECT_TRUE(check(vec, 2, std::vector<WeightedFloat>{WeightedFloat(55.5f, 100), WeightedFloat(77.7f, 20)})); EXPECT_TRUE(check(vec, 3, std::vector<WeightedFloat>{})); - EXPECT_TRUE(check(vec, 4, std::vector<WeightedFloat>{WeightedFloat(55.5f, 110)})); + EXPECT_TRUE(check(vec, 4, std::vector<WeightedFloat>{})); + EXPECT_TRUE(check(vec, 5, std::vector<WeightedFloat>{WeightedFloat(55.5f, 110)})); } { - BasicType bt(BasicType::STRING); - AttributePtr vec = create<std::string, StringAttribute>(5, "first", 100, "in1/wsstring", Config(bt, ct)); + auto vec = AttributeBuilder("in1/wsstring", Config(BasicType::STRING, ct)).fill_wset({{{"first", 100}}, {{"first", 100}}, {{"first", 100}}, {{"first", 100}}, {{"first", 100}}}).get(); auto first = StringFieldValue::make("first"); auto copyOfFirst = StringFieldValue::make("first"); auto second = StringFieldValue::make("second"); @@ -390,12 +358,12 @@ TEST_F("require that weighted set attributes are updated", Fixture) assign->add(*second, 20); f.applyWeightedSetUpdates(*vec, std::move(assign), std::move(first), std::move(copyOfFirst), std::move(second)); - EXPECT_EQUAL(5u, vec->getNumDocs()); - EXPECT_TRUE(check(vec, 0, std::vector<WeightedString>{WeightedString("second", 20)})); - EXPECT_TRUE(check(vec, 1, std::vector<WeightedString>{WeightedString("first", 100), WeightedString("second", 20)})); - EXPECT_TRUE(check(vec, 2, std::vector<WeightedString>{})); + EXPECT_EQUAL(6u, vec->getNumDocs()); + EXPECT_TRUE(check(vec, 1, std::vector<WeightedString>{WeightedString("second", 20)})); + EXPECT_TRUE(check(vec, 2, std::vector<WeightedString>{WeightedString("first", 100), WeightedString("second", 20)})); EXPECT_TRUE(check(vec, 3, std::vector<WeightedString>{})); - EXPECT_TRUE(check(vec, 4, std::vector<WeightedString>{WeightedString("first", 110)})); + EXPECT_TRUE(check(vec, 4, std::vector<WeightedString>{})); + EXPECT_TRUE(check(vec, 5, std::vector<WeightedString>{WeightedString("first", 110)})); } } diff --git a/searchcore/src/tests/proton/docsummary/docsummary_test.cpp b/searchcore/src/tests/proton/docsummary/docsummary_test.cpp index 020f4ff42c1..70f81d629d5 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary_test.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary_test.cpp @@ -107,7 +107,10 @@ namespace proton { class MockDocsumFieldWriterFactory : public search::docsummary::IDocsumFieldWriterFactory { public: - std::unique_ptr<DocsumFieldWriter> create_docsum_field_writer(const vespalib::string&, const vespalib::string&, const vespalib::string&) override { + std::unique_ptr<DocsumFieldWriter> create_docsum_field_writer(const vespalib::string&, + const vespalib::string&, + const vespalib::string&, + std::shared_ptr<search::MatchingElementsFields>) override { return {}; } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp index d39d2873edb..f6200bb14ee 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp @@ -200,6 +200,7 @@ AttributeInitializer::loadAttribute(const AttributeVectorSP &attr, attr->getBaseFileName().c_str()); return false; } else { + attr->set_reserved_doc_values(); attr->commit(CommitParam(serialNum)); EventLogger::loadAttributeComplete(_documentSubDbName, attr->getName(), timer.elapsed()); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int16ResultNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int16ResultNode.java index ae7d0a67b2f..b0f98685578 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int16ResultNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int16ResultNode.java @@ -80,7 +80,7 @@ public class Int16ResultNode extends NumericResultNode { @Override public void add(ResultNode rhs) { - value += rhs.getInteger(); + value += (short)rhs.getInteger(); } @Override @@ -90,7 +90,7 @@ public class Int16ResultNode extends NumericResultNode { @Override public void multiply(ResultNode rhs) { - value *= rhs.getInteger(); + value *= (short)rhs.getInteger(); } @Override @@ -101,7 +101,7 @@ public class Int16ResultNode extends NumericResultNode { @Override public void modulo(ResultNode rhs) { - value %= rhs.getInteger(); + value %= (short)rhs.getInteger(); } @Override diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int32ResultNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int32ResultNode.java index da31cbc236a..711b8f1bd3f 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int32ResultNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int32ResultNode.java @@ -80,7 +80,7 @@ public class Int32ResultNode extends NumericResultNode { @Override public void add(ResultNode rhs) { - value += rhs.getInteger(); + value += (int)rhs.getInteger(); } @Override @@ -90,7 +90,7 @@ public class Int32ResultNode extends NumericResultNode { @Override public void multiply(ResultNode rhs) { - value *= rhs.getInteger(); + value *= (int)rhs.getInteger(); } @Override @@ -101,7 +101,7 @@ public class Int32ResultNode extends NumericResultNode { @Override public void modulo(ResultNode rhs) { - value %= rhs.getInteger(); + value %= (int)rhs.getInteger(); } @Override @@ -122,7 +122,7 @@ public class Int32ResultNode extends NumericResultNode { @Override public Object getNumber() { - return Integer.valueOf(value); + return value; } @Override diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int8ResultNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int8ResultNode.java index ae53cf45a6f..d6706ce1dfe 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/Int8ResultNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/Int8ResultNode.java @@ -78,7 +78,7 @@ public class Int8ResultNode extends NumericResultNode { @Override public void add(ResultNode rhs) { - value += rhs.getInteger(); + value += (byte)rhs.getInteger(); } @Override @@ -88,7 +88,7 @@ public class Int8ResultNode extends NumericResultNode { @Override public void multiply(ResultNode rhs) { - value *= rhs.getInteger(); + value *= (byte)rhs.getInteger(); } @Override @@ -99,7 +99,7 @@ public class Int8ResultNode extends NumericResultNode { @Override public void modulo(ResultNode rhs) { - value %= rhs.getInteger(); + value %= (byte)rhs.getInteger(); } @Override diff --git a/searchlib/src/main/java/com/yahoo/searchlib/expression/RawResultNode.java b/searchlib/src/main/java/com/yahoo/searchlib/expression/RawResultNode.java index 5a0e056f254..d1dc46fc4d0 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/expression/RawResultNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/expression/RawResultNode.java @@ -18,8 +18,8 @@ public class RawResultNode extends SingleResultNode { // The global class identifier shared with C++. public static final int classId = registerClass(0x4000 + 54, RawResultNode.class); - private static RawResultNode negativeInfinity = new RawResultNode(); - private static PositiveInfinityResultNode positiveInfinity = new PositiveInfinityResultNode(); + private static final RawResultNode negativeInfinity = new RawResultNode(); + private static final PositiveInfinityResultNode positiveInfinity = new PositiveInfinityResultNode(); // The raw value of this node. private RawData value = null; @@ -147,7 +147,7 @@ public class RawResultNode extends SingleResultNode { @Override public Object getValue() { - return getString(); + return value; } @Override diff --git a/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/FieldMatchMetrics.java b/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/FieldMatchMetrics.java index 2b5efdb1ffe..5b6a53a7019 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/FieldMatchMetrics.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/ranking/features/fieldmatch/FieldMatchMetrics.java @@ -16,7 +16,7 @@ import static java.lang.Math.*; public final class FieldMatchMetrics implements Cloneable { /** The calculator creating this - given on initialization */ - private FieldMatchMetricsComputer source; + private final FieldMatchMetricsComputer source; /** The trace accumulated during execution - empty if no tracing */ private final Trace trace = new Trace(); @@ -75,7 +75,7 @@ public final class FieldMatchMetrics implements Cloneable { currentSequence=0; segmentStarts.clear(); - queryLength=source.getQuery().getTerms().length; + queryLength = source.getQuery().getTerms().length; } /** Are these metrics representing a complete match */ @@ -93,7 +93,7 @@ public final class FieldMatchMetrics implements Cloneable { */ public float get(String name) { try { - Method getter=getClass().getMethod("get" + name.substring(0,1).toUpperCase() + name.substring(1)); + Method getter = getClass().getMethod("get" + name.substring(0, 1).toUpperCase() + name.substring(1)); return ((Number)getter.invoke(this)).floatValue(); } catch (NoSuchMethodException e) { @@ -140,7 +140,7 @@ public final class FieldMatchMetrics implements Cloneable { * segment or out of order */ public float getAbsoluteProximity() { - if (pairs <1) return 0.1f; + if (pairs < 1) return 0.1f; return proximity/pairs; } @@ -151,7 +151,7 @@ public final class FieldMatchMetrics implements Cloneable { * following each other in sequence, and close to 0 if they are far from each other or out of order */ public float getUnweightedProximity() { - if (pairs <1) return 1f; + if (pairs < 1) return 1f; return unweightedProximity/pairs; } @@ -271,33 +271,33 @@ public final class FieldMatchMetrics implements Cloneable { * <code>queryCompleteness * ( 1 - fieldCompletenessImportance) + fieldCompletenessImportance * fieldCompleteness</code> */ public float getCompleteness() { - float fieldCompletenessImportance=source.getParameters().getFieldCompletenessImportance(); + float fieldCompletenessImportance = source.getParameters().getFieldCompletenessImportance(); return getQueryCompleteness() * ( 1 - fieldCompletenessImportance) + fieldCompletenessImportance*getFieldCompleteness(); } /** Returns how well the order of the terms agreed in segments: <code>1-outOfOrder/pairs</code> */ public float getOrderness() { - if (pairs ==0) return 1f; + if (pairs == 0) return 1f; return 1-(float)outOfOrder/pairs; } /** Returns the degree to which different terms are related (occurring in the same segment): <code>1-segments/(matches-1)</code> */ public float getRelatedness() { - if (matches==0) return 0; - if (matches==1) return 1; - return 1-(float)(segments-1)/(matches-1); + if (matches == 0) return 0; + if (matches == 1) return 1; + return 1 - (float)(segments - 1) / (matches - 1); } /** Returns <code>longestSequence/matches</code> */ public float getLongestSequenceRatio() { - if (matches==0) return 0; - return (float)longestSequence/matches; + if (matches == 0) return 0; + return (float)longestSequence / matches; } /** Returns the closeness of the segments in the field: <code>1-segmentDistance/fieldLength</code> */ public float getSegmentProximity() { - if (matches==0) return 0; - return 1-segmentDistance/source.getField().terms().size(); + if (matches == 0) return 0; + return 1 - segmentDistance / source.getField().terms().size(); } /** @@ -306,14 +306,14 @@ public final class FieldMatchMetrics implements Cloneable { * This is absoluteProximity/average connectedness. */ public float getProximity() { - float totalConnectedness=0; - for (int i=1; i<queryLength; i++) { - totalConnectedness+=Math.max(0.1,source.getQuery().getTerms()[i].getConnectedness()); + float totalConnectedness = 0; + for (int i = 1; i < queryLength; i++) { + totalConnectedness += (float)Math.max(0.1, source.getQuery().getTerms()[i].getConnectedness()); } - float averageConnectedness=0.1f; - if (queryLength>1) - averageConnectedness=totalConnectedness/(queryLength-1); - return getAbsoluteProximity()/averageConnectedness; + float averageConnectedness = 0.1f; + if (queryLength > 1) + averageConnectedness = totalConnectedness / (queryLength - 1); + return getAbsoluteProximity() / averageConnectedness; } /** @@ -378,7 +378,7 @@ public final class FieldMatchMetrics implements Cloneable { * not only when the metrics are complete, because this metric is used to choose segments during calculation.</p> */ float getSegmentationScore() { - if (segments==0) return 0; + if (segments == 0) return 0; return getAbsoluteProximity() * getExactness() / (segments * segments); } @@ -389,7 +389,7 @@ public final class FieldMatchMetrics implements Cloneable { /** Called once for every match */ void onMatch(int i, int j) { - if (matches>=source.getField().terms().size()) return; + if (matches >= source.getField().terms().size()) return; matches++; weight += (float)source.getQuery().getTerms()[i].getWeight() / source.getQuery().getTotalTermWeight(); significance += source.getQuery().getTerms()[i].getSignificance() / source.getQuery().getTotalSignificance(); @@ -418,42 +418,42 @@ public final class FieldMatchMetrics implements Cloneable { } /** Called once when this value is calculated, before onComplete */ - void setOccurrence(float occurrence) { this.occurrence=occurrence; } + void setOccurrence(float occurrence) { this.occurrence = occurrence; } /** Called once when this value is calculated, before onComplete */ - void setWeightedOccurrence(float weightedOccurrence) { this.weightedOccurrence=weightedOccurrence; } + void setWeightedOccurrence(float weightedOccurrence) { this.weightedOccurrence = weightedOccurrence; } /** Called once when this value is calculated, before onComplete */ - void setAbsoluteOccurrence(float absoluteOccurrence) { this.absoluteOccurrence=absoluteOccurrence; } + void setAbsoluteOccurrence(float absoluteOccurrence) { this.absoluteOccurrence = absoluteOccurrence; } /** Called once when this value is calculated, before onComplete */ - void setWeightedAbsoluteOccurrence(float weightedAbsoluteOccurrence) { this.weightedAbsoluteOccurrence=weightedAbsoluteOccurrence; } + void setWeightedAbsoluteOccurrence(float weightedAbsoluteOccurrence) { this.weightedAbsoluteOccurrence = weightedAbsoluteOccurrence; } /** Called once when this value is calculated, before onComplete */ - void setSignificantOccurrence(float significantOccurrence) { this.significantOccurrence =significantOccurrence; } + void setSignificantOccurrence(float significantOccurrence) { this.significantOccurrence = significantOccurrence; } /** Called once when matching is complete */ void onComplete() { // segment distance - calculated from sorted segment starts - if (segmentStarts.size()<=1) { - segmentDistance=0; + if (segmentStarts.size() <= 1) { + segmentDistance = 0; } else { Collections.sort(segmentStarts); - for (int i=1; i<segmentStarts.size(); i++) { - segmentDistance+=segmentStarts.get(i)-segmentStarts.get(i-1)+1; + for (int i = 1; i < segmentStarts.size(); i++) { + segmentDistance += segmentStarts.get(i) - segmentStarts.get(i - 1) + 1; } } - if (head==-1) head=0; - if (tail==-1) tail=0; + if (head == -1) head = 0; + if (tail == -1) tail = 0; } // Events on pairs ---------- /** Called when <i>any</i> pair is encountered */ void onPair(int i, int j, int previousJ) { - int distance = j-previousJ-1; + int distance = j - previousJ - 1; if (distance < 0) distance++; // Discontinuity where the two terms are in the same position if (abs(distance) > source.getParameters().getProximityLimit()) return; // Contribution=0 @@ -463,7 +463,7 @@ public final class FieldMatchMetrics implements Cloneable { unweightedProximity += pairProximity; float connectedness = source.getQuery().getTerms()[i].getConnectedness(); - proximity += pow(pairProximity, connectedness/0.1) * max(0.1, connectedness); + proximity += (float)pow(pairProximity, connectedness / 0.1) * (float)max(0.1, connectedness); pairs++; } @@ -498,8 +498,8 @@ public final class FieldMatchMetrics implements Cloneable { @Override public FieldMatchMetrics clone() { try { - FieldMatchMetrics clone=(FieldMatchMetrics)super.clone(); - clone.segmentStarts=new ArrayList<>(segmentStarts); + FieldMatchMetrics clone = (FieldMatchMetrics)super.clone(); + clone.segmentStarts = new ArrayList<>(segmentStarts); return clone; } catch (CloneNotSupportedException e) { @@ -514,19 +514,19 @@ public final class FieldMatchMetrics implements Cloneable { public String toStringDump() { try { - StringBuilder b=new StringBuilder(); + StringBuilder b = new StringBuilder(); for (Method m : this.getClass().getDeclaredMethods()) { if ( ! m.getName().startsWith("get")) continue; - if (m.getReturnType()!=Integer.TYPE && m.getReturnType()!=Float.TYPE) continue; - if ( m.getParameterTypes().length!=0 ) continue; + if (m.getReturnType() != Integer.TYPE && m.getReturnType() != Float.TYPE) continue; + if ( m.getParameterTypes().length != 0 ) continue; - Object value=m.invoke(this,new Object[0]); - b.append(m.getName().substring(3,4).toLowerCase() + m.getName().substring(4) + ": " + value + "\n"); + Object value = m.invoke(this, new Object[0]); + b.append(m.getName().substring(3, 4).toLowerCase() + m.getName().substring(4) + ": " + value + "\n"); } return b.toString(); } catch (Exception e) { - throw new RuntimeException("Programming error",e); + throw new RuntimeException("Programming error", e); } } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTNode.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTNode.java index 949e1f026f7..df721a4309e 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTNode.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/gbdtoptimization/GBDTNode.java @@ -70,13 +70,13 @@ public final class GBDTNode extends ExpressionNode { int offset = (int)nextValue - MAX_LEAF_VALUE; boolean comparisonIsTrue = false; if (offset < MAX_VARIABLES) { - comparisonIsTrue = context.getDouble(offset)<values[pc++]; + comparisonIsTrue = context.getDouble(offset) < values[pc++]; } - else if (offset < MAX_VARIABLES*2) { - comparisonIsTrue = context.getDouble(offset-MAX_VARIABLES)==values[pc++]; + else if (offset < MAX_VARIABLES * 2) { + comparisonIsTrue = context.getDouble(offset - MAX_VARIABLES) == values[pc++]; } - else if (offset<MAX_VARIABLES*3) { - double testValue = context.getDouble(offset-MAX_VARIABLES*2); + else if (offset < MAX_VARIABLES * 3) { + double testValue = context.getDouble(offset - MAX_VARIABLES * 2); int setValuesLeft = (int)values[pc++]; while (setValuesLeft > 0) { // test each value in the set setValuesLeft--; @@ -88,13 +88,13 @@ public final class GBDTNode extends ExpressionNode { pc += setValuesLeft; // jump to after the set } else { // offset<MAX_VARIABLES*4 - comparisonIsTrue = ! (context.getDouble(offset-MAX_VARIABLES*3)>=values[pc++]); + comparisonIsTrue = ! (context.getDouble(offset - MAX_VARIABLES * 3) >= values[pc++]); } if (comparisonIsTrue) pc++; // true branch - skip the jump value else - pc += values[pc]; // false branch - jump + pc += (int)values[pc]; // false branch - jump } else { // a leaf return nextValue; diff --git a/searchlib/src/test/java/com/yahoo/searchlib/gbdt/GbdtConverterTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/gbdt/GbdtConverterTestCase.java index d846d322720..8a33f320bb0 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/gbdt/GbdtConverterTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/gbdt/GbdtConverterTestCase.java @@ -3,14 +3,12 @@ package com.yahoo.searchlib.gbdt; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.parser.ParseException; -import org.junit.After; -import org.junit.Before; import org.junit.Test; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.io.UnsupportedEncodingException; -import java.security.Permission; +import java.nio.charset.StandardCharsets; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertEquals; @@ -21,36 +19,6 @@ import static org.junit.Assert.fail; */ public class GbdtConverterTestCase { - @Before - @SuppressWarnings("removal") - public void enableSecurityManager() { - System.setSecurityManager(new NoExitSecurityManager()); - } - - @After - @SuppressWarnings("removal") - public void disableSecurityManager() { - System.setSecurityManager(null); - } - - @Test - public void testOnlyOneArgumentIsAccepted() throws UnsupportedEncodingException { - assertError("Usage: GbdtConverter <filename>\n", new String[0]); - assertError("Usage: GbdtConverter <filename>\n", new String[] { "foo", "bar" }); - } - - @Test - public void testFileIsFound() throws UnsupportedEncodingException { - assertError("Could not find file 'not.found'.\n", new String[] { "not.found" }); - } - - @Test - public void testFileParsingExceptionIsCaught() throws UnsupportedEncodingException { - assertError("An error occurred while parsing the content of file 'src/test/files/gbdt_err.xml': " + - "Node 'Unknown' has no 'DecisionTree' children.\n", - new String[] { "src/test/files/gbdt_err.xml" }); - } - @Test public void testEmptyTreesAreIgnored() throws Exception { assertConvert("src/test/files/gbdt_empty_tree.xml", @@ -125,7 +93,7 @@ public class GbdtConverterTestCase { ByteArrayOutputStream out = new ByteArrayOutputStream(); System.setOut(new PrintStream(out)); GbdtConverter.main(new String[] { gbdtModelFile }); - String actualExpression = out.toString("UTF-8"); + String actualExpression = out.toString(StandardCharsets.UTF_8); assertEquals(expectedExpression, actualExpression); assertNotNull(new RankingExpression(actualExpression)); } @@ -138,26 +106,7 @@ public class GbdtConverterTestCase { fail(); } catch (ExitException e) { assertEquals(1, e.status); - assertEquals(expected, err.toString("UTF-8")); - } - } - - @SuppressWarnings("removal") - private static class NoExitSecurityManager extends SecurityManager { - - @Override - public void checkPermission(Permission perm) { - // allow anything - } - - @Override - public void checkPermission(Permission perm, Object context) { - // allow anything - } - - @Override - public void checkExit(int status) { - throw new ExitException(status); + assertEquals(expected, err.toString(StandardCharsets.UTF_8)); } } @@ -169,4 +118,5 @@ public class GbdtConverterTestCase { this.status = status; } } + } diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp index 7eb43faac18..3a1a5b457ef 100644 --- a/searchlib/src/tests/attribute/attribute_test.cpp +++ b/searchlib/src/tests/attribute/attribute_test.cpp @@ -912,8 +912,8 @@ AttributeTest::testSingle() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("sv-post-int32", cfg); ptr->updateStat(true); - EXPECT_EQ(339020u, ptr->getStatus().getAllocated()); - EXPECT_EQ(101852u, ptr->getStatus().getUsed()); + EXPECT_EQ(338972u, ptr->getStatus().getAllocated()); + EXPECT_EQ(101632u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testSingle<IntegerAttribute, AttributeVector::largeint_t, int32_t>(ptr, values); } @@ -934,8 +934,8 @@ AttributeTest::testSingle() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("sv-post-float", cfg); ptr->updateStat(true); - EXPECT_EQ(339020, ptr->getStatus().getAllocated()); - EXPECT_EQ(101852u, ptr->getStatus().getUsed()); + EXPECT_EQ(338972u, ptr->getStatus().getAllocated()); + EXPECT_EQ(101632u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testSingle<FloatingPointAttribute, double, float>(ptr, values); } @@ -947,8 +947,8 @@ AttributeTest::testSingle() { AttributePtr ptr = createAttribute("sv-string", Config(BasicType::STRING, CollectionType::SINGLE)); ptr->updateStat(true); - EXPECT_EQ(117256u + sizeof_large_string_entry, ptr->getStatus().getAllocated()); - EXPECT_EQ(53240u + sizeof_large_string_entry, ptr->getStatus().getUsed()); + EXPECT_EQ(116528u + sizeof_large_string_entry, ptr->getStatus().getAllocated()); + EXPECT_EQ(52844u + sizeof_large_string_entry, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testSingle<StringAttribute, string, string>(ptr, values); } @@ -957,8 +957,8 @@ AttributeTest::testSingle() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("sv-fs-string", cfg); ptr->updateStat(true); - EXPECT_EQ(345624u + sizeof_large_string_entry, ptr->getStatus().getAllocated()); - EXPECT_EQ(105176u + sizeof_large_string_entry, ptr->getStatus().getUsed()); + EXPECT_EQ(344848u + sizeof_large_string_entry, ptr->getStatus().getAllocated()); + EXPECT_EQ(104556u + sizeof_large_string_entry, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testSingle<StringAttribute, string, string>(ptr, values); } @@ -1089,8 +1089,8 @@ AttributeTest::testArray() { AttributePtr ptr = createAttribute("a-int32", Config(BasicType::INT32, CollectionType::ARRAY)); ptr->updateStat(true); - EXPECT_EQ(512056u, ptr->getStatus().getAllocated()); - EXPECT_EQ(504392u, ptr->getStatus().getUsed()); + EXPECT_EQ(487480u, ptr->getStatus().getAllocated()); + EXPECT_EQ(479720u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<IntegerAttribute, AttributeVector::largeint_t>(ptr, values); } @@ -1099,8 +1099,8 @@ AttributeTest::testArray() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("flags", cfg); ptr->updateStat(true); - EXPECT_EQ(512056u, ptr->getStatus().getAllocated()); - EXPECT_EQ(504392u, ptr->getStatus().getUsed()); + EXPECT_EQ(487480u, ptr->getStatus().getAllocated()); + EXPECT_EQ(479720u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<IntegerAttribute, AttributeVector::largeint_t>(ptr, values); } @@ -1109,8 +1109,8 @@ AttributeTest::testArray() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("a-fs-int32", cfg); ptr->updateStat(true); - EXPECT_EQ(868788u, ptr->getStatus().getAllocated()); - EXPECT_EQ(606264u, ptr->getStatus().getUsed()); + EXPECT_EQ(844116u, ptr->getStatus().getAllocated()); + EXPECT_EQ(581372u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<IntegerAttribute, AttributeVector::largeint_t>(ptr, values); } @@ -1128,8 +1128,8 @@ AttributeTest::testArray() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("a-fs-float", cfg); ptr->updateStat(true); - EXPECT_EQ(868788u, ptr->getStatus().getAllocated()); - EXPECT_EQ(606264u, ptr->getStatus().getUsed()); + EXPECT_EQ(844116u, ptr->getStatus().getAllocated()); + EXPECT_EQ(581372u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<FloatingPointAttribute, double>(ptr, values); } @@ -1140,8 +1140,8 @@ AttributeTest::testArray() { AttributePtr ptr = createAttribute("a-string", Config(BasicType::STRING, CollectionType::ARRAY)); ptr->updateStat(true); - EXPECT_EQ(625088u + sizeof_large_string_entry, ptr->getStatus().getAllocated()); - EXPECT_EQ(557632u + sizeof_large_string_entry, ptr->getStatus().getUsed()); + EXPECT_EQ(599784u + sizeof_large_string_entry, ptr->getStatus().getAllocated()); + EXPECT_EQ(532564u + sizeof_large_string_entry, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<StringAttribute, string>(ptr, values); } @@ -1150,8 +1150,8 @@ AttributeTest::testArray() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("afs-string", cfg); ptr->updateStat(true); - EXPECT_EQ(875392u + sizeof_large_string_entry, ptr->getStatus().getAllocated()); - EXPECT_EQ(609588u + sizeof_large_string_entry, ptr->getStatus().getUsed()); + EXPECT_EQ(849992u + sizeof_large_string_entry, ptr->getStatus().getAllocated()); + EXPECT_EQ(584296u + sizeof_large_string_entry, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<StringAttribute, string>(ptr, values); } @@ -1718,7 +1718,7 @@ AttributeTest::testStatus() ptr->commit(true); EXPECT_EQ(ptr->getStatus().getNumDocs(), 100u); EXPECT_EQ(ptr->getStatus().getNumValues(), 100u); - EXPECT_EQ(ptr->getStatus().getNumUniqueValues(), 1u); + EXPECT_EQ(ptr->getStatus().getNumUniqueValues(), 2u); size_t expUsed = 0; expUsed += 1 * InternalNodeSize + 1 * LeafNodeSize; // enum store tree expUsed += 1 * 32; // enum store (uniquevalues * bytes per entry) @@ -1741,7 +1741,7 @@ AttributeTest::testStatus() ptr->commit(true); EXPECT_EQ(ptr->getStatus().getNumDocs(), numDocs); EXPECT_EQ(ptr->getStatus().getNumValues(), numDocs*numValuesPerDoc); - EXPECT_EQ(ptr->getStatus().getNumUniqueValues(), numUniq); + EXPECT_EQ(ptr->getStatus().getNumUniqueValues(), numUniq + 1); size_t expUsed = 0; expUsed += 1 * InternalNodeSize + 1 * LeafNodeSize; // Approximate enum store tree expUsed += 272; // TODO Approximate... enum store (16 unique values, 17 bytes per entry) @@ -2145,12 +2145,12 @@ AttributeTest::test_default_value_ref_count_is_updated_after_shrink_lid_space() const auto & iattr = dynamic_cast<const search::IntegerAttributeTemplate<int32_t> &>(*attr); attr->addReservedDoc(); attr->addDocs(10); - EXPECT_EQ(11u, get_default_value_ref_count(*attr, iattr.defaultValue())); + EXPECT_EQ(12u, get_default_value_ref_count(*attr, iattr.defaultValue())); attr->compactLidSpace(6); - EXPECT_EQ(11u, get_default_value_ref_count(*attr, iattr.defaultValue())); + EXPECT_EQ(12u, get_default_value_ref_count(*attr, iattr.defaultValue())); attr->shrinkLidSpace(); EXPECT_EQ(6u, attr->getNumDocs()); - EXPECT_EQ(6u, get_default_value_ref_count(*attr, iattr.defaultValue())); + EXPECT_EQ(7u, get_default_value_ref_count(*attr, iattr.defaultValue())); } template <typename AttributeType> @@ -2170,7 +2170,7 @@ AttributeTest::requireThatAddressSpaceUsageIsReported(const Config &config, bool AddressSpaceUsage after = attrPtr->getAddressSpaceUsage(); if (attrPtr->hasEnum()) { LOG(info, "requireThatAddressSpaceUsageIsReported(%s): Has enum", attrName.c_str()); - EXPECT_EQ(before.enum_store_usage().used(), 1u); + EXPECT_EQ(before.enum_store_usage().used(), 2u); EXPECT_EQ(before.enum_store_usage().dead(), 1u); EXPECT_GT(after.enum_store_usage().used(), before.enum_store_usage().used()); EXPECT_GE(after.enum_store_usage().limit(), before.enum_store_usage().limit()); @@ -2334,6 +2334,10 @@ AttributeTest::test_paged_attribute(const vespalib::string& name, const vespalib size_t rounded_size = vespalib::round_up_to_page_size(1); size_t lid_mapping_size = 1200; size_t sv_maxlid = 1200; + if (rounded_size == 16_Ki) { + lid_mapping_size = 4200; + sv_maxlid = 1300; + } if (rounded_size == 64_Ki) { lid_mapping_size = 17000; sv_maxlid = 1500; diff --git a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp index 820f39089d1..5501c99652b 100644 --- a/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp +++ b/searchlib/src/tests/attribute/enumeratedsave/enumeratedsave_test.cpp @@ -183,7 +183,7 @@ MemAttr::bufEqual(const Buffer &lhs, const Buffer &rhs) const return false; if (lhs.get() == NULL) return true; - if (!EXPECT_TRUE(lhs->getDataLen() == rhs->getDataLen())) + if (!EXPECT_EQUAL(lhs->getDataLen(), rhs->getDataLen())) return false; if (!EXPECT_TRUE(vespalib::memcmp_safe(lhs->getData(), rhs->getData(), lhs->getDataLen()) == 0)) @@ -243,8 +243,9 @@ EnumeratedSaveTest::populate(IntegerAttribute &v, unsigned seed, int weight = 1; for(size_t i(0), m(v.getNumDocs()); i < m; i++) { v.clearDoc(i); - if (i == 9) + if (i == 9) { continue; + } if (i == 7) { if (v.hasMultiValue()) { v.append(i, -42, 27); @@ -270,7 +271,7 @@ EnumeratedSaveTest::populate(IntegerAttribute &v, unsigned seed, i + 1); } } else { - EXPECT_TRUE( v.update(i, lrand48() & mask) ); + EXPECT_TRUE( v.update(i, rnd.lrand48() & mask) ); } } v.commit(); @@ -288,8 +289,9 @@ EnumeratedSaveTest::populate(FloatingPointAttribute &v, unsigned seed, int weight = 1; for(size_t i(0), m(v.getNumDocs()); i < m; i++) { v.clearDoc(i); - if (i == 9) + if (i == 9) { continue; + } if (i == 7) { if (v.hasMultiValue()) { v.append(i, -42.0, 27); @@ -315,7 +317,7 @@ EnumeratedSaveTest::populate(FloatingPointAttribute &v, unsigned seed, i + 1); } } else { - EXPECT_TRUE( v.update(i, lrand48()) ); + EXPECT_TRUE( v.update(i, rnd.lrand48()) ); } } v.commit(); @@ -332,8 +334,9 @@ EnumeratedSaveTest::populate(StringAttribute &v, unsigned seed, int weight = 1; for(size_t i(0), m(v.getNumDocs()); i < m; i++) { v.clearDoc(i); - if (i == 9) + if (i == 9) { continue; + } if (i == 7) { if (v.hasMultiValue()) { v.append(i, "foo", 27); @@ -712,9 +715,9 @@ EnumeratedSaveTest::test(BasicType bt, CollectionType ct, Config check_cfg(cfg); check_cfg.setFastSearch(true); - checkLoad<VectorType, BufferType>(check_cfg, pref + "0_ee", v0); - checkLoad<VectorType, BufferType>(check_cfg, pref + "1_ee", v1); - checkLoad<VectorType, BufferType>(check_cfg, pref + "2_ee", v2); + TEST_DO((checkLoad<VectorType, BufferType>(check_cfg, pref + "0_ee", v0))); + TEST_DO((checkLoad<VectorType, BufferType>(check_cfg, pref + "1_ee", v1))); + TEST_DO((checkLoad<VectorType, BufferType>(check_cfg, pref + "2_ee", v2))); TEST_DO((testReload<VectorType, BufferType>(v0, v1, v2, mv0, mv1, mv2, diff --git a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp index b3c7516777c..2b01c266e80 100644 --- a/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp +++ b/searchlib/src/tests/attribute/enumstore/enumstore_test.cpp @@ -180,15 +180,35 @@ TYPED_TEST(FloatEnumStoreTest, numbers_can_be_inserted_and_retrieved) } } +TEST(EnumStoreTest, default_value_is_present) +{ + StringEnumStore ses(false, DictionaryConfig::Type::BTREE); + using EntryType = StringEnumStore::EntryType; + EntryType undefined = attribute::getUndefined<EntryType>(); + EnumIndex idx; + EXPECT_TRUE(ses.find_index(undefined, idx)); + EXPECT_TRUE(idx.valid()); + EXPECT_EQ(ses.get_default_value_ref().load_relaxed(), idx); + ses.clear_default_value_ref(); + EXPECT_FALSE(ses.find_index(undefined, idx)); + EXPECT_FALSE(ses.get_default_value_ref().load_relaxed().valid()); + ses.setup_default_value_ref(); + idx = EnumIndex(); + EXPECT_TRUE(ses.find_index(undefined, idx)); + EXPECT_TRUE(idx.valid()); + EXPECT_EQ(ses.get_default_value_ref().load_relaxed(), idx); +} + TEST(EnumStoreTest, test_find_folded_on_string_enum_store) { StringEnumStore ses(false, DictionaryConfig::Type::BTREE); + using EntryType = StringEnumStore::EntryType; std::vector<EnumIndex> indices; std::vector<std::string> unique({"", "one", "two", "TWO", "Two", "three"}); for (std::string &str : unique) { EnumIndex idx = ses.insert(str.c_str()); indices.push_back(idx); - EXPECT_EQ(1u, ses.get_ref_count(idx)); + EXPECT_EQ((str == attribute::getUndefined<EntryType>()) ? 2u : 1u, ses.get_ref_count(idx)); } ses.freeze_dictionary(); for (uint32_t i = 0; i < indices.size(); ++i) { @@ -233,13 +253,14 @@ void StringEnumStoreTest::testInsert(bool hasPostings) { StringEnumStore ses(hasPostings, DictionaryConfig::Type::BTREE); + using EntryType = StringEnumStore::EntryType; std::vector<EnumIndex> indices; std::vector<std::string> unique = {"", "add", "enumstore", "unique"}; for (const auto & i : unique) { EnumIndex idx = ses.insert(i.c_str()); - EXPECT_EQ(1u, ses.get_ref_count(idx)); + EXPECT_EQ((i == attribute::getUndefined<EntryType>()) ? 2u : 1u, ses.get_ref_count(idx)); indices.push_back(idx); EXPECT_TRUE(ses.find_index(i.c_str(), idx)); } @@ -253,7 +274,7 @@ StringEnumStoreTest::testInsert(bool hasPostings) EnumIndex idx; EXPECT_TRUE(ses.find_index(unique[i].c_str(), idx)); EXPECT_TRUE(idx == indices[i]); - EXPECT_EQ(1u, ses.get_ref_count(indices[i])); + EXPECT_EQ((i == 0) ? 2u : 1u, ses.get_ref_count(indices[i])); const char* value = nullptr; EXPECT_TRUE(ses.get_value(indices[i], value)); EXPECT_TRUE(strcmp(unique[i].c_str(), value) == 0); @@ -354,22 +375,22 @@ TEST(EnumStoreTest, address_space_usage_is_reported) NumericEnumStore store(false, DictionaryConfig::Type::BTREE); using vespalib::AddressSpace; - EXPECT_EQ(AddressSpace(1, 1, ADDRESS_LIMIT), store.get_values_address_space_usage()); - EnumIndex idx1 = store.insert(10); EXPECT_EQ(AddressSpace(2, 1, ADDRESS_LIMIT), store.get_values_address_space_usage()); - EnumIndex idx2 = store.insert(20); + EnumIndex idx1 = store.insert(10); // Address limit increases because buffer is re-sized. EXPECT_EQ(AddressSpace(3, 1, ADDRESS_LIMIT + 2), store.get_values_address_space_usage()); + EnumIndex idx2 = store.insert(20); + EXPECT_EQ(AddressSpace(4, 1, ADDRESS_LIMIT + 2), store.get_values_address_space_usage()); dec_ref_count(store, idx1); - EXPECT_EQ(AddressSpace(3, 2, ADDRESS_LIMIT + 2), store.get_values_address_space_usage()); + EXPECT_EQ(AddressSpace(4, 2, ADDRESS_LIMIT + 2), store.get_values_address_space_usage()); dec_ref_count(store, idx2); - EXPECT_EQ(AddressSpace(3, 3, ADDRESS_LIMIT + 2), store.get_values_address_space_usage()); + EXPECT_EQ(AddressSpace(4, 3, ADDRESS_LIMIT + 2), store.get_values_address_space_usage()); } TEST(EnumStoreTest, provided_memory_allocator_is_used) { AllocStats stats; - NumericEnumStore ses(false, DictionaryConfig::Type::BTREE, std::make_unique<MemoryAllocatorObserver>(stats)); + NumericEnumStore ses(false, DictionaryConfig::Type::BTREE, std::make_unique<MemoryAllocatorObserver>(stats), attribute::getUndefined<NumericEnumStore::EntryType>()); EXPECT_EQ(AllocStats(1, 0), stats); } @@ -539,6 +560,7 @@ TYPED_TEST_SUITE(LoaderTest, LoaderTestTypes); TYPED_TEST(LoaderTest, store_is_instantiated_with_enumerated_loader) { + this->store.clear_default_value_ref(); auto loader = this->store.make_enumerated_loader(); this->load_values(loader); loader.allocate_enums_histogram(); @@ -554,6 +576,7 @@ TYPED_TEST(LoaderTest, store_is_instantiated_with_enumerated_loader) TYPED_TEST(LoaderTest, store_is_instantiated_with_enumerated_postings_loader) { + this->store.clear_default_value_ref(); auto loader = this->store.make_enumerated_postings_loader(); this->load_values(loader); this->set_ref_count(0, 1, loader); @@ -568,6 +591,7 @@ TYPED_TEST(LoaderTest, store_is_instantiated_with_enumerated_postings_loader) TYPED_TEST(LoaderTest, store_is_instantiated_with_non_enumerated_loader) { + this->store.clear_default_value_ref(); auto loader = this->store.make_non_enumerated_loader(); using MyValues = LoaderTestValues<typename TypeParam::EnumStoreType>; loader.insert(MyValues::values[0], 100); @@ -610,6 +634,7 @@ public: void test_normalize_posting_lists(bool use_filter, bool one_filter); void test_foreach_posting_list(bool one_filter); static EntryRef fake_pidx() { return EntryRef(42); } + EnumIndex check_default_value_ref() const noexcept; }; template <typename EnumStoreTypeAndDictionaryType> @@ -775,6 +800,16 @@ EnumStoreDictionaryTest<EnumStoreTypeAndDictionaryType>::test_foreach_posting_li clear_sample_values(large_population); } +template <typename EnumStoreTypeAndDictionaryType> +EnumIndex +EnumStoreDictionaryTest<EnumStoreTypeAndDictionaryType>::check_default_value_ref() const noexcept +{ + EnumIndex default_value_ref = store.get_default_value_ref().load_relaxed(); + EXPECT_TRUE(default_value_ref.valid()); + EXPECT_EQ(attribute::getUndefined<EntryType>(), store.get_value(default_value_ref)); + return default_value_ref; +} + using EnumStoreDictionaryTestTypes = ::testing::Types<BTreeNumericEnumStore, HybridNumericEnumStore, HashNumericEnumStore>; TYPED_TEST_SUITE(EnumStoreDictionaryTest, EnumStoreDictionaryTestTypes); @@ -875,6 +910,7 @@ TYPED_TEST(EnumStoreDictionaryTest, compact_worst_works) updater.commit(); generation_t gen = 3; inc_generation(gen, this->store); + // Compact dictionary auto& dict = this->store.get_dictionary(); if (dict.get_has_btree_dictionary()) { EXPECT_LT(CompactionStrategy::DEAD_BYTES_SLACK, dict.get_btree_memory_usage().deadBytes()); @@ -902,8 +938,31 @@ TYPED_TEST(EnumStoreDictionaryTest, compact_worst_works) if (dict.get_has_hash_dictionary()) { EXPECT_GT(CompactionStrategy::DEAD_BYTES_SLACK, dict.get_hash_memory_usage().deadBytes()); } + auto old_default_value_ref = this->check_default_value_ref(); + // Compact values + EXPECT_LT(CompactionStrategy::DEAD_BYTES_SLACK, this->store.get_values_memory_usage().deadBytes()); + compaction_strategy = CompactionStrategy::make_compact_all_active_buffers_strategy(); + int compact_values_count = 0; + for (uint32_t i = 0; i < 2; ++i) { + this->store.update_stat(compaction_strategy); + auto remapper = this->store.consider_compact_values(compaction_strategy); + if (remapper) { + remapper->done(); + ++compact_values_count; + } else { + break; + } + EXPECT_FALSE(this->store.consider_compact_values(compaction_strategy)); + inc_generation(gen, this->store); + } + EXPECT_EQ(1, compact_values_count); + auto new_default_value_ref = this->check_default_value_ref(); + EXPECT_NE(old_default_value_ref, new_default_value_ref); + EXPECT_GT(CompactionStrategy::DEAD_BYTES_SLACK, this->store.get_values_memory_usage().deadBytes()); + std::vector<int32_t> exp_values; std::vector<int32_t> values; + exp_values.push_back(std::numeric_limits<int32_t>::min()); for (int32_t i = 0; i < 20; ++i) { exp_values.push_back(i); } diff --git a/searchlib/src/tests/attribute/extendattributes/extendattribute.cpp b/searchlib/src/tests/attribute/extendattributes/extendattribute.cpp index 8f056323733..3f775e99891 100644 --- a/searchlib/src/tests/attribute/extendattributes/extendattribute.cpp +++ b/searchlib/src/tests/attribute/extendattributes/extendattribute.cpp @@ -1,9 +1,25 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/gtest/gtest.h> +#include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/value_codec.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/searchlib/attribute/extendableattributes.h> #include <vespa/searchlib/attribute/single_raw_ext_attribute.h> +#include <vespa/searchlib/tensor/tensor_ext_attribute.h> +#include <vespa/searchlib/tensor/vector_bundle.h> +#include <vespa/vespalib/stllike/asciistream.h> +using search::attribute::Config; +using search::attribute::BasicType; +using search::attribute::CollectionType; using search::attribute::SingleRawExtAttribute; +using search::tensor::TensorExtAttribute; +using vespalib::eval::FastValueBuilderFactory; +using vespalib::eval::TensorSpec; +using vespalib::eval::Value; +using vespalib::eval::ValueType; namespace search { @@ -15,8 +31,46 @@ std::vector<char> as_vector(vespalib::ConstArrayRef<char> value) { return {value.data(), value.data() + value.size()}; } +std::vector<double> as_vector(vespalib::ConstArrayRef<double> value) { + return {value.data(), value.data() + value.size()}; +} + +vespalib::string vec_2d_spec("tensor(x[2])"); +vespalib::string vec_mixed_2d_spec("tensor(a{},x[2])"); + +TensorSpec +vec_2d(double x0, double x1) +{ + return TensorSpec(vec_2d_spec).add({{"x", 0}}, x0).add({{"x", 1}}, x1); +} + +TensorSpec +vec_mixed_2d(std::vector<std::vector<double>> val) +{ + TensorSpec spec(vec_mixed_2d_spec); + for (uint32_t a = 0; a < val.size(); ++a) { + vespalib::asciistream a_stream; + a_stream << a; + vespalib::string a_as_string = a_stream.str(); + for (uint32_t x = 0; x < val[a].size(); ++x) { + spec.add({{"a", a_as_string.c_str()},{"x", x}}, val[a][x]); + } + } + return spec; +} + +void add_doc(AttributeVector& attr, uint32_t exp_docid) +{ + uint32_t docid(0); + EXPECT_EQ(exp_docid, attr.getNumDocs()); + attr.addDoc(docid); + EXPECT_EQ(exp_docid, docid); + EXPECT_EQ(exp_docid + 1, attr.getNumDocs()); +} + class ExtendAttributeTest : public ::testing::Test { + std::vector<std::unique_ptr<Value>> _tensors; protected: ExtendAttributeTest() = default; ~ExtendAttributeTest() override = default; @@ -27,16 +81,22 @@ protected: template <typename Attribute> void testExtendString(Attribute & attr); void testExtendRaw(AttributeVector& attr); + void testExtendTensor(AttributeVector& attr); + const Value& create_tensor(const TensorSpec &spec); }; +const Value& +ExtendAttributeTest::create_tensor(const TensorSpec &spec) +{ + auto value = value_from_spec(spec, FastValueBuilderFactory::get()); + _tensors.emplace_back(std::move(value)); + return *_tensors.back(); +} + template <typename Attribute> void ExtendAttributeTest::testExtendInteger(Attribute & attr) { - uint32_t docId(0); - EXPECT_EQ(attr.getNumDocs(), 0u); - attr.addDoc(docId); - EXPECT_EQ(docId, 0u); - EXPECT_EQ(attr.getNumDocs(), 1u); + add_doc(attr, 0); attr.add(1, 10); EXPECT_EQ(attr.getInt(0), 1); attr.add(2, 20); @@ -51,9 +111,7 @@ void ExtendAttributeTest::testExtendInteger(Attribute & attr) EXPECT_EQ(v[1].getWeight(), 20); } } - attr.addDoc(docId); - EXPECT_EQ(docId, 1u); - EXPECT_EQ(attr.getNumDocs(), 2u); + add_doc(attr, 1); attr.add(3, 30); EXPECT_EQ(attr.getInt(1), 3); if (attr.hasMultiValue()) { @@ -69,11 +127,7 @@ void ExtendAttributeTest::testExtendInteger(Attribute & attr) template <typename Attribute> void ExtendAttributeTest::testExtendFloat(Attribute & attr) { - uint32_t docId(0); - EXPECT_EQ(attr.getNumDocs(), 0u); - attr.addDoc(docId); - EXPECT_EQ(docId, 0u); - EXPECT_EQ(attr.getNumDocs(), 1u); + add_doc(attr, 0); attr.add(1.7, 10); EXPECT_EQ(attr.getInt(0), 1); EXPECT_EQ(attr.getFloat(0), 1.7); @@ -89,9 +143,7 @@ void ExtendAttributeTest::testExtendFloat(Attribute & attr) EXPECT_EQ(v[1].getWeight(), 20); } } - attr.addDoc(docId); - EXPECT_EQ(docId, 1u); - EXPECT_EQ(attr.getNumDocs(), 2u); + add_doc(attr, 1); attr.add(3.6, 30); EXPECT_EQ(attr.getFloat(1), 3.6); if (attr.hasMultiValue()) { @@ -107,11 +159,7 @@ void ExtendAttributeTest::testExtendFloat(Attribute & attr) template <typename Attribute> void ExtendAttributeTest::testExtendString(Attribute & attr) { - uint32_t docId(0); - EXPECT_EQ(attr.getNumDocs(), 0u); - attr.addDoc(docId); - EXPECT_EQ(docId, 0u); - EXPECT_EQ(attr.getNumDocs(), 1u); + add_doc(attr, 0); attr.add("1.7", 10); auto buf = attr.get_raw(0); EXPECT_EQ(std::string(buf.data(), buf.size()), "1.7"); @@ -128,9 +176,7 @@ void ExtendAttributeTest::testExtendString(Attribute & attr) EXPECT_EQ(v[1].getWeight(), 20); } } - attr.addDoc(docId); - EXPECT_EQ(docId, 1u); - EXPECT_EQ(attr.getNumDocs(), 2u); + add_doc(attr, 1); attr.add("3.6", 30); buf = attr.get_raw(1); EXPECT_EQ(std::string(buf.data(), buf.size()), "3.6"); @@ -150,41 +196,77 @@ void ExtendAttributeTest::testExtendRaw(AttributeVector& attr) std::vector<char> zeros{10, 0, 0, 11}; auto* ext_attr = attr.getExtendInterface(); EXPECT_NE(nullptr, ext_attr); - uint32_t docId(0); - EXPECT_EQ(0u, attr.getNumDocs()); - attr.addDoc(docId); - EXPECT_EQ(0u, docId); - EXPECT_EQ(1u, attr.getNumDocs()); + add_doc(attr, 0); ext_attr->add(as_vector("1.7")); auto buf = attr.get_raw(0); EXPECT_EQ(as_vector("1.7"), as_vector(buf)); ext_attr->add(vespalib::ConstArrayRef<char>(as_vector("2.3"))); buf = attr.get_raw(0); EXPECT_EQ(as_vector("2.3"), as_vector(buf)); - attr.addDoc(docId); - EXPECT_EQ(1u, docId); - EXPECT_EQ(attr.getNumDocs(), 2u); + add_doc(attr, 1); ext_attr->add(as_vector("3.6")); buf = attr.get_raw(1); EXPECT_EQ(as_vector("3.6"), as_vector(buf)); buf = attr.get_raw(0); EXPECT_EQ(as_vector("2.3"), as_vector(buf)); - attr.addDoc(docId); - EXPECT_EQ(2u, docId); + add_doc(attr, 2); ext_attr->add(zeros); buf = attr.get_raw(2); EXPECT_EQ(zeros, as_vector(buf)); - attr.addDoc(docId); - EXPECT_EQ(3u, docId); + add_doc(attr, 3); buf = attr.get_raw(3); EXPECT_EQ(empty, as_vector(buf)); - attr.addDoc(docId); - EXPECT_EQ(4u, docId); + add_doc(attr, 4); ext_attr->add(empty); buf = attr.get_raw(4); EXPECT_EQ(empty, as_vector(buf)); } +void ExtendAttributeTest::testExtendTensor(AttributeVector& attr) +{ + std::vector<double> empty_cells{0.0, 0.0}; + std::vector<double> spec0_dense_cells{1.0, 2.0}; + std::vector<double> spec0_mixed_cells0{3.0, 4.0}; + std::vector<double> spec0_mixed_cells1{5.0, 6.0}; + bool dense = attr.getConfig().tensorType().is_dense(); + auto* ext_attr = attr.getExtendInterface(); + EXPECT_NE(nullptr, ext_attr); + auto* tensor_attr = attr.asTensorAttribute(); + EXPECT_NE(nullptr, tensor_attr); + add_doc(attr, 0); + TensorSpec spec0 = dense ? vec_2d(1.0, 2.0) : vec_mixed_2d({{3.0, 4.0}, {5.0, 6.0}}); + EXPECT_TRUE(ext_attr->add(create_tensor(spec0))); + auto tensor = tensor_attr->getTensor(0); + EXPECT_NE(nullptr, tensor.get()); + EXPECT_EQ(spec0, TensorSpec::from_value(*tensor)); + EXPECT_EQ(dense, tensor_attr->supports_extract_cells_ref()); + if (dense) { + EXPECT_EQ(spec0_dense_cells, as_vector(tensor_attr->extract_cells_ref(0).typify<double>())); + } + EXPECT_TRUE(tensor_attr->supports_get_tensor_ref()); + EXPECT_EQ(spec0, TensorSpec::from_value(tensor_attr->get_tensor_ref(0))); + EXPECT_FALSE(tensor_attr->supports_get_serialized_tensor_ref()); + auto vectors = tensor_attr->get_vectors(0); + if (dense) { + EXPECT_EQ(1, vectors.subspaces()); + EXPECT_EQ(spec0_dense_cells, as_vector(vectors.cells(0).typify<double>())); + EXPECT_EQ(spec0_dense_cells, as_vector(tensor_attr->get_vector(0, 0).typify<double>())); + EXPECT_EQ(empty_cells, as_vector(tensor_attr->get_vector(0, 1).typify<double>())); + } else { + EXPECT_EQ(2, vectors.subspaces()); + EXPECT_EQ(spec0_mixed_cells0, as_vector(vectors.cells(0).typify<double>())); + EXPECT_EQ(spec0_mixed_cells1, as_vector(vectors.cells(1).typify<double>())); + EXPECT_EQ(spec0_mixed_cells0, as_vector(tensor_attr->get_vector(0, 0).typify<double>())); + EXPECT_EQ(spec0_mixed_cells1, as_vector(tensor_attr->get_vector(0, 1).typify<double>())); + EXPECT_EQ(empty_cells, as_vector(tensor_attr->get_vector(0, 2).typify<double>())); + } + add_doc(attr, 1); + vectors = tensor_attr->get_vectors(1); + EXPECT_EQ(0, vectors.subspaces()); + EXPECT_EQ(empty_cells, as_vector(tensor_attr->get_vector(1, 0).typify<double>())); + EXPECT_EQ(nullptr, tensor_attr->getTensor(1).get()); +} + TEST_F(ExtendAttributeTest, single_integer_ext_attribute) { SingleIntegerExtAttribute siattr("si1"); @@ -255,6 +337,24 @@ TEST_F(ExtendAttributeTest, single_raw_ext_attribute) testExtendRaw(srattr); } +TEST_F(ExtendAttributeTest, tensor_ext_attribute_dense) +{ + Config cfg(BasicType::TENSOR, CollectionType::SINGLE); + cfg.setTensorType(ValueType::from_spec(vec_2d_spec)); + TensorExtAttribute tattr("td1", cfg); + EXPECT_TRUE(! tattr.hasMultiValue()); + testExtendTensor(tattr); +} + +TEST_F(ExtendAttributeTest, tensor_ext_attribute_mixed) +{ + Config cfg(BasicType::TENSOR, CollectionType::SINGLE); + cfg.setTensorType(ValueType::from_spec(vec_mixed_2d_spec)); + TensorExtAttribute tattr("tm1", cfg); + EXPECT_TRUE(! tattr.hasMultiValue()); + testExtendTensor(tattr); +} + } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp b/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp index df2a5ea2ac9..17e852c8b92 100644 --- a/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp +++ b/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp @@ -22,11 +22,11 @@ using vespalib::datastore::CompactionStrategy; using vespalib::alloc::test::MemoryAllocatorObserver; using AllocStats = MemoryAllocatorObserver::Stats; -template <typename EntryT> +template <typename ElemT> void -assertArray(const std::vector<EntryT> &exp, vespalib::ConstArrayRef<EntryT> values) +assertArray(const std::vector<ElemT> &exp, vespalib::ConstArrayRef<ElemT> values) { - EXPECT_EQ(exp, std::vector<EntryT>(values.cbegin(), values.cend())); + EXPECT_EQ(exp, std::vector<ElemT>(values.cbegin(), values.cend())); } template <class MvMapping> @@ -69,10 +69,10 @@ public: constexpr float ALLOC_GROW_FACTOR = 0.2; -template <typename EntryT> +template <typename ElemT> class MappingTestBase : public ::testing::Test { protected: - using MvMapping = search::attribute::MultiValueMapping<EntryT>; + using MvMapping = search::attribute::MultiValueMapping<ElemT>; using AttributeType = MyAttribute<MvMapping>; AllocStats _stats; std::unique_ptr<MvMapping> _mvMapping; @@ -82,8 +82,8 @@ protected: using generation_t = vespalib::GenerationHandler::generation_t; public: - using ArrayRef = vespalib::ArrayRef<EntryT>; - using ConstArrayRef = vespalib::ConstArrayRef<EntryT>; + using ArrayRef = vespalib::ArrayRef<ElemT>; + using ConstArrayRef = vespalib::ConstArrayRef<ElemT>; MappingTestBase() : _stats(), _mvMapping(), @@ -99,9 +99,9 @@ public: _attr = std::make_unique<AttributeType>(*_mvMapping); _maxSmallArraySize = maxSmallArraySize; } - void setup(uint32_t maxSmallArraySize, size_t minArrays, size_t maxArrays, size_t numArraysForNewBuffer, bool enable_free_lists = true) { + void setup(uint32_t maxSmallArraySize, size_t min_entries, size_t max_entries, size_t num_entries_for_new_buffer, bool enable_free_lists = true) { ArrayStoreConfig config(maxSmallArraySize, - ArrayStoreConfig::AllocSpec(minArrays, maxArrays, numArraysForNewBuffer, ALLOC_GROW_FACTOR)); + ArrayStoreConfig::AllocSpec(min_entries, max_entries, num_entries_for_new_buffer, ALLOC_GROW_FACTOR)); config.enable_free_lists(enable_free_lists); _mvMapping = std::make_unique<MvMapping>(config, vespalib::GrowStrategy(), std::make_unique<MemoryAllocatorObserver>(_stats)); _attr = std::make_unique<AttributeType>(*_mvMapping); @@ -109,12 +109,12 @@ public: } ~MappingTestBase() { } - void set(uint32_t docId, const std::vector<EntryT> &values) { _mvMapping->set(docId, values); } + void set(uint32_t docId, const std::vector<ElemT> &values) { _mvMapping->set(docId, values); } ConstArrayRef get(uint32_t docId) { return _mvMapping->get(docId); } ArrayRef get_writable(uint32_t docId) { return _mvMapping->get_writable(docId); } - void assertGet(uint32_t docId, const std::vector<EntryT> &exp) { + void assertGet(uint32_t docId, const std::vector<ElemT> &exp) { ConstArrayRef act = get(docId); - EXPECT_EQ(exp, std::vector<EntryT>(act.cbegin(), act.cend())); + EXPECT_EQ(exp, std::vector<ElemT>(act.cbegin(), act.cend())); } void assign_generation(generation_t current_gen) { _mvMapping->assign_generation(current_gen); } void reclaim_memory(generation_t oldest_used_gen) { _mvMapping->reclaim_memory(oldest_used_gen); } diff --git a/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp b/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp index a19b093ef4d..2804e3f74e4 100644 --- a/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp +++ b/searchlib/src/tests/attribute/stringattribute/stringattribute_test.cpp @@ -8,7 +8,6 @@ #include <vespa/searchlib/attribute/enumstore.hpp> #include <vespa/searchlib/attribute/single_string_enum_search_context.h> -#include <vespa/searchlib/attribute/singlestringpostattribute.h> #include <vespa/searchlib/attribute/multistringpostattribute.hpp> #include <vespa/log/log.h> diff --git a/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp b/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp index 896fd27f90e..757f74bdaff 100644 --- a/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp +++ b/searchlib/src/tests/expression/attributenode/attribute_node_test.cpp @@ -54,8 +54,7 @@ namespace { vespalib::string stringValue(const ResultNode &result, const IAttributeVector &attr) { if (result.inherits(EnumResultNode::classId)) { auto enumHandle = result.getEnum(); - auto &stringAttr = dynamic_cast<const StringAttribute &>(attr); - return vespalib::string(stringAttr.getFromEnum(enumHandle)); + return vespalib::string(attr.getStringFromEnum(enumHandle)); } char buf[100]; BufferRef bref(&buf[0], sizeof(buf)); @@ -199,8 +198,9 @@ AttributeManagerFixture::buildIntegerArrayAttribute(const vespalib::string &name } -struct Fixture +class Fixture { +public: AttributeManagerFixture attrs; AttributeContext context; Fixture(); @@ -208,11 +208,13 @@ struct Fixture std::unique_ptr<AttributeNode> makeNode(const vespalib::string &attributeName, bool useEnumOptimiation = false, bool preserveAccurateTypes = false); void assertInts(std::vector<IAttributeVector::largeint_t> expVals, const vespalib::string &attributteName, bool preserveAccurateTypes = false); void assertBools(std::vector<bool> expVals, const vespalib::string &attributteName, bool preserveAccurateTypes = false); - void assertStrings(std::vector<vespalib::string> expVals, const vespalib::string &attributteName, bool useEnumOptimization = false); + void assertStrings(std::vector<vespalib::string> expVals, const vespalib::string &attributteName); void assertFloats(std::vector<double> expVals, const vespalib::string &attributteName); void assertIntArrays(std::vector<std::vector<IAttributeVector::largeint_t>> expVals, const vespalib::string &attributteName, bool preserveAccurateTypes = false); void assertStringArrays(std::vector<std::vector<vespalib::string>> expVals, const vespalib::string &attributteName, bool useEnumOptimization = false); void assertFloatArrays(std::vector<std::vector<double>> expVals, const vespalib::string &attributteName); +private: + void assertStrings(std::vector<vespalib::string> expVals, const vespalib::string &attributteName, bool useEnumOptimization); }; Fixture::Fixture() @@ -280,6 +282,11 @@ Fixture::assertBools(std::vector<bool> expVals, const vespalib::string &attribut } } +void +Fixture::assertStrings(std::vector<vespalib::string> expVals, const vespalib::string &attributeName) { + assertStrings(expVals, attributeName, false); + assertStrings(expVals, attributeName, true); +} void Fixture::assertStrings(std::vector<vespalib::string> expVals, const vespalib::string &attributeName, bool useEnumOptimization) @@ -293,6 +300,9 @@ Fixture::assertStrings(std::vector<vespalib::string> expVals, const vespalib::st const auto &result = *node->getResult(); if (useEnumOptimization) { ASSERT_TRUE(result.inherits(EnumResultNode::classId)); + search::enumstore::EnumHandle enumVal(0); + ASSERT_TRUE(node->getAttribute()->findEnum(expDocVal.c_str(), enumVal)); + EXPECT_EQUAL(result.getEnum(), enumVal); } else { ASSERT_TRUE(result.inherits(StringResultNode::classId)); } @@ -404,7 +414,6 @@ TEST_F("test single values", Fixture) TEST_DO(f.assertInts({ 10, getUndefined<int8_t>()}, "ifield")); TEST_DO(f.assertInts({ 10, getUndefined<int8_t>()}, "ifield", true)); TEST_DO(f.assertStrings({ "n1", "" }, "sfield")); - TEST_DO(f.assertStrings({ "n1", "" }, "sfield", true)); TEST_DO(f.assertFloats({ 110.0, getUndefined<double>() }, "ffield")); } diff --git a/searchlib/src/tests/features/prod_features.cpp b/searchlib/src/tests/features/prod_features.cpp index 4ebc94ccb8b..dc64c3328e4 100644 --- a/searchlib/src/tests/features/prod_features.cpp +++ b/searchlib/src/tests/features/prod_features.cpp @@ -1252,7 +1252,7 @@ Test::testDotProduct() { // test funky syntax dotproduct::wset::EnumVector out(sv); WeightedSetParser::parse("( a: 1, b:2 ,c: , :3)", out); - EXPECT_EQUAL(out.getVector().size(), 3u); + EXPECT_EQUAL(out.getVector().size(), 4u); EXPECT_TRUE(sv->findEnum("a", e)); EXPECT_EQUAL(out.getVector()[0].first, e); EXPECT_EQUAL(out.getVector()[0].second, 1); @@ -1262,6 +1262,9 @@ Test::testDotProduct() EXPECT_TRUE(sv->findEnum("c", e)); EXPECT_EQUAL(out.getVector()[2].first, e); EXPECT_EQUAL(out.getVector()[2].second, 0); + EXPECT_TRUE(sv->findEnum("", e)); + EXPECT_EQUAL(out.getVector()[3].first, e); + EXPECT_EQUAL(out.getVector()[3].second, 3); } { // strings not in attribute vector dotproduct::wset::EnumVector out(sv); diff --git a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp index b57a2f42ea7..69478c09a25 100644 --- a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp +++ b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp @@ -1032,14 +1032,14 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working) auto beforeStats = getFeatureStoreMemStats(_fic); LOG(info, - "Before feature compaction: allocElems=%zu, usedElems=%zu" - ", deadElems=%zu, holdElems=%zu" + "Before feature compaction: alloc_entries=%zu, used_entries=%zu" + ", dead_entries=%zu, hold_entries=%zu" ", freeBuffers=%u, activeBuffers=%u" ", holdBuffers=%u", - beforeStats._allocElems, - beforeStats._usedElems, - beforeStats._deadElems, - beforeStats._holdElems, + beforeStats._alloc_entries, + beforeStats._used_entries, + beforeStats._dead_entries, + beforeStats._hold_entries, beforeStats._freeBuffers, beforeStats._activeBuffers, beforeStats._holdBuffers); @@ -1052,14 +1052,14 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working) myCommit(_fic, *_pushThreads); auto duringStats = getFeatureStoreMemStats(_fic); LOG(info, - "During feature compaction: allocElems=%zu, usedElems=%zu" - ", deadElems=%zu, holdElems=%zu" + "During feature compaction: alloc_entries=%zu, used_entries=%zu" + ", dead_entries=%zu, hold_entries=%zu" ", freeBuffers=%u, activeBuffers=%u" ", holdBuffers=%u", - duringStats._allocElems, - duringStats._usedElems, - duringStats._deadElems, - duringStats._holdElems, + duringStats._alloc_entries, + duringStats._used_entries, + duringStats._dead_entries, + duringStats._hold_entries, duringStats._freeBuffers, duringStats._activeBuffers, duringStats._holdBuffers); @@ -1067,14 +1067,14 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working) myCommit(_fic, *_pushThreads); auto afterStats = getFeatureStoreMemStats(_fic); LOG(info, - "After feature compaction: allocElems=%zu, usedElems=%zu" - ", deadElems=%zu, holdElems=%zu" + "After feature compaction: alloc_entries=%zu, used_entries=%zu" + ", dead_entries=%zu, hold_entries=%zu" ", freeBuffers=%u, activeBuffers=%u" ", holdBuffers=%u", - afterStats._allocElems, - afterStats._usedElems, - afterStats._deadElems, - afterStats._holdElems, + afterStats._alloc_entries, + afterStats._used_entries, + afterStats._dead_entries, + afterStats._hold_entries, afterStats._freeBuffers, afterStats._activeBuffers, afterStats._holdBuffers); diff --git a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp index 8073fb8d232..e3b9cf9702d 100644 --- a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp +++ b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp @@ -462,8 +462,8 @@ TEST(MemoryIndexTest, require_that_num_docs_and_doc_id_limit_is_returned) TEST(MemoryIndexTest, require_that_we_understand_the_memory_footprint) { - constexpr size_t BASE_ALLOCATED = 361032u; - constexpr size_t BASE_USED = 151188u; + constexpr size_t BASE_ALLOCATED = 360936u; + constexpr size_t BASE_USED = 150804u; { MySetup setup; Index index(setup); diff --git a/searchlib/src/tests/predicate/document_features_store_test.cpp b/searchlib/src/tests/predicate/document_features_store_test.cpp index 4ac4bdc32f0..d30df9dba6e 100644 --- a/searchlib/src/tests/predicate/document_features_store_test.cpp +++ b/searchlib/src/tests/predicate/document_features_store_test.cpp @@ -165,17 +165,17 @@ TEST("require that both features and ranges are removed by 'remove'") { TEST("require that both features and ranges counts towards memory usage") { DocumentFeaturesStore features_store(10); - EXPECT_EQUAL(50136u, features_store.getMemoryUsage().usedBytes()); + EXPECT_EQUAL(50064u, features_store.getMemoryUsage().usedBytes()); PredicateTreeAnnotations annotations; annotations.features.push_back(PredicateHash::hash64("foo=100-199")); features_store.insert(annotations, doc_id); - EXPECT_EQUAL(50144u, features_store.getMemoryUsage().usedBytes()); + EXPECT_EQUAL(50072u, features_store.getMemoryUsage().usedBytes()); annotations.features.clear(); annotations.range_features.push_back({"foo", 100, 199}); features_store.insert(annotations, doc_id + 1); - EXPECT_EQUAL(50240u, features_store.getMemoryUsage().usedBytes()); + EXPECT_EQUAL(50168u, features_store.getMemoryUsage().usedBytes()); } TEST("require that DocumentFeaturesStore can be serialized") { @@ -205,17 +205,17 @@ TEST("require that serialization cleans up wordstore") { PredicateTreeAnnotations annotations; annotations.range_features.push_back({"foo", 100, 199}); features_store.insert(annotations, doc_id); - EXPECT_EQUAL(50232u, features_store.getMemoryUsage().usedBytes()); + EXPECT_EQUAL(50160u, features_store.getMemoryUsage().usedBytes()); annotations.range_features.push_back({"bar", 100, 199}); features_store.insert(annotations, doc_id + 1); - EXPECT_EQUAL(50620u, features_store.getMemoryUsage().usedBytes()); + EXPECT_EQUAL(50548u, features_store.getMemoryUsage().usedBytes()); features_store.remove(doc_id + 1); - EXPECT_EQUAL(50572u, features_store.getMemoryUsage().usedBytes()); + EXPECT_EQUAL(50500u, features_store.getMemoryUsage().usedBytes()); vespalib::DataBuffer buffer; features_store.serialize(buffer); DocumentFeaturesStore features_store2(buffer); - EXPECT_EQUAL(50232u, features_store2.getMemoryUsage().usedBytes()); + EXPECT_EQUAL(50160u, features_store2.getMemoryUsage().usedBytes()); } diff --git a/searchlib/src/tests/query/streaming_query_test.cpp b/searchlib/src/tests/query/streaming_query_test.cpp index f354f635def..2c202d9131b 100644 --- a/searchlib/src/tests/query/streaming_query_test.cpp +++ b/searchlib/src/tests/query/streaming_query_test.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchlib/query/streaming/query.h> +#include <vespa/searchlib/query/streaming/nearest_neighbor_query_node.h> #include <vespa/searchlib/query/tree/querybuilder.h> #include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/query/tree/stackdumpcreator.h> @@ -804,6 +805,42 @@ TEST("testSameElementEvaluate") { EXPECT_TRUE(sameElem->evaluate()); } +TEST("test_nearest_neighbor_query_node") +{ + QueryBuilder<SimpleQueryNodeTypes> builder; + constexpr double distance_threshold = 35.5; + constexpr int32_t id = 42; + constexpr int32_t weight = 1; + constexpr uint32_t target_num_hits = 100; + constexpr bool allow_approximate = false; + constexpr uint32_t explore_additional_hits = 800; + constexpr double raw_score = 0.5; + builder.add_nearest_neighbor_term("qtensor", "field", id, Weight(weight), target_num_hits, allow_approximate, explore_additional_hits, distance_threshold); + auto build_node = builder.build(); + auto stack_dump = StackDumpCreator::create(*build_node); + QueryNodeResultFactory empty; + Query q(empty, stack_dump); + auto* qterm = dynamic_cast<QueryTerm *>(&q.getRoot()); + EXPECT_TRUE(qterm != nullptr); + auto* node = dynamic_cast<NearestNeighborQueryNode *>(&q.getRoot()); + EXPECT_TRUE(node != nullptr); + EXPECT_EQUAL(node, qterm->as_nearest_neighbor_query_node()); + EXPECT_EQUAL("qtensor", node->get_query_tensor_name()); + EXPECT_EQUAL("field", node->getIndex()); + EXPECT_EQUAL(id, static_cast<int32_t>(node->uniqueId())); + EXPECT_EQUAL(weight, node->weight().percent()); + EXPECT_EQUAL(distance_threshold, node->get_distance_threshold()); + EXPECT_FALSE(node->get_raw_score().has_value()); + EXPECT_FALSE(node->evaluate()); + node->set_raw_score(raw_score); + EXPECT_TRUE(node->get_raw_score().has_value()); + EXPECT_EQUAL(raw_score, node->get_raw_score().value()); + EXPECT_TRUE(node->evaluate()); + node->reset(); + EXPECT_FALSE(node->get_raw_score().has_value()); + EXPECT_FALSE(node->evaluate()); +} + TEST("Control the size of query terms") { EXPECT_EQUAL(112u, sizeof(QueryTermSimple)); EXPECT_EQUAL(128u, sizeof(QueryTermUCS4)); diff --git a/searchlib/src/vespa/searchcommon/common/undefinedvalues.h b/searchlib/src/vespa/searchcommon/common/undefinedvalues.h index bbe3198a8dc..a080648c054 100644 --- a/searchlib/src/vespa/searchcommon/common/undefinedvalues.h +++ b/searchlib/src/vespa/searchcommon/common/undefinedvalues.h @@ -24,6 +24,10 @@ inline constexpr double getUndefined<double>() { return -std::numeric_limits<double>::quiet_NaN(); } +template <> +inline constexpr const char* getUndefined<const char*>() { + return ""; +} // for all signed integers template <typename T> diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp index e6b584d29b2..f4ab447ed51 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.cpp @@ -353,6 +353,8 @@ AttributeVector::load(vespalib::Executor * executor) { bool loaded = onLoad(executor); if (loaded) { commit(); + incGeneration(); + updateStat(true); } _loaded = loaded; return _loaded; @@ -440,21 +442,18 @@ AttributeVector::addReservedDoc() addDoc(docId); // Reserved assert(docId == 0u); assert(docId < getNumDocs()); + set_reserved_doc_values(); +} + +void +AttributeVector::set_reserved_doc_values() +{ + uint32_t docId = 0; + if (docId >= getNumDocs()) { + return; + } clearDoc(docId); commit(); - FloatingPointAttribute * vec = dynamic_cast<FloatingPointAttribute *>(this); - if (vec) { - if (hasMultiValue()) { - bool appendedUndefined = vec->append(0, attribute::getUndefined<double>(), 1); - assert(appendedUndefined); - (void) appendedUndefined; - } else { - bool updatedUndefined = vec->update(0, attribute::getUndefined<double>()); - assert(updatedUndefined); - (void) updatedUndefined; - } - commit(); - } } attribute::IPostingListAttributeBase *AttributeVector::getIPostingListAttributeBase() { return nullptr; } diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index 3d14622ca02..b58060a05bf 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -38,6 +38,8 @@ namespace vespalib::alloc { class Alloc; } +namespace vespalib::eval { struct Value; } + namespace search { template <typename T> class ComponentGuard; @@ -86,6 +88,7 @@ public: virtual bool add(double, int32_t = 1) { return false; } virtual bool add(const char *, int32_t = 1) { return false; } virtual bool add(vespalib::ConstArrayRef<char>, int32_t = 1) { return false; } + virtual bool add(const vespalib::eval::Value&, int32_t = 1) { return false; } virtual ~IExtendAttribute() = default; }; @@ -475,6 +478,10 @@ public: * Add reserved initial document with docId 0 and undefined value. */ void addReservedDoc(); + /** + * set undefined values for reserved document 0. + */ + void set_reserved_doc_values(); bool getEnumeratedSave() const { return _hasEnum; } virtual attribute::IPostingListAttributeBase * getIPostingListAttributeBase(); diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.cpp b/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.cpp index eeaa3e9539f..c1345b4f770 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.cpp +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.cpp @@ -93,6 +93,7 @@ EnumeratedLoader::build_dictionary() { _store.get_dictionary().build(_indexes); release_enum_indexes(); + _store.setup_default_value_ref(); } EnumeratedPostingsLoader::EnumeratedPostingsLoader(IEnumStore& store) @@ -131,6 +132,13 @@ EnumeratedPostingsLoader::build_dictionary() _store.get_dictionary().build_with_payload(_indexes, _posting_indexes); release_enum_indexes(); EntryRefVector().swap(_posting_indexes); + _store.setup_default_value_ref(); +} + +void +EnumeratedPostingsLoader::build_empty_dictionary() +{ + _store.setup_default_value_ref(); } } diff --git a/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.h b/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.h index 2a72fcac628..937ceb91628 100644 --- a/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.h +++ b/searchlib/src/vespa/searchlib/attribute/enum_store_loaders.h @@ -85,6 +85,7 @@ public: void set_ref_count(Index idx, uint32_t ref_count); vespalib::ArrayRef<EntryRef> initialize_empty_posting_indexes(); void build_dictionary(); + void build_empty_dictionary(); }; } diff --git a/searchlib/src/vespa/searchlib/attribute/enumattribute.h b/searchlib/src/vespa/searchlib/attribute/enumattribute.h index f0ff23a06b4..4753dbe65f9 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/enumattribute.h @@ -50,13 +50,12 @@ protected: /* * Iterate through the change vector and find new unique values. - * Perform compaction if necessary and insert the new unique values into the EnumStore. + * Insert the new unique values into the EnumStore. */ void insertNewUniqueValues(EnumStoreBatchUpdater& updater); virtual void considerAttributeChange(const Change & c, EnumStoreBatchUpdater & inserter) = 0; vespalib::MemoryUsage getEnumStoreValuesMemoryUsage() const override; void populate_address_space_usage(AddressSpaceUsage& usage) const override; - void cache_change_data_entry_ref(const Change& c) const; public: EnumAttribute(const vespalib::string & baseFileName, const AttributeVector::Config & cfg); ~EnumAttribute(); diff --git a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp index c5188b89129..66d555df3cb 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumattribute.hpp @@ -15,7 +15,7 @@ EnumAttribute<B>:: EnumAttribute(const vespalib::string &baseFileName, const AttributeVector::Config &cfg) : B(baseFileName, cfg), - _enumStore(cfg.fastSearch(), cfg.get_dictionary_config(), this->get_memory_allocator()) + _enumStore(cfg.fastSearch(), cfg.get_dictionary_config(), this->get_memory_allocator(), this->_defaultValue._data.raw()) { this->setEnum(true); } @@ -50,6 +50,7 @@ void EnumAttribute<B>::load_enum_store(LoadedVector& loaded) loader.set_ref_count_for_last_value(prevRefCount); } loader.build_dictionary(); + _enumStore.setup_default_value_ref(); } } @@ -85,15 +86,4 @@ EnumAttribute<B>::populate_address_space_usage(AddressSpaceUsage& usage) const usage.set(AddressSpaceComponents::enum_store, _enumStore.get_values_address_space_usage()); } -template <typename B> -void -EnumAttribute<B>::cache_change_data_entry_ref(const Change& c) const -{ - EnumIndex new_idx; - _enumStore.find_index(c._data.raw(), new_idx); - c.set_entry_ref(new_idx.ref()); -} - } // namespace search - - diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.h b/searchlib/src/vespa/searchlib/attribute/enumstore.h index 266437fafa1..f6467194d74 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.h +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.h @@ -28,6 +28,9 @@ namespace search { * It uses an instance of vespalib::datastore::UniqueStore to store the actual values. * It also exposes the dictionary used for fast lookups into the set of unique values. * + * The default value is always present except for a short time window + * during attribute vector load. + * * @tparam EntryType The type of the entries/values stored. * It has special handling of type 'const char *' for strings. */ @@ -55,6 +58,8 @@ private: ComparatorType _comparator; ComparatorType _foldedComparator; enumstore::EnumStoreCompactionSpec _compaction_spec; + EntryType _default_value; + AtomicIndex _default_value_ref; EnumStoreT(const EnumStoreT & rhs) = delete; EnumStoreT & operator=(const EnumStoreT & rhs) = delete; @@ -75,7 +80,7 @@ private: std::unique_ptr<EntryComparator> allocate_optionally_folded_comparator(bool folded) const; ComparatorType make_optionally_folded_comparator(bool folded) const; public: - EnumStoreT(bool has_postings, const search::DictionaryConfig& dict_cfg, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator); + EnumStoreT(bool has_postings, const search::DictionaryConfig& dict_cfg, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator, EntryType default_value); EnumStoreT(bool has_postings, const search::DictionaryConfig & dict_cfg); ~EnumStoreT() override; @@ -201,6 +206,9 @@ public: bool find_index(EntryType value, Index& idx) const; void free_unused_values() override; void free_unused_values(IndexList to_remove); + void clear_default_value_ref() override; + void setup_default_value_ref() override; + const AtomicIndex& get_default_value_ref() const noexcept { return _default_value_ref; } vespalib::MemoryUsage update_stat(const CompactionStrategy& compaction_strategy) override; std::unique_ptr<EnumIndexRemapper> consider_compact_values(const CompactionStrategy& compaction_strategy) override; std::unique_ptr<EnumIndexRemapper> compact_worst_values(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy) override; diff --git a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp index bc767a296eb..c0eebee8e94 100644 --- a/searchlib/src/vespa/searchlib/attribute/enumstore.hpp +++ b/searchlib/src/vespa/searchlib/attribute/enumstore.hpp @@ -17,6 +17,7 @@ #include <vespa/vespalib/datastore/unique_store.hpp> #include <vespa/vespalib/datastore/unique_store_string_allocator.hpp> #include <vespa/vespalib/util/array.hpp> +#include <vespa/searchcommon/common/undefinedvalues.h> #include <vespa/searchlib/util/bufferwriter.h> #include <vespa/vespalib/datastore/compaction_strategy.h> @@ -72,23 +73,26 @@ EnumStoreT<EntryT>::load_unique_value(const void* src, size_t available, Index& } template <typename EntryT> -EnumStoreT<EntryT>::EnumStoreT(bool has_postings, const DictionaryConfig& dict_cfg, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator) +EnumStoreT<EntryT>::EnumStoreT(bool has_postings, const DictionaryConfig& dict_cfg, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator, EntryType default_value) : _store(std::move(memory_allocator)), _dict(), _is_folded(dict_cfg.getMatch() == DictionaryConfig::Match::UNCASED), _comparator(_store.get_data_store()), _foldedComparator(make_optionally_folded_comparator(is_folded())), - _compaction_spec() + _compaction_spec(), + _default_value(default_value), + _default_value_ref() { _store.set_dictionary(make_enum_store_dictionary(*this, has_postings, dict_cfg, allocate_comparator(), allocate_optionally_folded_comparator(is_folded()))); _dict = static_cast<IEnumStoreDictionary*>(&_store.get_dictionary()); + setup_default_value_ref(); } template <typename EntryT> EnumStoreT<EntryT>::EnumStoreT(bool has_postings, const DictionaryConfig& dict_cfg) - : EnumStoreT<EntryT>(has_postings, dict_cfg, {}) + : EnumStoreT<EntryT>(has_postings, dict_cfg, {}, attribute::getUndefined<EntryType>()) { } @@ -215,6 +219,33 @@ EnumStoreT<EntryT>::insert(EntryType value) return _store.add(value).ref(); } + +template <typename EntryT> +void +EnumStoreT<EntryT>::clear_default_value_ref() +{ + auto ref = _default_value_ref.load_relaxed(); + if (ref.valid()) { + auto updater = make_batch_updater(); + updater.dec_ref_count(ref); + _default_value_ref.store_relaxed(Index()); + updater.commit(); + } +} + +template <typename EntryT> +void +EnumStoreT<EntryT>::setup_default_value_ref() +{ + if (!_default_value_ref.load_relaxed().valid()) { + auto updater = make_batch_updater(); + auto ref = updater.insert(_default_value); + updater.inc_ref_count(ref); + _default_value_ref.store_relaxed(ref); + updater.commit(); + } +} + template <typename EntryT> vespalib::MemoryUsage EnumStoreT<EntryT>::update_stat(const CompactionStrategy& compaction_strategy) @@ -236,7 +267,14 @@ template <typename EntryT> std::unique_ptr<IEnumStore::EnumIndexRemapper> EnumStoreT<EntryT>::compact_worst_values(CompactionSpec compaction_spec, const CompactionStrategy& compaction_strategy) { - return _store.compact_worst(compaction_spec, compaction_strategy); + auto remapper = _store.compact_worst(compaction_spec, compaction_strategy); + if (remapper) { + auto ref = _default_value_ref.load_relaxed(); + if (ref.valid() && remapper->get_entry_ref_filter().has(ref)) { + _default_value_ref.store_release(remapper->remap(ref)); + } + } + return remapper; } template <typename EntryT> diff --git a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h index 2157db3e5ed..aa9fd549b60 100644 --- a/searchlib/src/vespa/searchlib/attribute/i_enum_store.h +++ b/searchlib/src/vespa/searchlib/attribute/i_enum_store.h @@ -74,6 +74,8 @@ public: virtual std::unique_ptr<Enumerator> make_enumerator() = 0; virtual std::unique_ptr<vespalib::datastore::EntryComparator> allocate_comparator() const = 0; + virtual void clear_default_value_ref() = 0; + virtual void setup_default_value_ref() = 0; }; } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h index bd624e9f388..4fce64aa762 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h @@ -12,18 +12,18 @@ namespace search::attribute { /** * Class for mapping from document id to an array of values. */ -template <typename EntryT, typename RefT = vespalib::datastore::EntryRefT<19> > +template <typename ElemT, typename RefT = vespalib::datastore::EntryRefT<19> > class MultiValueMapping : public MultiValueMappingBase { public: - using MultiValueType = EntryT; + using MultiValueType = ElemT; using RefType = RefT; - using ReadView = MultiValueMappingReadView<EntryT, RefT>; + using ReadView = MultiValueMappingReadView<ElemT, RefT>; private: - using ArrayRef = vespalib::ArrayRef<EntryT>; - using ArrayStore = vespalib::datastore::ArrayStore<EntryT, RefT>; + using ArrayRef = vespalib::ArrayRef<ElemT>; + using ArrayStore = vespalib::datastore::ArrayStore<ElemT, RefT>; using generation_t = vespalib::GenerationHandler::generation_t; - using ConstArrayRef = vespalib::ConstArrayRef<EntryT>; + using ConstArrayRef = vespalib::ConstArrayRef<ElemT>; ArrayStore _store; public: @@ -73,7 +73,7 @@ public: static vespalib::datastore::ArrayStoreConfig optimizedConfigForHugePage(size_t maxSmallArraySize, size_t hugePageSize, size_t smallPageSize, - size_t minNumArraysForNewBuffer, + size_t min_num_entries_for_new_buffer, float allocGrowFactor, bool enable_free_lists); }; diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp index b486fa60265..ab68bea58cc 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp @@ -7,8 +7,8 @@ namespace search::attribute { -template <typename EntryT, typename RefT> -MultiValueMapping<EntryT,RefT>::MultiValueMapping(const vespalib::datastore::ArrayStoreConfig &storeCfg, +template <typename ElemT, typename RefT> +MultiValueMapping<ElemT,RefT>::MultiValueMapping(const vespalib::datastore::ArrayStoreConfig &storeCfg, const vespalib::GrowStrategy &gs, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator) : MultiValueMappingBase(gs, ArrayStore::getGenerationHolderLocation(_store), memory_allocator), @@ -16,12 +16,12 @@ MultiValueMapping<EntryT,RefT>::MultiValueMapping(const vespalib::datastore::Arr { } -template <typename EntryT, typename RefT> -MultiValueMapping<EntryT,RefT>::~MultiValueMapping() = default; +template <typename ElemT, typename RefT> +MultiValueMapping<ElemT,RefT>::~MultiValueMapping() = default; -template <typename EntryT, typename RefT> +template <typename ElemT, typename RefT> void -MultiValueMapping<EntryT,RefT>::set(uint32_t docId, ConstArrayRef values) +MultiValueMapping<ElemT,RefT>::set(uint32_t docId, ConstArrayRef values) { _indices.ensure_size(docId + 1); EntryRef oldRef(_indices[docId].load_relaxed()); @@ -31,18 +31,18 @@ MultiValueMapping<EntryT,RefT>::set(uint32_t docId, ConstArrayRef values) _store.remove(oldRef); } -template <typename EntryT, typename RefT> +template <typename ElemT, typename RefT> vespalib::MemoryUsage -MultiValueMapping<EntryT,RefT>::update_stat(const CompactionStrategy& compaction_strategy) +MultiValueMapping<ElemT,RefT>::update_stat(const CompactionStrategy& compaction_strategy) { auto retval = _store.update_stat(compaction_strategy); retval.merge(_indices.getMemoryUsage()); return retval; } -template <typename EntryT, typename RefT> +template <typename ElemT, typename RefT> void -MultiValueMapping<EntryT,RefT>::compact_worst(const CompactionStrategy& compaction_strategy) +MultiValueMapping<ElemT,RefT>::compact_worst(const CompactionStrategy& compaction_strategy) { vespalib::datastore::ICompactionContext::UP compactionContext(_store.compact_worst(compaction_strategy)); if (compactionContext) { @@ -50,29 +50,29 @@ MultiValueMapping<EntryT,RefT>::compact_worst(const CompactionStrategy& compacti } } -template <typename EntryT, typename RefT> +template <typename ElemT, typename RefT> vespalib::MemoryUsage -MultiValueMapping<EntryT,RefT>::getArrayStoreMemoryUsage() const +MultiValueMapping<ElemT,RefT>::getArrayStoreMemoryUsage() const { return _store.getMemoryUsage(); } -template <typename EntryT, typename RefT> +template <typename ElemT, typename RefT> vespalib::AddressSpace -MultiValueMapping<EntryT, RefT>::getAddressSpaceUsage() const { +MultiValueMapping<ElemT, RefT>::getAddressSpaceUsage() const { return _store.addressSpaceUsage(); } -template <typename EntryT, typename RefT> +template <typename ElemT, typename RefT> vespalib::datastore::ArrayStoreConfig -MultiValueMapping<EntryT, RefT>::optimizedConfigForHugePage(size_t maxSmallArraySize, +MultiValueMapping<ElemT, RefT>::optimizedConfigForHugePage(size_t maxSmallArraySize, size_t hugePageSize, size_t smallPageSize, - size_t minNumArraysForNewBuffer, + size_t min_num_entries_for_new_buffer, float allocGrowFactor, bool enable_free_lists) { - auto result = ArrayStore::optimizedConfigForHugePage(maxSmallArraySize, hugePageSize, smallPageSize, minNumArraysForNewBuffer, allocGrowFactor); + auto result = ArrayStore::optimizedConfigForHugePage(maxSmallArraySize, hugePageSize, smallPageSize, min_num_entries_for_new_buffer, allocGrowFactor); result.enable_free_lists(enable_free_lists); return result; } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_read_view.h b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_read_view.h index 116e069e8b4..41138ff0890 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_read_view.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_read_view.h @@ -11,12 +11,12 @@ namespace search::attribute { /** * Class for mapping from document id to an array of values as reader. */ -template <typename EntryT, typename RefT = vespalib::datastore::EntryRefT<19> > +template <typename ElemT, typename RefT = vespalib::datastore::EntryRefT<19> > class MultiValueMappingReadView { using AtomicEntryRef = vespalib::datastore::AtomicEntryRef; using Indices = vespalib::ConstArrayRef<AtomicEntryRef>; - using ArrayStore = vespalib::datastore::ArrayStore<EntryT, RefT>; + using ArrayStore = vespalib::datastore::ArrayStore<ElemT, RefT>; Indices _indices; const ArrayStore* _store; @@ -31,7 +31,7 @@ public: _store(store) { } - vespalib::ConstArrayRef<EntryT> get(uint32_t doc_id) const { return _store->get(_indices[doc_id].load_acquire()); } + vespalib::ConstArrayRef<ElemT> get(uint32_t doc_id) const { return _store->get(_indices[doc_id].load_acquire()); } bool valid() const noexcept { return _store != nullptr; } }; diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp index edfea23f48d..59c1216829d 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericenumattribute.hpp @@ -97,6 +97,10 @@ MultiValueNumericEnumAttribute<B, M>::onLoad(vespalib::Executor *) return false; } + this->_enumStore.clear_default_value_ref(); + this->commit(); + this->incGeneration(); + this->setCreateSerialNum(attrReader.getCreateSerialNum()); if (attrReader.getEnumerated()) { diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp index ea449300aef..1009fa2fb5f 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp @@ -51,7 +51,7 @@ template <typename B, typename M> MultiValueNumericPostingAttribute<B, M>::~MultiValueNumericPostingAttribute() { this->disableFreeLists(); - this->disableElemHoldList(); + this->disable_entry_hold_list(); clearAllPostings(); } diff --git a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp index a63862126fa..7b11fcd59f4 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringattribute.hpp @@ -42,7 +42,6 @@ MultiValueStringAttributeT<B, M>::freezeEnumDictionary() this->getEnumStore().freeze_dictionary(); } - template <typename B, typename M> std::unique_ptr<attribute::SearchContext> MultiValueStringAttributeT<B, M>::getSearch(QueryTermSimpleUP qTerm, diff --git a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp index cd46bbb5a8a..19840b5a474 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp @@ -27,7 +27,7 @@ template <typename B, typename T> MultiValueStringPostingAttributeT<B, T>::~MultiValueStringPostingAttributeT() { this->disableFreeLists(); - this->disableElemHoldList(); + this->disable_entry_hold_list(); clearAllPostings(); } diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp index 6ef3b575c3e..01e68949f92 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.cpp @@ -49,6 +49,7 @@ PostingListAttributeBase<P>::handle_load_posting_lists_and_update_enum_store(enu PostingChange<P> postings; const auto& loaded_enums = loader.get_loaded_enums(); if (loaded_enums.empty()) { + loader.build_empty_dictionary(); return; } uint32_t preve = 0; diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h index 29440b6ce43..ecf7a46f21e 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h @@ -58,7 +58,7 @@ protected: void updatePostings(PostingMap &changePost, const vespalib::datastore::EntryComparator &cmp); void clearAllPostings(); void disableFreeLists() { _postingList.disableFreeLists(); } - void disableElemHoldList() { _postingList.disableElemHoldList(); } + void disable_entry_hold_list() { _postingList.disable_entry_hold_list(); } void handle_load_posting_lists_and_update_enum_store(enumstore::EnumeratedPostingsLoader& loader); bool forwardedOnAddDoc(DocId doc, size_t wantSize, size_t wantCapacity); diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp index 94720212faf..2703201b292 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp @@ -231,7 +231,7 @@ PostingStore<DataT>::dropBitVector(EntryRef &ref) (void) tree; (void) docFreq; _bvs.erase(ref.ref()); - _store.holdElem(iRef, 1); + _store.hold_entry(iRef); _status.decBitVectors(); _bvExtraBytes -= bv->writer().extraByteSize(); ref = ref2; @@ -267,7 +267,7 @@ PostingStore<DataT>::makeBitVector(EntryRef &ref) if (_enableOnlyBitVector) { BTreeType *tree = getWTreeEntry(iRef); tree->clear(_allocator); - _store.holdElem(ref, 1); + _store.hold_entry(ref); } else { bve->_tree = ref; } @@ -590,19 +590,19 @@ PostingStore<DataT>::clear(const EntryRef ref) assert(isBTree(iRef2)); BTreeType *tree = getWTreeEntry(iRef2); tree->clear(_allocator); - _store.holdElem(iRef2, 1); + _store.hold_entry(iRef2); } _bvs.erase(ref.ref()); _status.decBitVectors(); _bvExtraBytes -= bve->_bv->writer().extraByteSize(); - _store.holdElem(ref, 1); + _store.hold_entry(ref); } else { BTreeType *tree = getWTreeEntry(iRef); tree->clear(_allocator); - _store.holdElem(ref, 1); + _store.hold_entry(ref); } } else { - _store.holdElem(ref, clusterSize); + _store.hold_entry(ref); } } diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.h b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.h index aac9a7b5416..7f36238ec6a 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.h @@ -67,14 +67,14 @@ protected: void considerAttributeChange(const Change & c, EnumStoreBatchUpdater & inserter) override; // implemented by single value numeric enum attribute. - virtual void considerUpdateAttributeChange(const Change & c) { (void) c; } + virtual void considerUpdateAttributeChange(DocId, const Change&) { } virtual void considerArithmeticAttributeChange(const Change & c, EnumStoreBatchUpdater & inserter) { (void) c; (void) inserter; } virtual void applyValueChanges(EnumStoreBatchUpdater& updater) ; virtual void applyArithmeticValueChange(const Change& c, EnumStoreBatchUpdater& updater) { (void) c; (void) updater; } - void updateEnumRefCounts(const Change& c, EnumIndex newIdx, EnumIndex oldIdx, EnumStoreBatchUpdater& updater); + void updateEnumRefCounts(DocId doc, EnumIndex newIdx, EnumIndex oldIdx, EnumStoreBatchUpdater& updater); virtual void freezeEnumDictionary() { this->getEnumStore().freeze_dictionary(); diff --git a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp index f4f2b777abd..95976609940 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singleenumattribute.hpp @@ -146,7 +146,7 @@ SingleValueEnumAttribute<B>::considerUpdateAttributeChange(const Change & c, Enu } else { c.set_entry_ref(idx.ref()); } - considerUpdateAttributeChange(c); // for numeric + considerUpdateAttributeChange(c._doc, c); // for numeric } template <typename B> @@ -158,9 +158,7 @@ SingleValueEnumAttribute<B>::considerAttributeChange(const Change & c, EnumStore } else if (c._type >= ChangeBase::ADD && c._type <= ChangeBase::DIV) { considerArithmeticAttributeChange(c, inserter); // for numeric } else if (c._type == ChangeBase::CLEARDOC) { - Change clearDoc(this->_defaultValue); - clearDoc._doc = c._doc; - considerUpdateAttributeChange(clearDoc, inserter); + considerUpdateAttributeChange(c._doc, this->_defaultValue); } } @@ -175,7 +173,7 @@ SingleValueEnumAttribute<B>::applyUpdateValueChange(const Change& c, EnumStoreBa } else { this->_enumStore.find_index(c._data.raw(), newIdx); } - updateEnumRefCounts(c, newIdx, oldIdx, updater); + updateEnumRefCounts(c._doc, newIdx, oldIdx, updater); } template <typename B> @@ -183,30 +181,26 @@ void SingleValueEnumAttribute<B>::applyValueChanges(EnumStoreBatchUpdater& updater) { ValueModifier valueGuard(this->getValueModifier()); - // This avoids searching for the defaultValue in the enum store for each CLEARDOC in the change vector. - this->cache_change_data_entry_ref(this->_defaultValue); for (const auto& change : this->_changes.getInsertOrder()) { if (change._type == ChangeBase::UPDATE) { applyUpdateValueChange(change, updater); } else if (change._type >= ChangeBase::ADD && change._type <= ChangeBase::DIV) { applyArithmeticValueChange(change, updater); } else if (change._type == ChangeBase::CLEARDOC) { - Change clearDoc(this->_defaultValue); - clearDoc._doc = change._doc; - applyUpdateValueChange(clearDoc, updater); + EnumIndex oldIdx = _enumIndices[change._doc].load_relaxed(); + EnumIndex newIdx = this->_enumStore.get_default_value_ref().load_relaxed(); + updateEnumRefCounts(change._doc, newIdx, oldIdx, updater); } } - // We must clear the cached entry ref as the defaultValue might be located in another data buffer on later invocations. - this->_defaultValue.clear_entry_ref(); } template <typename B> void -SingleValueEnumAttribute<B>::updateEnumRefCounts(const Change& c, EnumIndex newIdx, EnumIndex oldIdx, +SingleValueEnumAttribute<B>::updateEnumRefCounts(DocId doc, EnumIndex newIdx, EnumIndex oldIdx, EnumStoreBatchUpdater& updater) { updater.inc_ref_count(newIdx); - _enumIndices[c._doc].store_release(newIdx); + _enumIndices[doc].store_release(newIdx); if (oldIdx.valid()) { updater.dec_ref_count(oldIdx); } diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp index a105d980986..c75ee0aacb5 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericattribute.hpp @@ -134,8 +134,9 @@ SingleValueNumericAttribute<B>::onLoad(vespalib::Executor *) PrimitiveReader<T> attrReader(*this); bool ok(attrReader.getHasLoadData()); - if (!ok) + if (!ok) { return false; + } this->setCreateSerialNum(attrReader.getCreateSerialNum()); diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.h b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.h index 5b0e1c6131e..4eeb6ceda57 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.h @@ -43,7 +43,7 @@ private: protected: // from SingleValueEnumAttribute - void considerUpdateAttributeChange(const Change & c) override; + void considerUpdateAttributeChange(DocId doc, const Change & c) override; void considerArithmeticAttributeChange(const Change & c, EnumStoreBatchUpdater & inserter) override; void applyArithmeticValueChange(const Change& c, EnumStoreBatchUpdater& updater) override; diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp index 52ea0a53533..b840a0516b2 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericenumattribute.hpp @@ -15,9 +15,9 @@ namespace search { template <typename B> void -SingleValueNumericEnumAttribute<B>::considerUpdateAttributeChange(const Change & c) +SingleValueNumericEnumAttribute<B>::considerUpdateAttributeChange(DocId doc, const Change & c) { - _currDocValues[c._doc] = c._data.get(); + _currDocValues[doc] = c._data.get(); } template <typename B> @@ -53,7 +53,7 @@ SingleValueNumericEnumAttribute<B>::applyArithmeticValueChange(const Change& c, T newValue = this->template applyArithmetic<T, typename Change::DataType>(get(c._doc), c._data.getArithOperand(), c._type); this->_enumStore.find_index(newValue, newIdx); - this->updateEnumRefCounts(c, newIdx, oldIdx, updater); + this->updateEnumRefCounts(c._doc, newIdx, oldIdx, updater); } template <typename B> @@ -117,6 +117,10 @@ SingleValueNumericEnumAttribute<B>::onLoad(vespalib::Executor *) return false; } + this->_enumStore.clear_default_value_ref(); + this->commit(); + this->incGeneration(); + this->setCreateSerialNum(attrReader.getCreateSerialNum()); if (attrReader.getEnumerated()) { diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp index 1775774171d..e353d03a9e8 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp @@ -13,7 +13,7 @@ template <typename B> SingleValueNumericPostingAttribute<B>::~SingleValueNumericPostingAttribute() { this->disableFreeLists(); - this->disableElemHoldList(); + this->disable_entry_hold_list(); clearAllPostings(); } @@ -89,8 +89,6 @@ SingleValueNumericPostingAttribute<B>::applyValueChanges(EnumStoreBatchUpdater& // used to make sure several arithmetic operations on the same document in a single commit works std::map<DocId, EnumIndex> currEnumIndices; - // This avoids searching for the defaultValue in the enum store for each CLEARDOC in the change vector. - this->cache_change_data_entry_ref(this->_defaultValue); for (const auto& change : this->_changes.getInsertOrder()) { auto enumIter = currEnumIndices.find(change._doc); EnumIndex oldIdx; @@ -111,13 +109,9 @@ SingleValueNumericPostingAttribute<B>::applyValueChanges(EnumStoreBatchUpdater& currEnumIndices[change._doc] = newIdx; } } else if(change._type == ChangeBase::CLEARDOC) { - Change clearDoc(this->_defaultValue); - clearDoc._doc = change._doc; - applyUpdateValueChange(clearDoc, enumStore, currEnumIndices); + currEnumIndices[change._doc] = enumStore.get_default_value_ref().load_relaxed(); } } - // We must clear the cached entry ref as the defaultValue might be located in another data buffer on later invocations. - this->_defaultValue.clear_entry_ref(); makePostingChange(enumStore.get_comparator(), currEnumIndices, changePost); diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp index 82a4393fc91..69fe6435a03 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlestringattribute.hpp @@ -40,7 +40,6 @@ SingleValueStringAttributeT<B>::freezeEnumDictionary() this->getEnumStore().freeze_dictionary(); } - template <typename B> std::unique_ptr<attribute::SearchContext> SingleValueStringAttributeT<B>::getSearch(QueryTermSimpleUP qTerm, diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp index eef72984e79..5b5214f6d3e 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp @@ -27,7 +27,7 @@ template <typename B> SingleValueStringPostingAttributeT<B>::~SingleValueStringPostingAttributeT() { this->disableFreeLists(); - this->disableElemHoldList(); + this->disable_entry_hold_list(); clearAllPostings(); } @@ -98,8 +98,6 @@ SingleValueStringPostingAttributeT<B>::applyValueChanges(EnumStoreBatchUpdater& // used to make sure several arithmetic operations on the same document in a single commit works std::map<DocId, EnumIndex> currEnumIndices; - // This avoids searching for the defaultValue in the enum store for each CLEARDOC in the change vector. - this->cache_change_data_entry_ref(this->_defaultValue); for (const auto& change : this->_changes.getInsertOrder()) { auto enumIter = currEnumIndices.find(change._doc); EnumIndex oldIdx; @@ -111,12 +109,9 @@ SingleValueStringPostingAttributeT<B>::applyValueChanges(EnumStoreBatchUpdater& if (change._type == ChangeBase::UPDATE) { applyUpdateValueChange(change, enumStore, currEnumIndices); } else if (change._type == ChangeBase::CLEARDOC) { - this->_defaultValue._doc = change._doc; - applyUpdateValueChange(this->_defaultValue, enumStore, currEnumIndices); + currEnumIndices[change._doc] = enumStore.get_default_value_ref().load_relaxed(); } } - // We must clear the cached entry ref as the defaultValue might be located in another data buffer on later invocations. - this->_defaultValue.clear_entry_ref(); makePostingChange(enumStore.get_folded_comparator(), dictionary, currEnumIndices, changePost); diff --git a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp index 80967affaa7..b37318d470e 100644 --- a/searchlib/src/vespa/searchlib/attribute/stringbase.cpp +++ b/searchlib/src/vespa/searchlib/attribute/stringbase.cpp @@ -223,6 +223,10 @@ StringAttribute::onLoad(vespalib::Executor *) return false; } + getEnumStoreBase()->clear_default_value_ref(); + commit(); + incGeneration(); + setCreateSerialNum(attrReader.getCreateSerialNum()); assert(attrReader.getEnumerated()); diff --git a/searchlib/src/vespa/searchlib/attribute/stringbase.h b/searchlib/src/vespa/searchlib/attribute/stringbase.h index ed8777d6bab..98a3316947b 100644 --- a/searchlib/src/vespa/searchlib/attribute/stringbase.h +++ b/searchlib/src/vespa/searchlib/attribute/stringbase.h @@ -53,16 +53,16 @@ public: largeint_t getInt(DocId doc) const override { return strtoll(get(doc), nullptr, 0); } double getFloat(DocId doc) const override; vespalib::ConstArrayRef<char> get_raw(DocId) const override; + static const char * defaultValue() { return ""; } protected: StringAttribute(const vespalib::string & name); StringAttribute(const vespalib::string & name, const Config & c); ~StringAttribute() override; - static const char * defaultValue() { return ""; } using Change = ChangeTemplate<StringChangeData>; using ChangeVector = ChangeVectorT<Change>; using EnumEntryType = const char*; ChangeVector _changes; - Change _defaultValue; + const Change _defaultValue; bool onLoad(vespalib::Executor *executor) override; bool onLoadEnumerated(ReaderBase &attrReader); diff --git a/searchlib/src/vespa/searchlib/expression/attribute_map_lookup_node.cpp b/searchlib/src/vespa/searchlib/expression/attribute_map_lookup_node.cpp index b3cc126efdb..52ee467fac8 100644 --- a/searchlib/src/vespa/searchlib/expression/attribute_map_lookup_node.cpp +++ b/searchlib/src/vespa/searchlib/expression/attribute_map_lookup_node.cpp @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "attribute_map_lookup_node.h" -#include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vespalib/util/exceptions.h> +#include <vespa/searchlib/attribute/stringbase.h> #include <vespa/searchcommon/attribute/attributecontent.h> #include <vespa/searchcommon/attribute/iattributecontext.h> -#include <vespa/searchcommon/common/undefinedvalues.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/util/exceptions.h> using search::attribute::AttributeContent; using search::attribute::IAttributeVector; @@ -36,8 +36,6 @@ public: namespace { -vespalib::string indirectKeyMarker("attribute("); - class BadKeyHandler : public AttributeMapLookupNode::KeyHandler { public: @@ -220,7 +218,6 @@ IAttributeVector::largeint_t getUndefinedValue(BasicType::Type basicType) return getUndefined<int32_t>(); case BasicType::INT64: return getUndefined<int64_t>(); - break; default: return 0; } @@ -330,7 +327,6 @@ AttributeMapLookupNode::onPrepare(bool preserveAccurateTypes) break; default: throw std::runtime_error("This is no valid integer attribute " + attribute->getName()); - break; } } else { prepareIntValues<Int64ResultNode>(std::move(keyHandler), *attribute, undefinedValue); @@ -342,7 +338,11 @@ AttributeMapLookupNode::onPrepare(bool preserveAccurateTypes) } else if (attribute->isStringType()) { if (_useEnumOptimization) { auto resultNode = std::make_unique<EnumResultNode>(); - _handler = std::make_unique<EnumValueHandler>(std::move(keyHandler), *attribute, *resultNode, EnumHandle()); + const StringAttribute & sattr = dynamic_cast<const StringAttribute &>(*attribute); + EnumHandle undefined(0); + bool found = attribute->findEnum(sattr.defaultValue(), undefined); + assert(found); + _handler = std::make_unique<EnumValueHandler>(std::move(keyHandler), *attribute, *resultNode, undefined); setResultType(std::move(resultNode)); } else { auto resultNode = std::make_unique<StringResultNode>(); diff --git a/searchlib/src/vespa/searchlib/memoryindex/feature_store.cpp b/searchlib/src/vespa/searchlib/memoryindex/feature_store.cpp index 72f4a7ae579..4bc7f5b1144 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/feature_store.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/feature_store.cpp @@ -36,7 +36,7 @@ EntryRef FeatureStore::addFeatures(const uint8_t *src, uint64_t byteLen) { uint32_t pad = Aligner::pad(byteLen); - auto result = _store.rawAllocator<uint8_t>(_typeId).alloc(byteLen + pad, DECODE_SAFETY); + auto result = _store.rawAllocator<uint8_t>(_typeId).alloc((byteLen + pad) / buffer_array_size, DECODE_SAFETY_ENTRIES); uint8_t *dst = result.data; memcpy(dst, src, byteLen); dst += byteLen; @@ -113,7 +113,7 @@ FeatureStore::add_features_guard_bytes() { uint32_t len = DECODE_SAFETY; uint32_t pad = Aligner::pad(len); - auto result = _store.rawAllocator<uint8_t>(_typeId).alloc(len + pad); + auto result = _store.rawAllocator<uint8_t>(_typeId).alloc((len + pad) / buffer_array_size); memset(result.data, 0, len + pad); } diff --git a/searchlib/src/vespa/searchlib/memoryindex/feature_store.h b/searchlib/src/vespa/searchlib/memoryindex/feature_store.h index 53588fa2894..1e48189987e 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/feature_store.h +++ b/searchlib/src/vespa/searchlib/memoryindex/feature_store.h @@ -29,6 +29,7 @@ private: using PosOccFieldsParams = bitcompression::PosOccFieldsParams; static constexpr uint32_t DECODE_SAFETY = 16; + static constexpr uint32_t DECODE_SAFETY_ENTRIES = 16 / buffer_array_size; DataStoreType _store; @@ -117,7 +118,7 @@ public: * overrun beyond the compressed data either goes into other features * already written or into the guard area. * - * If buffer type is changed to have a nonzero numArraysForNewBuffer then + * If buffer type is changed to have a nonzero num_entries_for_new_buffer then * extra logic to add guard bytes is needed when switching primary buffer * to avoid issues if the buffer is resumed as primary buffer later on. */ diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp index 4be3031303e..8dd76a90b14 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp @@ -55,9 +55,9 @@ template <bool interleaved_features> FieldIndex<interleaved_features>::~FieldIndex() { _postingListStore.disableFreeLists(); - _postingListStore.disableElemHoldList(); + _postingListStore.disable_entry_hold_list(); _dict.disableFreeLists(); - _dict.disableElemHoldList(); + _dict.disable_entry_hold_list(); // XXX: Kludge for (DictionaryTree::Iterator it = _dict.begin(); it.valid(); ++it) { diff --git a/searchlib/src/vespa/searchlib/memoryindex/word_store.cpp b/searchlib/src/vespa/searchlib/memoryindex/word_store.cpp index 3e4c38ceb0e..e330dc83055 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/word_store.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/word_store.cpp @@ -27,7 +27,7 @@ WordStore::addWord(const vespalib::stringref word) { size_t wordSize = word.size() + 1; size_t bufferSize = wordSize + Aligner::pad(wordSize); - auto result = _store.rawAllocator<char>(_typeId).alloc(bufferSize); + auto result = _store.rawAllocator<char>(_typeId).alloc(bufferSize / buffer_array_size); char *be = result.data; for (size_t i = 0; i < word.size(); ++i) { *be++ = word[i]; diff --git a/searchlib/src/vespa/searchlib/predicate/document_features_store.cpp b/searchlib/src/vespa/searchlib/predicate/document_features_store.cpp index 604a467a6e6..a6a82ec09f8 100644 --- a/searchlib/src/vespa/searchlib/predicate/document_features_store.cpp +++ b/searchlib/src/vespa/searchlib/predicate/document_features_store.cpp @@ -102,7 +102,7 @@ DocumentFeaturesStore::DocumentFeaturesStore(DataBuffer &buffer) DocumentFeaturesStore::~DocumentFeaturesStore() { _word_index.disableFreeLists(); - _word_index.disableElemHoldList(); + _word_index.disable_entry_hold_list(); _word_index.getAllocator().freeze(); _word_index.clear(); } diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_interval_store.cpp b/searchlib/src/vespa/searchlib/predicate/predicate_interval_store.cpp index af809b2fa69..af5aae6e519 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_interval_store.cpp +++ b/searchlib/src/vespa/searchlib/predicate/predicate_interval_store.cpp @@ -95,7 +95,7 @@ PredicateIntervalStore::remove(EntryRef ref) { // BufferState &state = _store.getBufferState(buffer_id); // uint32_t type_id = state.getTypeId(); // uint32_t size = type_id <= MAX_ARRAY_SIZE ? type_id : 1; - // _store.holdElem(ref, size); + // _store.hold_entries(ref, size); } } diff --git a/searchlib/src/vespa/searchlib/predicate/simple_index.hpp b/searchlib/src/vespa/searchlib/predicate/simple_index.hpp index c6f640d72ed..9320488f88e 100644 --- a/searchlib/src/vespa/searchlib/predicate/simple_index.hpp +++ b/searchlib/src/vespa/searchlib/predicate/simple_index.hpp @@ -41,7 +41,7 @@ SimpleIndex<Posting, Key, DocId>::insertIntoVectorPosting(vespalib::datastore::E template <typename Posting, typename Key, typename DocId> SimpleIndex<Posting, Key, DocId>::~SimpleIndex() { _btree_posting_lists.disableFreeLists(); - _btree_posting_lists.disableElemHoldList(); + _btree_posting_lists.disable_entry_hold_list(); for (auto it = _dictionary.begin(); it.valid(); ++it) { vespalib::datastore::EntryRef ref(it.getData()); @@ -51,13 +51,13 @@ SimpleIndex<Posting, Key, DocId>::~SimpleIndex() { } _vector_posting_lists.disableFreeLists(); - _vector_posting_lists.disableElemHoldList(); + _vector_posting_lists.disable_entry_hold_list(); _vector_posting_lists.clear(); _vector_posting_lists.getAllocator().freeze(); _vector_posting_lists.getAllocator().reclaim_all_memory(); _dictionary.disableFreeLists(); - _dictionary.disableElemHoldList(); + _dictionary.disable_entry_hold_list(); _dictionary.clear(); _dictionary.getAllocator().freeze(); _dictionary.getAllocator().reclaim_all_memory(); diff --git a/searchlib/src/vespa/searchlib/query/query_term_simple.h b/searchlib/src/vespa/searchlib/query/query_term_simple.h index 74728ab1f2e..a79e33dba32 100644 --- a/searchlib/src/vespa/searchlib/query/query_term_simple.h +++ b/searchlib/src/vespa/searchlib/query/query_term_simple.h @@ -23,7 +23,8 @@ public: SUFFIXTERM = 4, REGEXP = 5, GEO_LOCATION = 6, - FUZZYTERM = 7 + FUZZYTERM = 7, + NEAREST_NEIGHBOR = 8 }; template <typename N> @@ -65,6 +66,7 @@ public: bool isRegex() const { return (_type == Type::REGEXP); } bool isGeoLoc() const { return (_type == Type::GEO_LOCATION); } bool isFuzzy() const { return (_type == Type::FUZZYTERM); } + bool is_nearest_neighbor() const noexcept { return (_type == Type::NEAREST_NEIGHBOR); } bool empty() const { return _term.empty(); } virtual void visitMembers(vespalib::ObjectVisitor &visitor) const; vespalib::string getClassName() const; diff --git a/searchlib/src/vespa/searchlib/query/streaming/CMakeLists.txt b/searchlib/src/vespa/searchlib/query/streaming/CMakeLists.txt index 27f9870dc18..c71b838fb37 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/query/streaming/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(searchlib_query_streaming OBJECT SOURCES + nearest_neighbor_query_node.cpp query.cpp querynode.cpp querynoderesultbase.cpp diff --git a/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.cpp b/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.cpp new file mode 100644 index 00000000000..d1c37cd6dcd --- /dev/null +++ b/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.cpp @@ -0,0 +1,36 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "nearest_neighbor_query_node.h" + +namespace search::streaming { + +NearestNeighborQueryNode::NearestNeighborQueryNode(std::unique_ptr<QueryNodeResultBase> resultBase, const string& term, const string& index, int32_t id, search::query::Weight weight, double distance_threshold) + : QueryTerm(std::move(resultBase), term, index, Type::NEAREST_NEIGHBOR), + _distance_threshold(distance_threshold), + _raw_score() +{ + setUniqueId(id); + setWeight(weight); +} + +NearestNeighborQueryNode::~NearestNeighborQueryNode() = default; + +bool +NearestNeighborQueryNode::evaluate() const +{ + return _raw_score.has_value(); +} + +void +NearestNeighborQueryNode::reset() +{ + _raw_score.reset(); +} + +NearestNeighborQueryNode* +NearestNeighborQueryNode::as_nearest_neighbor_query_node() noexcept +{ + return this; +} + +} diff --git a/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.h b/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.h new file mode 100644 index 00000000000..0beb130c53d --- /dev/null +++ b/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.h @@ -0,0 +1,35 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "queryterm.h" +#include <optional> + +namespace search::streaming { + +/* + * Nearest neighbor query node. + */ +class NearestNeighborQueryNode: public QueryTerm { +private: + double _distance_threshold; + // When this value is set it also indicates a match + std::optional<double> _raw_score; + +public: + NearestNeighborQueryNode(std::unique_ptr<QueryNodeResultBase> resultBase, const string& term, const string& index, int32_t id, search::query::Weight weight, double distance_threshold); + NearestNeighborQueryNode(const NearestNeighborQueryNode &) = delete; + NearestNeighborQueryNode & operator = (const NearestNeighborQueryNode &) = delete; + NearestNeighborQueryNode(NearestNeighborQueryNode &&) = delete; + NearestNeighborQueryNode & operator = (NearestNeighborQueryNode &&) = delete; + ~NearestNeighborQueryNode() override; + bool evaluate() const override; + void reset() override; + NearestNeighborQueryNode* as_nearest_neighbor_query_node() noexcept override; + const vespalib::string& get_query_tensor_name() const { return getTermString(); } + double get_distance_threshold() const { return _distance_threshold; } + void set_raw_score(double value) { _raw_score = value; } + const std::optional<double>& get_raw_score() const noexcept { return _raw_score; } +}; + +} diff --git a/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp b/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp index 6d59886a4f5..226cb92c894 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp +++ b/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "query.h" +#include "nearest_neighbor_query_node.h" #include <vespa/searchlib/parsequery/stackdumpiterator.h> #include <charconv> #include <vespa/log/log.h> @@ -77,6 +78,9 @@ QueryNode::Build(const QueryNode * parent, const QueryNodeResultFactory & factor queryRep.getIndexName(), QueryTerm::Type::GEO_LOCATION); break; + case ParseItem::ITEM_NEAREST_NEIGHBOR: + qn = build_nearest_neighbor_query_node(factory, queryRep); + break; case ParseItem::ITEM_NUMTERM: case ParseItem::ITEM_TERM: case ParseItem::ITEM_PREFIXTERM: @@ -191,4 +195,20 @@ const HitList & QueryNode::evaluateHits(HitList & hl) const return hl; } +std::unique_ptr<QueryNode> +QueryNode::build_nearest_neighbor_query_node(const QueryNodeResultFactory& factory, SimpleQueryStackDumpIterator& query_rep) +{ + vespalib::stringref query_tensor_name = query_rep.getTerm(); + vespalib::stringref field_name = query_rep.getIndexName(); + int32_t id = query_rep.getUniqueId(); + search::query::Weight weight = query_rep.GetWeight(); + double distance_threshold = query_rep.getDistanceThreshold(); + return std::make_unique<NearestNeighborQueryNode>(factory.create(), + query_tensor_name, + field_name, + id, + weight, + distance_threshold); +} + } diff --git a/searchlib/src/vespa/searchlib/query/streaming/querynode.h b/searchlib/src/vespa/searchlib/query/streaming/querynode.h index 574a3c16ca3..c3fa2b63f69 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/querynode.h +++ b/searchlib/src/vespa/searchlib/query/streaming/querynode.h @@ -28,6 +28,7 @@ using ConstQueryTermList = std::vector<const QueryTerm *>; */ class QueryNode { + static std::unique_ptr<QueryNode> build_nearest_neighbor_query_node(const QueryNodeResultFactory& factory, SimpleQueryStackDumpIterator& queryRep); public: using UP = std::unique_ptr<QueryNode>; @@ -54,7 +55,7 @@ class QueryNode virtual size_t depth() const { return 1; } /// Return the width of this tree. virtual size_t width() const { return 1; } - static UP Build(const QueryNode * parent, const QueryNodeResultFactory & org, SimpleQueryStackDumpIterator & queryRep, bool allowRewrite); + static UP Build(const QueryNode * parent, const QueryNodeResultFactory& factory, SimpleQueryStackDumpIterator & queryRep, bool allowRewrite); }; /// A list conating the QuerNode objects. With copy/assignment. diff --git a/searchlib/src/vespa/searchlib/query/streaming/queryterm.cpp b/searchlib/src/vespa/searchlib/query/streaming/queryterm.cpp index 83f4410a520..11557bf1dcc 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/queryterm.cpp +++ b/searchlib/src/vespa/searchlib/query/streaming/queryterm.cpp @@ -92,4 +92,10 @@ void QueryTerm::add(unsigned pos, unsigned context, uint32_t elemId, int32_t wei _hitList.emplace_back(pos, context, elemId, weight_); } +NearestNeighborQueryNode* +QueryTerm::as_nearest_neighbor_query_node() noexcept +{ + return nullptr; +} + } diff --git a/searchlib/src/vespa/searchlib/query/streaming/queryterm.h b/searchlib/src/vespa/searchlib/query/streaming/queryterm.h index dd9f56b11e1..51987225692 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/queryterm.h +++ b/searchlib/src/vespa/searchlib/query/streaming/queryterm.h @@ -12,6 +12,8 @@ namespace search::streaming { +class NearestNeighborQueryNode; + /** This is a leaf in the Query tree. All terms are leafs. A QueryTerm has the index for where to find the term. The term is a string, @@ -57,7 +59,7 @@ public: QueryTerm & operator = (const QueryTerm &) = delete; QueryTerm(QueryTerm &&) = delete; QueryTerm & operator = (QueryTerm &&) = delete; - ~QueryTerm(); + ~QueryTerm() override; bool evaluate() const override; const HitList & evaluateHits(HitList & hl) const override; void reset() override; @@ -87,6 +89,7 @@ public: const string & getIndex() const override { return _index; } void setFuzzyMaxEditDistance(uint32_t fuzzyMaxEditDistance) { _fuzzyMaxEditDistance = fuzzyMaxEditDistance; } void setFuzzyPrefixLength(uint32_t fuzzyPrefixLength) { _fuzzyPrefixLength = fuzzyPrefixLength; } + virtual NearestNeighborQueryNode* as_nearest_neighbor_query_node() noexcept; protected: using QueryNodeResultBaseContainer = std::unique_ptr<QueryNodeResultBase>; string _index; diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_search.cpp b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_search.cpp index 292605127fb..1a7e91b2d1a 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_search.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/wand/parallel_weak_and_search.cpp @@ -136,27 +136,25 @@ createWand(const wand::Terms &terms, using WandType = ParallelWeakAndSearchImpl<VectorizedIteratorTerms, FutureHeap, PastHeap, IS_STRICT>; if (should_monitor_wand()) { wand::Terms termsWithMonitoring = insertMonitoringSearchIterator(terms); - MonitoringSearchIterator::UP monitoringIterator = - MonitoringSearchIterator::UP(new MonitoringSearchIterator - (make_string("PWAND(%u,%" PRId64 "),strict=%u", - matchParams.scores.getScoresToTrack(), - matchParams.scoreThreshold, - IS_STRICT), - SearchIterator::UP(new WandType(rankParams.rootMatchData, - VectorizedIteratorTerms(termsWithMonitoring, - DotProductScorer(), - matchParams.docIdLimit, - std::move(rankParams.childrenMatchData)), - matchParams)), - false)); + auto monitoringIterator = std::make_unique<MonitoringSearchIterator>( + make_string("PWAND(%u,%" PRId64 "),strict=%u", + matchParams.scores.getScoresToTrack(), + matchParams.scoreThreshold, IS_STRICT), + std::make_unique<WandType>(rankParams.rootMatchData, + VectorizedIteratorTerms(termsWithMonitoring, + DotProductScorer(), + matchParams.docIdLimit, + std::move(rankParams.childrenMatchData)), + matchParams), + false); return std::make_unique<MonitoringDumpIterator>(std::move(monitoringIterator)); } return std::make_unique<WandType>(rankParams.rootMatchData, - VectorizedIteratorTerms(terms, - DotProductScorer(), - matchParams.docIdLimit, - std::move(rankParams.childrenMatchData)), - matchParams); + VectorizedIteratorTerms(terms, + DotProductScorer(), + matchParams.docIdLimit, + std::move(rankParams.childrenMatchData)), + matchParams); } } // namespace search::queryeval::wand::<unnamed> diff --git a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt index c8c5d4d4257..313863d8dcb 100644 --- a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt @@ -40,6 +40,7 @@ vespa_add_library(searchlib_tensor OBJECT tensor_buffer_store.cpp tensor_buffer_type_mapper.cpp tensor_deserialize.cpp + tensor_ext_attribute.cpp tensor_store.cpp DEPENDS ) diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp index c373f6bdcd0..c51d0ec7fd3 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp @@ -62,10 +62,10 @@ DenseTensorStore::BufferType::BufferType(const TensorSizeCalc &tensorSizeCalc, s DenseTensorStore::BufferType::~BufferType() = default; void -DenseTensorStore::BufferType::cleanHold(void *buffer, size_t offset, - ElemCount numElems, CleanContext) +DenseTensorStore::BufferType::clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext) { - memset(static_cast<char *>(buffer) + offset, 0, numElems); + auto num_elems = num_entries * getArraySize(); + memset(static_cast<char *>(buffer) + offset * getArraySize(), 0, num_elems); } const vespalib::alloc::MemoryAllocator* @@ -107,7 +107,7 @@ DenseTensorStore::allocRawBuffer() { size_t bufSize = getBufSize(); size_t alignedBufSize = _tensorSizeCalc.alignedSize(); - auto result = _concreteStore.freeListRawAllocator<char>(0u).alloc(alignedBufSize); + auto result = _concreteStore.freeListRawAllocator<char>(0u).alloc(1); clearPadAreaAfterBuffer(result.data, bufSize, alignedBufSize); return result; } @@ -118,7 +118,7 @@ DenseTensorStore::holdTensor(EntryRef ref) if (!ref.valid()) { return; } - _concreteStore.holdElem(ref, _tensorSizeCalc.alignedSize()); + _concreteStore.hold_entry(ref); } TensorStore::EntryRef diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h index 9e326e0ab1e..0dd483e7f08 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h @@ -44,7 +44,7 @@ public: public: BufferType(const TensorSizeCalc &tensorSizeCalc, std::shared_ptr<vespalib::alloc::MemoryAllocator> allocator); ~BufferType() override; - void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override; + void clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx) override; const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override; }; private: diff --git a/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.cpp index fa13ab6303c..8526138fd31 100644 --- a/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.cpp @@ -30,11 +30,11 @@ DirectTensorStore::TensorBufferType::TensorBufferType() } void -DirectTensorStore::TensorBufferType::cleanHold(void* buffer, size_t offset, ElemCount num_elems, CleanContext clean_ctx) +DirectTensorStore::TensorBufferType::clean_hold(void* buffer, size_t offset, EntryCount num_entries, CleanContext clean_ctx) { TensorSP* elem = static_cast<TensorSP*>(buffer) + offset; const auto& empty = empty_entry(); - for (size_t i = 0; i < num_elems; ++i) { + for (size_t i = 0; i < num_entries; ++i) { clean_ctx.extraBytesCleaned((*elem)->get_memory_usage().allocatedBytes()); *elem = empty; ++elem; @@ -69,7 +69,7 @@ DirectTensorStore::holdTensor(EntryRef ref) } const auto& tensor = _tensor_store.getEntry(ref); assert(tensor); - _tensor_store.holdElem(ref, 1, tensor->get_memory_usage().allocatedBytes()); + _tensor_store.hold_entry(ref, tensor->get_memory_usage().allocatedBytes()); } EntryRef diff --git a/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h b/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h index 01084e89776..1230494fe41 100644 --- a/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h +++ b/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h @@ -20,7 +20,7 @@ namespace search::tensor { */ class DirectTensorStore : public TensorStore { private: - // Note: Must use SP (instead of UP) because of fallbackCopy() and initializeReservedElements() in BufferType, + // Note: Must use SP (instead of UP) because of fallback_copy() and initialize_reserved_entries() in BufferType, // and implementation of move(). using TensorSP = std::shared_ptr<vespalib::eval::Value>; using TensorStoreType = vespalib::datastore::DataStore<TensorSP>; @@ -32,7 +32,7 @@ private: using CleanContext = typename ParentType::CleanContext; public: TensorBufferType(); - void cleanHold(void* buffer, size_t offset, ElemCount num_elems, CleanContext clean_ctx) override; + void clean_hold(void* buffer, size_t offset, EntryCount num_entries, CleanContext clean_ctx) override; }; TensorStoreType _tensor_store; diff --git a/searchlib/src/vespa/searchlib/tensor/large_subspaces_buffer_type.cpp b/searchlib/src/vespa/searchlib/tensor/large_subspaces_buffer_type.cpp index cb074348f08..5d3b2206703 100644 --- a/searchlib/src/vespa/searchlib/tensor/large_subspaces_buffer_type.cpp +++ b/searchlib/src/vespa/searchlib/tensor/large_subspaces_buffer_type.cpp @@ -12,7 +12,7 @@ using vespalib::alloc::MemoryAllocator; namespace search::tensor { LargeSubspacesBufferType::LargeSubspacesBufferType(const AllocSpec& spec, std::shared_ptr<MemoryAllocator> memory_allocator, TensorBufferTypeMapper& type_mapper) noexcept - : ParentType(1u, spec.minArraysInBuffer, spec.maxArraysInBuffer, spec.numArraysForNewBuffer, spec.allocGrowFactor), + : ParentType(1u, spec.min_entries_in_buffer, spec.max_entries_in_buffer, spec.num_entries_for_new_buffer, spec.allocGrowFactor), _memory_allocator(std::move(memory_allocator)), _ops(type_mapper.get_tensor_buffer_operations()) { @@ -21,10 +21,10 @@ LargeSubspacesBufferType::LargeSubspacesBufferType(const AllocSpec& spec, std::s LargeSubspacesBufferType::~LargeSubspacesBufferType() = default; void -LargeSubspacesBufferType::cleanHold(void* buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) +LargeSubspacesBufferType::clean_hold(void* buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx) { auto elem = static_cast<ArrayType*>(buffer) + offset; - for (size_t i = 0; i < numElems; ++i) { + for (size_t i = 0; i < num_entries; ++i) { if (!elem->empty()) { cleanCtx.extraBytesCleaned(elem->size()); _ops.reclaim_labels({elem->data(), elem->size()}); @@ -35,10 +35,10 @@ LargeSubspacesBufferType::cleanHold(void* buffer, size_t offset, ElemCount numEl } void -LargeSubspacesBufferType::destroyElements(void *buffer, ElemCount numElems) +LargeSubspacesBufferType::destroy_entries(void *buffer, EntryCount num_entries) { auto elem = static_cast<ArrayType*>(buffer); - for (size_t i = 0; i < numElems; ++i) { + for (size_t i = 0; i < num_entries; ++i) { if (!elem->empty()) { _ops.reclaim_labels({elem->data(), elem->size()}); ArrayType().swap(*elem); @@ -48,11 +48,11 @@ LargeSubspacesBufferType::destroyElements(void *buffer, ElemCount numElems) } void -LargeSubspacesBufferType::fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems) +LargeSubspacesBufferType::fallback_copy(void *newBuffer, const void *oldBuffer, EntryCount num_entries) { auto old_elems = static_cast<const ArrayType*>(oldBuffer); auto new_elems = static_cast<ArrayType*>(newBuffer); - for (size_t i = 0; i < numElems; ++i) { + for (size_t i = 0; i < num_entries; ++i) { auto& old_elem = old_elems[i]; new (new_elems + i) ArrayType(old_elem); if (!old_elem.empty()) { @@ -62,12 +62,12 @@ LargeSubspacesBufferType::fallbackCopy(void *newBuffer, const void *oldBuffer, E } void -LargeSubspacesBufferType::initializeReservedElements(void *buffer, ElemCount reservedElements) +LargeSubspacesBufferType::initialize_reserved_entries(void *buffer, EntryCount reserved_entries) { - auto new_elems = static_cast<ArrayType*>(buffer); + auto new_entries = static_cast<ArrayType*>(buffer); const auto& empty = empty_entry(); - for (size_t i = 0; i < reservedElements; ++i) { - new (new_elems + i) ArrayType(empty); + for (size_t i = 0; i < reserved_entries; ++i) { + new (new_entries + i) ArrayType(empty); } } diff --git a/searchlib/src/vespa/searchlib/tensor/large_subspaces_buffer_type.h b/searchlib/src/vespa/searchlib/tensor/large_subspaces_buffer_type.h index cfab8ef20af..8cce08e9d81 100644 --- a/searchlib/src/vespa/searchlib/tensor/large_subspaces_buffer_type.h +++ b/searchlib/src/vespa/searchlib/tensor/large_subspaces_buffer_type.h @@ -30,10 +30,10 @@ class LargeSubspacesBufferType : public vespalib::datastore::BufferType<vespalib public: LargeSubspacesBufferType(const AllocSpec& spec, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator, TensorBufferTypeMapper& type_mapper) noexcept; ~LargeSubspacesBufferType() override; - void cleanHold(void* buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override; - void destroyElements(void *buffer, ElemCount numElems) override; - void fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems) override; - void initializeReservedElements(void *buffer, ElemCount reservedElements) override; + void clean_hold(void* buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx) override; + void destroy_entries(void *buffer, EntryCount num_entries) override; + void fallback_copy(void *newBuffer, const void *oldBuffer, EntryCount num_entries) override; + void initialize_reserved_entries(void *buffer, EntryCount reserved_entries) override; const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override; }; diff --git a/searchlib/src/vespa/searchlib/tensor/small_subspaces_buffer_type.cpp b/searchlib/src/vespa/searchlib/tensor/small_subspaces_buffer_type.cpp index 6a71388a3b9..7b54182f062 100644 --- a/searchlib/src/vespa/searchlib/tensor/small_subspaces_buffer_type.cpp +++ b/searchlib/src/vespa/searchlib/tensor/small_subspaces_buffer_type.cpp @@ -10,7 +10,7 @@ using vespalib::alloc::MemoryAllocator; namespace search::tensor { SmallSubspacesBufferType::SmallSubspacesBufferType(uint32_t array_size, const AllocSpec& spec, std::shared_ptr<MemoryAllocator> memory_allocator, TensorBufferTypeMapper& type_mapper) noexcept - : ParentType(array_size, spec.minArraysInBuffer, spec.maxArraysInBuffer, spec.numArraysForNewBuffer, spec.allocGrowFactor), + : ParentType(array_size, spec.min_entries_in_buffer, spec.max_entries_in_buffer, spec.num_entries_for_new_buffer, spec.allocGrowFactor), _memory_allocator(std::move(memory_allocator)), _ops(type_mapper.get_tensor_buffer_operations()) { @@ -19,45 +19,45 @@ SmallSubspacesBufferType::SmallSubspacesBufferType(uint32_t array_size, const Al SmallSubspacesBufferType::~SmallSubspacesBufferType() = default; void -SmallSubspacesBufferType::cleanHold(void* buffer, size_t offset, ElemCount numElems, CleanContext) +SmallSubspacesBufferType::clean_hold(void* buffer, size_t offset, EntryCount num_entries, CleanContext) { - char* elem = static_cast<char *>(buffer) + offset; - while (numElems >= getArraySize()) { + char* elem = static_cast<char *>(buffer) + offset * getArraySize(); + while (num_entries >= 1) { _ops.reclaim_labels(vespalib::ArrayRef<char>(elem, getArraySize())); elem += getArraySize(); - numElems -= getArraySize(); + --num_entries; } } void -SmallSubspacesBufferType::destroyElements(void *buffer, ElemCount numElems) +SmallSubspacesBufferType::destroy_entries(void *buffer, EntryCount num_entries) { char* elem = static_cast<char *>(buffer); - while (numElems >= getArraySize()) { + while (num_entries >= 1) { _ops.reclaim_labels(vespalib::ArrayRef<char>(elem, getArraySize())); elem += getArraySize(); - numElems -= getArraySize(); + --num_entries; } } void -SmallSubspacesBufferType::fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems) +SmallSubspacesBufferType::fallback_copy(void *newBuffer, const void *oldBuffer, EntryCount num_entries) { - if (numElems > 0) { - memcpy(newBuffer, oldBuffer, numElems); + if (num_entries > 0) { + memcpy(newBuffer, oldBuffer, num_entries * getArraySize()); } const char *elem = static_cast<const char *>(oldBuffer); - while (numElems >= getArraySize()) { + while (num_entries >= 1) { _ops.copied_labels(unconstify(vespalib::ConstArrayRef<char>(elem, getArraySize()))); elem += getArraySize(); - numElems -= getArraySize(); + --num_entries; } } void -SmallSubspacesBufferType::initializeReservedElements(void *buffer, ElemCount reservedElements) +SmallSubspacesBufferType::initialize_reserved_entries(void *buffer, EntryCount reserved_entries) { - memset(buffer, 0, reservedElements); + memset(buffer, 0, reserved_entries * getArraySize()); } const vespalib::alloc::MemoryAllocator* diff --git a/searchlib/src/vespa/searchlib/tensor/small_subspaces_buffer_type.h b/searchlib/src/vespa/searchlib/tensor/small_subspaces_buffer_type.h index 5622e9970b8..2f287ef1f3d 100644 --- a/searchlib/src/vespa/searchlib/tensor/small_subspaces_buffer_type.h +++ b/searchlib/src/vespa/searchlib/tensor/small_subspaces_buffer_type.h @@ -30,10 +30,10 @@ public: SmallSubspacesBufferType& operator=(SmallSubspacesBufferType&&) noexcept = delete; SmallSubspacesBufferType(uint32_t array_size, const AllocSpec& spec, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator, TensorBufferTypeMapper& type_mapper) noexcept; ~SmallSubspacesBufferType() override; - void cleanHold(void* buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override; - void destroyElements(void *buffer, ElemCount numElems) override; - void fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems) override; - void initializeReservedElements(void *buffer, ElemCount reservedElements) override; + void clean_hold(void* buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx) override; + void destroy_entries(void *buffer, EntryCount num_entries) override; + void fallback_copy(void *newBuffer, const void *oldBuffer, EntryCount num_entries) override; + void initialize_reserved_entries(void *buffer, EntryCount reserved_entries) override; const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override; }; diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_ext_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_ext_attribute.cpp new file mode 100644 index 00000000000..19c8cf6053b --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/tensor_ext_attribute.cpp @@ -0,0 +1,181 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "tensor_ext_attribute.h" +#include "serialized_tensor_ref.h" +#include "vector_bundle.h" +#include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/value_codec.h> +#include <vespa/searchcommon/attribute/config.h> + +#include <vespa/log/log.h> +LOG_SETUP(".searchlib.tensor.tensor_ext_attribute"); + +using vespalib::eval::FastValueBuilderFactory; +using vespalib::eval::TensorSpec; +using vespalib::eval::TypedCells; +using vespalib::eval::Value; +using vespalib::eval::ValueType; + +namespace search::tensor { + +namespace { + +std::unique_ptr<Value> +create_empty_tensor(const ValueType& type) +{ + const auto &factory = FastValueBuilderFactory::get(); + TensorSpec empty_spec(type.to_spec()); + return vespalib::eval::value_from_spec(empty_spec, factory); +} + +} + +TensorExtAttribute::TensorExtAttribute(const vespalib::string& name, const Config& cfg) + : NotImplementedAttribute(name, cfg), + ITensorAttribute(), + IExtendAttribute(), + _subspace_type(cfg.tensorType()), + _empty(_subspace_type), + _empty_tensor(create_empty_tensor(cfg.tensorType())) +{ +} + +TensorExtAttribute::~TensorExtAttribute() = default; + +const ITensorAttribute* +TensorExtAttribute::asTensorAttribute() const +{ + return this; +} + +void +TensorExtAttribute::onCommit() +{ + LOG_ABORT("should not be reached"); +} + +void +TensorExtAttribute::onUpdateStat() +{ +} + +bool +TensorExtAttribute::addDoc(DocId& docId) +{ + docId = _data.size(); + _data.emplace_back(nullptr); + incNumDocs(); + setCommittedDocIdLimit(getNumDocs()); + return true; +} + +bool +TensorExtAttribute::add(const vespalib::eval::Value& v, int32_t) +{ + _data.back() = &v; + return true; +} + +IExtendAttribute* +TensorExtAttribute::getExtendInterface() +{ + return this; +} + +TypedCells +TensorExtAttribute::get_vector(uint32_t docid, uint32_t subspace) const +{ + auto vectors = get_vectors(docid); + return (subspace < vectors.subspaces()) ? vectors.cells(subspace) : _empty.cells(); +} + +VectorBundle +TensorExtAttribute::get_vectors(uint32_t docid) const +{ + auto tensor = _data[docid]; + if (tensor == nullptr) { + return VectorBundle(); + } + return VectorBundle(tensor->cells().data, tensor->index().size(), _subspace_type); +} + +std::unique_ptr<Value> +TensorExtAttribute::getTensor(uint32_t docid) const +{ + auto tensor = _data[docid]; + if (tensor == nullptr) { + return {}; + } + return FastValueBuilderFactory::get().copy(*tensor); +} + +std::unique_ptr<Value> +TensorExtAttribute::getEmptyTensor() const +{ + return FastValueBuilderFactory::get().copy(*_empty_tensor); +} + +TypedCells +TensorExtAttribute::extract_cells_ref(uint32_t docid) const +{ + return get_vector(docid, 0); +} + +const vespalib::eval::Value& +TensorExtAttribute::get_tensor_ref(uint32_t docid) const +{ + auto tensor = _data[docid]; + return (tensor == nullptr) ? *_empty_tensor : *tensor; +} + +SerializedTensorRef +TensorExtAttribute::get_serialized_tensor_ref(uint32_t) const +{ + notImplemented(); +} + +bool +TensorExtAttribute::supports_extract_cells_ref() const +{ + return getConfig().tensorType().is_dense(); +} + +bool +TensorExtAttribute::supports_get_tensor_ref() const +{ + return true; +} + +bool +TensorExtAttribute::supports_get_serialized_tensor_ref() const +{ + return false; +} + +const ValueType& +TensorExtAttribute::getTensorType() const +{ + return getConfig().tensorType(); +} + +TensorExtAttribute::DistanceMetric +TensorExtAttribute::distance_metric() const +{ + return getConfig().distance_metric(); +} + +uint32_t +TensorExtAttribute::get_num_docs() const +{ + return _data.size(); +} + +void +TensorExtAttribute::get_state(const vespalib::slime::Inserter& inserter) const +{ + (void) inserter; +} + +} diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_ext_attribute.h b/searchlib/src/vespa/searchlib/tensor/tensor_ext_attribute.h new file mode 100644 index 00000000000..a58426cd146 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/tensor_ext_attribute.h @@ -0,0 +1,54 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_tensor_attribute.h" +#include "empty_subspace.h" +#include "subspace_type.h" +#include <vespa/searchlib/attribute/not_implemented_attribute.h> +#include <vespa/vespalib/stllike/allocator.h> + +namespace search::tensor { + +/** + * Attribute vector storing a pointer to single tensor value per + * document in streaming search. The tensor is not owned by this + * attribute vector. + */ +class TensorExtAttribute : public NotImplementedAttribute, + public ITensorAttribute, + public IExtendAttribute +{ + std::vector<const vespalib::eval::Value*> _data; + SubspaceType _subspace_type; + EmptySubspace _empty; + std::unique_ptr<vespalib::eval::Value> _empty_tensor; +public: + TensorExtAttribute(const vespalib::string& name, const Config& cfg); + ~TensorExtAttribute() override; + const ITensorAttribute* asTensorAttribute() const override; + void onCommit() override; + void onUpdateStat() override; + bool addDoc(DocId& docId) override; + bool add(const vespalib::eval::Value& v, int32_t) override; + IExtendAttribute* getExtendInterface() override; + // DocVectorAccess API + vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const override; + VectorBundle get_vectors(uint32_t docid) const override; + + // ITensorAttribute API + std::unique_ptr<vespalib::eval::Value> getTensor(uint32_t docid) const override; + std::unique_ptr<vespalib::eval::Value> getEmptyTensor() const override; + vespalib::eval::TypedCells extract_cells_ref(uint32_t docid) const override; + const vespalib::eval::Value& get_tensor_ref(uint32_t docid) const override; + SerializedTensorRef get_serialized_tensor_ref(uint32_t docid) const override; + bool supports_extract_cells_ref() const override; + bool supports_get_tensor_ref() const override; + bool supports_get_serialized_tensor_ref() const override; + const vespalib::eval::ValueType & getTensorType() const override; + search::attribute::DistanceMetric distance_metric() const override; + uint32_t get_num_docs() const override; + void get_state(const vespalib::slime::Inserter& inserter) const override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/test/attribute_builder.cpp b/searchlib/src/vespa/searchlib/test/attribute_builder.cpp index cc84355385d..ce8c86367b1 100644 --- a/searchlib/src/vespa/searchlib/test/attribute_builder.cpp +++ b/searchlib/src/vespa/searchlib/test/attribute_builder.cpp @@ -7,6 +7,7 @@ #include <vespa/searchlib/attribute/attributevector.h> #include <vespa/searchlib/attribute/floatbase.h> #include <vespa/searchlib/attribute/integerbase.h> +#include <vespa/searchlib/attribute/single_raw_attribute.h> #include <vespa/searchlib/attribute/stringbase.h> #include <vespa/searchlib/tensor/tensor_attribute.h> #include <cassert> @@ -158,6 +159,13 @@ AttributeBuilder::fill_wset(std::initializer_list<WeightedStringList> values) } AttributeBuilder& +AttributeBuilder::fill(std::initializer_list<vespalib::ConstArrayRef<char>> values) +{ + fill_helper<SingleRawAttribute, vespalib::ConstArrayRef<char>>(_attr, values); + return *this; +} + +AttributeBuilder& AttributeBuilder::fill_tensor(const std::vector<vespalib::string>& values) { add_docs(_attr, values.size()); diff --git a/searchlib/src/vespa/searchlib/test/attribute_builder.h b/searchlib/src/vespa/searchlib/test/attribute_builder.h index 339af4e22f5..e6730b74249 100644 --- a/searchlib/src/vespa/searchlib/test/attribute_builder.h +++ b/searchlib/src/vespa/searchlib/test/attribute_builder.h @@ -4,6 +4,7 @@ #include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/util/arrayref.h> #include <memory> #include <utility> #include <vector> @@ -52,6 +53,9 @@ public: AttributeBuilder& fill_array(std::initializer_list<StringList> values); AttributeBuilder& fill_wset(std::initializer_list<WeightedStringList> values); + // Fill function for raw attributes + AttributeBuilder& fill(std::initializer_list<vespalib::ConstArrayRef<char>> values); + /** * Fill this tensor attribute with the given tensor values. * diff --git a/searchlib/src/vespa/searchlib/test/make_attribute_map_lookup_node.cpp b/searchlib/src/vespa/searchlib/test/make_attribute_map_lookup_node.cpp index a030dcb0640..2edd00ad5a5 100644 --- a/searchlib/src/vespa/searchlib/test/make_attribute_map_lookup_node.cpp +++ b/searchlib/src/vespa/searchlib/test/make_attribute_map_lookup_node.cpp @@ -13,7 +13,7 @@ vespalib::string indirectKeyMarker("attribute("); } std::unique_ptr<AttributeNode> -makeAttributeMapLookupNode(const vespalib::string attributeName) +makeAttributeMapLookupNode(vespalib::stringref attributeName) { vespalib::asciistream keyName; vespalib::asciistream valueName; diff --git a/searchlib/src/vespa/searchlib/test/make_attribute_map_lookup_node.h b/searchlib/src/vespa/searchlib/test/make_attribute_map_lookup_node.h index f56cea962d3..baa074b4004 100644 --- a/searchlib/src/vespa/searchlib/test/make_attribute_map_lookup_node.h +++ b/searchlib/src/vespa/searchlib/test/make_attribute_map_lookup_node.h @@ -9,6 +9,6 @@ namespace search::expression { class AttributeNode; } namespace search::expression::test { std::unique_ptr<AttributeNode> -makeAttributeMapLookupNode(const vespalib::string attributeName); +makeAttributeMapLookupNode(vespalib::stringref attributeName); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp index d6f06c9161e..6ac4fea2921 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp @@ -23,8 +23,7 @@ namespace search::docsummary { DocsumFieldWriterFactory::DocsumFieldWriterFactory(bool use_v8_geo_positions, const IDocsumEnvironment& env, const IQueryTermFilterFactory& query_term_filter_factory) : _use_v8_geo_positions(use_v8_geo_positions), _env(env), - _query_term_filter_factory(query_term_filter_factory), - _matching_elems_fields(std::make_shared<MatchingElementsFields>()) + _query_term_filter_factory(query_term_filter_factory) { } @@ -58,7 +57,8 @@ throw_missing_source(const vespalib::string& command) std::unique_ptr<DocsumFieldWriter> DocsumFieldWriterFactory::create_docsum_field_writer(const vespalib::string& field_name, const vespalib::string& command, - const vespalib::string& source) + const vespalib::string& source, + std::shared_ptr<MatchingElementsFields> matching_elems_fields) { std::unique_ptr<DocsumFieldWriter> fieldWriter; if (command == command::dynamic_teaser) { @@ -116,9 +116,9 @@ DocsumFieldWriterFactory::create_docsum_field_writer(const vespalib::string& fie if (has_attribute_manager()) { auto attr_ctx = getEnvironment().getAttributeManager()->createContext(); if (attr_ctx->getAttribute(source_field) != nullptr) { - fieldWriter = AttributeDFWFactory::create(*getEnvironment().getAttributeManager(), source_field, true, _matching_elems_fields); + fieldWriter = AttributeDFWFactory::create(*getEnvironment().getAttributeManager(), source_field, true, matching_elems_fields); } else { - fieldWriter = AttributeCombinerDFW::create(source_field, *attr_ctx, true, _matching_elems_fields); + fieldWriter = AttributeCombinerDFW::create(source_field, *attr_ctx, true, matching_elems_fields); } throw_if_nullptr(fieldWriter, command); } @@ -126,7 +126,7 @@ DocsumFieldWriterFactory::create_docsum_field_writer(const vespalib::string& fie const vespalib::string& source_field = source.empty() ? field_name : source; if (has_attribute_manager()) { auto attr_ctx = getEnvironment().getAttributeManager()->createContext(); - fieldWriter = MatchedElementsFilterDFW::create(source_field,*attr_ctx, _matching_elems_fields); + fieldWriter = MatchedElementsFilterDFW::create(source_field,*attr_ctx, matching_elems_fields); throw_if_nullptr(fieldWriter, command); } } else if (command == command::documentid) { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.h b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.h index e50fb85cca6..7175f043701 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.h @@ -20,7 +20,6 @@ class DocsumFieldWriterFactory : public IDocsumFieldWriterFactory const IDocsumEnvironment& _env; const IQueryTermFilterFactory& _query_term_filter_factory; protected: - std::shared_ptr<MatchingElementsFields> _matching_elems_fields; const IDocsumEnvironment& getEnvironment() const noexcept { return _env; } bool has_attribute_manager() const noexcept; public: @@ -28,7 +27,8 @@ public: ~DocsumFieldWriterFactory() override; std::unique_ptr<DocsumFieldWriter> create_docsum_field_writer(const vespalib::string& field_name, const vespalib::string& command, - const vespalib::string& source) override; + const vespalib::string& source, + std::shared_ptr<MatchingElementsFields> matching_elems_fields) override; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_field_writer_factory.h b/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_field_writer_factory.h index 6a5cd691857..bc2ebe3c40c 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_field_writer_factory.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_field_writer_factory.h @@ -5,6 +5,8 @@ #include <memory> #include <vespa/vespalib/stllike/string.h> +namespace search { class MatchingElementsFields; } + namespace search::docsummary { class DocsumFieldWriter; @@ -21,7 +23,8 @@ public: */ virtual std::unique_ptr<DocsumFieldWriter> create_docsum_field_writer(const vespalib::string& field_name, const vespalib::string& command, - const vespalib::string& source) = 0; + const vespalib::string& source, + std::shared_ptr<MatchingElementsFields> matching_elems_fields) = 0; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp index f620dcb1df5..eddb67f5822 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp @@ -5,6 +5,7 @@ #include "docsum_field_writer_factory.h" #include "resultclass.h" #include <vespa/config-summary.h> +#include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/exceptions.h> #include <atomic> @@ -124,6 +125,7 @@ ResultConfig::readConfig(const SummaryConfig &cfg, const char *configId, IDocsum break; } resClass->set_omit_summary_features(cfg_class.omitsummaryfeatures); + auto matching_elems_fields = std::make_shared<MatchingElementsFields>(); for (const auto & field : cfg_class.fields) { const char *fieldname = field.name.c_str(); vespalib::string command = field.command; @@ -134,7 +136,8 @@ ResultConfig::readConfig(const SummaryConfig &cfg, const char *configId, IDocsum try { docsum_field_writer = docsum_field_writer_factory.create_docsum_field_writer(fieldname, command, - source_name); + source_name, + matching_elems_fields); } catch (const vespalib::IllegalArgumentException& ex) { LOG(error, "Exception during setup of summary result class '%s': field='%s', command='%s', source='%s': %s", cfg_class.name.c_str(), fieldname, command.c_str(), source_name.c_str(), ex.getMessage().c_str()); diff --git a/security-utils/src/main/java/com/yahoo/security/SideChannelSafe.java b/security-utils/src/main/java/com/yahoo/security/SideChannelSafe.java index 1f160d94c6a..bd085f6f624 100644 --- a/security-utils/src/main/java/com/yahoo/security/SideChannelSafe.java +++ b/security-utils/src/main/java/com/yahoo/security/SideChannelSafe.java @@ -46,7 +46,7 @@ public class SideChannelSafe { // differed in any byte compared between the two arrays. byte accu = 0; for (int i = 0; i < lhs.length; ++i) { - accu |= (lhs[i] ^ rhs[i]); + accu |= (byte)(lhs[i] ^ rhs[i]); } return (accu == 0); } diff --git a/security-utils/src/test/java/com/yahoo/security/SharedKeyTest.java b/security-utils/src/test/java/com/yahoo/security/SharedKeyTest.java index 26627e9a5fa..90b8beb461f 100644 --- a/security-utils/src/test/java/com/yahoo/security/SharedKeyTest.java +++ b/security-utils/src/test/java/com/yahoo/security/SharedKeyTest.java @@ -285,12 +285,12 @@ public class SharedKeyTest { String plaintext = "...hello world?"; byte[] encrypted = streamEncryptString(plaintext, myShared); // Corrupt MAC tag in ciphertext - encrypted[encrypted.length - 1] ^= 0x80; + encrypted[encrypted.length - 1] ^= (byte)0x80; // We don't necessarily know _which_ exception is thrown, but one _should_ be thrown! assertThrows(Exception.class, () -> doOutputStreamCipherDecrypt(myShared, encrypted)); // Also try with corrupted ciphertext (pre MAC tag) - encrypted[encrypted.length - 1] ^= 0x80; // Flip MAC bit back to correct state - encrypted[encrypted.length - 17] ^= 0x80; // Pre 128-bit MAC tag + encrypted[encrypted.length - 1] ^= (byte)0x80; // Flip MAC bit back to correct state + encrypted[encrypted.length - 17] ^= (byte)0x80; // Pre 128-bit MAC tag assertThrows(Exception.class, () -> doOutputStreamCipherDecrypt(myShared, encrypted)); } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java index 3a9c2b1f1e7..d76b4ecc0b5 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.duper; +import ai.vespa.http.DomainName; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.HostName; import java.util.logging.Level; import com.yahoo.vespa.service.monitor.DuperModelListener; @@ -26,8 +26,8 @@ public class DuperModel { private static Logger logger = Logger.getLogger(DuperModel.class.getName()); private final Map<ApplicationId, ApplicationInfo> applicationsById = new HashMap<>(); - private final Map<HostName, ApplicationId> idsByHostname = new HashMap<>(); - private final Map<ApplicationId, HashSet<HostName>> hostnamesById = new HashMap<>(); + private final Map<DomainName, ApplicationId> idsByHostname = new HashMap<>(); + private final Map<ApplicationId, HashSet<DomainName>> hostnamesById = new HashMap<>(); private final List<DuperModelListener> listeners = new ArrayList<>(); private boolean isComplete = false; @@ -67,7 +67,7 @@ public class DuperModel { return Optional.ofNullable(applicationsById.get(applicationId)); } - public Optional<ApplicationInfo> getApplicationInfo(HostName hostName) { + public Optional<ApplicationInfo> getApplicationInfo(DomainName hostName) { return Optional.ofNullable(idsByHostname.get(hostName)).map(applicationsById::get); } @@ -76,12 +76,12 @@ public class DuperModel { } /** Note: Returns an empty set for unknown application. */ - public Set<HostName> getHostnames(ApplicationId applicationId) { - HashSet<HostName> set = hostnamesById.get(applicationId); + public Set<DomainName> getHostnames(ApplicationId applicationId) { + HashSet<DomainName> set = hostnamesById.get(applicationId); return set == null ? Set.of() : Set.copyOf(set); } - public Optional<ApplicationId> getApplicationId(HostName hostname) { + public Optional<ApplicationId> getApplicationId(DomainName hostname) { return Optional.ofNullable(idsByHostname.get(hostname)); } @@ -103,7 +103,7 @@ public class DuperModel { } public void remove(ApplicationId applicationId) { - Set<HostName> hostnames = hostnamesById.remove(applicationId); + Set<DomainName> hostnames = hostnamesById.remove(applicationId); if (hostnames != null) { hostnames.forEach(idsByHostname::remove); } @@ -117,11 +117,11 @@ public class DuperModel { /** Update hostnamesById and idsByHostname based on a new applicationInfo. */ private void updateHostnameVsIdMaps(ApplicationInfo applicationInfo, ApplicationId id) { - Set<HostName> removedHosts = new HashSet<>(hostnamesById.computeIfAbsent(id, k -> new HashSet<>())); + Set<DomainName> removedHosts = new HashSet<>(hostnamesById.computeIfAbsent(id, k -> new HashSet<>())); applicationInfo.getModel().getHosts().stream() .map(HostInfo::getHostname) - .map(HostName::of) + .map(DomainName::of) .forEach(hostname -> { if (!removedHosts.remove(hostname)) { // hostname has been added @@ -135,7 +135,7 @@ public class DuperModel { logger.log(Level.WARNING, hostname + " has been reassigned from " + previousId.toFullString() + " to " + id.toFullString()); - Set<HostName> previousHostnames = hostnamesById.get(previousId); + Set<DomainName> previousHostnames = hostnamesById.get(previousId); if (previousHostnames != null) { previousHostnames.remove(hostname); } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java index b8c980a8760..400c4f4b907 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java @@ -1,14 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.duper; -import com.yahoo.component.annotation.Inject; +import ai.vespa.http.DomainName; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.SuperModel; import com.yahoo.config.model.api.SuperModelListener; import com.yahoo.config.model.api.SuperModelProvider; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.HostName; import com.yahoo.vespa.service.monitor.CriticalRegion; import com.yahoo.vespa.service.monitor.DuperModelInfraApi; import com.yahoo.vespa.service.monitor.DuperModelListener; @@ -133,7 +133,7 @@ public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi } @Override - public void infraApplicationActivated(ApplicationId applicationId, List<HostName> hostnames) { + public void infraApplicationActivated(ApplicationId applicationId, List<DomainName> hostnames) { InfraApplication application = supportedInfraApplications.get(applicationId); if (application == null) { throw new IllegalArgumentException("There is no infrastructure application with ID '" + applicationId + "'"); @@ -172,7 +172,7 @@ public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi return lockedSupplier(() -> duperModel.getApplicationInfo(applicationId)); } - public Optional<ApplicationInfo> getApplicationInfo(HostName hostname) { + public Optional<ApplicationInfo> getApplicationInfo(DomainName hostname) { return lockedSupplier(() -> duperModel.getApplicationInfo(hostname)); } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java index e82e2cc0265..07f9d7bddc5 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.duper; +import ai.vespa.http.DomainName; import com.yahoo.component.Version; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.HostInfo; @@ -9,7 +10,6 @@ import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; import com.yahoo.vespa.applicationmodel.ClusterId; @@ -93,12 +93,12 @@ public abstract class InfraApplication implements InfraApplicationApi { return new TenantId(application.id().tenant().value()); } - public ApplicationInfo makeApplicationInfo(List<HostName> hostnames) { + public ApplicationInfo makeApplicationInfo(List<DomainName> hostnames) { List<HostInfo> hostInfos = hostnames.stream().map(this::makeHostInfo).toList(); return new ApplicationInfo(application.id(), 0, new HostsModel(hostInfos)); } - private HostInfo makeHostInfo(HostName hostname) { + private HostInfo makeHostInfo(DomainName hostname) { PortInfo portInfo = new PortInfo(healthPort, StateV1HealthModel.HTTP_HEALTH_PORT_TAGS); Map<String, String> properties = new HashMap<>(); @@ -116,7 +116,7 @@ public abstract class InfraApplication implements InfraApplicationApi { return new HostInfo(hostname.value(), Collections.singletonList(serviceInfo)); } - public ConfigId configIdFor(HostName hostname) { + public ConfigId configIdFor(DomainName hostname) { // Not necessarily unique, but service monitor doesn't require it to be unique. return new ConfigId(String.format("%s/%s", clusterSpecId.value(), prefixTo(hostname.value(), '.'))); } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java index a654a4557ef..1d832cc0eec 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.health; -import com.yahoo.config.provision.HostName; +import ai.vespa.http.DomainName; import com.yahoo.vespa.service.executor.RunletExecutor; import com.yahoo.vespa.service.monitor.ServiceId; @@ -22,7 +22,7 @@ class StateV1HealthEndpoint implements HealthEndpoint { private final RunletExecutor executor; StateV1HealthEndpoint(ServiceId serviceId, - HostName hostname, + DomainName hostname, int port, Duration delay, Duration requestTimeout, diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java index 3391bbcf82a..3969ef141bd 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.health; +import ai.vespa.http.DomainName; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.PortInfo; import com.yahoo.config.model.api.ServiceInfo; -import com.yahoo.config.provision.HostName; import com.yahoo.vespa.service.executor.RunletExecutor; import com.yahoo.vespa.service.model.ApplicationInstanceGenerator; import com.yahoo.vespa.service.monitor.ServiceId; @@ -45,7 +45,7 @@ public class StateV1HealthModel implements AutoCloseable { Map<ServiceId, HealthEndpoint> endpoints = new HashMap<>(); for (HostInfo hostInfo : application.getModel().getHosts()) { - HostName hostname = HostName.of(hostInfo.getHostname()); + DomainName hostname = DomainName.of(hostInfo.getHostname()); for (ServiceInfo serviceInfo : hostInfo.getServices()) { ServiceId serviceId = ApplicationInstanceGenerator.getServiceId(application, serviceInfo); for (PortInfo portInfo : serviceInfo.getPorts()) { diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java index 815f8de26b4..53db7cc135c 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.model; +import ai.vespa.http.DomainName; import com.yahoo.component.annotation.Inject; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.provision.ApplicationId; @@ -68,7 +69,7 @@ public class ServiceMonitorImpl implements ServiceMonitor, AntiServiceMonitor { @Override public Optional<ApplicationInstanceReference> getApplicationInstanceReference(HostName hostname) { - return duperModelManager.getApplicationInfo(toConfigProvisionHostName(hostname)) + return duperModelManager.getApplicationInfo(DomainName.of(hostname.s())) .map(ApplicationInfo::getApplicationId) .map(modelGenerator::toApplicationInstanceReference); } @@ -113,11 +114,7 @@ public class ServiceMonitorImpl implements ServiceMonitor, AntiServiceMonitor { } private Optional<ApplicationInfo> getApplicationInfo(HostName hostname) { - return duperModelManager.getApplicationInfo(toConfigProvisionHostName(hostname)); + return duperModelManager.getApplicationInfo(DomainName.of(hostname.s())); } - /** The duper model uses HostName from config.provision. */ - private static com.yahoo.config.provision.HostName toConfigProvisionHostName(HostName hostname) { - return com.yahoo.config.provision.HostName.of(hostname.s()); - } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java index 91782d5a582..783f0a27e51 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.monitor; +import ai.vespa.http.DomainName; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.HostName; import java.util.List; import java.util.Optional; @@ -24,7 +24,7 @@ public interface DuperModelInfraApi { boolean infraApplicationIsActive(ApplicationId applicationId); /** Update the DuperModel: A supported infrastructure application has been (re)activated or is active. */ - void infraApplicationActivated(ApplicationId applicationId, List<HostName> hostnames); + void infraApplicationActivated(ApplicationId applicationId, List<DomainName> hostnames); /** Update the DuperModel: A supported infrastructure application has been removed or is not active. */ void infraApplicationRemoved(ApplicationId applicationId); diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java index 6cf7d4d9f26..ad7f0f188c0 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java @@ -1,17 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.duper; +import ai.vespa.http.DomainName; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.SuperModel; import com.yahoo.config.model.api.SuperModelListener; import com.yahoo.config.model.api.SuperModelProvider; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.HostName; import org.junit.Test; import org.mockito.ArgumentCaptor; import java.util.List; -import java.util.stream.Collectors; import java.util.stream.Stream; import static com.yahoo.vespa.service.duper.DuperModelManager.configServerApplication; @@ -64,7 +63,7 @@ public class DuperModelManagerTest { makeManager(false); ApplicationId id = proxyHostApplication.getApplicationId(); - List<HostName> proxyHostHosts = Stream.of("proxyhost1", "proxyhost2").map(HostName::of).toList(); + List<DomainName> proxyHostHosts = Stream.of("proxyhost1", "proxyhost2").map(DomainName::of).toList(); verify(duperModel, times(0)).add(any()); manager.infraApplicationActivated(id, proxyHostHosts); verify(duperModel, times(1)).add(any()); @@ -91,12 +90,12 @@ public class DuperModelManagerTest { } private void testEnabledConfigServerLikeInfraApplication(ApplicationId firstId, ApplicationId secondId) { - List<HostName> hostnames1 = Stream.of("node11", "node12").map(HostName::of).toList(); + List<DomainName> hostnames1 = Stream.of("node11", "node12").map(DomainName::of).toList(); manager.infraApplicationActivated(firstId, hostnames1); verify(duperModel, times(1)).add(any()); // Adding the second config server like application will be ignored - List<HostName> hostnames2 = Stream.of("node21", "node22").map(HostName::of).toList(); + List<DomainName> hostnames2 = Stream.of("node21", "node22").map(DomainName::of).toList(); assertThrows(IllegalArgumentException.class, () -> manager.infraApplicationActivated(secondId, hostnames2)); verify(duperModel, times(1)).add(any()); diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java index 73a49ca8717..568dc3640e2 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.duper; +import ai.vespa.http.DomainName; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.Model; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.HostName; import com.yahoo.vespa.service.monitor.DuperModelListener; import org.junit.Before; import org.junit.Test; @@ -14,7 +14,6 @@ import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; import java.util.stream.Stream; import static org.junit.Assert.assertEquals; @@ -34,12 +33,12 @@ public class DuperModelTest { private final ApplicationId id1 = ApplicationId.fromSerializedForm("tenant:app1:default"); private final ApplicationInfo application1 = mock(ApplicationInfo.class); - private final HostName hostname1_1 = HostName.of("hostname1-1"); - private final HostName hostname1_2 = HostName.of("hostname1-2"); + private final DomainName hostname1_1 = DomainName.of("hostname1-1"); + private final DomainName hostname1_2 = DomainName.of("hostname1-2"); private final ApplicationId id2 = ApplicationId.fromSerializedForm("tenant:app2:default"); private final ApplicationInfo application2 = mock(ApplicationInfo.class); - private final HostName hostname2_1 = HostName.of("hostname2-1"); + private final DomainName hostname2_1 = DomainName.of("hostname2-1"); private final DuperModelListener listener1 = mock(DuperModelListener.class); @@ -49,7 +48,7 @@ public class DuperModelTest { setUpApplication(id2, application2, hostname2_1); } - private void setUpApplication(ApplicationId id, ApplicationInfo info, HostName... hostnames) { + private void setUpApplication(ApplicationId id, ApplicationInfo info, DomainName... hostnames) { when(info.getApplicationId()).thenReturn(id); Model model = mock(Model.class); @@ -129,7 +128,7 @@ public class DuperModelTest { addAndVerifyApplication1("host1"); addAndVerifyApplication1("host1", "host2"); addAndVerifyApplication1("host2", "host3"); - assertEquals(Optional.empty(), duperModel.getApplicationId(HostName.of("host1"))); + assertEquals(Optional.empty(), duperModel.getApplicationId(DomainName.of("host1"))); duperModel.remove(id1); assertEquals(0, duperModel.numberOfApplications()); @@ -138,7 +137,7 @@ public class DuperModelTest { } private void addAndVerifyApplication1(String... hostnameStrings) { - HostName[] hostnameArray = Stream.of(hostnameStrings).map(HostName::of).toArray(HostName[]::new); + DomainName[] hostnameArray = Stream.of(hostnameStrings).map(DomainName::of).toArray(DomainName[]::new); setUpApplication(id1, application1, hostnameArray); duperModel.add(application1); diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java index d99e882985f..9f46b5dfe8f 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java @@ -1,13 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.health; +import ai.vespa.http.DomainName; import com.yahoo.config.model.api.ApplicationInfo; -import com.yahoo.config.provision.HostName; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.service.duper.ConfigServerApplication; -import com.yahoo.vespa.service.monitor.ServiceId; import com.yahoo.vespa.service.monitor.ConfigserverUtil; +import com.yahoo.vespa.service.monitor.ServiceId; import org.junit.Test; import java.util.HashMap; @@ -87,7 +87,7 @@ public class ApplicationHealthMonitorTest { return new ServiceId(configServerApplication.getApplicationId(), configServerApplication.getClusterId(), configServerApplication.getServiceType(), - configServerApplication.configIdFor(HostName.of(hostname))); + configServerApplication.configIdFor(DomainName.of(hostname))); } private ServiceStatus getStatus(ApplicationHealthMonitor monitor, String hostname) { @@ -95,7 +95,7 @@ public class ApplicationHealthMonitorTest { configServerApplication.getApplicationId(), configServerApplication.getClusterId(), configServerApplication.getServiceType(), - configServerApplication.configIdFor(HostName.of(hostname))) + configServerApplication.configIdFor(DomainName.of(hostname))) .serviceStatus(); } }
\ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java index f7ee64b52a6..f9129b5e4f4 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.health; +import ai.vespa.http.DomainName; import com.yahoo.config.model.api.ApplicationInfo; -import com.yahoo.config.provision.HostName; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.service.duper.ControllerHostApplication; @@ -14,7 +14,6 @@ import org.junit.Before; import org.junit.Test; import java.util.List; -import java.util.stream.Collectors; import java.util.stream.Stream; import static org.junit.Assert.assertEquals; @@ -48,7 +47,7 @@ public class HealthMonitorManagerTest { public void infrastructureApplication() { ProxyHostApplication proxyHostApplication = new ProxyHostApplication(); when(duperModel.isSupportedInfraApplication(proxyHostApplication.getApplicationId())).thenReturn(true); - List<HostName> hostnames = Stream.of("proxyhost1", "proxyhost2").map(HostName::of).toList(); + List<DomainName> hostnames = Stream.of("proxyhost1", "proxyhost2").map(DomainName::of).toList(); ApplicationInfo proxyHostApplicationInfo = proxyHostApplication.makeApplicationInfo(hostnames); manager.applicationActivated(proxyHostApplicationInfo); @@ -77,7 +76,7 @@ public class HealthMonitorManagerTest { infraApplication.getApplicationId(), infraApplication.getClusterId(), infraApplication.getServiceType(), - infraApplication.configIdFor(HostName.of(hostname))).serviceStatus(); + infraApplication.configIdFor(DomainName.of(hostname))).serviceStatus(); assertEquals(expected, actual); @@ -85,7 +84,7 @@ public class HealthMonitorManagerTest { infraApplication.getApplicationId(), infraApplication.getClusterId(), infraApplication.getServiceType(), - infraApplication.configIdFor(HostName.of(hostname))); + infraApplication.configIdFor(DomainName.of(hostname))); } }
\ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java index 18b56f76e88..01b7930a74f 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.health; +import ai.vespa.http.DomainName; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.PortInfo; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.HostName; import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; @@ -18,7 +18,6 @@ import org.junit.Test; import java.time.Duration; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import java.util.stream.Stream; import static org.junit.Assert.assertEquals; @@ -38,7 +37,7 @@ public class StateV1HealthModelTest { private final Duration requestTimeout = Duration.ofSeconds(2); private final Duration keepAlive = Duration.ofSeconds(3); private final ProxyHostApplication proxyHostApplication = new ProxyHostApplication(); - private final List<HostName> hostnames = Stream.of("host1", "host2").map(HostName::of).toList(); + private final List<DomainName> hostnames = Stream.of("host1", "host2").map(DomainName::of).toList(); private final ApplicationInfo proxyHostApplicationInfo = proxyHostApplication.makeApplicationInfo(hostnames); private final StateV1HealthModel model = new StateV1HealthModel(healthStaleness, requestTimeout, keepAlive, executor); diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java index a2532590f52..e85a00d732c 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.model; +import ai.vespa.http.DomainName; import com.yahoo.config.model.api.ApplicationInfo; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ServiceStatus; @@ -12,7 +12,6 @@ import com.yahoo.vespa.service.monitor.ServiceStatusProvider; import org.junit.Test; import java.util.List; -import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -34,7 +33,7 @@ public class ApplicationInstanceGeneratorTest { when(statusProvider.getStatus(any(), any(), any(), any())).thenReturn(new ServiceStatusInfo(ServiceStatus.NOT_CHECKED)); Zone zone = mock(Zone.class); ApplicationInfo configServer = configServerApplication.makeApplicationInfo( - configServerList.stream().map(HostName::of).toList()); + configServerList.stream().map(DomainName::of).toList()); ApplicationInstance applicationInstance = new ApplicationInstanceGenerator(configServer, zone) .makeApplicationInstance(statusProvider); diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceHostListenerAdapterTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceHostListenerAdapterTest.java index c93e4dfb7fe..23422640af9 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceHostListenerAdapterTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceHostListenerAdapterTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.model; +import ai.vespa.http.DomainName; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.Model; @@ -100,7 +101,7 @@ public class ServiceHostListenerAdapterTest { } private Optional<ApplicationInfo> getDuperModelApplicationInfo(String hostname) { - return duperModel.getApplicationInfo(com.yahoo.config.provision.HostName.of(hostname)); + return duperModel.getApplicationInfo(DomainName.of(hostname)); } private void removeAndVerify(ApplicationId id, boolean listenerInvoked) { diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ConfigserverUtil.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ConfigserverUtil.java index 498a9dfc15e..d2480cd6520 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ConfigserverUtil.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ConfigserverUtil.java @@ -1,18 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.monitor; +import ai.vespa.http.DomainName; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.model.api.ApplicationInfo; -import com.yahoo.config.provision.HostName; import com.yahoo.vespa.service.duper.ConfigServerApplication; -import java.util.stream.Collectors; -import java.util.stream.Stream; +import java.util.List; /** * @author hakon */ public class ConfigserverUtil { + /** Create a ConfigserverConfig with the given settings. */ public static ConfigserverConfig create( boolean multitenant, @@ -31,16 +31,10 @@ public class ConfigserverUtil { return create(true, "cfg1", "cfg2", "cfg3"); } - public static ApplicationInfo makeConfigServerApplicationInfo( - String configServerHostname1, - String configServerHostname2, - String configServerHostname3) { - return new ConfigServerApplication().makeApplicationInfo( - Stream.of(configServerHostname1, configServerHostname2, configServerHostname3) - .map(HostName::of).toList()); - } - public static ApplicationInfo makeExampleConfigServer() { - return makeConfigServerApplicationInfo("cfg1", "cfg2", "cfg3"); + return new ConfigServerApplication().makeApplicationInfo(List.of(DomainName.of("cfg1"), + DomainName.of("cfg2"), + DomainName.of("cfg3"))); } + } diff --git a/storage/src/tests/persistence/persistencetestutils.h b/storage/src/tests/persistence/persistencetestutils.h index 94ae7b9fb53..e60260f3ee8 100644 --- a/storage/src/tests/persistence/persistencetestutils.h +++ b/storage/src/tests/persistence/persistencetestutils.h @@ -150,6 +150,18 @@ public: _replySender, MockBucketLock::make(bucket, _mock_bucket_locks), std::move(cmd)); } + template <typename T> + requires std::is_base_of_v<api::StorageReply, T> + [[nodiscard]] std::shared_ptr<T> + fetch_single_reply(MessageTracker::UP tracker) { + if (tracker && tracker->hasReply()) { + tracker->sendReply(); // Forward to queue so we can fetch it below + } + std::shared_ptr<api::StorageMessage> msg; + _replySender.queue.getNext(msg, 60s); + return std::dynamic_pointer_cast<T>(msg); + } + api::ReturnCode fetchResult(const MessageTracker::UP & tracker) { if (tracker) { diff --git a/storage/src/tests/persistence/testandsettest.cpp b/storage/src/tests/persistence/testandsettest.cpp index 5be1c7cd92a..1aa359de634 100644 --- a/storage/src/tests/persistence/testandsettest.cpp +++ b/storage/src/tests/persistence/testandsettest.cpp @@ -1,16 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // @author Vegard Sjonfjell -#include <vespa/storage/persistence/persistencehandler.h> #include <tests/persistence/persistencetestutils.h> #include <vespa/document/test/make_document_bucket.h> -#include <vespa/documentapi/messagebus/messages/testandsetcondition.h> #include <vespa/document/fieldvalue/fieldvalues.h> #include <vespa/document/update/documentupdate.h> #include <vespa/document/update/assignvalueupdate.h> #include <vespa/document/fieldset/fieldsets.h> +#include <vespa/documentapi/messagebus/messages/testandsetcondition.h> #include <vespa/persistence/spi/test.h> #include <vespa/persistence/spi/persistenceprovider.h> #include <vespa/persistence/spi/docentry.h> +#include <vespa/storage/persistence/persistencehandler.h> #include <functional> using std::unique_ptr; @@ -19,6 +19,7 @@ using std::shared_ptr; using storage::spi::test::makeSpiBucket; using document::test::makeDocumentBucket; using document::StringFieldValue; +using documentapi::TestAndSetCondition; using namespace ::testing; namespace storage { @@ -34,15 +35,18 @@ struct TestAndSetTest : PersistenceTestUtils { const StringFieldValue OLD_CONTENT{"Some old content"}; const StringFieldValue NEW_CONTENT{"Freshly pressed and squeezed content"}; const document::Bucket BUCKET = makeDocumentBucket(BUCKET_ID); + const TestAndSetCondition MATCHING_CONDITION{"testdoctype1.hstringval=\"*woofy dog*\""}; unique_ptr<PersistenceHandler> persistenceHandler; const AsyncHandler * asyncHandler; + const SimpleMessageHandler* simple_handler; shared_ptr<document::Document> testDoc; document::DocumentId testDocId; TestAndSetTest() : persistenceHandler(), - asyncHandler(nullptr) + asyncHandler(nullptr), + simple_handler(nullptr) {} void SetUp() override { @@ -54,6 +58,7 @@ struct TestAndSetTest : PersistenceTestUtils { testDoc = createTestDocument(); testDocId = testDoc->getId(); asyncHandler = &_persistenceHandler->asyncHandler(); + simple_handler = &_persistenceHandler->simpleMessageHandler(); } void TearDown() override { @@ -68,6 +73,8 @@ struct TestAndSetTest : PersistenceTestUtils { document::Document::SP retrieveTestDocument(); void setTestCondition(api::TestAndSetCommand & command); void putTestDocument(bool matchingHeader, api::Timestamp timestamp); + std::shared_ptr<api::GetReply> invoke_conditional_get(); + void feed_remove_entry_with_timestamp(api::Timestamp timestamp); void assertTestDocumentFoundAndMatchesContent(const document::FieldValue & value); static std::string expectedDocEntryString( @@ -247,6 +254,59 @@ TEST_F(TestAndSetTest, conditional_put_to_non_existing_document_should_fail) { EXPECT_EQ("", dumpBucket(BUCKET_ID)); } +TEST_F(TestAndSetTest, conditional_get_returns_doc_metadata_on_match) { + const api::Timestamp timestamp = 12345; + putTestDocument(true, timestamp); + auto reply = invoke_conditional_get(); + + ASSERT_EQ(reply->getResult(), api::ReturnCode()); + EXPECT_EQ(reply->getLastModifiedTimestamp(), timestamp); + EXPECT_TRUE(reply->condition_matched()); + EXPECT_FALSE(reply->is_tombstone()); + // Checking reply->wasFound() is tempting but doesn't make sense here, as that checks for + // the presence of a document object, which metadata-only gets by definition do not return. +} + +TEST_F(TestAndSetTest, conditional_get_returns_doc_metadata_on_mismatch) { + const api::Timestamp timestamp = 12345; + putTestDocument(false, timestamp); + auto reply = invoke_conditional_get(); + + ASSERT_EQ(reply->getResult(), api::ReturnCode()); + EXPECT_EQ(reply->getLastModifiedTimestamp(), timestamp); + EXPECT_FALSE(reply->condition_matched()); + EXPECT_FALSE(reply->is_tombstone()); +} + +TEST_F(TestAndSetTest, conditional_get_for_non_existing_document_returns_zero_timestamp) { + auto reply = invoke_conditional_get(); + + ASSERT_EQ(reply->getResult(), api::ReturnCode()); + EXPECT_EQ(reply->getLastModifiedTimestamp(), 0); + EXPECT_FALSE(reply->condition_matched()); + EXPECT_FALSE(reply->is_tombstone()); +} + +TEST_F(TestAndSetTest, conditional_get_for_non_existing_document_with_explicit_tombstone_returns_tombstone_timestamp) { + api::Timestamp timestamp = 56789; + feed_remove_entry_with_timestamp(timestamp); + auto reply = invoke_conditional_get(); + + ASSERT_EQ(reply->getResult(), api::ReturnCode()); + EXPECT_EQ(reply->getLastModifiedTimestamp(), timestamp); + EXPECT_FALSE(reply->condition_matched()); + EXPECT_TRUE(reply->is_tombstone()); +} + +TEST_F(TestAndSetTest, conditional_get_requires_metadata_only_fieldset) { + auto get = std::make_shared<api::GetCommand>(BUCKET, testDocId, document::AllFields::NAME); + get->set_condition(MATCHING_CONDITION); + // Note: uses fetchResult instead of fetch_single_reply due to implicit failure signalling via tracker instance. + auto result = fetchResult(simple_handler->handleGet(*get, createTracker(get, BUCKET))); + ASSERT_EQ(result, api::ReturnCode(api::ReturnCode::ILLEGAL_PARAMETERS, + "Conditional Get operations must be metadata-only")); +} + document::Document::SP TestAndSetTest::createTestDocument() { @@ -270,7 +330,7 @@ TestAndSetTest::retrieveTestDocument() auto tracker = _persistenceHandler->simpleMessageHandler().handleGet(*get, createTracker(get, BUCKET)); assert(tracker->getResult() == api::ReturnCode::Result::OK); - auto & reply = static_cast<api::GetReply &>(tracker->getReply()); + auto& reply = dynamic_cast<api::GetReply&>(tracker->getReply()); assert(reply.wasFound()); return reply.getDocument(); @@ -278,7 +338,7 @@ TestAndSetTest::retrieveTestDocument() void TestAndSetTest::setTestCondition(api::TestAndSetCommand & command) { - command.setCondition(documentapi::TestAndSetCondition("testdoctype1.hstringval=\"*woofy dog*\"")); + command.setCondition(MATCHING_CONDITION); } void TestAndSetTest::putTestDocument(bool matchingHeader, api::Timestamp timestamp) { @@ -290,6 +350,17 @@ void TestAndSetTest::putTestDocument(bool matchingHeader, api::Timestamp timesta fetchResult(asyncHandler->handlePut(*put, createTracker(put, BUCKET))); } +std::shared_ptr<api::GetReply> TestAndSetTest::invoke_conditional_get() { + auto get = std::make_shared<api::GetCommand>(BUCKET, testDocId, document::NoFields::NAME); + get->set_condition(MATCHING_CONDITION); + return fetch_single_reply<api::GetReply>(simple_handler->handleGet(*get, createTracker(get, BUCKET))); +} + +void TestAndSetTest::feed_remove_entry_with_timestamp(api::Timestamp timestamp) { + auto remove = std::make_shared<api::RemoveCommand>(BUCKET, testDocId, timestamp); + (void)fetchResult(asyncHandler->handleRemove(*remove, createTracker(remove, BUCKET))); +} + void TestAndSetTest::assertTestDocumentFoundAndMatchesContent(const document::FieldValue & value) { auto doc = retrieveTestDocument(); diff --git a/storage/src/tests/storageapi/mbusprot/storageprotocoltest.cpp b/storage/src/tests/storageapi/mbusprot/storageprotocoltest.cpp index d3036a2fad3..6d8c3585726 100644 --- a/storage/src/tests/storageapi/mbusprot/storageprotocoltest.cpp +++ b/storage/src/tests/storageapi/mbusprot/storageprotocoltest.cpp @@ -848,7 +848,7 @@ TEST_P(StorageProtocolTest, track_memory_footprint_for_some_messages) { EXPECT_EQ(144u + sizeof(vespalib::string), sizeof(PutCommand)); EXPECT_EQ(144u + sizeof(vespalib::string), sizeof(UpdateCommand)); EXPECT_EQ(224u + sizeof(vespalib::string), sizeof(RemoveCommand)); - EXPECT_EQ(296u, sizeof(GetCommand)); + EXPECT_EQ(296u + sizeof(documentapi::TestAndSetCondition), sizeof(GetCommand)); } } // storage::api diff --git a/storage/src/vespa/storage/bucketdb/btree_lockable_map.hpp b/storage/src/vespa/storage/bucketdb/btree_lockable_map.hpp index 1c0f9ec7a59..e24e97ff2a4 100644 --- a/storage/src/vespa/storage/bucketdb/btree_lockable_map.hpp +++ b/storage/src/vespa/storage/bucketdb/btree_lockable_map.hpp @@ -49,7 +49,7 @@ struct BTreeLockableMap<T>::ValueTraits { return store.addEntry(value).ref(); } static void remove_by_wrapped_value(DataStoreType& store, uint64_t value) noexcept { - store.holdElem(entry_ref_from_value(value), 1); + store.hold_entry(entry_ref_from_value(value)); } static ValueType unwrap_from_key_value(const DataStoreType& store, [[maybe_unused]] uint64_t key, uint64_t value) { return store.getEntry(entry_ref_from_value(value)); diff --git a/storage/src/vespa/storage/persistence/asynchandler.cpp b/storage/src/vespa/storage/persistence/asynchandler.cpp index e20c0475556..60c6d507416 100644 --- a/storage/src/vespa/storage/persistence/asynchandler.cpp +++ b/storage/src/vespa/storage/persistence/asynchandler.cpp @@ -358,7 +358,11 @@ bool AsyncHandler::tasConditionMatches(const api::TestAndSetCommand & cmd, MessageTracker & tracker, spi::Context & context, bool missingDocumentImpliesMatch) const { try { - TestAndSetHelper helper(_env, _spi, _bucketIdFactory, cmd, missingDocumentImpliesMatch); + TestAndSetHelper helper(_env, _spi, _bucketIdFactory, + cmd.getCondition(), + cmd.getBucket(), cmd.getDocumentId(), + cmd.getDocumentType(), + missingDocumentImpliesMatch); auto code = helper.retrieveAndMatch(context); if (code.failed()) { diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index 92b9e832a23..1d29a8795d5 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -1054,7 +1054,7 @@ FileStorHandlerImpl::Stripe::waitUntilNoLocks() const { std::unique_lock guard(*_lock); while (!_lockedBuckets.empty()) { - _cond->wait(guard); + _cond->wait_for(guard, 100ms); } } @@ -1062,7 +1062,7 @@ void FileStorHandlerImpl::Stripe::waitInactive(const AbortBucketOperationsCommand& cmd) const { std::unique_lock guard(*_lock); while (hasActive(guard, cmd)) { - _cond->wait(guard); + _cond->wait_for(guard, 100ms); } } diff --git a/storage/src/vespa/storage/persistence/persistencehandler.cpp b/storage/src/vespa/storage/persistence/persistencehandler.cpp index 8d71cc9308b..69f910d0910 100644 --- a/storage/src/vespa/storage/persistence/persistencehandler.cpp +++ b/storage/src/vespa/storage/persistence/persistencehandler.cpp @@ -24,7 +24,7 @@ PersistenceHandler::PersistenceHandler(vespalib::ISequencedTaskExecutor & sequen cfg.commonMergeChainOptimalizationMinimumSize), _asyncHandler(_env, provider, bucketOwnershipNotifier, sequencedExecutor, component.getBucketIdFactory()), _splitJoinHandler(_env, provider, bucketOwnershipNotifier, cfg.enableMultibitSplitOptimalization), - _simpleHandler(_env, provider) + _simpleHandler(_env, provider, component.getBucketIdFactory()) { } diff --git a/storage/src/vespa/storage/persistence/simplemessagehandler.cpp b/storage/src/vespa/storage/persistence/simplemessagehandler.cpp index e83d460f47a..ea929bf8620 100644 --- a/storage/src/vespa/storage/persistence/simplemessagehandler.cpp +++ b/storage/src/vespa/storage/persistence/simplemessagehandler.cpp @@ -2,6 +2,7 @@ #include "simplemessagehandler.h" #include "persistenceutil.h" +#include "testandsethelper.h" #include <vespa/persistence/spi/persistenceprovider.h> #include <vespa/persistence/spi/docentry.h> #include <vespa/storageapi/message/bucket.h> @@ -45,21 +46,45 @@ getFieldSet(const document::FieldSetRepo & repo, vespalib::stringref name, Messa } } -SimpleMessageHandler::SimpleMessageHandler(const PersistenceUtil& env, spi::PersistenceProvider& spi) +SimpleMessageHandler::SimpleMessageHandler(const PersistenceUtil& env, + spi::PersistenceProvider& spi, + const document::BucketIdFactory& bucket_id_factory) : _env(env), - _spi(spi) + _spi(spi), + _bucket_id_factory(bucket_id_factory) { } MessageTracker::UP +SimpleMessageHandler::handle_conditional_get(api::GetCommand& cmd, MessageTracker::UP tracker) const +{ + if (cmd.getFieldSet() == document::NoFields::NAME) { + TestAndSetHelper tas_helper(_env, _spi, _bucket_id_factory, cmd.condition(), + cmd.getBucket(), cmd.getDocumentId(), nullptr); + auto result = tas_helper.fetch_and_match_raw(tracker->context()); + tracker->setReply(std::make_shared<api::GetReply>(cmd, nullptr, result.timestamp, false, + result.is_tombstone(), result.is_match())); + } else { + tracker->fail(api::ReturnCode::ILLEGAL_PARAMETERS, "Conditional Get operations must be metadata-only"); + } + return tracker; +} + +MessageTracker::UP SimpleMessageHandler::handleGet(api::GetCommand& cmd, MessageTracker::UP tracker) const { auto& metrics = _env._metrics.get; tracker->setMetric(metrics); metrics.request_size.addValue(cmd.getApproxByteSize()); + if (cmd.has_condition()) { + return handle_conditional_get(cmd, std::move(tracker)); + } + auto fieldSet = getFieldSet(_env.getFieldSetRepo(), cmd.getFieldSet(), *tracker); - if ( ! fieldSet) { return tracker; } + if (!fieldSet) { + return tracker; + } tracker->context().setReadConsistency(api_read_consistency_to_spi(cmd.internal_read_consistency())); spi::GetResult result = _spi.get(_env.getBucket(cmd.getDocumentId(), cmd.getBucket()), @@ -70,7 +95,7 @@ SimpleMessageHandler::handleGet(api::GetCommand& cmd, MessageTracker::UP tracker metrics.notFound.inc(); } tracker->setReply(std::make_shared<api::GetReply>(cmd, result.getDocumentPtr(), result.getTimestamp(), - false, result.is_tombstone())); + false, result.is_tombstone(), false)); } return tracker; diff --git a/storage/src/vespa/storage/persistence/simplemessagehandler.h b/storage/src/vespa/storage/persistence/simplemessagehandler.h index 009fd6dff52..a5a19772556 100644 --- a/storage/src/vespa/storage/persistence/simplemessagehandler.h +++ b/storage/src/vespa/storage/persistence/simplemessagehandler.h @@ -7,6 +7,8 @@ #include <vespa/storage/common/bucketmessages.h> #include <vespa/storageapi/message/persistence.h> +namespace document { class BucketIdFactory; } + namespace storage { namespace spi { struct PersistenceProvider; } @@ -19,7 +21,9 @@ class PersistenceUtil; */ class SimpleMessageHandler : public Types { public: - SimpleMessageHandler(const PersistenceUtil&, spi::PersistenceProvider&); + SimpleMessageHandler(const PersistenceUtil&, + spi::PersistenceProvider&, + const document::BucketIdFactory&); MessageTrackerUP handleGet(api::GetCommand& cmd, MessageTrackerUP tracker) const; MessageTrackerUP handleRevert(api::RevertCommand& cmd, MessageTrackerUP tracker) const; MessageTrackerUP handleCreateIterator(CreateIteratorCommand& cmd, MessageTrackerUP tracker) const; @@ -27,8 +31,11 @@ public: MessageTrackerUP handleReadBucketList(ReadBucketList& cmd, MessageTrackerUP tracker) const; MessageTrackerUP handleReadBucketInfo(ReadBucketInfo& cmd, MessageTrackerUP tracker) const; private: - const PersistenceUtil & _env; - spi::PersistenceProvider & _spi; + MessageTrackerUP handle_conditional_get(api::GetCommand& cmd, MessageTrackerUP tracker) const; + + const PersistenceUtil& _env; + spi::PersistenceProvider& _spi; + const document::BucketIdFactory& _bucket_id_factory; }; } // storage diff --git a/storage/src/vespa/storage/persistence/testandsethelper.cpp b/storage/src/vespa/storage/persistence/testandsethelper.cpp index 393dac09f72..1cda9427761 100644 --- a/storage/src/vespa/storage/persistence/testandsethelper.cpp +++ b/storage/src/vespa/storage/persistence/testandsethelper.cpp @@ -31,69 +31,91 @@ void TestAndSetHelper::parseDocumentSelection(const document::DocumentTypeRepo & document::select::Parser parser(documentTypeRepo, bucketIdFactory); try { - _docSelectionUp = parser.parse(_cmd.getCondition().getSelection()); + _docSelectionUp = parser.parse(_condition.getSelection()); } catch (const document::select::ParsingFailedException & e) { throw TestAndSetException(api::ReturnCode(api::ReturnCode::ILLEGAL_PARAMETERS, "Failed to parse test and set condition: "s + e.getMessage())); } } spi::GetResult TestAndSetHelper::retrieveDocument(const document::FieldSet & fieldSet, spi::Context & context) { - return _spi.get(_env.getBucket(_docId, _cmd.getBucket()), fieldSet, _cmd.getDocumentId(), context); + return _spi.get(_env.getBucket(_docId, _bucket), fieldSet, _docId, context); } -TestAndSetHelper::TestAndSetHelper(const PersistenceUtil & env, const spi::PersistenceProvider & spi, - const document::BucketIdFactory & bucketFactory, - const api::TestAndSetCommand & cmd, bool missingDocumentImpliesMatch) +TestAndSetHelper::TestAndSetHelper(const PersistenceUtil& env, + const spi::PersistenceProvider& spi, + const document::BucketIdFactory& bucket_id_factory, + const documentapi::TestAndSetCondition& condition, + document::Bucket bucket, + document::DocumentId doc_id, + const document::DocumentType* doc_type_ptr, + bool missingDocumentImpliesMatch) : _env(env), _spi(spi), - _cmd(cmd), - _docId(cmd.getDocumentId()), - _docTypePtr(_cmd.getDocumentType()), + _condition(condition), + _bucket(bucket), + _docId(std::move(doc_id)), + _docTypePtr(doc_type_ptr), _missingDocumentImpliesMatch(missingDocumentImpliesMatch) { const auto & repo = _env.getDocumentTypeRepo(); resolveDocumentType(repo); - parseDocumentSelection(repo, bucketFactory); + parseDocumentSelection(repo, bucket_id_factory); } TestAndSetHelper::~TestAndSetHelper() = default; -api::ReturnCode -TestAndSetHelper::retrieveAndMatch(spi::Context & context) { - // Walk document selection tree to build a minimal field set +TestAndSetHelper::Result +TestAndSetHelper::fetch_and_match_raw(spi::Context& context) { + // Walk document selection tree to build a minimal field set FieldVisitor fieldVisitor(*_docTypePtr); try { _docSelectionUp->visit(fieldVisitor); } catch (const document::FieldNotFoundException& e) { - return api::ReturnCode(api::ReturnCode::ILLEGAL_PARAMETERS, - vespalib::make_string("Condition field '%s' could not be found, or is an imported field. " - "Imported fields are not supported in conditional mutations.", - e.getFieldName().c_str())); + throw TestAndSetException(api::ReturnCode( + api::ReturnCode::ILLEGAL_PARAMETERS, + vespalib::make_string("Condition field '%s' could not be found, or is an imported field. " + "Imported fields are not supported in conditional mutations.", + e.getFieldName().c_str()))); } - - // Retrieve document auto result = retrieveDocument(fieldVisitor.getFieldSet(), context); - // If document exists, match it with selection if (result.hasDocument()) { auto docPtr = result.getDocumentPtr(); if (_docSelectionUp->contains(*docPtr) != document::select::Result::True) { - return api::ReturnCode(api::ReturnCode::TEST_AND_SET_CONDITION_FAILED, - vespalib::make_string("Condition did not match document nodeIndex=%d bucket=%" PRIx64 " %s", - _env._nodeIndex, _cmd.getBucketId().getRawId(), - _cmd.hasBeenRemapped() ? "remapped" : "")); + return {result.getTimestamp(), Result::ConditionOutcome::IsNotMatch}; } - // Document matches - return api::ReturnCode(); - } else if (_missingDocumentImpliesMatch) { - return api::ReturnCode(); + return {result.getTimestamp(), Result::ConditionOutcome::IsMatch}; } + return {result.getTimestamp(), result.is_tombstone() ? Result::ConditionOutcome::IsTombstone + : Result::ConditionOutcome::DocNotFound}; +} - return api::ReturnCode(api::ReturnCode::TEST_AND_SET_CONDITION_FAILED, - vespalib::make_string("Document does not exist nodeIndex=%d bucket=%" PRIx64 " %s", - _env._nodeIndex, _cmd.getBucketId().getRawId(), - _cmd.hasBeenRemapped() ? "remapped" : "")); +api::ReturnCode +TestAndSetHelper::to_api_return_code(const Result& result) const { + switch (result.condition_outcome) { + case Result::ConditionOutcome::IsNotMatch: + return {api::ReturnCode::TEST_AND_SET_CONDITION_FAILED, + vespalib::make_string("Condition did not match document nodeIndex=%d bucket=%" PRIx64, + _env._nodeIndex, _bucket.getBucketId().getRawId())}; + case Result::ConditionOutcome::IsTombstone: + case Result::ConditionOutcome::DocNotFound: + if (!_missingDocumentImpliesMatch) { + return {api::ReturnCode::TEST_AND_SET_CONDITION_FAILED, + vespalib::make_string("Document does not exist nodeIndex=%d bucket=%" PRIx64, + _env._nodeIndex, _bucket.getBucketId().getRawId())}; + } + [[fallthrough]]; // as match + case Result::ConditionOutcome::IsMatch: + return {}; // OK + } + abort(); +} + +api::ReturnCode +TestAndSetHelper::retrieveAndMatch(spi::Context & context) { + auto result = fetch_and_match_raw(context); + return to_api_return_code(result); } } // storage diff --git a/storage/src/vespa/storage/persistence/testandsethelper.h b/storage/src/vespa/storage/persistence/testandsethelper.h index 82710e523c4..31b1cc79a54 100644 --- a/storage/src/vespa/storage/persistence/testandsethelper.h +++ b/storage/src/vespa/storage/persistence/testandsethelper.h @@ -25,9 +25,8 @@ class PersistenceUtil; class TestAndSetException : public std::runtime_error { api::ReturnCode _code; - public: - TestAndSetException(api::ReturnCode code) + explicit TestAndSetException(api::ReturnCode code) : std::runtime_error(code.getMessage()), _code(std::move(code)) {} @@ -36,11 +35,12 @@ public: }; class TestAndSetHelper { - const PersistenceUtil &_env; - const spi::PersistenceProvider &_spi; - const api::TestAndSetCommand &_cmd; + const PersistenceUtil& _env; + const spi::PersistenceProvider& _spi; + const documentapi::TestAndSetCondition& _condition; + const document::Bucket _bucket; const document::DocumentId _docId; - const document::DocumentType * _docTypePtr; + const document::DocumentType* _docTypePtr; std::unique_ptr<document::select::Node> _docSelectionUp; bool _missingDocumentImpliesMatch; @@ -50,10 +50,44 @@ class TestAndSetHelper { spi::GetResult retrieveDocument(const document::FieldSet & fieldSet, spi::Context & context); public: - TestAndSetHelper(const PersistenceUtil & env, const spi::PersistenceProvider & _spi, - const document::BucketIdFactory & bucketIdFactory, - const api::TestAndSetCommand & cmd, bool missingDocumentImpliesMatch = false); + struct Result { + enum class ConditionOutcome { + DocNotFound, + IsMatch, + IsNotMatch, + IsTombstone + }; + + api::Timestamp timestamp = 0; + ConditionOutcome condition_outcome = ConditionOutcome::IsNotMatch; + + [[nodiscard]] bool doc_not_found() const noexcept { + return condition_outcome == ConditionOutcome::DocNotFound; + } + [[nodiscard]] bool is_match() const noexcept { + return condition_outcome == ConditionOutcome::IsMatch; + } + [[nodiscard]] bool is_not_match() const noexcept { + return condition_outcome == ConditionOutcome::IsNotMatch; + } + [[nodiscard]] bool is_tombstone() const noexcept { + return condition_outcome == ConditionOutcome::IsTombstone; + } + }; + + TestAndSetHelper(const PersistenceUtil& env, + const spi::PersistenceProvider& _spi, + const document::BucketIdFactory& bucket_id_factory, + const documentapi::TestAndSetCondition& condition, + document::Bucket bucket, + document::DocumentId doc_id, + const document::DocumentType* doc_type_ptr, + bool missingDocumentImpliesMatch = false); ~TestAndSetHelper(); + + Result fetch_and_match_raw(spi::Context& context); + api::ReturnCode to_api_return_code(const Result& result) const; + api::ReturnCode retrieveAndMatch(spi::Context & context); }; diff --git a/storage/src/vespa/storageapi/message/persistence.cpp b/storage/src/vespa/storageapi/message/persistence.cpp index 41a53449b67..1b09639fd9b 100644 --- a/storage/src/vespa/storageapi/message/persistence.cpp +++ b/storage/src/vespa/storageapi/message/persistence.cpp @@ -222,7 +222,8 @@ GetReply::GetReply(const GetCommand& cmd, const DocumentSP& doc, Timestamp lastModified, bool had_consistent_replicas, - bool is_tombstone) + bool is_tombstone, + bool condition_matched) : BucketInfoReply(cmd), _docId(cmd.getDocumentId()), _fieldSet(cmd.getFieldSet()), @@ -230,7 +231,8 @@ GetReply::GetReply(const GetCommand& cmd, _beforeTimestamp(cmd.getBeforeTimestamp()), _lastModifiedTime(lastModified), _had_consistent_replicas(had_consistent_replicas), - _is_tombstone(is_tombstone) + _is_tombstone(is_tombstone), + _condition_matched(condition_matched) { } diff --git a/storage/src/vespa/storageapi/message/persistence.h b/storage/src/vespa/storageapi/message/persistence.h index d1709c46a6e..d010c295ca7 100644 --- a/storage/src/vespa/storageapi/message/persistence.h +++ b/storage/src/vespa/storageapi/message/persistence.h @@ -185,9 +185,10 @@ public: * timestamp. */ class GetCommand : public BucketInfoCommand { - document::DocumentId _docId; - Timestamp _beforeTimestamp; - vespalib::string _fieldSet; + document::DocumentId _docId; + Timestamp _beforeTimestamp; + vespalib::string _fieldSet; + TestAndSetCondition _condition; InternalReadConsistency _internal_read_consistency; public: GetCommand(const document::Bucket &bucket, const document::DocumentId&, @@ -198,6 +199,9 @@ public: Timestamp getBeforeTimestamp() const { return _beforeTimestamp; } const vespalib::string& getFieldSet() const { return _fieldSet; } void setFieldSet(vespalib::stringref fieldSet) { _fieldSet = fieldSet; } + [[nodiscard]] bool has_condition() const noexcept { return _condition.isPresent(); } + [[nodiscard]] const TestAndSetCondition& condition() const noexcept { return _condition; } + void set_condition(TestAndSetCondition cond) { _condition = std::move(cond); } InternalReadConsistency internal_read_consistency() const noexcept { return _internal_read_consistency; } @@ -229,12 +233,14 @@ class GetReply : public BucketInfoReply { Timestamp _lastModifiedTime; bool _had_consistent_replicas; bool _is_tombstone; + bool _condition_matched; public: explicit GetReply(const GetCommand& cmd, const DocumentSP& doc = DocumentSP(), Timestamp lastModified = 0, bool had_consistent_replicas = false, - bool is_tombstone = false); + bool is_tombstone = false, + bool condition_matched = false); ~GetReply() override; @@ -247,6 +253,7 @@ public: [[nodiscard]] bool had_consistent_replicas() const noexcept { return _had_consistent_replicas; } [[nodiscard]] bool is_tombstone() const noexcept { return _is_tombstone; } + [[nodiscard]] bool condition_matched() const noexcept { return _condition_matched; } bool wasFound() const { return (_doc.get() != nullptr); } void print(std::ostream& out, bool verbose, const std::string& indent) const override; diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index d0e3d1f038b..2caf2de1d0b 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -7,6 +7,7 @@ #include <vespa/persistence/spi/docentry.h> #include <vespa/document/datatype/positiondatatype.h> #include <vespa/document/datatype/documenttype.h> +#include <vespa/document/datatype/tensor_data_type.h> #include <vespa/document/datatype/weightedsetdatatype.h> #include <vespa/document/datatype/mapdatatype.h> #include <vespa/searchlib/aggregation/modifiers.h> @@ -14,6 +15,7 @@ #include <vespa/searchlib/common/packets.h> #include <vespa/searchlib/uca/ucaconverter.h> #include <vespa/searchlib/features/setup.h> +#include <vespa/searchlib/tensor/tensor_ext_attribute.h> #include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/geo/zcurve.h> #include <vespa/vespalib/objects/nbostream.h> @@ -99,6 +101,16 @@ createMultiValueAttribute(const vespalib::string & name, const document::FieldVa return {}; } +const document::TensorDataType* +get_tensor_type(const document::FieldValue& fv) +{ + auto tfv = dynamic_cast<const document::TensorFieldValue*>(&fv); + if (tfv == nullptr) { + return nullptr; + } + return dynamic_cast<const document::TensorDataType*>(tfv->getDataType()); +} + AttributeVector::SP createAttribute(const vespalib::string & name, const document::FieldValue & fv) { @@ -111,6 +123,12 @@ createAttribute(const vespalib::string & name, const document::FieldValue & fv) return std::make_shared<search::SingleStringExtAttribute>(name); } else if (fv.isA(document::FieldValue::Type::RAW)) { return std::make_shared<search::attribute::SingleRawExtAttribute>(name); + } else if (fv.isA(document::FieldValue::Type::TENSOR) && get_tensor_type(fv) != nullptr) { + search::attribute::Config cfg(search::attribute::BasicType::TENSOR, search::attribute::CollectionType::SINGLE); + auto tdt = get_tensor_type(fv); + assert(tdt != nullptr); + cfg.setTensorType(tdt->getTensorType()); + return std::make_shared<search::tensor::TensorExtAttribute>(name, cfg); } else { LOG(debug, "Can not make an attribute out of %s of type '%s'.", name.c_str(), fv.className()); } @@ -444,6 +462,14 @@ SearchVisitor::AttributeInserter::onPrimitive(uint32_t, const Content & c) } else if (_attribute.is_raw_type()) { auto raw_value = value.getAsRaw(); attr.add(vespalib::ConstArrayRef<char>(raw_value.first, raw_value.second), c.getWeight()); + } else if (_attribute.isTensorType()) { + auto tfvalue = dynamic_cast<const document::TensorFieldValue*>(&value); + if (tfvalue != nullptr) { + auto tensor = tfvalue->getAsTensorPtr(); + if (tensor != nullptr) { + attr.add(*tensor, c.getWeight()); + } + } } else { assert(false && "We got an attribute vector that is of an unknown type"); } diff --git a/streamingvisitors/src/vespa/vsm/config/vsmfields.def b/streamingvisitors/src/vespa/vsm/config/vsmfields.def index 5e943c9274d..a7bd1f03f85 100644 --- a/streamingvisitors/src/vespa/vsm/config/vsmfields.def +++ b/streamingvisitors/src/vespa/vsm/config/vsmfields.def @@ -12,7 +12,7 @@ searchall int default=1 fieldspec[].name string ## The search method for a given field. Note: same field in 2 different document types must match on type if not a random result might be expected. -fieldspec[].searchmethod enum { NONE, BOOL, AUTOUTF8, UTF8, SSE2UTF8, INT8, INT16, INT32, INT64, FLOAT16, FLOAT, DOUBLE, GEOPOS } default=AUTOUTF8 +fieldspec[].searchmethod enum { NONE, BOOL, AUTOUTF8, UTF8, SSE2UTF8, INT8, INT16, INT32, INT64, FLOAT16, FLOAT, DOUBLE, GEOPOS, NEAREST_NEIGHBOR } default=AUTOUTF8 fieldspec[].arg1 string default="" ## Maximum number of chars to search per field. diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.cpp b/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.cpp index 36873b713aa..b103d7d85b2 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.cpp @@ -48,7 +48,8 @@ DocsumFieldWriterFactory::~DocsumFieldWriterFactory() = default; std::unique_ptr<DocsumFieldWriter> DocsumFieldWriterFactory::create_docsum_field_writer(const vespalib::string& field_name, const vespalib::string& command, - const vespalib::string& source) + const vespalib::string& source, + std::shared_ptr<MatchingElementsFields> matching_elems_fields) { std::unique_ptr<DocsumFieldWriter> fieldWriter; using namespace search::docsummary; @@ -65,10 +66,10 @@ DocsumFieldWriterFactory::create_docsum_field_writer(const vespalib::string& fie } else if ((command == command::matched_attribute_elements_filter) || (command == command::matched_elements_filter)) { vespalib::string source_field = source.empty() ? field_name : source; - populate_fields(*_matching_elems_fields, _vsm_fields_config, source_field); - fieldWriter = MatchedElementsFilterDFW::create(source_field, _matching_elems_fields); + populate_fields(*matching_elems_fields, _vsm_fields_config, source_field); + fieldWriter = MatchedElementsFilterDFW::create(source_field, matching_elems_fields); } else { - return search::docsummary::DocsumFieldWriterFactory::create_docsum_field_writer(field_name, command, source); + return search::docsummary::DocsumFieldWriterFactory::create_docsum_field_writer(field_name, command, source, matching_elems_fields); } return fieldWriter; } diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.h b/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.h index 078c466d3d2..ac5cae8c49d 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.h +++ b/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.h @@ -21,7 +21,8 @@ public: std::unique_ptr<search::docsummary::DocsumFieldWriter> create_docsum_field_writer(const vespalib::string& field_name, const vespalib::string& command, - const vespalib::string& source) override; + const vespalib::string& source, + std::shared_ptr<search::MatchingElementsFields> matching_elems_fields) override; }; } diff --git a/vdslib/src/main/java/com/yahoo/vdslib/VisitorStatistics.java b/vdslib/src/main/java/com/yahoo/vdslib/VisitorStatistics.java index fda456ace05..14b44ede12a 100644 --- a/vdslib/src/main/java/com/yahoo/vdslib/VisitorStatistics.java +++ b/vdslib/src/main/java/com/yahoo/vdslib/VisitorStatistics.java @@ -2,6 +2,7 @@ package com.yahoo.vdslib; public class VisitorStatistics { + int bucketsVisited = 0; long documentsVisited = 0; long bytesVisited = 0; @@ -20,8 +21,8 @@ public class VisitorStatistics { public void setBucketsVisited(int bucketsVisited) { this.bucketsVisited = bucketsVisited; } /** - * @return the number of documents matching the document selection in the backend and that - * has been passed to the client-specified visitor instance (dumpvisitor, searchvisitor etc). + * Returns the number of documents matching the document selection in the backend that + * has been passed to the client-specified visitor instance (dumpvisitor, searchvisitor etc). */ public long getDocumentsVisited() { return documentsVisited; } public void setDocumentsVisited(long documentsVisited) { this.documentsVisited = documentsVisited; } @@ -30,9 +31,9 @@ public class VisitorStatistics { public void setBytesVisited(long bytesVisited) { this.bytesVisited = bytesVisited; } /** - * @return Number of documents returned to the visitor client by the backend. This number may - * be lower than that returned by getDocumentsVisited() since the client-specified visitor - * instance may further have filtered the set of documents returned by the backend. + * Returns the number of documents returned to the visitor client by the backend. This number may + * be lower than that returned by getDocumentsVisited() since the client-specified visitor + * instance may further have filtered the set of documents returned by the backend. */ public long getDocumentsReturned() { return documentsReturned; } public void setDocumentsReturned(long documentsReturned) { this.documentsReturned = documentsReturned; } @@ -40,15 +41,14 @@ public class VisitorStatistics { public long getBytesReturned() { return bytesReturned; } public void setBytesReturned(long bytesReturned) { this.bytesReturned = bytesReturned; } + @Override public String toString() { - String out = + return "Buckets visited: " + bucketsVisited + "\n" + "Documents visited: " + documentsVisited + "\n" + "Bytes visited: " + bytesVisited + "\n" + "Documents returned: " + documentsReturned + "\n" + "Bytes returned: " + bytesReturned + "\n"; - - return out; } } diff --git a/vdslib/src/main/java/com/yahoo/vdslib/distribution/Distribution.java b/vdslib/src/main/java/com/yahoo/vdslib/distribution/Distribution.java index 9a451ac56ec..a83e2a4f89c 100644 --- a/vdslib/src/main/java/com/yahoo/vdslib/distribution/Distribution.java +++ b/vdslib/src/main/java/com/yahoo/vdslib/distribution/Distribution.java @@ -187,11 +187,11 @@ public class Distribution { } private int getStorageSeed(BucketId bucket, ClusterState state) { - int seed = (int) lastNBits(bucket.getRawId(), state.getDistributionBitCount()); + int seed = (int)lastNBits(bucket.getRawId(), state.getDistributionBitCount()); if (bucket.getUsedBits() > 33) { int usedBits = bucket.getUsedBits() - 1; - seed ^= lastNBits(bucket.getRawId() >> 32, usedBits - 32) << 6; + seed ^= (int)lastNBits(bucket.getRawId() >> 32, usedBits - 32) << 6; } return seed; } diff --git a/vdslib/src/tests/distribution/distributiontest.cpp b/vdslib/src/tests/distribution/distributiontest.cpp index a10aca45b33..b5c756aece9 100644 --- a/vdslib/src/tests/distribution/distributiontest.cpp +++ b/vdslib/src/tests/distribution/distributiontest.cpp @@ -17,8 +17,9 @@ #include <vespa/vespalib/util/size_literals.h> #include <gmock/gmock.h> #include <chrono> -#include <thread> #include <fstream> +#include <iterator> +#include <thread> using namespace ::testing; @@ -434,7 +435,8 @@ TEST(DistributionTest, test_distribution) _distribution[i].second = distr.getIdealStorageNodes( systemState, document::BucketId(26, i)); sort(_distribution[i].second.begin(), _distribution[i].second.end()); - unique(_distribution[i].second.begin(), _distribution[i].second.end()); + auto unique_nodes = std::distance(_distribution[i].second.begin(), unique(_distribution[i].second.begin(), _distribution[i].second.end())); + _distribution[i].second.resize(unique_nodes); for (unsigned j = 0; j < _distribution[i].second.size(); j++) { _nodeCount[_distribution[i].second[j]]++; diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index 95f1179b1a3..b5841d1c9e4 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -182,7 +182,7 @@ org.glassfish.jersey.media:jersey-media-multipart:2.25 org.hdrhistogram:HdrHistogram:2.1.12 org.iq80.snappy:snappy:0.4 org.javassist:javassist:3.20.0-GA -org.json:json:20220320 +org.json:json:20230227 org.junit.jupiter:junit-jupiter-api:5.8.1 org.junit.jupiter:junit-jupiter-engine:5.8.1 org.junit.platform:junit-platform-commons:1.8.1 diff --git a/vespa-feed-client-api/pom.xml b/vespa-feed-client-api/pom.xml index 5509c339eee..0782bb6b28e 100644 --- a/vespa-feed-client-api/pom.xml +++ b/vespa-feed-client-api/pom.xml @@ -42,6 +42,12 @@ <configuration> <release>${vespaClients.jdk.releaseVersion}</release> <showDeprecation>true</showDeprecation> + <compilerArgs> <!-- Remove (to use default) when not compiling for 8 --> + <arg>-Xlint:all</arg> + <arg>-Xlint:-rawtypes</arg> + <arg>-Xlint:-unchecked</arg> + <arg>-Xlint:-serial</arg> + </compilerArgs> </configuration> </plugin> <plugin> diff --git a/vespa-feed-client-cli/pom.xml b/vespa-feed-client-cli/pom.xml index 46679906fc4..b917a39b675 100644 --- a/vespa-feed-client-cli/pom.xml +++ b/vespa-feed-client-cli/pom.xml @@ -53,6 +53,12 @@ <configuration> <release>${vespaClients.jdk.releaseVersion}</release> <showDeprecation>true</showDeprecation> + <compilerArgs> <!-- Remove (to use default) when not compiling for 8 --> + <arg>-Xlint:all</arg> + <arg>-Xlint:-rawtypes</arg> + <arg>-Xlint:-unchecked</arg> + <arg>-Xlint:-serial</arg> + </compilerArgs> </configuration> </plugin> <plugin> diff --git a/vespa-feed-client/pom.xml b/vespa-feed-client/pom.xml index 01b9b00b8a0..b6440653a78 100644 --- a/vespa-feed-client/pom.xml +++ b/vespa-feed-client/pom.xml @@ -65,6 +65,12 @@ <goal>compile</goal> </goals> <configuration> + <compilerArgs> <!-- Remove (to use default) when not compiling for 8 --> + <arg>-Xlint:all</arg> + <arg>-Xlint:-rawtypes</arg> + <arg>-Xlint:-unchecked</arg> + <arg>-Xlint:-serial</arg> + </compilerArgs> <release>${vespaClients.jdk.releaseVersion}</release> <showDeprecation>true</showDeprecation> </configuration> diff --git a/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/ApacheClusterTest.java b/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/ApacheClusterTest.java index 9195b5ab858..cf9a36f2aa8 100644 --- a/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/ApacheClusterTest.java +++ b/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/ApacheClusterTest.java @@ -35,7 +35,7 @@ class ApacheClusterTest { final WireMockExtension server = new WireMockExtension(); @Test - void testClient() throws IOException, ExecutionException, InterruptedException, TimeoutException { + void testClient() throws Exception { for (Compression compression : Compression.values()) { try (ApacheCluster cluster = new ApacheCluster(new FeedClientBuilderImpl(List.of(URI.create("http://localhost:" + server.port()))) .setCompression(compression))) { @@ -48,25 +48,28 @@ class ApacheClusterTest { Map.of("name1", () -> "value1", "name2", () -> "value2"), "content".getBytes(UTF_8), - Duration.ofSeconds(20)), + Duration.ofSeconds(10)), vessel); - HttpResponse response = vessel.get(15, TimeUnit.SECONDS); - assertEquals("{}", new String(response.body(), UTF_8)); - assertEquals(200, response.code()); - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - try (OutputStream zip = new GZIPOutputStream(buffer)) { zip.write("content".getBytes(UTF_8)); } - server.verify(1, anyRequestedFor(anyUrl())); - RequestPatternBuilder expected = postRequestedFor(urlEqualTo("/path")).withHeader("name1", equalTo("value1")) - .withHeader("name2", equalTo("value2")) - .withHeader("Content-Type", equalTo("application/json; charset=UTF-8")) - .withRequestBody(equalTo("content")); - expected = switch (compression) { - case auto, none -> expected.withoutHeader("Content-Encoding"); - case gzip -> expected.withHeader("Content-Encoding", equalTo("gzip")); + AutoCloseable verifyResponse = () -> { + HttpResponse response = vessel.get(15, TimeUnit.SECONDS); + assertEquals("{}", new String(response.body(), UTF_8)); + assertEquals(200, response.code()); }; - server.verify(1, expected); - server.resetRequests(); + AutoCloseable verifyServer = () -> { + server.verify(1, anyRequestedFor(anyUrl())); + RequestPatternBuilder expected = postRequestedFor(urlEqualTo("/path")).withHeader("name1", equalTo("value1")) + .withHeader("name2", equalTo("value2")) + .withHeader("Content-Type", equalTo("application/json; charset=UTF-8")) + .withRequestBody(equalTo("content")); + expected = switch (compression) { + case auto, none -> expected.withoutHeader("Content-Encoding"); + case gzip -> expected.withHeader("Content-Encoding", equalTo("gzip")); + }; + server.verify(1, expected); + server.resetRequests(); + }; + try (verifyServer; verifyResponse) { } } } } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java index fe1e2b46830..5126f7e0f43 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java @@ -526,23 +526,18 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { parameters = getProperty(request, TIMEOUT, timeoutMillisParser).map(clock.instant()::plusMillis) .map(parameters::withDeadline) .orElse(parameters); - for (String name : names) switch (name) { - case CLUSTER: - parameters = getProperty(request, CLUSTER).map(cluster -> resolveCluster(Optional.of(cluster), clusters).name()) - .map(parameters::withRoute) - .orElse(parameters); - break; - case FIELD_SET: - parameters = getProperty(request, FIELD_SET).map(parameters::withFieldSet) - .orElse(parameters); - break; - case ROUTE: - parameters = getProperty(request, ROUTE).map(parameters::withRoute) - .orElse(parameters); - break; - default: - throw new IllegalArgumentException("Unrecognized document operation parameter name '" + name + "'"); - } + for (String name : names) + parameters = switch (name) { + case CLUSTER -> + getProperty(request, CLUSTER) + .map(cluster -> resolveCluster(Optional.of(cluster), clusters).name()) + .map(parameters::withRoute) + .orElse(parameters); + case FIELD_SET -> getProperty(request, FIELD_SET).map(parameters::withFieldSet).orElse(parameters); + case ROUTE -> getProperty(request, ROUTE).map(parameters::withRoute).orElse(parameters); + default -> + throw new IllegalArgumentException("Unrecognized document operation parameter name '" + name + "'"); + }; return parameters; } @@ -630,10 +625,6 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { private boolean first = true; private ContentChannel channel; - private JsonResponse(ResponseHandler handler) throws IOException { - this(handler, null); - } - private JsonResponse(ResponseHandler handler, HttpRequest request) throws IOException { this.handler = handler; this.request = request; @@ -642,11 +633,6 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { } /** Creates a new JsonResponse with path and id fields written. */ - static JsonResponse create(DocumentPath path, ResponseHandler handler) throws IOException { - return create(path, handler, null); - } - - /** Creates a new JsonResponse with path and id fields written. */ static JsonResponse create(DocumentPath path, ResponseHandler handler, HttpRequest request) throws IOException { JsonResponse response = new JsonResponse(handler, request); response.writePathId(path.rawPath()); @@ -749,23 +735,17 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { } private boolean tensorShortForm() { - if (request != null && - request.parameters().containsKey("format.tensors") && - ( request.parameters().get("format.tensors").contains("long") - || request.parameters().get("format.tensors").contains("long-value"))) { - return false; - } - return true; // default + return request == null || + !request.parameters().containsKey("format.tensors") || + (!request.parameters().get("format.tensors").contains("long") + && !request.parameters().get("format.tensors").contains("long-value"));// default } private boolean tensorDirectValues() { - if (request != null && - request.parameters().containsKey("format.tensors") && - ( request.parameters().get("format.tensors").contains("short-value") - || request.parameters().get("format.tensors").contains("long-value"))) { - return true; - } - return false; // TODO: Flip default on Vespa 9 + return request != null && + request.parameters().containsKey("format.tensors") && + (request.parameters().get("format.tensors").contains("short-value") + || request.parameters().get("format.tensors").contains("long-value"));// TODO: Flip default on Vespa 9 } synchronized void writeSingleDocument(Document document) throws IOException { @@ -1168,9 +1148,8 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { // ------------------------------------------------- Visits ------------------------------------------------ private VisitorParameters parseGetParameters(HttpRequest request, DocumentPath path, boolean streamed) { - int wantedDocumentCount = Math.min(streamed ? Integer.MAX_VALUE : 1 << 10, - getProperty(request, WANTED_DOCUMENT_COUNT, integerParser) - .orElse(streamed ? Integer.MAX_VALUE : 1)); + int wantedDocumentCount = getProperty(request, WANTED_DOCUMENT_COUNT, integerParser) + .orElse(streamed ? Integer.MAX_VALUE : 1); if (wantedDocumentCount <= 0) throw new IllegalArgumentException("wantedDocumentCount must be positive"); @@ -1226,7 +1205,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { getProperty(request, TRACELEVEL, integerParser).ifPresent(parameters::setTraceLevel); - getProperty(request, CONTINUATION).map(ProgressToken::fromSerializedString).ifPresent(parameters::setResumeToken); + getProperty(request, CONTINUATION, ProgressToken::fromSerializedString).ifPresent(parameters::setResumeToken); parameters.setPriority(DocumentProtocol.Priority.NORMAL_4); getProperty(request, FROM_TIMESTAMP, unsignedLongParser).ifPresent(parameters::setFromTimestamp); @@ -1546,11 +1525,11 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { private static Map<String, StorageCluster> parseClusters(ClusterListConfig clusters, AllClustersBucketSpacesConfig buckets) { return clusters.storage().stream() - .collect(toUnmodifiableMap(storage -> storage.name(), + .collect(toUnmodifiableMap(ClusterListConfig.Storage::name, storage -> new StorageCluster(storage.name(), buckets.cluster(storage.name()) .documentType().entrySet().stream() - .collect(toMap(entry -> entry.getKey(), + .collect(toMap(Map.Entry::getKey, entry -> entry.getValue().bucketSpace()))))); } diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java index 851a0949266..7696fd2196c 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java @@ -217,7 +217,7 @@ public class DocumentV1ApiTest { access.expect(parameters -> { assertEquals("content", parameters.getRoute().toString()); assertEquals("default", parameters.getBucketSpace()); - assertEquals(1024, parameters.getMaxTotalHits()); + assertEquals(1025, parameters.getMaxTotalHits()); assertEquals(100, ((StaticThrottlePolicy) parameters.getThrottlePolicy()).getMaxPendingCount()); assertEquals("[id]", parameters.getFieldSet()); assertEquals("(all the things)", parameters.getDocumentSelection()); diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java index 33e46ebc75f..31f4038c16e 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java @@ -37,15 +37,17 @@ public abstract class Maintainer implements Runnable { private final AtomicBoolean shutDown = new AtomicBoolean(); private final boolean ignoreCollision; private final Clock clock; + private final Double successFactorBaseline; public Maintainer(String name, Duration interval, Clock clock, JobControl jobControl, - JobMetrics jobMetrics, List<String> clusterHostnames, boolean ignoreCollision) { + JobMetrics jobMetrics, List<String> clusterHostnames, boolean ignoreCollision, Double successFactorBaseline) { this.name = name; this.interval = requireInterval(interval); this.jobControl = Objects.requireNonNull(jobControl); this.jobMetrics = Objects.requireNonNull(jobMetrics); this.ignoreCollision = ignoreCollision; this.clock = clock; + this.successFactorBaseline = successFactorBaseline; var startedAt = clock.instant(); Objects.requireNonNull(clusterHostnames); Duration initialDelay = staggeredDelay(interval, startedAt, HostName.getLocalhost(), clusterHostnames) @@ -55,6 +57,10 @@ public abstract class Maintainer implements Runnable { jobControl.started(name(), this); } + public Maintainer(String name, Duration interval, Clock clock, JobControl jobControl, + JobMetrics jobMetrics, List<String> clusterHostnames, boolean ignoreCollision) { + this(name, interval, clock, jobControl, jobMetrics, clusterHostnames, ignoreCollision, 1.0); + } @Override public void run() { lockAndMaintain(false); @@ -91,16 +97,16 @@ public abstract class Maintainer implements Runnable { /** * Called once each time this maintenance job should run. * - * @return the degree to which the run was successful - a number between 0 (no success), to 1 (complete success). + * @return the degree to which the run was deviated from the successFactorBaseline - a number between -1 (no success), to 0 (complete success). * Note that this indicates whether something is wrong, so e.g if the call did nothing because it should do - * nothing, 1.0 should be returned. + * nothing, 0.0 should be returned. */ protected abstract double maintain(); - /** Convenience methods to convert attempts and failures into a success factor */ - protected final double asSuccessFactor(int attempts, int failures) { + /** Convenience methods to convert attempts and failures into a success factor, and return */ + protected final double asSuccessFactorDeviation(int attempts, int failures) { double factor = attempts == 0 ? 1.0 : 1 - (double)failures / attempts; - return new BigDecimal(factor).setScale(2, RoundingMode.HALF_UP).doubleValue(); + return new BigDecimal(factor - successFactorBaseline).setScale(2, RoundingMode.HALF_UP).doubleValue(); } /** Returns the interval at which this job is set to run */ @@ -111,14 +117,14 @@ public abstract class Maintainer implements Runnable { if (!force && !jobControl.isActive(name())) return; log.log(Level.FINE, () -> "Running " + this.getClass().getSimpleName()); - double successFactor = 0; + double successFactorDeviation = 0; long startTime = clock.millis(); try (var lock = jobControl.lockJob(name())) { - successFactor = maintain(); + successFactorDeviation = maintain(); } catch (UncheckedTimeoutException e) { if (ignoreCollision) - successFactor = 1; + successFactorDeviation = 0; else log.log(Level.WARNING, this + " collided with another run. Will retry in " + interval); } @@ -127,7 +133,7 @@ public abstract class Maintainer implements Runnable { } finally { long endTime = clock.millis(); - jobMetrics.completed(name(), successFactor, endTime - startTime); + jobMetrics.completed(name(), successFactorDeviation, endTime - startTime); } log.log(Level.FINE, () -> "Finished " + this.getClass().getSimpleName()); } diff --git a/vespajlib/src/main/java/com/yahoo/slime/BinaryEncoder.java b/vespajlib/src/main/java/com/yahoo/slime/BinaryEncoder.java index f12496f7a76..e0b4fb2c672 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/BinaryEncoder.java +++ b/vespajlib/src/main/java/com/yahoo/slime/BinaryEncoder.java @@ -28,7 +28,7 @@ final class BinaryEncoder implements ArrayTraverser, ObjectSymbolTraverser { byte next = (byte)(value & 0x7f); value >>>= 7; // unsigned shift while (value != 0) { - next |= 0x80; + next |= (byte)0x80; out.put(next); next = (byte)(value & 0x7f); value >>>= 7; diff --git a/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java b/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java index f9230ab6df6..6acd0679da2 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java +++ b/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java @@ -166,81 +166,32 @@ public class SlimeUtils { false); } - private static class Equal { - protected final Inspector rhsInspector; - - protected boolean equal = true; - - public Equal(Inspector rhsInspector) { this.rhsInspector = rhsInspector; } - - public boolean isEqual() { return equal; } - } - - private static class EqualArray extends Equal implements ArrayTraverser { - public EqualArray(Inspector rhsInspector) { super(rhsInspector); } - - @Override - public void entry(int idx, Inspector inspector) { - if (equal) { - equal = inspector.equalTo(rhsInspector.entry(idx)); - } - } - } - - private static class EqualObject extends Equal implements ObjectTraverser { - public EqualObject(Inspector rhsInspector) { super(rhsInspector); } - - @Override - public void field(String name, Inspector inspector) { - if (equal) { - equal = inspector.equalTo(rhsInspector.field(name)); - } - } - } - public static boolean equalTo(Inspector a, Inspector b) { - boolean equal = a.type() == b.type(); - - if (equal) { - switch (a.type()) { - case NIX: - equal = a.valid() == b.valid(); - break; - case BOOL: - equal = a.asBool() == b.asBool(); - break; - case LONG: - equal = a.asLong() == b.asLong(); - break; - case DOUBLE: - equal = Double.compare(a.asDouble(), b.asDouble()) == 0; - break; - case STRING: - equal = a.asString().equals(b.asString()); - break; - case DATA: - equal = Arrays.equals(a.asData(), b.asData()); - break; - case ARRAY: - { - var traverser = new EqualArray(b); - a.traverse(traverser); - equal = traverser.isEqual() && (a.entries() == b.entries()); - } - break; - case OBJECT: - { - var traverser = new EqualObject(b); - a.traverse(traverser); - equal = traverser.isEqual() && (a.fields() == b.fields()); + if (a.type() != b.type()) return false; + + switch (a.type()) { + case NIX: return a.valid() == b.valid(); + case BOOL: return a.asBool() == b.asBool(); + case LONG: return a.asLong() == b.asLong(); + case DOUBLE: return Double.compare(a.asDouble(), b.asDouble()) == 0; + case STRING: return a.asString().equals(b.asString()); + case DATA: return Arrays.equals(a.asData(), b.asData()); + case ARRAY: { + if (a.entries() != b.entries()) return false; + for (int i = 0; i < a.entries(); i++) { + if (!equalTo(a.entry(i), b.entry(i))) return false; } - break; - default: - assert(false); - break; + return true; } + case OBJECT: { + if (a.fields() != b.fields()) return false; + boolean[] equal = new boolean[]{ true }; + a.traverse((String key, Inspector value) -> { + if (equal[0] && !equalTo(value, b.field(key))) equal[0] = false; + }); + return equal[0]; + } + default: throw new IllegalStateException("Unexpected type: " + a.type()); } - - return equal; } } diff --git a/vespajlib/src/main/java/com/yahoo/tensor/functions/ReduceJoin.java b/vespajlib/src/main/java/com/yahoo/tensor/functions/ReduceJoin.java index 11996b6a23d..de1c30e6414 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/functions/ReduceJoin.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/functions/ReduceJoin.java @@ -333,7 +333,7 @@ public class ReduceJoin<NAMETYPE extends Name> extends CompositeTensorFunction<N private final long[] bounds; private final long[] iterator; - private int remaining; + private long remaining; MultiDimensionIterator(TensorType type) { bounds = new long[type.dimensions().size()]; diff --git a/vespajlib/src/test/java/com/yahoo/io/FatalErrorHandlerTestCase.java b/vespajlib/src/test/java/com/yahoo/io/FatalErrorHandlerTestCase.java deleted file mode 100644 index dab91b6a995..00000000000 --- a/vespajlib/src/test/java/com/yahoo/io/FatalErrorHandlerTestCase.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.io; - -import static org.junit.Assert.*; - -import java.security.Permission; - -import org.junit.After; -import org.junit.Before; -import org.junit.Test; - -/** - * Just to remove noise from the coverage report. - * - * @author Steinar Knutsen - */ -public class FatalErrorHandlerTestCase { - @SuppressWarnings("removal") - private static final class AvoidExiting extends SecurityManager { - - @Override - public void checkPermission(Permission perm) { - } - - @Override - public void checkExit(int status) { - throw new SecurityException(); - } - - } - - private FatalErrorHandler h; - - @Before - @SuppressWarnings("removal") - public void setUp() throws Exception { - h = new FatalErrorHandler(); - System.setSecurityManager(new AvoidExiting()); - } - - @After - @SuppressWarnings("removal") - public void tearDown() throws Exception { - System.setSecurityManager(null); - } - - @Test - public final void testHandle() { - boolean caught = false; - try { - h.handle(new Throwable(), "abc"); - } catch (SecurityException e) { - caught = true; - } - assertTrue(caught); - } - -} diff --git a/vespajlib/src/test/java/com/yahoo/slime/ArrayValueTestCase.java b/vespajlib/src/test/java/com/yahoo/slime/ArrayValueTestCase.java index c9ff86e7c2e..730c3909b4c 100644 --- a/vespajlib/src/test/java/com/yahoo/slime/ArrayValueTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/slime/ArrayValueTestCase.java @@ -2,11 +2,11 @@ package com.yahoo.slime; import org.junit.Test; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.not; -import static org.hamcrest.CoreMatchers.sameInstance; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertNotSame; import java.util.List; import java.util.ArrayList; @@ -20,17 +20,17 @@ public class ArrayValueTestCase { @Test public void testSymbolTableForwarding() { SymbolTable names = new SymbolTable(); - assertThat(names.symbols(), is(0)); + assertEquals(0, names.symbols()); new ArrayValue(names).addArray().addObject().setLong("foo", 3); - assertThat(names.symbols(), is(1)); + assertEquals(1, names.symbols()); } @Test public void testOutOfBoundsAccess() { var array = makeArray(); array.addBool(true); - assertThat(array.entry(-1).valid(), is(false)); - assertThat(array.entry(1).valid(), is(false)); + assertFalse(array.entry(-1).valid()); + assertFalse(array.entry(1).valid()); } @Test @@ -44,8 +44,8 @@ public class ArrayValueTestCase { var e1 = array.entry(i); var e2 = array.entry(i); var e3 = added.get(i); - assertThat(e1, sameInstance(e2)); - assertThat(e1, sameInstance(e3)); + assertSame(e2, e1); + assertSame(e3, e1); } } @@ -61,12 +61,12 @@ public class ArrayValueTestCase { var e1 = array.entry(i); var e2 = array.entry(i); var e3 = added.get(i); - assertThat(e1, not(sameInstance(e2))); - assertThat(e1, not(sameInstance(e3))); - assertThat(e1.equalTo(e2), is(true)); - assertThat(e1.equalTo(e3), is(true)); - assertThat(e1.type(), is(Type.LONG)); - assertThat(e1.asLong(), is(expect)); + assertNotSame(e2, e1); + assertNotSame(e3, e1); + assertTrue(e1.equalTo(e2)); + assertTrue(e1.equalTo(e3)); + assertEquals(Type.LONG, e1.type()); + assertEquals(expect, e1.asLong()); } } @@ -82,12 +82,12 @@ public class ArrayValueTestCase { var e1 = array.entry(i); var e2 = array.entry(i); var e3 = added.get(i); - assertThat(e1, not(sameInstance(e2))); - assertThat(e1, not(sameInstance(e3))); - assertThat(e1.equalTo(e2), is(true)); - assertThat(e1.equalTo(e3), is(true)); - assertThat(e1.type(), is(Type.DOUBLE)); - assertThat(e1.asDouble(), is(expect)); + assertNotSame(e2, e1); + assertNotSame(e3, e1); + assertTrue(e1.equalTo(e2)); + assertTrue(e1.equalTo(e3)); + assertEquals(Type.DOUBLE, e1.type()); + assertEquals(expect, e1.asDouble(), 0.0); } } @@ -109,20 +109,20 @@ public class ArrayValueTestCase { case ARRAY: added.add(array.addArray()); break; case OBJECT: added.add(array.addObject()); break; } - assertThat(array.entries(), is(65)); - assertThat(array.entry(64).type(), is(type)); - assertThat(added.get(64), sameInstance(array.entry(64))); + assertEquals(65, array.entries()); + assertEquals(type, array.entry(64).type()); + assertSame(array.entry(64), added.get(64)); for (int i = 0; i < 64; ++i) { var e1 = array.entry(i); var e2 = array.entry(i); var e3 = added.get(i); long expect = i; - assertThat(e1, sameInstance(e2)); - assertThat(e1, not(sameInstance(e3))); - assertThat(e1.equalTo(e2), is(true)); - assertThat(e1.equalTo(e3), is(true)); - assertThat(e1.type(), is(Type.LONG)); - assertThat(e1.asLong(), is(expect)); + assertSame(e2, e1); + assertNotSame(e3, e1); + assertTrue(e1.equalTo(e2)); + assertTrue(e1.equalTo(e3)); + assertEquals(Type.LONG, e1.type()); + assertEquals(expect, e1.asLong()); } } } @@ -146,20 +146,20 @@ public class ArrayValueTestCase { case ARRAY: added.add(array.addArray()); break; case OBJECT: added.add(array.addObject()); break; } - assertThat(array.entries(), is(65)); - assertThat(array.entry(64).type(), is(type)); - assertThat(added.get(64), sameInstance(array.entry(64))); + assertEquals(65, array.entries()); + assertEquals(type, array.entry(64).type()); + assertSame(array.entry(64), added.get(64)); for (int i = 0; i < 64; ++i) { var e1 = array.entry(i); var e2 = array.entry(i); var e3 = added.get(i); double expect = i; - assertThat(e1, sameInstance(e2)); - assertThat(e1, not(sameInstance(e3))); - assertThat(e1.equalTo(e2), is(true)); - assertThat(e1.equalTo(e3), is(true)); - assertThat(e1.type(), is(Type.DOUBLE)); - assertThat(e1.asDouble(), is(expect)); + assertSame(e2, e1); + assertNotSame(e3, e1); + assertTrue(e1.equalTo(e2)); + assertTrue(e1.equalTo(e3)); + assertEquals(Type.DOUBLE, e1.type()); + assertEquals(expect, e1.asDouble(), 0.0); } } } @@ -179,9 +179,9 @@ public class ArrayValueTestCase { case ARRAY: added = array.addArray(); break; case OBJECT: added = array.addObject(); break; } - assertThat(array.entries(), is(1)); - assertThat(array.entry(0).type(), is(type)); - assertThat(added, sameInstance(array.entry(0))); + assertEquals(1, array.entries()); + assertEquals(type, array.entry(0).type()); + assertSame(array.entry(0), added); } } } diff --git a/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java index db001a9276b..5f96247b77e 100644 --- a/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java @@ -12,9 +12,10 @@ import static com.yahoo.slime.BinaryFormat.decode_zigzag; import static com.yahoo.slime.BinaryFormat.encode_double; import static com.yahoo.slime.BinaryFormat.encode_type_and_meta; import static com.yahoo.slime.BinaryFormat.encode_zigzag; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; - +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; public class BinaryFormatTestCase { @@ -36,50 +37,50 @@ public class BinaryFormatTestCase { BinaryEncoder bof = new BinaryEncoder(output); bof.encode_cmpr_int(value); byte[] actual = output.toArray(); - assertThat(actual, is(expect)); + assertArrayEquals(expect, actual); BinaryDecoder bif = new BinaryDecoder(); bif.in = new BufferedInput(expect); int got = bif.in.read_cmpr_int(); - assertThat(got, is(value)); - assertThat(bif.in.failed(), is(false)); + assertEquals(value, got); + assertFalse(bif.in.failed()); bif = new BinaryDecoder(); bif.in = new BufferedInput(expect); got = bif.in.skip_cmpr_int(); - assertThat(got, is(expect.length - 1)); - assertThat(bif.in.getPosition(), is(expect.length)); - assertThat(bif.in.failed(), is(false)); + assertEquals(expect.length - 1, got); + assertEquals(expect.length, bif.in.getPosition()); + assertFalse(bif.in.failed()); - assertThat(BinaryView.peek_cmpr_int_for_testing(expect, 0), is(value)); - assertThat(BinaryView.skip_cmpr_int_for_testing(expect, 0), is(expect.length)); + assertEquals(value, BinaryView.peek_cmpr_int_for_testing(expect, 0)); + assertEquals(expect.length, BinaryView.skip_cmpr_int_for_testing(expect, 0)); } void verify_read_cmpr_int_fails(byte[] data) { BinaryDecoder bif = new BinaryDecoder(); bif.in = new BufferedInput(data); int got = bif.in.read_cmpr_int(); - assertThat(got, is(0)); - assertThat(bif.in.failed(), is(true)); + assertEquals(0, got); + assertTrue(bif.in.failed()); bif = new BinaryDecoder(); bif.in = new BufferedInput(data); got = bif.in.skip_cmpr_int(); - assertThat(got, is(data.length - 1)); - assertThat(bif.in.getPosition(), is(data.length)); - assertThat(bif.in.failed(), is(false)); + assertEquals(data.length - 1, got); + assertEquals(data.length, bif.in.getPosition()); + assertFalse(bif.in.failed()); - assertThat(BinaryView.skip_cmpr_int_for_testing(data, 0), is(data.length)); + assertEquals(data.length, BinaryView.skip_cmpr_int_for_testing(data, 0)); } // was verifyBasic void verifyEncoding(Slime slime, byte[] expect) { - assertThat(BinaryFormat.encode(slime), is(expect)); - assertThat(slime.get().equalTo(BinaryView.inspect(expect)), is(true)); + assertArrayEquals(expect, BinaryFormat.encode(slime)); + assertTrue(slime.get().equalTo(BinaryView.inspect(expect))); Compressor compressor = new Compressor(CompressionType.LZ4, 3, 2, 0); Compressor.Compression result = BinaryFormat.encode_and_compress(slime, compressor); byte [] decompressed = compressor.decompress(result); - assertThat(decompressed, is(expect)); + assertArrayEquals(expect, decompressed); verifyMultiEncode(expect); } @@ -90,65 +91,65 @@ public class BinaryFormatTestCase { for (int i = 0; i < 5; ++i) { Slime slime = BinaryFormat.decode(buffers[i]); buffers[i+1] = BinaryFormat.encode(slime); - assertThat(buffers[i+1], is(expect)); + assertArrayEquals(expect, buffers[i+1]); } } @Test public void testZigZagConversion() { - assertThat(encode_zigzag(0), is(0L)); - assertThat(decode_zigzag(encode_zigzag(0)), is(0L)); + assertEquals(0L, encode_zigzag(0)); + assertEquals(0L, decode_zigzag(encode_zigzag(0))); - assertThat(encode_zigzag(-1), is(1L)); - assertThat(decode_zigzag(encode_zigzag(-1)), is(-1L)); + assertEquals(1L, encode_zigzag(-1)); + assertEquals(-1L, decode_zigzag(encode_zigzag(-1))); - assertThat(encode_zigzag(1), is(2L)); - assertThat(decode_zigzag(encode_zigzag(1)), is(1L)); + assertEquals(2L, encode_zigzag(1)); + assertEquals(1L, decode_zigzag(encode_zigzag(1))); - assertThat(encode_zigzag(-2), is(3L)); - assertThat(decode_zigzag(encode_zigzag(-2)), is(-2L)); + assertEquals(3L, encode_zigzag(-2)); + assertEquals(-2L, decode_zigzag(encode_zigzag(-2))); - assertThat(encode_zigzag(2), is(4L)); - assertThat(decode_zigzag(encode_zigzag(2)), is(2L)); + assertEquals(4L, encode_zigzag(2)); + assertEquals(2L, decode_zigzag(encode_zigzag(2))); - assertThat(encode_zigzag(-1000), is(1999L)); - assertThat(decode_zigzag(encode_zigzag(-1000)), is(-1000L)); + assertEquals(1999L, encode_zigzag(-1000)); + assertEquals(-1000L, decode_zigzag(encode_zigzag(-1000))); - assertThat(encode_zigzag(1000), is(2000L)); - assertThat(decode_zigzag(encode_zigzag(1000)), is(1000L)); + assertEquals(2000L, encode_zigzag(1000)); + assertEquals(1000L, decode_zigzag(encode_zigzag(1000))); - assertThat(encode_zigzag(-0x8000000000000000L), is(-1L)); - assertThat(decode_zigzag(encode_zigzag(-0x8000000000000000L)), is(-0x8000000000000000L)); + assertEquals(-1L, encode_zigzag(-0x8000000000000000L)); + assertEquals(-0x8000000000000000L, decode_zigzag(encode_zigzag(-0x8000000000000000L))); - assertThat(encode_zigzag(0x7fffffffffffffffL), is(-2L)); - assertThat(decode_zigzag(encode_zigzag(0x7fffffffffffffffL)), is(0x7fffffffffffffffL)); + assertEquals(-2L, encode_zigzag(0x7fffffffffffffffL)); + assertEquals(0x7fffffffffffffffL, decode_zigzag(encode_zigzag(0x7fffffffffffffffL))); } @Test public void testDoubleConversion() { - assertThat(encode_double(0.0), is(0L)); - assertThat(decode_double(encode_double(0.0)), is(0.0)); + assertEquals(0L, encode_double(0.0)); + assertEquals(0.0, decode_double(encode_double(0.0)), 0.0); - assertThat(encode_double(1.0), is(0x3ff0000000000000L)); - assertThat(decode_double(encode_double(1.0)), is(1.0)); + assertEquals(0x3ff0000000000000L, encode_double(1.0)); + assertEquals(1.0, decode_double(encode_double(1.0)), 0.0); - assertThat(encode_double(-1.0), is(0xbff0000000000000L)); - assertThat(decode_double(encode_double(-1.0)), is(-1.0)); + assertEquals(0xbff0000000000000L, encode_double(-1.0)); + assertEquals(-1.0, decode_double(encode_double(-1.0)), 0.0); - assertThat(encode_double(2.0), is(0x4000000000000000L)); - assertThat(decode_double(encode_double(2.0)), is(2.0)); + assertEquals(0x4000000000000000L, encode_double(2.0)); + assertEquals(2.0, decode_double(encode_double(2.0)), 0.0); - assertThat(encode_double(-2.0), is(0xc000000000000000L)); - assertThat(decode_double(encode_double(-2.0)), is(-2.0)); + assertEquals(0xc000000000000000L, encode_double(-2.0)); + assertEquals(-2.0, decode_double(encode_double(-2.0)), 0.0); - assertThat(encode_double(-0.0), is(0x8000000000000000L)); - assertThat(decode_double(encode_double(-0.0)), is(-0.0)); + assertEquals(0x8000000000000000L, encode_double(-0.0)); + assertEquals(-0.0, decode_double(encode_double(-0.0)), 0.0); - assertThat(encode_double(3.5), is(0x400c000000000000L)); - assertThat(decode_double(encode_double(3.5)), is(3.5)); + assertEquals(0x400c000000000000L, encode_double(3.5)); + assertEquals(3.5, decode_double(encode_double(3.5)), 0.0); - assertThat(encode_double(65535.875), is(0x40EFFFFC00000000L)); - assertThat(decode_double(encode_double(65535.875)), is(65535.875)); + assertEquals(0x40EFFFFC00000000L, encode_double(65535.875)); + assertEquals(65535.875, decode_double(encode_double(65535.875)), 0.0); } @Test @@ -156,8 +157,8 @@ public class BinaryFormatTestCase { for (byte type = 0; type < TYPE_LIMIT; ++type) { for (int meta = 0; meta < META_LIMIT; ++meta) { byte mangled = encode_type_and_meta(type, meta); - assertThat(decode_type(mangled).ID, is(type)); - assertThat(decode_meta(mangled), is(meta)); + assertEquals(type, decode_type(mangled).ID); + assertEquals(meta, decode_meta(mangled)); } } } @@ -241,7 +242,7 @@ public class BinaryFormatTestCase { BinaryEncoder encoder = new BinaryEncoder(actual); encoder.write_type_and_size(type, size); } - assertThat(actual.toArray(), is(expect.toArray())); + assertArrayEquals(expect.toArray(), actual.toArray()); byte[] got = expect.toArray(); BinaryDecoder bif = new BinaryDecoder(); @@ -249,12 +250,12 @@ public class BinaryFormatTestCase { byte b = bif.in.getByte(); Type decodedType = decode_type(b); int decodedSize = bif.in.read_size(decode_meta(b)); - assertThat(decodedType.ID, is(type)); - assertThat(decodedSize, is(size)); - assertThat(bif.in.getConsumedSize(), is(got.length)); - assertThat(bif.in.failed(), is(false)); + assertEquals(type, decodedType.ID); + assertEquals(size, decodedSize); + assertEquals(got.length, bif.in.getConsumedSize()); + assertFalse(bif.in.failed()); - assertThat(BinaryView.extract_children_for_testing(got, 0), is(size)); + assertEquals(size, BinaryView.extract_children_for_testing(got, 0)); } } @@ -292,21 +293,21 @@ public class BinaryFormatTestCase { bof.write_type_and_bytes_le(type, bits); } byte[] actual = output.toArray(); - assertThat(actual, is(expect)); + assertArrayEquals(expect, actual); // test input: BinaryDecoder bif = new BinaryDecoder(); bif.in = new BufferedInput(expect); int size = decode_meta(bif.in.getByte()); long decodedBits = (hi != 0) ? bif.read_bytes_be(size) : bif.read_bytes_le(size); - assertThat(decodedBits, is(bits)); - assertThat(bif.in.getConsumedSize(), is(expect.length)); - assertThat(bif.in.failed(), is(false)); + assertEquals(bits, decodedBits); + assertEquals(expect.length, bif.in.getConsumedSize()); + assertFalse(bif.in.failed()); if (hi != 0) { - assertThat(encode_double(BinaryView.extract_double_for_testing(expect, 0)), is(bits)); + assertEquals(bits, encode_double(BinaryView.extract_double_for_testing(expect, 0))); } else { - assertThat(encode_zigzag(BinaryView.extract_long_for_testing(expect, 0)), is(bits)); + assertEquals(bits, encode_zigzag(BinaryView.extract_long_for_testing(expect, 0))); } } } @@ -322,7 +323,7 @@ public class BinaryFormatTestCase { expect.put((byte)0); // nix byte[] actual = BinaryFormat.encode(slime); - assertThat(actual, is(expect.toArray())); + assertArrayEquals(expect.toArray(), actual); verifyMultiEncode(expect.toArray()); } @@ -428,7 +429,7 @@ public class BinaryFormatTestCase { System.arraycopy(expect, 0, overlappingBuffer, 1, expect.length); overlappingBuffer[overlappingBuffer.length - 1] = 0; Slime copy = BinaryFormat.decode(overlappingBuffer, 1, expect.length); - assertThat(BinaryFormat.encode(slime), is(BinaryFormat.encode(copy))); + assertArrayEquals(BinaryFormat.encode(copy), BinaryFormat.encode(slime)); } @Test @@ -550,18 +551,17 @@ public class BinaryFormatTestCase { BinaryDecoder decoder = new BinaryDecoder(); Slime slime = decoder.decode(data); int consumed = decoder.in.getConsumedSize(); - assertThat(consumed, is(data.length)); + assertEquals(data.length, consumed); Cursor c = slime.get(); - assertThat(c.valid(), is(true)); - assertThat(c.type(), is(Type.OBJECT)); - assertThat(c.children(), is(5)); - assertThat(c.field("b").asBool(), is(true)); - assertThat(c.field("c").asLong(), is(5L)); - assertThat(c.field("d").asDouble(), is(3.5)); - assertThat(c.field("e").asString(), is("string")); + assertTrue(c.valid()); + assertEquals(Type.OBJECT, c.type()); + assertEquals(5, c.children()); + assertTrue(c.field("b").asBool()); + assertEquals(5L, c.field("c").asLong()); + assertEquals(3.5, c.field("d").asDouble(), 0.0); + assertEquals("string", c.field("e").asString()); byte[] expd = { 'd', 'a', 't', 'a' }; - assertThat(c.field("f").asData(), is(expd)); - assertThat(c.entry(5).valid(), is(false)); // not ARRAY + assertArrayEquals(expd, c.field("f").asData()); + assertFalse(c.entry(5).valid()); // not ARRAY } - } diff --git a/vespajlib/src/test/java/com/yahoo/slime/BinaryViewTest.java b/vespajlib/src/test/java/com/yahoo/slime/BinaryViewTest.java index 568124369d4..920a25b96c9 100644 --- a/vespajlib/src/test/java/com/yahoo/slime/BinaryViewTest.java +++ b/vespajlib/src/test/java/com/yahoo/slime/BinaryViewTest.java @@ -140,99 +140,43 @@ public class BinaryViewTest { } } - class MyVisitor implements Visitor { - enum Called { NONE, INVALID, NIX, BOOL, LONG, DOUBLE, UTF8, DATA, ARRAY, OBJECT } - Called called = Called.NONE; - boolean boolValue; - long longValue; - double doubleValue; - byte[] bytes; - Inspector stuff; - @Override public void visitInvalid() { - assertEquals(ctx, Called.NONE, called); - called = Called.INVALID; - } - @Override public void visitNix() { - assertEquals(ctx, Called.NONE, called); - called = Called.NIX; - } - @Override public void visitBool(boolean bit) { - assertEquals(ctx, Called.NONE, called); - called = Called.BOOL; - boolValue = bit; - } - @Override public void visitLong(long l) { - assertEquals(ctx, Called.NONE, called); - called = Called.LONG; - longValue = l; - } - @Override public void visitDouble(double d) { - assertEquals(ctx, Called.NONE, called); - called = Called.DOUBLE; - doubleValue = d; - } - @Override public void visitString(String str) { - fail(ctx + ", strings are never utf-16 in binary view"); - } - @Override public void visitString(byte[] utf8) { - assertEquals(ctx, Called.NONE, called); - called = Called.UTF8; - bytes = utf8; - } - @Override public void visitData(byte[] data) { - assertEquals(ctx, Called.NONE, called); - called = Called.DATA; - bytes = data; - } - @Override public void visitArray(Inspector arr) { - assertEquals(ctx, Called.NONE, called); - called = Called.ARRAY; - stuff = arr; - } - @Override public void visitObject(Inspector obj) { - assertEquals(ctx, Called.NONE, called); - called = Called.OBJECT; - stuff = obj; - } - }; - void checkVisitor(Inspector view) { - var visitor = new MyVisitor(); + var visitor = new MockVisitor(ctx); view.accept(visitor); if (!view.valid()) { - assertEquals(ctx, MyVisitor.Called.INVALID, visitor.called); + assertEquals(ctx, MockVisitor.Called.INVALID, visitor.called); return; } switch (view.type()) { case NIX: - assertEquals(ctx, MyVisitor.Called.NIX, visitor.called); + assertEquals(ctx, MockVisitor.Called.NIX, visitor.called); break; case BOOL: - assertEquals(ctx, MyVisitor.Called.BOOL, visitor.called); + assertEquals(ctx, MockVisitor.Called.BOOL, visitor.called); assertEquals(ctx, view.asBool(), visitor.boolValue); break; case LONG: - assertEquals(ctx, MyVisitor.Called.LONG, visitor.called); + assertEquals(ctx, MockVisitor.Called.LONG, visitor.called); assertEquals(ctx, view.asLong(), visitor.longValue); break; case DOUBLE: - assertEquals(ctx, MyVisitor.Called.DOUBLE, visitor.called); + assertEquals(ctx, MockVisitor.Called.DOUBLE, visitor.called); assertEquals(ctx, view.asDouble(), visitor.doubleValue, 0.0); break; case STRING: - assertEquals(ctx, MyVisitor.Called.UTF8, visitor.called); + assertEquals(ctx, MockVisitor.Called.UTF8, visitor.called); assertArrayEquals(ctx, view.asUtf8(), visitor.bytes); break; case DATA: - assertEquals(ctx, MyVisitor.Called.DATA, visitor.called); + assertEquals(ctx, MockVisitor.Called.DATA, visitor.called); assertArrayEquals(ctx, view.asData(), visitor.bytes); break; case ARRAY: - assertEquals(ctx, MyVisitor.Called.ARRAY, visitor.called); + assertEquals(ctx, MockVisitor.Called.ARRAY, visitor.called); assertSame(ctx, view, visitor.stuff); break; case OBJECT: - assertEquals(ctx, MyVisitor.Called.OBJECT, visitor.called); + assertEquals(ctx, MockVisitor.Called.OBJECT, visitor.called); assertSame(ctx, view, visitor.stuff); break; default: diff --git a/vespajlib/src/test/java/com/yahoo/slime/MockVisitor.java b/vespajlib/src/test/java/com/yahoo/slime/MockVisitor.java new file mode 100644 index 00000000000..a4354adbcd0 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/slime/MockVisitor.java @@ -0,0 +1,69 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +import static org.junit.Assert.assertEquals; + +class MockVisitor implements Visitor { + enum Called { NONE, INVALID, NIX, BOOL, LONG, DOUBLE, STRING, UTF8, DATA, ARRAY, OBJECT } + Called called = Called.NONE; + boolean boolValue; + long longValue; + double doubleValue; + String string; + byte[] bytes; + Inspector stuff; + String ctx; + + MockVisitor(String context) { + ctx = context; + } + MockVisitor() { this(""); } + @Override public void visitInvalid() { + assertEquals(ctx, Called.NONE, called); + called = Called.INVALID; + } + @Override public void visitNix() { + assertEquals(ctx, Called.NONE, called); + called = Called.NIX; + } + @Override public void visitBool(boolean bit) { + assertEquals(ctx, Called.NONE, called); + called = Called.BOOL; + boolValue = bit; + } + @Override public void visitLong(long l) { + assertEquals(ctx, Called.NONE, called); + called = Called.LONG; + longValue = l; + } + @Override public void visitDouble(double d) { + assertEquals(ctx, Called.NONE, called); + called = Called.DOUBLE; + doubleValue = d; + } + @Override public void visitString(String str) { + assertEquals(ctx, Called.NONE, called); + called = Called.STRING; + string = str; + } + @Override public void visitString(byte[] utf8) { + assertEquals(ctx, Called.NONE, called); + called = Called.UTF8; + bytes = utf8; + } + @Override public void visitData(byte[] data) { + assertEquals(ctx, Called.NONE, called); + called = Called.DATA; + bytes = data; + } + @Override public void visitArray(Inspector arr) { + assertEquals(ctx, Called.NONE, called); + called = Called.ARRAY; + stuff = arr; + } + @Override public void visitObject(Inspector obj) { + assertEquals(ctx, Called.NONE, called); + called = Called.OBJECT; + stuff = obj; + } +} diff --git a/vespajlib/src/test/java/com/yahoo/slime/SlimeTestCase.java b/vespajlib/src/test/java/com/yahoo/slime/SlimeTestCase.java index 6ee8fb6c7e2..0a5c69b2d73 100644 --- a/vespajlib/src/test/java/com/yahoo/slime/SlimeTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/slime/SlimeTestCase.java @@ -2,10 +2,11 @@ package com.yahoo.slime; import org.junit.Test; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.sameInstance; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; public class SlimeTestCase { @@ -21,25 +22,25 @@ public class SlimeTestCase { public void testTypeIds() { System.out.println("testing type identifiers..."); - assertThat(Type.NIX.ID, is((byte)0)); - assertThat(Type.BOOL.ID, is((byte)1)); - assertThat(Type.LONG.ID, is((byte)2)); - assertThat(Type.DOUBLE.ID, is((byte)3)); - assertThat(Type.STRING.ID, is((byte)4)); - assertThat(Type.DATA.ID, is((byte)5)); - assertThat(Type.ARRAY.ID, is((byte)6)); - assertThat(Type.OBJECT.ID, is((byte)7)); + assertEquals((byte)0, Type.NIX.ID); + assertEquals((byte)1, Type.BOOL.ID); + assertEquals((byte)2, Type.LONG.ID); + assertEquals((byte)3, Type.DOUBLE.ID); + assertEquals((byte)4, Type.STRING.ID); + assertEquals((byte)5, Type.DATA.ID); + assertEquals((byte)6, Type.ARRAY.ID); + assertEquals((byte)7, Type.OBJECT.ID); - assertThat(Type.values().length, is(8)); + assertEquals(8, Type.values().length); - assertThat(Type.values()[0], sameInstance(Type.NIX)); - assertThat(Type.values()[1], sameInstance(Type.BOOL)); - assertThat(Type.values()[2], sameInstance(Type.LONG)); - assertThat(Type.values()[3], sameInstance(Type.DOUBLE)); - assertThat(Type.values()[4], sameInstance(Type.STRING)); - assertThat(Type.values()[5], sameInstance(Type.DATA)); - assertThat(Type.values()[6], sameInstance(Type.ARRAY)); - assertThat(Type.values()[7], sameInstance(Type.OBJECT)); + assertSame(Type.NIX, Type.values()[0]); + assertSame(Type.BOOL, Type.values()[1]); + assertSame(Type.LONG, Type.values()[2]); + assertSame(Type.DOUBLE, Type.values()[3]); + assertSame(Type.STRING, Type.values()[4]); + assertSame(Type.DATA, Type.values()[5]); + assertSame(Type.ARRAY, Type.values()[6]); + assertSame(Type.OBJECT, Type.values()[7]); } @Test @@ -50,41 +51,41 @@ public class SlimeTestCase { for (int i = 0; i < 2; i++) { if (i == 0) { cur = slime.get(); - assertThat(cur.valid(), is(true)); + assertTrue(cur.valid()); } else { cur = NixValue.invalid(); - assertThat(cur.valid(), is(false)); + assertFalse(cur.valid()); } - assertThat(cur.type(), is(Type.NIX)); - assertThat(cur.children(), is(0)); - assertThat(cur.asBool(), is(false)); - assertThat(cur.asLong(), is((long)0)); - assertThat(cur.asDouble(), is(0.0)); - assertThat(cur.asString(), is("")); - assertThat(cur.asData(), is(new byte[0])); - assertThat(cur.entry(0).valid(), is(false)); - assertThat(cur.field(0).valid(), is(false)); - assertThat(cur.field("foo").valid(), is(false)); + assertEquals(Type.NIX, cur.type()); + assertEquals(0, cur.children()); + assertFalse(cur.asBool()); + assertEquals(0L, cur.asLong()); + assertEquals(0.0, cur.asDouble(), 0.0); + assertEquals("", cur.asString()); + assertArrayEquals(new byte[0], cur.asData()); + assertFalse(cur.entry(0).valid()); + assertFalse(cur.field(0).valid()); + assertFalse(cur.field("foo").valid()); } Inspector insp; for (int i = 0; i < 2; i++) { if (i == 0) { insp = slime.get(); - assertThat(insp.valid(), is(true)); + assertTrue(insp.valid()); } else { insp = NixValue.invalid(); - assertThat(insp.valid(), is(false)); + assertFalse(insp.valid()); } - assertThat(insp.type(), is(Type.NIX)); - assertThat(insp.children(), is(0)); - assertThat(insp.asBool(), is(false)); - assertThat(insp.asLong(), is((long)0)); - assertThat(insp.asDouble(), is(0.0)); - assertThat(insp.asString(), is("")); - assertThat(insp.asData(), is(new byte[0])); - assertThat(insp.entry(0).valid(), is(false)); - assertThat(insp.field(0).valid(), is(false)); - assertThat(insp.field("foo").valid(), is(false)); + assertEquals(Type.NIX, insp.type()); + assertEquals(0, insp.children()); + assertFalse(insp.asBool()); + assertEquals(0L, insp.asLong()); + assertEquals(0.0, insp.asDouble(), 0.0); + assertEquals("", insp.asString()); + assertArrayEquals(new byte[0], insp.asData()); + assertFalse(insp.entry(0).valid()); + assertFalse(insp.field(0).valid()); + assertFalse(insp.field("foo").valid()); } } @@ -96,63 +97,63 @@ public class SlimeTestCase { System.out.println("testing boolean value"); slime.setBool(true); Inspector insp = slime.get(); - assertThat(insp.valid(), is(true)); - assertThat(insp.type(), sameInstance(Type.BOOL)); - assertThat(insp.asBool(), is(true)); + assertTrue(insp.valid()); + assertSame(Type.BOOL, insp.type()); + assertTrue(insp.asBool()); Cursor cur = slime.get(); - assertThat(cur.valid(), is(true)); - assertThat(cur.type(), sameInstance(Type.BOOL)); - assertThat(cur.asBool(), is(true)); + assertTrue(cur.valid()); + assertSame(Type.BOOL, cur.type()); + assertTrue(cur.asBool()); System.out.println("testing long value"); slime.setLong(42); cur = slime.get(); insp = slime.get(); - assertThat(cur.valid(), is(true)); - assertThat(insp.valid(), is(true)); - assertThat(cur.type(), sameInstance(Type.LONG)); - assertThat(insp.type(), sameInstance(Type.LONG)); - assertThat(cur.asLong(), is((long)42)); - assertThat(insp.asLong(), is((long)42)); + assertTrue(cur.valid()); + assertTrue(insp.valid()); + assertSame(Type.LONG, cur.type()); + assertSame(Type.LONG, insp.type()); + assertEquals(42L, cur.asLong()); + assertEquals(42L, insp.asLong()); System.out.println("testing double value"); slime.setDouble(4.2); cur = slime.get(); insp = slime.get(); - assertThat(cur.valid(), is(true)); - assertThat(insp.valid(), is(true)); - assertThat(cur.type(), sameInstance(Type.DOUBLE)); - assertThat(insp.type(), sameInstance(Type.DOUBLE)); - assertThat(cur.asDouble(), is(4.2)); - assertThat(insp.asDouble(), is(4.2)); + assertTrue(cur.valid()); + assertTrue(insp.valid()); + assertSame(Type.DOUBLE, cur.type()); + assertSame(Type.DOUBLE, insp.type()); + assertEquals(4.2, cur.asDouble(), 0.0); + assertEquals(4.2, insp.asDouble(), 0.0); System.out.println("testing string value"); slime.setString("fortytwo"); cur = slime.get(); insp = slime.get(); - assertThat(cur.valid(), is(true)); - assertThat(insp.valid(), is(true)); - assertThat(cur.type(), sameInstance(Type.STRING)); - assertThat(insp.type(), sameInstance(Type.STRING)); - assertThat(cur.asString(), is("fortytwo")); - assertThat(insp.asString(), is("fortytwo")); + assertTrue(cur.valid()); + assertTrue(insp.valid()); + assertSame(Type.STRING, cur.type()); + assertSame(Type.STRING, insp.type()); + assertEquals("fortytwo", cur.asString()); + assertEquals("fortytwo", insp.asString()); System.out.println("testing data value"); byte[] data = { (byte)4, (byte)2 }; slime.setData(data); cur = slime.get(); insp = slime.get(); - assertThat(cur.valid(), is(true)); - assertThat(insp.valid(), is(true)); - assertThat(cur.type(), sameInstance(Type.DATA)); - assertThat(insp.type(), sameInstance(Type.DATA)); - assertThat(cur.asData(), is(data)); - assertThat(insp.asData(), is(data)); + assertTrue(cur.valid()); + assertTrue(insp.valid()); + assertSame(Type.DATA, cur.type()); + assertSame(Type.DATA, insp.type()); + assertArrayEquals(data, cur.asData()); + assertArrayEquals(data, insp.asData()); data[0] = 10; data[1] = 20; byte[] data2 = { 10, 20 }; - assertThat(cur.asData(), is(data2)); - assertThat(insp.asData(), is(data2)); + assertArrayEquals(data2, cur.asData()); + assertArrayEquals(data2, insp.asData()); } @Test @@ -160,13 +161,13 @@ public class SlimeTestCase { System.out.println("testing array values..."); Slime slime = new Slime(); Cursor c = slime.setArray(); - assertThat(c.valid(), is(true)); - assertThat(c.type(), is(Type.ARRAY)); - assertThat(c.children(), is(0)); + assertTrue(c.valid()); + assertEquals(Type.ARRAY, c.type()); + assertEquals(0, c.children()); Inspector i = slime.get(); - assertThat(i.valid(), is(true)); - assertThat(i.type(), is(Type.ARRAY)); - assertThat(i.children(), is(0)); + assertTrue(i.valid()); + assertEquals(Type.ARRAY, i.type()); + assertEquals(0, i.children()); c.addNix(); c.addBool(true); c.addLong(5); @@ -174,23 +175,23 @@ public class SlimeTestCase { c.addString("string"); byte[] data = { (byte)'d', (byte)'a', (byte)'t', (byte)'a' }; c.addData(data); - assertThat(c.children(), is(6)); - assertThat(c.entry(0).valid(), is(true)); - assertThat(c.entry(1).asBool(), is(true)); - assertThat(c.entry(2).asLong(), is((long)5)); - assertThat(c.entry(3).asDouble(), is(3.5)); - assertThat(c.entry(4).asString(), is("string")); - assertThat(c.entry(5).asData(), is(data)); - assertThat(c.field(5).valid(), is(false)); // not OBJECT + assertEquals(6, c.children()); + assertTrue(c.entry(0).valid()); + assertTrue(c.entry(1).asBool()); + assertEquals(5L, c.entry(2).asLong()); + assertEquals(3.5, c.entry(3).asDouble(), 0.0); + assertEquals("string", c.entry(4).asString()); + assertArrayEquals(data, c.entry(5).asData()); + assertFalse(c.field(5).valid()); // not OBJECT - assertThat(i.children(), is(6)); - assertThat(i.entry(0).valid(), is(true)); - assertThat(i.entry(1).asBool(), is(true)); - assertThat(i.entry(2).asLong(), is((long)5)); - assertThat(i.entry(3).asDouble(), is(3.5)); - assertThat(i.entry(4).asString(), is("string")); - assertThat(i.entry(5).asData(), is(data)); - assertThat(i.field(5).valid(), is(false)); // not OBJECT + assertEquals(6, i.children()); + assertTrue(i.entry(0).valid()); + assertTrue(i.entry(1).asBool()); + assertEquals(5L, i.entry(2).asLong()); + assertEquals(3.5, i.entry(3).asDouble(), 0.0); + assertEquals("string", i.entry(4).asString()); + assertArrayEquals(data, i.entry(5).asData()); + assertFalse(i.field(5).valid()); // not OBJECT } @Test @@ -199,13 +200,13 @@ public class SlimeTestCase { Slime slime = new Slime(); Cursor c = slime.setObject(); - assertThat(c.valid(), is(true)); - assertThat(c.type(), is(Type.OBJECT)); - assertThat(c.children(), is(0)); + assertTrue(c.valid()); + assertEquals(Type.OBJECT, c.type()); + assertEquals(0, c.children()); Inspector i = slime.get(); - assertThat(i.valid(), is(true)); - assertThat(i.type(), is(Type.OBJECT)); - assertThat(i.children(), is(0)); + assertTrue(i.valid()); + assertEquals(Type.OBJECT, i.type()); + assertEquals(0, i.children()); c.setNix("a"); c.setBool("b", true); @@ -215,23 +216,23 @@ public class SlimeTestCase { byte[] data = { (byte)'d', (byte)'a', (byte)'t', (byte)'a' }; c.setData("f", data); - assertThat(c.children(), is(6)); - assertThat(c.field("a").valid(), is(true)); - assertThat(c.field("b").asBool(), is(true)); - assertThat(c.field("c").asLong(), is((long)5)); - assertThat(c.field("d").asDouble(), is(3.5)); - assertThat(c.field("e").asString(), is("string")); - assertThat(c.field("f").asData(), is(data)); - assertThat(c.entry(4).valid(), is(false)); // not ARRAY + assertEquals(6, c.children()); + assertTrue(c.field("a").valid()); + assertTrue(c.field("b").asBool()); + assertEquals(5L, c.field("c").asLong()); + assertEquals(3.5, c.field("d").asDouble(), 0.0); + assertEquals("string", c.field("e").asString()); + assertArrayEquals(data, c.field("f").asData()); + assertFalse(c.entry(4).valid()); // not ARRAY - assertThat(i.children(), is(6)); - assertThat(i.field("a").valid(), is(true)); - assertThat(i.field("b").asBool(), is(true)); - assertThat(i.field("c").asLong(), is((long)5)); - assertThat(i.field("d").asDouble(), is(3.5)); - assertThat(i.field("e").asString(), is("string")); - assertThat(i.field("f").asData(), is(data)); - assertThat(i.entry(4).valid(), is(false)); // not ARRAY + assertEquals(6, i.children()); + assertTrue(i.field("a").valid()); + assertTrue(i.field("b").asBool()); + assertEquals(5L, i.field("c").asLong()); + assertEquals(3.5, i.field("d").asDouble(), 0.0); + assertEquals("string", i.field("e").asString()); + assertArrayEquals(data, i.field("f").asData()); + assertFalse(i.entry(4).valid()); // not ARRAY } @Test @@ -240,12 +241,12 @@ public class SlimeTestCase { { Slime slime = new Slime(); Cursor c = slime.setArray(); - assertThat(c.addLong(5).asLong(), is((long)5)); + assertEquals(5L, c.addLong(5).asLong()); } { Slime slime = new Slime(); Cursor c = slime.setObject(); - assertThat(c.setLong("a", 5).asLong(), is((long)5)); + assertEquals(5L, c.setLong("a", 5).asLong()); } } @@ -256,10 +257,10 @@ public class SlimeTestCase { Slime slime = new Slime(); Cursor c = slime.setLong(10); Inspector i1 = c; - assertThat(i1.asLong(), is((long)10)); + assertEquals(10L, i1.asLong()); Inspector i2 = slime.get(); - assertThat(i2.asLong(), is((long)10)); + assertEquals(10L, i2.asLong()); } @Test @@ -275,14 +276,14 @@ public class SlimeTestCase { c3.setLong("answer", 42); } Inspector i = slime.get(); - assertThat(i.field("bar").asLong(), is((long)10)); - assertThat(i.field("foo").entry(0).asLong(), is((long)20)); - assertThat(i.field("foo").entry(1).field("answer").asLong(), is((long)42)); + assertEquals(10L, i.field("bar").asLong()); + assertEquals(20L, i.field("foo").entry(0).asLong()); + assertEquals(42L, i.field("foo").entry(1).field("answer").asLong()); Cursor c = slime.get(); - assertThat(c.field("bar").asLong(), is((long)10)); - assertThat(c.field("foo").entry(0).asLong(), is((long)20)); - assertThat(c.field("foo").entry(1).field("answer").asLong(), is((long)42)); + assertEquals(10L, c.field("bar").asLong()); + assertEquals(20L, c.field("foo").entry(0).asLong()); + assertEquals(42L, c.field("foo").entry(1).field("answer").asLong()); } @Test @@ -293,19 +294,19 @@ public class SlimeTestCase { Cursor c = slime.setObject(); for (int i = 0; i < n; i++) { String str = ("" + i + "_str_" + i); - assertThat(slime.lookup(str), is(SymbolTable.INVALID)); - assertThat(c.field(str).type(), sameInstance(Type.NIX)); + assertEquals(SymbolTable.INVALID, slime.lookup(str)); + assertSame(Type.NIX, c.field(str).type()); switch (i % 2) { - case 0: assertThat((int)c.setLong(str, i).asLong(), is(i)); break; - case 1: assertThat(slime.insert(str), is(i)); break; + case 0: assertEquals(i, (int)c.setLong(str, i).asLong()); break; + case 1: assertEquals(i, slime.insert(str)); break; } } for (int i = 0; i < n; i++) { String str = ("" + i + "_str_" + i); - assertThat(slime.lookup(str), is(i)); + assertEquals(i, slime.lookup(str)); switch (i % 2) { - case 0: assertThat((int)c.field(str).asLong(), is(i)); break; - case 1: assertThat((int)c.field(str).asLong(), is(0)); break; + case 0: assertEquals(i, (int)c.field(str).asLong()); break; + case 1: assertEquals(0, (int)c.field(str).asLong()); break; } } } @@ -317,12 +318,12 @@ public class SlimeTestCase { Slime slime = new Slime(); Cursor c = slime.setArray(); for (int i = 0; i < n; i++) { - assertThat((int)c.addLong(i).asLong(), is(i)); + assertEquals(i, (int)c.addLong(i).asLong()); } for (int i = 0; i < n; i++) { - assertThat((int)c.entry(i).asLong(), is(i)); + assertEquals(i, (int)c.entry(i).asLong()); } - assertThat((int)c.entry(n).asLong(), is(0)); + assertEquals(0, (int)c.entry(n).asLong()); } @Test @@ -332,8 +333,8 @@ public class SlimeTestCase { c1.addLong(20); Cursor c2 = c1.addObject(); c2.setLong("answer", 42); - assertThat(slime.get().toString(), is("[20,{\"answer\":42}]")); + assertEquals("[20,{\"answer\":42}]", slime.get().toString()); c1.addString("\u2008"); - assertThat(slime.get().toString(), is("[20,{\"answer\":42},\"\u2008\"]")); + assertEquals("[20,{\"answer\":42},\"\u2008\"]", slime.get().toString()); } } diff --git a/vespajlib/src/test/java/com/yahoo/slime/VisitorTestCase.java b/vespajlib/src/test/java/com/yahoo/slime/VisitorTestCase.java index 916cddac56b..cc4bf5d16e2 100644 --- a/vespajlib/src/test/java/com/yahoo/slime/VisitorTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/slime/VisitorTestCase.java @@ -2,100 +2,97 @@ package com.yahoo.slime; import org.junit.Test; -import org.mockito.Mockito; - -import static org.hamcrest.CoreMatchers.sameInstance; -import static org.mockito.hamcrest.MockitoHamcrest.argThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertSame; public class VisitorTestCase { @Test public void testVisitInvalid() { - Visitor visitor = Mockito.mock(Visitor.class); + var visitor = new MockVisitor(); Inspector inspector = new Slime().get().field("invalid"); inspector.accept(visitor); - Mockito.verify(visitor).visitInvalid(); - Mockito.verifyNoMoreInteractions(visitor); + assertEquals(MockVisitor.Called.INVALID, visitor.called); } @Test public void testVisitNix() { - Visitor visitor = Mockito.mock(Visitor.class); + var visitor = new MockVisitor(); Inspector inspector = new Slime().get(); inspector.accept(visitor); - Mockito.verify(visitor).visitNix(); - Mockito.verifyNoMoreInteractions(visitor); + assertEquals(MockVisitor.Called.NIX, visitor.called); } @Test public void testVisitBool() { - Visitor visitor = Mockito.mock(Visitor.class); + var visitor = new MockVisitor(); Inspector inspector = new Slime().setBool(true); inspector.accept(visitor); - Mockito.verify(visitor).visitBool(true); - Mockito.verifyNoMoreInteractions(visitor); + assertEquals(MockVisitor.Called.BOOL, visitor.called); + assertEquals(true, visitor.boolValue); } @Test public void testVisitLong() { - Visitor visitor = Mockito.mock(Visitor.class); + var visitor = new MockVisitor(); Inspector inspector = new Slime().setLong(123); inspector.accept(visitor); - Mockito.verify(visitor).visitLong(123); - Mockito.verifyNoMoreInteractions(visitor); + assertEquals(MockVisitor.Called.LONG, visitor.called); + assertEquals(123, visitor.longValue); } @Test public void testVisitDouble() { - Visitor visitor = Mockito.mock(Visitor.class); + var visitor = new MockVisitor(); Inspector inspector = new Slime().setDouble(123.0); inspector.accept(visitor); - Mockito.verify(visitor).visitDouble(123.0); - Mockito.verifyNoMoreInteractions(visitor); + assertEquals(MockVisitor.Called.DOUBLE, visitor.called); + assertEquals(123.0, visitor.doubleValue, 0.0); } @Test public void testVisitStringUtf16() { - Visitor visitor = Mockito.mock(Visitor.class); + var visitor = new MockVisitor(); Inspector inspector = new Slime().setString("abc"); inspector.accept(visitor); - Mockito.verify(visitor).visitString("abc"); - Mockito.verifyNoMoreInteractions(visitor); + assertEquals(MockVisitor.Called.STRING, visitor.called); + assertEquals("abc", visitor.string); } @Test public void testVisitStringUtf8() { - Visitor visitor = Mockito.mock(Visitor.class); + var visitor = new MockVisitor(); Inspector inspector = new Slime().setString(new byte[] {65,66,67}); inspector.accept(visitor); - Mockito.verify(visitor).visitString(new byte[] {65,66,67}); - Mockito.verifyNoMoreInteractions(visitor); + assertEquals(MockVisitor.Called.UTF8, visitor.called); + assertArrayEquals(new byte[] {65,66,67}, visitor.bytes); } @Test public void testVisitData() { - Visitor visitor = Mockito.mock(Visitor.class); + var visitor = new MockVisitor(); Inspector inspector = new Slime().setData(new byte[] {1,2,3}); inspector.accept(visitor); - Mockito.verify(visitor).visitData(new byte[] {1,2,3}); - Mockito.verifyNoMoreInteractions(visitor); + assertEquals(MockVisitor.Called.DATA, visitor.called); + assertArrayEquals(new byte[] {1,2,3}, visitor.bytes); } @Test public void testVisitArray() { - Visitor visitor = Mockito.mock(Visitor.class); + var visitor = new MockVisitor(); Inspector inspector = new Slime().setArray(); inspector.accept(visitor); - Mockito.verify(visitor).visitArray(argThat(sameInstance(inspector))); - Mockito.verifyNoMoreInteractions(visitor); + assertEquals(MockVisitor.Called.ARRAY, visitor.called); + assertSame(inspector, visitor.stuff); } @Test public void testVisitObject() { - Visitor visitor = Mockito.mock(Visitor.class); + var visitor = new MockVisitor(); Inspector inspector = new Slime().setObject(); inspector.accept(visitor); - Mockito.verify(visitor).visitObject(argThat(sameInstance(inspector))); - Mockito.verifyNoMoreInteractions(visitor); + assertEquals(MockVisitor.Called.OBJECT, visitor.called); + assertSame(inspector, visitor.stuff); } } diff --git a/vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp b/vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp index 44decb9bf91..31113f2b4f2 100644 --- a/vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp +++ b/vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp @@ -53,7 +53,7 @@ public: ~RealIntStore(); EntryRef add(uint32_t value) { return _store.addEntry(value); } AtomicEntryRef add_relaxed(uint32_t value) { return AtomicEntryRef(add(value)); } - void hold(const AtomicEntryRef& ref) { _store.holdElem(ref.load_relaxed(), 1); } + void hold(const AtomicEntryRef& ref) { _store.hold_entry(ref.load_relaxed()); } EntryRef move(EntryRef ref); void assign_generation(generation_t current_gen) { _store.assign_generation(current_gen); } void reclaim_memory(generation_t gen) { _store.reclaim_memory(gen); } diff --git a/vespalib/src/tests/btree/btree_test.cpp b/vespalib/src/tests/btree/btree_test.cpp index b8da9ea6042..afad3523fa3 100644 --- a/vespalib/src/tests/btree/btree_test.cpp +++ b/vespalib/src/tests/btree/btree_test.cpp @@ -1065,7 +1065,7 @@ adjustAllocatedBytes(size_t nodeCount, size_t nodeSize) TEST_F(BTreeTest, require_that_memory_usage_is_calculated) { constexpr size_t BASE_ALLOCATED = 28744u; - constexpr size_t BASE_USED = 24984; + constexpr size_t BASE_USED = 24936; typedef BTreeNodeAllocator<int32_t, int8_t, btree::NoAggregated, MyTraits::INTERNAL_SLOTS, MyTraits::LEAF_SLOTS> NodeAllocator; diff --git a/vespalib/src/tests/datastore/array_store/array_store_test.cpp b/vespalib/src/tests/datastore/array_store/array_store_test.cpp index ccc3ab88c31..1df03f6eb0a 100644 --- a/vespalib/src/tests/datastore/array_store/array_store_test.cpp +++ b/vespalib/src/tests/datastore/array_store/array_store_test.cpp @@ -29,16 +29,16 @@ constexpr float ALLOC_GROW_FACTOR = 0.2; } -template <typename TestT, typename EntryT, typename RefT = EntryRefT<19> > +template <typename TestT, typename ElemT, typename RefT = EntryRefT<19> > struct ArrayStoreTest : public TestT { using EntryRefType = RefT; - using ArrayStoreType = ArrayStore<EntryT, RefT>; + using ArrayStoreType = ArrayStore<ElemT, RefT>; using LargeArray = typename ArrayStoreType::LargeArray; using ConstArrayRef = typename ArrayStoreType::ConstArrayRef; - using EntryVector = std::vector<EntryT>; - using value_type = EntryT; - using ReferenceStore = vespalib::hash_map<EntryRef, EntryVector>; + using ElemVector = std::vector<ElemT>; + using value_type = ElemT; + using ReferenceStore = vespalib::hash_map<EntryRef, ElemVector>; AllocStats stats; ArrayStoreType store; @@ -61,11 +61,11 @@ struct ArrayStoreTest : public TestT add_using_allocate(false) {} ~ArrayStoreTest() override; - void assertAdd(const EntryVector &input) { + void assertAdd(const ElemVector &input) { EntryRef ref = add(input); assertGet(ref, input); } - EntryRef add(const EntryVector &input) { + EntryRef add(const ElemVector &input) { EntryRef result; if (add_using_allocate) { result = store.allocate(input.size()); @@ -82,16 +82,16 @@ struct ArrayStoreTest : public TestT refStore.insert(std::make_pair(result, input)); return result; } - void assertGet(EntryRef ref, const EntryVector &exp) const { + void assertGet(EntryRef ref, const ElemVector &exp) const { ConstArrayRef act = store.get(ref); - EXPECT_EQ(exp, EntryVector(act.begin(), act.end())); + EXPECT_EQ(exp, ElemVector(act.begin(), act.end())); } void remove(EntryRef ref) { ASSERT_EQ(1u, refStore.count(ref)); store.remove(ref); refStore.erase(ref); } - void remove(const EntryVector &input) { + void remove(const ElemVector &input) { remove(getEntryRef(input)); } uint32_t getBufferId(EntryRef ref) const { @@ -99,14 +99,14 @@ struct ArrayStoreTest : public TestT } void assertBufferState(EntryRef ref, const MemStats& expStats) { EXPECT_EQ(expStats._used, store.bufferState(ref).size()); - EXPECT_EQ(expStats._hold, store.bufferState(ref).stats().hold_elems()); - EXPECT_EQ(expStats._dead, store.bufferState(ref).stats().dead_elems()); + EXPECT_EQ(expStats._hold, store.bufferState(ref).stats().hold_entries()); + EXPECT_EQ(expStats._dead, store.bufferState(ref).stats().dead_entries()); } void assert_buffer_stats(EntryRef ref, const TestBufferStats& exp_stats) { const auto& state = store.bufferState(ref); EXPECT_EQ(exp_stats._used, state.size()); - EXPECT_EQ(exp_stats._hold, state.stats().hold_elems()); - EXPECT_EQ(exp_stats._dead, state.stats().dead_elems()); + EXPECT_EQ(exp_stats._hold, state.stats().hold_entries()); + EXPECT_EQ(exp_stats._dead, state.stats().dead_entries()); EXPECT_EQ(exp_stats._extra_used, state.stats().extra_used_bytes()); EXPECT_EQ(exp_stats._extra_hold, state.stats().extra_hold_bytes()); } @@ -121,7 +121,7 @@ struct ArrayStoreTest : public TestT assertGet(elem.first, elem.second); } } - void assert_ref_reused(const EntryVector& first, const EntryVector& second, bool should_reuse) { + void assert_ref_reused(const ElemVector& first, const ElemVector& second, bool should_reuse) { EntryRef ref1 = add(first); remove(ref1); reclaim_memory(); @@ -129,7 +129,7 @@ struct ArrayStoreTest : public TestT EXPECT_EQ(should_reuse, (ref2 == ref1)); assertGet(ref2, second); } - EntryRef getEntryRef(const EntryVector &input) { + EntryRef getEntryRef(const ElemVector &input) { for (auto itr = refStore.begin(); itr != refStore.end(); ++itr) { if (itr->second == input) { return itr->first; @@ -160,12 +160,12 @@ struct ArrayStoreTest : public TestT } refStore = compactedRefStore; } - size_t entrySize() const { return sizeof(EntryT); } + size_t elem_size() const { return sizeof(ElemT); } size_t largeArraySize() const { return sizeof(LargeArray); } }; -template <typename TestT, typename EntryT, typename RefT> -ArrayStoreTest<TestT, EntryT, RefT>::~ArrayStoreTest() = default; +template <typename TestT, typename ElemT, typename RefT> +ArrayStoreTest<TestT, ElemT, RefT>::~ArrayStoreTest() = default; struct TestParam { bool add_using_allocate; @@ -214,8 +214,8 @@ TEST_P(NumberStoreTest, control_static_sizes) { EXPECT_EQ(240u + sizeof_deque, sizeof(NumberStoreTest::ArrayStoreType::DataStoreType)); EXPECT_EQ(104u, sizeof(NumberStoreTest::ArrayStoreType::SmallBufferType)); MemoryUsage usage = store.getMemoryUsage(); - EXPECT_EQ(202120u, usage.allocatedBytes()); - EXPECT_EQ(197752u, usage.usedBytes()); + EXPECT_EQ(202116u, usage.allocatedBytes()); + EXPECT_EQ(197656u, usage.usedBytes()); } TEST_P(NumberStoreTest, add_and_get_small_arrays_of_trivial_type) @@ -246,15 +246,15 @@ TEST_F(StringStoreTest, add_and_get_large_arrays_of_non_trivial_type) assertAdd({"ddd", "eee", "ffff", "gggg", "hhhh"}); } -TEST_P(NumberStoreTest, elements_are_put_on_hold_when_a_small_array_is_removed) +TEST_P(NumberStoreTest, entries_are_put_on_hold_when_a_small_array_is_removed) { EntryRef ref = add({1,2,3}); - assertBufferState(ref, MemStats().used(3).hold(0)); + assertBufferState(ref, MemStats().used(1).hold(0)); store.remove(ref); - assertBufferState(ref, MemStats().used(3).hold(3)); + assertBufferState(ref, MemStats().used(1).hold(1)); } -TEST_P(NumberStoreTest, elements_are_put_on_hold_when_a_large_array_is_removed) +TEST_P(NumberStoreTest, entries_are_put_on_hold_when_a_large_array_is_removed) { EntryRef ref = add({1,2,3,4}); // Note: The first buffer has the first element reserved -> we expect 2 elements used here. @@ -319,7 +319,7 @@ test_compaction(NumberStoreBasicTest &f) f.remove(f.add({5,5})); f.reclaim_memory(); f.assertBufferState(size1Ref, MemStats().used(1).dead(0)); - f.assertBufferState(size2Ref, MemStats().used(4).dead(2)); + f.assertBufferState(size2Ref, MemStats().used(2).dead(1)); f.assertBufferState(size3Ref, MemStats().used(2).dead(1)); // Note: First element is reserved uint32_t size1BufferId = f.getBufferId(size1Ref); uint32_t size2BufferId = f.getBufferId(size2Ref); @@ -363,8 +363,8 @@ void testCompaction(NumberStoreTest &f, bool compactMemory, bool compactAddressS f.remove(f.add({7})); f.reclaim_memory(); f.assertBufferState(size1Ref, MemStats().used(3).dead(2)); - f.assertBufferState(size2Ref, MemStats().used(2).dead(0)); - f.assertBufferState(size3Ref, MemStats().used(6).dead(3)); + f.assertBufferState(size2Ref, MemStats().used(1).dead(0)); + f.assertBufferState(size3Ref, MemStats().used(2).dead(1)); uint32_t size1BufferId = f.getBufferId(size1Ref); uint32_t size2BufferId = f.getBufferId(size2Ref); uint32_t size3BufferId = f.getBufferId(size3Ref); @@ -434,22 +434,22 @@ TEST_P(NumberStoreTest, used_onHold_and_dead_memory_usage_is_tracked_for_small_a { MemStats exp(store.getMemoryUsage()); add({1,2,3}); - assertMemoryUsage(exp.used(entrySize() * 3)); + assertMemoryUsage(exp.used(elem_size() * 3)); remove({1,2,3}); - assertMemoryUsage(exp.hold(entrySize() * 3)); + assertMemoryUsage(exp.hold(elem_size() * 3)); reclaim_memory(); - assertMemoryUsage(exp.holdToDead(entrySize() * 3)); + assertMemoryUsage(exp.holdToDead(elem_size() * 3)); } TEST_P(NumberStoreTest, used_onHold_and_dead_memory_usage_is_tracked_for_large_arrays) { MemStats exp(store.getMemoryUsage()); add({1,2,3,4}); - assertMemoryUsage(exp.used(largeArraySize() + entrySize() * 4)); + assertMemoryUsage(exp.used(largeArraySize() + elem_size() * 4)); remove({1,2,3,4}); - assertMemoryUsage(exp.hold(largeArraySize() + entrySize() * 4)); + assertMemoryUsage(exp.hold(largeArraySize() + elem_size() * 4)); reclaim_memory(); - assertMemoryUsage(exp.decUsed(entrySize() * 4).decHold(largeArraySize() + entrySize() * 4). + assertMemoryUsage(exp.decUsed(elem_size() * 4).decHold(largeArraySize() + elem_size() * 4). dead(largeArraySize())); } diff --git a/vespalib/src/tests/datastore/array_store_config/array_store_config_test.cpp b/vespalib/src/tests/datastore/array_store_config/array_store_config_test.cpp index fcf1acd5cd3..171e7216638 100644 --- a/vespalib/src/tests/datastore/array_store_config/array_store_config_test.cpp +++ b/vespalib/src/tests/datastore/array_store_config/array_store_config_test.cpp @@ -22,32 +22,32 @@ struct Fixture Fixture(uint32_t maxSmallArrayTypeId, size_t hugePageSize, size_t smallPageSize, - size_t minNumArraysForNewBuffer) + size_t min_num_entries_for_new_buffer) : cfg(ArrayStoreConfig::optimizeForHugePage(maxSmallArrayTypeId, [](size_t type_id) noexcept { return type_id; }, hugePageSize, smallPageSize, sizeof(int), EntryRefType::offsetSize(), - minNumArraysForNewBuffer, + min_num_entries_for_new_buffer, ALLOC_GROW_FACTOR)) { } - void assertSpec(uint32_t type_id, uint32_t numArraysForNewBuffer) { + void assertSpec(uint32_t type_id, uint32_t num_entries_for_new_buffer) { assertSpec(type_id, AllocSpec(0, EntryRefType::offsetSize(), - numArraysForNewBuffer, ALLOC_GROW_FACTOR)); + num_entries_for_new_buffer, ALLOC_GROW_FACTOR)); } void assertSpec(uint32_t type_id, const AllocSpec &expSpec) { const auto& actSpec = cfg.spec_for_type_id(type_id); - EXPECT_EQUAL(expSpec.minArraysInBuffer, actSpec.minArraysInBuffer); - EXPECT_EQUAL(expSpec.maxArraysInBuffer, actSpec.maxArraysInBuffer); - EXPECT_EQUAL(expSpec.numArraysForNewBuffer, actSpec.numArraysForNewBuffer); + EXPECT_EQUAL(expSpec.min_entries_in_buffer, actSpec.min_entries_in_buffer); + EXPECT_EQUAL(expSpec.max_entries_in_buffer, actSpec.max_entries_in_buffer); + EXPECT_EQUAL(expSpec.num_entries_for_new_buffer, actSpec.num_entries_for_new_buffer); EXPECT_EQUAL(expSpec.allocGrowFactor, actSpec.allocGrowFactor); } }; AllocSpec -makeSpec(size_t minArraysInBuffer, - size_t maxArraysInBuffer, - size_t numArraysForNewBuffer) +makeSpec(size_t min_entries_in_buffer, + size_t max_entries_in_buffer, + size_t num_entries_for_new_buffer) { - return AllocSpec(minArraysInBuffer, maxArraysInBuffer, numArraysForNewBuffer, ALLOC_GROW_FACTOR); + return AllocSpec(min_entries_in_buffer, max_entries_in_buffer, num_entries_for_new_buffer, ALLOC_GROW_FACTOR); } constexpr size_t KB = 1024; diff --git a/vespalib/src/tests/datastore/buffer_stats/buffer_stats_test.cpp b/vespalib/src/tests/datastore/buffer_stats/buffer_stats_test.cpp index 09b2590a5f3..fec8d5949f8 100644 --- a/vespalib/src/tests/datastore/buffer_stats/buffer_stats_test.cpp +++ b/vespalib/src/tests/datastore/buffer_stats/buffer_stats_test.cpp @@ -9,10 +9,10 @@ using namespace vespalib::datastore; TEST(BufferStatsTest, buffer_stats_to_memory_stats) { InternalBufferStats buf; - buf.set_alloc_elems(17); + buf.set_alloc_entries(17); buf.pushed_back(7); - buf.set_dead_elems(5); - buf.set_hold_elems(3); + buf.set_dead_entries(5); + buf.set_hold_entries(3); buf.inc_extra_used_bytes(13); buf.inc_extra_hold_bytes(11); @@ -20,10 +20,10 @@ TEST(BufferStatsTest, buffer_stats_to_memory_stats) constexpr size_t es = 8; buf.add_to_mem_stats(es, mem); - EXPECT_EQ(17, mem._allocElems); - EXPECT_EQ(7, mem._usedElems); - EXPECT_EQ(5, mem._deadElems); - EXPECT_EQ(3, mem._holdElems); + EXPECT_EQ(17, mem._alloc_entries); + EXPECT_EQ(7, mem._used_entries); + EXPECT_EQ(5, mem._dead_entries); + EXPECT_EQ(3, mem._hold_entries); EXPECT_EQ(17 * es + 13, mem._allocBytes); EXPECT_EQ(7 * es + 13, mem._usedBytes); EXPECT_EQ(5 * es, mem._deadBytes); diff --git a/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp b/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp index de7d899e68a..9f7535a3676 100644 --- a/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp +++ b/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp @@ -7,40 +7,40 @@ using namespace vespalib::datastore; using IntBufferType = BufferType<int>; constexpr uint32_t ARRAYS_SIZE(4); -constexpr uint32_t MAX_ARRAYS(128); -constexpr uint32_t NUM_ARRAYS_FOR_NEW_BUFFER(0); +constexpr uint32_t MAX_ENTRIES(128); +constexpr uint32_t NUM_ENTRIES_FOR_NEW_BUFFER(0); struct Setup { - uint32_t _minArrays; - std::atomic<ElemCount> _usedElems; - ElemCount _neededElems; - std::atomic<ElemCount> _deadElems; + uint32_t _min_entries; + std::atomic<EntryCount> _used_entries; + EntryCount _needed_entries; + std::atomic<EntryCount> _dead_entries; uint32_t _bufferId; float _allocGrowFactor; bool _resizing; Setup() - : _minArrays(0), - _usedElems(0), - _neededElems(0), - _deadElems(0), + : _min_entries(0), + _used_entries(0), + _needed_entries(0), + _dead_entries(0), _bufferId(1), _allocGrowFactor(0.5), _resizing(false) {} Setup(const Setup& rhs) noexcept; - Setup &minArrays(uint32_t value) { _minArrays = value; return *this; } - Setup &used(size_t value) { _usedElems = value; return *this; } - Setup &needed(size_t value) { _neededElems = value; return *this; } - Setup &dead(size_t value) { _deadElems = value; return *this; } + Setup &min_entries(uint32_t value) { _min_entries = value; return *this; } + Setup &used(size_t value) { _used_entries = value; return *this; } + Setup &needed(size_t value) { _needed_entries = value; return *this; } + Setup &dead(size_t value) { _dead_entries = value; return *this; } Setup &bufferId(uint32_t value) { _bufferId = value; return *this; } Setup &resizing(bool value) { _resizing = value; return *this; } }; Setup::Setup(const Setup& rhs) noexcept - : _minArrays(rhs._minArrays), - _usedElems(rhs._usedElems.load(std::memory_order_relaxed)), - _neededElems(rhs._neededElems), - _deadElems(rhs._deadElems.load(std::memory_order_relaxed)), + : _min_entries(rhs._min_entries), + _used_entries(rhs._used_entries.load(std::memory_order_relaxed)), + _needed_entries(rhs._needed_entries), + _dead_entries(rhs._dead_entries.load(std::memory_order_relaxed)), _bufferId(rhs._bufferId), _allocGrowFactor(rhs._allocGrowFactor), _resizing(rhs._resizing) @@ -53,7 +53,7 @@ struct Fixture { int buffer[ARRAYS_SIZE]; Fixture(const Setup &setup_) : setups(), - bufferType(ARRAYS_SIZE, setup_._minArrays, MAX_ARRAYS, NUM_ARRAYS_FOR_NEW_BUFFER, setup_._allocGrowFactor), + bufferType(ARRAYS_SIZE, setup_._min_entries, MAX_ENTRIES, NUM_ENTRIES_FOR_NEW_BUFFER, setup_._allocGrowFactor), buffer() { setups.reserve(4); @@ -61,121 +61,121 @@ struct Fixture { } ~Fixture() { for (auto& setup : setups) { - bufferType.onHold(setup._bufferId, &setup._usedElems, &setup._deadElems); - bufferType.onFree(setup._usedElems); + bufferType.on_hold(setup._bufferId, &setup._used_entries, &setup._dead_entries); + bufferType.on_free(setup._used_entries); } } Setup& curr_setup() { return setups.back(); } void add_setup(const Setup& setup_in) { - // The buffer type stores pointers to ElemCount (from Setup) and we must ensure these do not move in memory. + // The buffer type stores pointers to EntryCount (from Setup) and we must ensure these do not move in memory. assert(setups.size() < setups.capacity()); setups.push_back(setup_in); } void onActive() { - bufferType.onActive(curr_setup()._bufferId, &curr_setup()._usedElems, &curr_setup()._deadElems, &buffer[0]); + bufferType.on_active(curr_setup()._bufferId, &curr_setup()._used_entries, &curr_setup()._dead_entries, &buffer[0]); } - size_t arraysToAlloc() { - return bufferType.calcArraysToAlloc(curr_setup()._bufferId, curr_setup()._neededElems, curr_setup()._resizing); + size_t entries_to_alloc() { + return bufferType.calc_entries_to_alloc(curr_setup()._bufferId, curr_setup()._needed_entries, curr_setup()._resizing); } - void assertArraysToAlloc(size_t exp) { + void assert_entries_to_alloc(size_t exp) { onActive(); - EXPECT_EQUAL(exp, arraysToAlloc()); + EXPECT_EQUAL(exp, entries_to_alloc()); } }; void -assertArraysToAlloc(size_t exp, const Setup &setup) +assert_entries_to_alloc(size_t exp, const Setup &setup) { Fixture f(setup); - f.assertArraysToAlloc(exp); + f.assert_entries_to_alloc(exp); } -TEST("require that complete arrays are allocated") +TEST("require that entries are allocated") { - TEST_DO(assertArraysToAlloc(1, Setup().needed(1))); - TEST_DO(assertArraysToAlloc(1, Setup().needed(2))); - TEST_DO(assertArraysToAlloc(1, Setup().needed(3))); - TEST_DO(assertArraysToAlloc(1, Setup().needed(4))); - TEST_DO(assertArraysToAlloc(2, Setup().needed(5))); + TEST_DO(assert_entries_to_alloc(1, Setup().needed(1))); + TEST_DO(assert_entries_to_alloc(2, Setup().needed(2))); + TEST_DO(assert_entries_to_alloc(3, Setup().needed(3))); + TEST_DO(assert_entries_to_alloc(4, Setup().needed(4))); + TEST_DO(assert_entries_to_alloc(5, Setup().needed(5))); } -TEST("require that reserved elements are taken into account when not resizing") +TEST("require that reserved entries are taken into account when not resizing") { - TEST_DO(assertArraysToAlloc(2, Setup().needed(1).bufferId(0))); - TEST_DO(assertArraysToAlloc(2, Setup().needed(4).bufferId(0))); - TEST_DO(assertArraysToAlloc(3, Setup().needed(5).bufferId(0))); + TEST_DO(assert_entries_to_alloc(2, Setup().needed(1).bufferId(0))); + TEST_DO(assert_entries_to_alloc(5, Setup().needed(4).bufferId(0))); + TEST_DO(assert_entries_to_alloc(6, Setup().needed(5).bufferId(0))); } -TEST("require that arrays to alloc is based on currently used elements (no resizing)") +TEST("require that entries to alloc is based on currently used entries (no resizing)") { - TEST_DO(assertArraysToAlloc(2, Setup().used(4 * 4).needed(4))); - TEST_DO(assertArraysToAlloc(4, Setup().used(8 * 4).needed(4))); + TEST_DO(assert_entries_to_alloc(2, Setup().used(4).needed(1))); + TEST_DO(assert_entries_to_alloc(4, Setup().used(8).needed(1))); } -TEST("require that arrays to alloc is based on currently used elements (with resizing)") +TEST("require that entries to alloc is based on currently used entries (with resizing)") { - TEST_DO(assertArraysToAlloc(4 + 2, Setup().used(4 * 4).needed(4).resizing(true))); - TEST_DO(assertArraysToAlloc(8 + 4, Setup().used(8 * 4).needed(4).resizing(true))); - TEST_DO(assertArraysToAlloc(4 + 3, Setup().used(4 * 4).needed(3 * 4).resizing(true))); + TEST_DO(assert_entries_to_alloc(4 + 2, Setup().used(4).needed(1).resizing(true))); + TEST_DO(assert_entries_to_alloc(8 + 4, Setup().used(8).needed(1).resizing(true))); + TEST_DO(assert_entries_to_alloc(4 + 3, Setup().used(4).needed(3).resizing(true))); } -TEST("require that arrays to alloc always contain elements needed") +TEST("require that entries to alloc always contain entries needed") { - TEST_DO(assertArraysToAlloc(2, Setup().used(4 * 4).needed(2 * 4))); - TEST_DO(assertArraysToAlloc(3, Setup().used(4 * 4).needed(3 * 4))); - TEST_DO(assertArraysToAlloc(4, Setup().used(4 * 4).needed(4 * 4))); + TEST_DO(assert_entries_to_alloc(2, Setup().used(4).needed(2))); + TEST_DO(assert_entries_to_alloc(3, Setup().used(4).needed(3))); + TEST_DO(assert_entries_to_alloc(4, Setup().used(4).needed(4))); } -TEST("require that arrays to alloc is capped to max arrays") +TEST("require that entries to alloc is capped to max entries") { - TEST_DO(assertArraysToAlloc(127, Setup().used(254 * 4).needed(4))); - TEST_DO(assertArraysToAlloc(128, Setup().used(256 * 4).needed(4))); - TEST_DO(assertArraysToAlloc(128, Setup().used(258 * 4).needed(8))); + TEST_DO(assert_entries_to_alloc(127, Setup().used(254).needed(1))); + TEST_DO(assert_entries_to_alloc(128, Setup().used(256).needed(1))); + TEST_DO(assert_entries_to_alloc(128, Setup().used(258).needed(2))); } TEST("require that arrays to alloc is capped to min arrays") { - TEST_DO(assertArraysToAlloc(16, Setup().used(30 * 4).needed(4).minArrays(16))); - TEST_DO(assertArraysToAlloc(16, Setup().used(32 * 4).needed(4).minArrays(16))); - TEST_DO(assertArraysToAlloc(17, Setup().used(34 * 4).needed(4).minArrays(16))); + TEST_DO(assert_entries_to_alloc(16, Setup().used(30).needed(1).min_entries(16))); + TEST_DO(assert_entries_to_alloc(16, Setup().used(32).needed(1).min_entries(16))); + TEST_DO(assert_entries_to_alloc(17, Setup().used(34).needed(1).min_entries(16))); } -TEST("arrays to alloc considers used elements across all active buffers of same type (no resizing)") +TEST("entries to alloc considers used entries across all active buffers of same type (no resizing)") { - Fixture f(Setup().used(6 * 4)); - f.assertArraysToAlloc(6 * 0.5); - f.add_setup(Setup().used(8 * 4).bufferId(2)); - f.assertArraysToAlloc((6 + 8) * 0.5); - f.add_setup(Setup().used(10 * 4).bufferId(3)); - f.assertArraysToAlloc((6 + 8 + 10) * 0.5); + Fixture f(Setup().used(6)); + f.assert_entries_to_alloc(6 * 0.5); + f.add_setup(Setup().used(8).bufferId(2)); + f.assert_entries_to_alloc((6 + 8) * 0.5); + f.add_setup(Setup().used(10).bufferId(3)); + f.assert_entries_to_alloc((6 + 8 + 10) * 0.5); } -TEST("arrays to alloc considers used elements across all active buffers of same type when resizing") +TEST("entries to alloc considers used entries across all active buffers of same type when resizing") { - Fixture f(Setup().used(6 * 4)); - f.assertArraysToAlloc(6 * 0.5); - f.add_setup(Setup().used(8 * 4).resizing(true).bufferId(2)); - f.assertArraysToAlloc(8 + (6 + 8) * 0.5); + Fixture f(Setup().used(6)); + f.assert_entries_to_alloc(6 * 0.5); + f.add_setup(Setup().used(8).resizing(true).bufferId(2)); + f.assert_entries_to_alloc(8 + (6 + 8) * 0.5); } -TEST("arrays to alloc considers (and subtracts) dead elements across all active buffers of same type (no resizing)") +TEST("entries to alloc considers (and subtracts) dead entries across all active buffers of same type (no resizing)") { - Fixture f(Setup().used(6 * 4).dead(2 * 4)); - f.assertArraysToAlloc((6 - 2) * 0.5); - f.add_setup(Setup().used(12 * 4).dead(4 * 4).bufferId(2)); - f.assertArraysToAlloc((6 - 2 + 12 - 4) * 0.5); - f.add_setup(Setup().used(20 * 4).dead(6 * 4).bufferId(3)); - f.assertArraysToAlloc((6 - 2 + 12 - 4 + 20 - 6) * 0.5); + Fixture f(Setup().used(6).dead(2)); + f.assert_entries_to_alloc((6 - 2) * 0.5); + f.add_setup(Setup().used(12).dead(4).bufferId(2)); + f.assert_entries_to_alloc((6 - 2 + 12 - 4) * 0.5); + f.add_setup(Setup().used(20).dead(6).bufferId(3)); + f.assert_entries_to_alloc((6 - 2 + 12 - 4 + 20 - 6) * 0.5); } TEST("arrays to alloc considers (and subtracts) dead elements across all active buffers of same type when resizing") { - Fixture f(Setup().used(6 * 4).dead(2 * 4)); - f.assertArraysToAlloc((6 - 2) * 0.5); - f.add_setup(Setup().used(12 * 4).dead(4 * 4).resizing(true).bufferId(2)); - f.assertArraysToAlloc(12 + (6 - 2 + 12 - 4) * 0.5); + Fixture f(Setup().used(6).dead(2)); + f.assert_entries_to_alloc((6 - 2) * 0.5); + f.add_setup(Setup().used(12).dead(4).resizing(true).bufferId(2)); + f.assert_entries_to_alloc(12 + (6 - 2 + 12 - 4) * 0.5); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/datastore/datastore/datastore_test.cpp b/vespalib/src/tests/datastore/datastore/datastore_test.cpp index df347267c7e..9e27ed37dd3 100644 --- a/vespalib/src/tests/datastore/datastore/datastore_test.cpp +++ b/vespalib/src/tests/datastore/datastore/datastore_test.cpp @@ -26,8 +26,8 @@ public: void holdBuffer(uint32_t bufferId) { ParentType::holdBuffer(bufferId); } - void holdElem(EntryRef ref, uint64_t len) { - ParentType::holdElem(ref, len); + void hold_entry(EntryRef ref) { + ParentType::hold_entry(ref); } void assign_generation(generation_t current_gen) { ParentType::assign_generation(current_gen); @@ -35,8 +35,8 @@ public: void reclaim_entry_refs(generation_t oldest_used_gen) override { ParentType::reclaim_entry_refs(oldest_used_gen); } - void ensureBufferCapacity(size_t sizeNeeded) { - ParentType::ensureBufferCapacity(0, sizeNeeded); + void ensure_buffer_capacity(size_t entries_needed) { + ParentType::ensure_buffer_capacity(0, entries_needed); } void enableFreeLists() { ParentType::enableFreeLists(); @@ -66,10 +66,10 @@ class GrowStore BufferType<DataType> _type; uint32_t _typeId; public: - GrowStore(size_t arraySize, size_t minArrays, size_t maxArrays, size_t numArraysForNewBuffer) + GrowStore(size_t arraySize, size_t min_entries, size_t max_entries, size_t num_entries_for_new_buffer) : _store(), - _firstType(1, 1, maxArrays, 0, ALLOC_GROW_FACTOR), - _type(arraySize, minArrays, maxArrays, numArraysForNewBuffer, ALLOC_GROW_FACTOR), + _firstType(1, 1, max_entries, 0, ALLOC_GROW_FACTOR), + _type(arraySize, min_entries, max_entries, num_entries_for_new_buffer, ALLOC_GROW_FACTOR), _typeId(0) { (void) _store.addType(&_firstType); @@ -87,7 +87,7 @@ public: while (sizes.size() < bufs) { RefType iRef = (_type.getArraySize() == 1) ? (_store.template allocator<DataType>(_typeId).alloc().ref) : - (_store.template allocator<DataType>(_typeId).allocArray(_type.getArraySize()).ref); + (_store.template allocator<DataType>(_typeId).allocArray().ref); int bufferId = iRef.bufferId(); if (bufferId != prevBufferId) { if (prevBufferId >= 0) { @@ -126,7 +126,7 @@ public: while (buffers.size() < bufs) { RefType iRef = (_type.getArraySize() == 1) ? (_store.template allocator<DataType>(_typeId).alloc().ref) : - (_store.template allocator<DataType>(_typeId).allocArray(_type.getArraySize()).ref); + (_store.template allocator<DataType>(_typeId).allocArray().ref); int buffer_id = iRef.bufferId(); if (buffers.empty() || buffers.back() != buffer_id) { buffers.push_back(buffer_id); @@ -143,10 +143,10 @@ void assertMemStats(const MemoryStats &exp, const MemoryStats &act) { - EXPECT_EQ(exp._allocElems, act._allocElems); - EXPECT_EQ(exp._usedElems, act._usedElems); - EXPECT_EQ(exp._deadElems, act._deadElems); - EXPECT_EQ(exp._holdElems, act._holdElems); + EXPECT_EQ(exp._alloc_entries, act._alloc_entries); + EXPECT_EQ(exp._used_entries, act._used_entries); + EXPECT_EQ(exp._dead_entries, act._dead_entries); + EXPECT_EQ(exp._hold_entries, act._hold_entries); EXPECT_EQ(exp._freeBuffers, act._freeBuffers); EXPECT_EQ(exp._activeBuffers, act._activeBuffers); EXPECT_EQ(exp._holdBuffers, act._holdBuffers); @@ -304,13 +304,13 @@ TEST(DataStoreTest, require_that_we_can_hold_and_trim_elements) { MyStore s; MyRef r1 = s.addEntry(1); - s.holdElem(r1, 1); + s.hold_entry(r1); s.assign_generation(10); MyRef r2 = s.addEntry(2); - s.holdElem(r2, 1); + s.hold_entry(r2); s.assign_generation(20); MyRef r3 = s.addEntry(3); - s.holdElem(r3, 1); + s.hold_entry(r3); s.assign_generation(30); EXPECT_EQ(1, s.getEntry(r1)); EXPECT_EQ(2, s.getEntry(r2)); @@ -358,11 +358,11 @@ TEST(DataStoreTest, require_that_we_can_use_free_lists) MyStore s; s.enableFreeLists(); auto r1 = s.addEntry(1); - s.holdElem(r1, 1); + s.hold_entry(r1); s.assign_generation(10); auto r2 = s.addEntry(2); expect_successive_refs(r1, r2); - s.holdElem(r2, 1); + s.hold_entry(r2); s.assign_generation(20); s.reclaim_entry_refs(11); auto r3 = s.addEntry(3); // reuse r1 @@ -389,21 +389,21 @@ TEST(DataStoreTest, require_that_we_can_use_free_lists_with_raw_allocator) s.enableFreeLists(); auto allocator = s.freeListRawAllocator<int>(grow_store.typeId()); - auto h1 = allocator.alloc(3); - auto h2 = allocator.alloc(3); + auto h1 = allocator.alloc(1); + auto h2 = allocator.alloc(1); expect_successive_handles(h1, h2); - s.holdElem(h1.ref, 3); - s.holdElem(h2.ref, 3); + s.hold_entry(h1.ref); + s.hold_entry(h2.ref); s.assign_generation(10); s.reclaim_entry_refs(11); - auto h3 = allocator.alloc(3); // reuse h2.ref from free list + auto h3 = allocator.alloc(1); // reuse h2.ref from free list EXPECT_EQ(h2, h3); - auto h4 = allocator.alloc(3); // reuse h1.ref from free list + auto h4 = allocator.alloc(1); // reuse h1.ref from free list EXPECT_EQ(h1, h4); - auto h5 = allocator.alloc(3); + auto h5 = allocator.alloc(1); expect_successive_handles(h2, h5); expect_successive_handles(h3, h5); } @@ -412,10 +412,10 @@ TEST(DataStoreTest, require_that_memory_stats_are_calculated) { MyStore s; MemoryStats m; - m._allocElems = MyRef::offsetSize(); - m._usedElems = 1; // ref = 0 is reserved - m._deadElems = 1; // ref = 0 is reserved - m._holdElems = 0; + m._alloc_entries = MyRef::offsetSize(); + m._used_entries = 1; // ref = 0 is reserved + m._dead_entries = 1; // ref = 0 is reserved + m._hold_entries = 0; m._activeBuffers = 1; m._freeBuffers = MyRef::numBuffers() - 1; m._holdBuffers = 0; @@ -423,7 +423,7 @@ TEST(DataStoreTest, require_that_memory_stats_are_calculated) // add entry MyRef r = s.addEntry(10); - m._usedElems++; + m._used_entries++; assertMemStats(m, s.getMemStats()); // hold buffer @@ -431,9 +431,9 @@ TEST(DataStoreTest, require_that_memory_stats_are_calculated) s.addEntry(30); s.holdBuffer(r.bufferId()); s.assign_generation(100); - m._usedElems += 2; - m._holdElems = m._usedElems; - m._deadElems = 0; + m._used_entries += 2; + m._hold_entries = m._used_entries; + m._dead_entries = 0; m._activeBuffers--; m._holdBuffers++; assertMemStats(m, s.getMemStats()); @@ -441,17 +441,17 @@ TEST(DataStoreTest, require_that_memory_stats_are_calculated) // new active buffer s.switch_primary_buffer(); s.addEntry(40); - m._allocElems += MyRef::offsetSize(); - m._usedElems++; + m._alloc_entries += MyRef::offsetSize(); + m._used_entries++; m._activeBuffers++; m._freeBuffers--; // trim hold buffer s.reclaim_memory(101); - m._allocElems -= MyRef::offsetSize(); - m._usedElems = 1; - m._deadElems = 0; - m._holdElems = 0; + m._alloc_entries -= MyRef::offsetSize(); + m._used_entries = 1; + m._dead_entries = 0; + m._hold_entries = 0; m._freeBuffers = MyRef::numBuffers() - 1; m._holdBuffers = 0; assertMemStats(m, s.getMemStats()); @@ -466,7 +466,7 @@ TEST(DataStoreTest, require_that_memory_stats_are_calculated) { // increase extra hold bytes auto prev_stats = s.getMemStats(); - s.get_active_buffer_state().hold_elems(0, 30); + s.get_active_buffer_state().hold_entries(0, 30); auto curr_stats = s.getMemStats(); EXPECT_EQ(prev_stats._holdBytes + 30, curr_stats._holdBytes); } @@ -475,7 +475,7 @@ TEST(DataStoreTest, require_that_memory_stats_are_calculated) TEST(DataStoreTest, require_that_memory_usage_is_calculated) { constexpr size_t BASE_ALLOCATED = 4228; - constexpr size_t BASE_USED = 308; + constexpr size_t BASE_USED = 284; MyStore s; MyRef r = s.addEntry(10); s.addEntry(20); @@ -494,7 +494,7 @@ TEST(DataStoreTest, require_that_memory_usage_is_calculated) TEST(DataStoreTest, require_that_we_can_disable_elemement_hold_list) { constexpr size_t BASE_ALLOCATED = 4228; - constexpr size_t BASE_USED = 308; + constexpr size_t BASE_USED = 284; MyStore s; MyRef r1 = s.addEntry(10); MyRef r2 = s.addEntry(20); @@ -505,14 +505,14 @@ TEST(DataStoreTest, require_that_we_can_disable_elemement_hold_list) EXPECT_EQ(4 * sizeof(int) + BASE_USED, m.usedBytes()); EXPECT_EQ(1 * sizeof(int), m.deadBytes()); EXPECT_EQ(0 * sizeof(int), m.allocatedBytesOnHold()); - s.holdElem(r1, 1); + s.hold_entry(r1); m = s.getMemoryUsage(); EXPECT_EQ(MyRef::offsetSize() * sizeof(int) + BASE_ALLOCATED, m.allocatedBytes()); EXPECT_EQ(4 * sizeof(int) + BASE_USED, m.usedBytes()); EXPECT_EQ(1 * sizeof(int), m.deadBytes()); EXPECT_EQ(1 * sizeof(int), m.allocatedBytesOnHold()); - s.disableElemHoldList(); - s.holdElem(r2, 1); + s.disable_entry_hold_list(); + s.hold_entry(r2); m = s.getMemoryUsage(); EXPECT_EQ(MyRef::offsetSize() * sizeof(int) + BASE_ALLOCATED, m.allocatedBytes()); EXPECT_EQ(4 * sizeof(int) + BASE_USED, m.usedBytes()); @@ -529,11 +529,11 @@ namespace { void assertGrowStats(GrowthStats expSizes, GrowthStats expFirstBufSizes, size_t expInitMemUsage, - size_t minArrays, size_t numArraysForNewBuffer, size_t maxArrays = 128) + size_t min_entries, size_t num_entries_for_new_buffer, size_t max_entries = 128) { - EXPECT_EQ(expSizes, IntGrowStore(1, minArrays, maxArrays, numArraysForNewBuffer).getGrowthStats(expSizes.size())); - EXPECT_EQ(expFirstBufSizes, IntGrowStore(1, minArrays, maxArrays, numArraysForNewBuffer).getFirstBufGrowStats()); - EXPECT_EQ(expInitMemUsage, IntGrowStore(1, minArrays, maxArrays, numArraysForNewBuffer).getMemoryUsage().allocatedBytes()); + EXPECT_EQ(expSizes, IntGrowStore(1, min_entries, max_entries, num_entries_for_new_buffer).getGrowthStats(expSizes.size())); + EXPECT_EQ(expFirstBufSizes, IntGrowStore(1, min_entries, max_entries, num_entries_for_new_buffer).getFirstBufGrowStats()); + EXPECT_EQ(expInitMemUsage, IntGrowStore(1, min_entries, max_entries, num_entries_for_new_buffer).getMemoryUsage().allocatedBytes()); } } @@ -574,10 +574,10 @@ namespace { template <typename DataType> void assertGrowStats(GrowthStats expSizes, uint32_t arraySize) { - uint32_t minArrays = 2048; - uint32_t maxArrays = RefType15::offsetSize(); - uint32_t numArraysForNewBuffer = 2048; - GrowStore<DataType, RefType15> store(arraySize, minArrays, maxArrays, numArraysForNewBuffer); + uint32_t min_entries = 2048; + uint32_t max_entries = RefType15::offsetSize(); + uint32_t num_entries_for_new_buffer = 2048; + GrowStore<DataType, RefType15> store(arraySize, min_entries, max_entries, num_entries_for_new_buffer); EXPECT_EQ(expSizes, store.getGrowthStats(expSizes.size())); } @@ -594,14 +594,14 @@ TEST(DataStoreTest, require_that_offset_in_EntryRefT_is_within_bounds_when_alloc * 3) Round up bytes to alloc to match the underlying allocator (power of 2 if less than huge page size): * After this we might end up with more bytes than the offset in EntryRef can handle. In this case this is 32768. * 4) Cap bytes to alloc to the max offset EntryRef can handle. - * The max bytes to alloc is: maxArrays * arraySize * elementSize. + * The max bytes to alloc is: max_entries * arraySize * elementSize. */ - assertGrowStats<uint8_t>({8192,16384,16384,65536,65536,98304,98304,98304,98304,98304,98304,98304}, 3); - assertGrowStats<uint8_t>({16384,16384,65536,65536,131072,131072,163840,163840,163840,163840,163840,163840}, 5); - assertGrowStats<uint8_t>({16384,32768,32768,131072,131072,229376,229376,229376,229376,229376,229376,229376}, 7); - assertGrowStats<uint32_t>({8192,16384,16384,65536,65536,98304,98304,98304,98304,98304,98304,98304}, 3); - assertGrowStats<uint32_t>({16384,16384,65536,65536,131072,131072,163840,163840,163840,163840,163840,163840}, 5); - assertGrowStats<uint32_t>({16384,32768,32768,131072,131072,229376,229376,229376,229376,229376,229376,229376}, 7); + assertGrowStats<uint8_t>({2730,5461,5461,21845,21845,32768,32768,32768,32768,32768,32768,32768}, 3); + assertGrowStats<uint8_t>({3276,3276,13107,13107,26214,26214,32768,32768,32768,32768,32768,32768}, 5); + assertGrowStats<uint8_t>({2340,4681,4681,18724,18724,32768,32768,32768,32768,32768,32768,32768}, 7); + assertGrowStats<uint32_t>({2730,5461,5461,21845,21845,32768,32768,32768,32768,32768,32768,32768}, 3); + assertGrowStats<uint32_t>({3276,3276,13107,13107,26214,26214,32768,32768,32768,32768,32768,32768}, 5); + assertGrowStats<uint32_t>({2340,4681,4681,18724,18724,32768,32768,32768,32768,32768,32768,32768}, 7); } namespace { @@ -667,9 +667,9 @@ TEST(DataStoreTest, can_reuse_active_buffer_as_primary_buffer) TEST(DataStoreTest, control_static_sizes) { EXPECT_EQ(88, sizeof(BufferTypeBase)); EXPECT_EQ(24, sizeof(FreeList)); - EXPECT_EQ(56, sizeof(BufferFreeList)); + EXPECT_EQ(48, sizeof(BufferFreeList)); EXPECT_EQ(1, sizeof(BufferState::State)); - EXPECT_EQ(144, sizeof(BufferState)); + EXPECT_EQ(120, sizeof(BufferState)); BufferState bs; EXPECT_EQ(0, bs.size()); } @@ -685,11 +685,11 @@ void test_free_element_to_held_buffer(bool before_hold_buffer) EXPECT_EQ(1u, s.primary_buffer_id()); if (before_hold_buffer) { - s.holdElem(ref, 1); + s.hold_entry(ref); } s.holdBuffer(0); // hold last buffer if (!before_hold_buffer) { - ASSERT_DEATH({ s.holdElem(ref, 1); }, "isActive\\(\\)"); + ASSERT_DEATH({ s.hold_entry(ref); }, "isActive\\(\\)"); } s.assign_generation(100); s.reclaim_memory(101); diff --git a/vespalib/src/tests/datastore/free_list/free_list_test.cpp b/vespalib/src/tests/datastore/free_list/free_list_test.cpp index 44e11b2316b..ec14d0dd28c 100644 --- a/vespalib/src/tests/datastore/free_list/free_list_test.cpp +++ b/vespalib/src/tests/datastore/free_list/free_list_test.cpp @@ -8,20 +8,18 @@ using namespace vespalib::datastore; using MyEntryRef = EntryRefT<8, 4>; -constexpr uint32_t array_size = 6; struct FreeListTest : public testing::Test { FreeList list; - std::atomic<ElemCount> dead_elems; + std::atomic<EntryCount> dead_entries; std::vector<BufferFreeList> bufs; FreeListTest() : list(), bufs() { for (size_t i = 0; i < 3; ++i) { - bufs.emplace_back(dead_elems); - bufs.back().set_array_size(array_size); + bufs.emplace_back(dead_entries); } } void TearDown() override { @@ -126,13 +124,13 @@ TEST_F(FreeListTest, buffer_free_list_can_be_disabled_and_detached_when_not_curr EXPECT_TRUE(list.empty()); } -TEST_F(FreeListTest, dead_elems_count_is_updated_when_popping_an_entry) +TEST_F(FreeListTest, dead_entries_count_is_updated_when_popping_an_entry) { enable(0); push_entry({10, 0}); - dead_elems.store(18, std::memory_order_relaxed); + dead_entries.store(18, std::memory_order_relaxed); pop_entry(); - EXPECT_EQ(18 - array_size, dead_elems.load(std::memory_order_relaxed)); + EXPECT_EQ(17, dead_entries.load(std::memory_order_relaxed)); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp index 5ccf9a8908c..a09a7213bf5 100644 --- a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp +++ b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp @@ -96,8 +96,8 @@ struct TestBase : public ::testing::Test { } void assertBufferState(EntryRef ref, const TestBufferStats expStats) const { EXPECT_EQ(expStats._used, store.bufferState(ref).size()); - EXPECT_EQ(expStats._hold, store.bufferState(ref).stats().hold_elems()); - EXPECT_EQ(expStats._dead, store.bufferState(ref).stats().dead_elems()); + EXPECT_EQ(expStats._hold, store.bufferState(ref).stats().hold_entries()); + EXPECT_EQ(expStats._dead, store.bufferState(ref).stats().dead_entries()); } void assertStoreContent() const { for (const auto &elem : refStore) { @@ -147,10 +147,7 @@ struct TestBase : public ::testing::Test { auto getBuilder(uint32_t uniqueValuesHint) { return store.getBuilder(uniqueValuesHint); } auto getEnumerator(bool sort_unique_values) { return store.getEnumerator(sort_unique_values); } size_t get_reserved(EntryRef ref) { - return store.bufferState(ref).getTypeHandler()->getReservedElements(getBufferId(ref)); - } - size_t get_array_size(EntryRef ref) { - return store.bufferState(ref).getArraySize(); + return store.bufferState(ref).getTypeHandler()->get_reserved_entries(getBufferId(ref)); } }; @@ -309,29 +306,27 @@ TYPED_TEST(TestBase, can_add_and_get_values) } } -TYPED_TEST(TestBase, elements_are_put_on_hold_when_value_is_removed) +TYPED_TEST(TestBase, entries_are_put_on_hold_when_value_is_removed) { EntryRef ref = this->add(this->values()[0]); size_t reserved = this->get_reserved(ref); - size_t array_size = this->get_array_size(ref); - this->assertBufferState(ref, TestBufferStats().used(array_size + reserved).hold(0).dead(reserved)); + this->assertBufferState(ref, TestBufferStats().used(1 + reserved).hold(0).dead(reserved)); this->store.remove(ref); - this->assertBufferState(ref, TestBufferStats().used(array_size + reserved).hold(array_size).dead(reserved)); + this->assertBufferState(ref, TestBufferStats().used(1 + reserved).hold(1).dead(reserved)); } -TYPED_TEST(TestBase, elements_are_reference_counted) +TYPED_TEST(TestBase, entries_are_reference_counted) { EntryRef ref = this->add(this->values()[0]); EntryRef ref2 = this->add(this->values()[0]); EXPECT_EQ(ref.ref(), ref2.ref()); - // Note: The first buffer have the first element reserved -> we expect 2 elements used here. + // Note: The first buffer have the first entry reserved -> we expect 2 entries used here. size_t reserved = this->get_reserved(ref); - size_t array_size = this->get_array_size(ref); - this->assertBufferState(ref, TestBufferStats().used(array_size + reserved).hold(0).dead(reserved)); + this->assertBufferState(ref, TestBufferStats().used(1 + reserved).hold(0).dead(reserved)); this->store.remove(ref); - this->assertBufferState(ref, TestBufferStats().used(array_size + reserved).hold(0).dead(reserved)); + this->assertBufferState(ref, TestBufferStats().used(1 + reserved).hold(0).dead(reserved)); this->store.remove(ref); - this->assertBufferState(ref, TestBufferStats().used(array_size + reserved).hold(array_size).dead(reserved)); + this->assertBufferState(ref, TestBufferStats().used(1 + reserved).hold(1).dead(reserved)); } TEST_F(SmallOffsetNumberTest, new_underlying_buffer_is_allocated_when_current_is_full) @@ -360,8 +355,7 @@ TYPED_TEST(TestBase, store_can_be_compacted) this->remove(this->add(this->values()[2])); this->reclaim_memory(); size_t reserved = this->get_reserved(val0Ref); - size_t array_size = this->get_array_size(val0Ref); - this->assertBufferState(val0Ref, TestBufferStats().used(reserved + 3 * array_size).dead(reserved + array_size)); + this->assertBufferState(val0Ref, TestBufferStats().used(reserved + 3).dead(reserved + 1)); uint32_t val1BufferId = this->getBufferId(val0Ref); EXPECT_EQ(2u, this->refStore.size()); @@ -389,8 +383,7 @@ TYPED_TEST(TestBase, store_can_be_instantiated_with_builder) EntryRef val0Ref = builder.mapEnumValueToEntryRef(1); EntryRef val1Ref = builder.mapEnumValueToEntryRef(2); size_t reserved = this->get_reserved(val0Ref); - size_t array_size = this->get_array_size(val0Ref); - this->assertBufferState(val0Ref, TestBufferStats().used(2 * array_size + reserved).dead(reserved)); // Note: First element is reserved + this->assertBufferState(val0Ref, TestBufferStats().used(2 + reserved).dead(reserved)); // Note: First entry is reserved EXPECT_TRUE(val0Ref.valid()); EXPECT_TRUE(val1Ref.valid()); EXPECT_NE(val0Ref.ref(), val1Ref.ref()); @@ -472,13 +465,13 @@ TEST_F(DoubleTest, nan_is_handled) TEST_F(DoubleTest, control_memory_usage) { static constexpr size_t sizeof_deque = vespalib::datastore::DataStoreBase::sizeof_entry_ref_hold_list_deque; EXPECT_EQ(368u + sizeof_deque, sizeof(store)); - EXPECT_EQ(144u, sizeof(BufferState)); + EXPECT_EQ(120u, sizeof(BufferState)); EXPECT_EQ(28740u, store.get_values_memory_usage().allocatedBytes()); - EXPECT_EQ(24804u, store.get_values_memory_usage().usedBytes()); + EXPECT_EQ(24780u, store.get_values_memory_usage().usedBytes()); EXPECT_EQ(126952u, store.get_dictionary_memory_usage().allocatedBytes()); - EXPECT_EQ(25248u, store.get_dictionary_memory_usage().usedBytes()); + EXPECT_EQ(25200u, store.get_dictionary_memory_usage().usedBytes()); EXPECT_EQ(155692u, store.getMemoryUsage().allocatedBytes()); - EXPECT_EQ(50052, store.getMemoryUsage().usedBytes()); + EXPECT_EQ(49980u, store.getMemoryUsage().usedBytes()); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/datastore/unique_store_string_allocator/unique_store_string_allocator_test.cpp b/vespalib/src/tests/datastore/unique_store_string_allocator/unique_store_string_allocator_test.cpp index 7d4451556c8..8ea7f807f56 100644 --- a/vespalib/src/tests/datastore/unique_store_string_allocator/unique_store_string_allocator_test.cpp +++ b/vespalib/src/tests/datastore/unique_store_string_allocator/unique_store_string_allocator_test.cpp @@ -62,8 +62,8 @@ struct TestBase : public ::testing::Test { void assert_buffer_state(EntryRef ref, const TestBufferStats expStats) { auto & stats = buffer_state(ref).stats(); EXPECT_EQ(expStats._used, buffer_state(ref).size()); - EXPECT_EQ(expStats._hold, stats.hold_elems()); - EXPECT_EQ(expStats._dead, stats.dead_elems()); + EXPECT_EQ(expStats._hold, stats.hold_entries()); + EXPECT_EQ(expStats._dead, stats.dead_entries()); EXPECT_EQ(expStats._extra_used, stats.extra_used_bytes()); EXPECT_EQ(expStats._extra_hold, stats.extra_hold_bytes()); } @@ -83,14 +83,14 @@ TEST_F(StringTest, can_add_and_get_values) assert_add(spaces1000.c_str()); } -TEST_F(StringTest, elements_are_put_on_hold_when_value_is_removed) +TEST_F(StringTest, entries_are_put_on_hold_when_value_is_removed) { EntryRef ref = add(small.c_str()); - assert_buffer_state(ref, TestBufferStats().used(16).hold(0).dead(0)); + assert_buffer_state(ref, TestBufferStats().used(1).hold(0).dead(0)); remove(ref); - assert_buffer_state(ref, TestBufferStats().used(16).hold(16).dead(0)); + assert_buffer_state(ref, TestBufferStats().used(1).hold(1).dead(0)); reclaim_memory(); - assert_buffer_state(ref, TestBufferStats().used(16).hold(0).dead(16)); + assert_buffer_state(ref, TestBufferStats().used(1).hold(0).dead(1)); } TEST_F(StringTest, extra_bytes_used_is_tracked) @@ -139,7 +139,7 @@ TEST_F(StringTest, free_list_is_used_when_enabled) EntryRef ref4 = add(spaces1000.c_str()); EXPECT_EQ(ref1, ref3); EXPECT_EQ(ref2, ref4); - assert_buffer_state(ref1, TestBufferStats().used(16).hold(0).dead(0)); + assert_buffer_state(ref1, TestBufferStats().used(1).hold(0).dead(0)); assert_buffer_state(ref2, TestBufferStats().used(2).hold(0).dead(1).extra_used(1001)); } @@ -155,7 +155,7 @@ TEST_F(StringTest, free_list_is_not_used_when_disabled) EntryRef ref4 = add(spaces1000.c_str()); EXPECT_NE(ref1, ref3); EXPECT_NE(ref2, ref4); - assert_buffer_state(ref1, TestBufferStats().used(32).hold(0).dead(16)); + assert_buffer_state(ref1, TestBufferStats().used(2).hold(0).dead(1)); assert_buffer_state(ref2, TestBufferStats().used(3).hold(0).dead(2).extra_used(1001)); } @@ -173,7 +173,7 @@ TEST_F(StringTest, free_list_is_never_used_for_move_on_compact) EntryRef ref6 = move_on_compact(ref2); EXPECT_NE(ref5, ref3); EXPECT_NE(ref6, ref4); - assert_buffer_state(ref1, TestBufferStats().used(48).hold(0).dead(16)); + assert_buffer_state(ref1, TestBufferStats().used(3).hold(0).dead(1)); assert_buffer_state(ref2, TestBufferStats().used(4).hold(0).dead(2).extra_used(2002)); } diff --git a/vespalib/src/tests/signalhandler/CMakeLists.txt b/vespalib/src/tests/signalhandler/CMakeLists.txt index 4f78eb2e82d..88be14f994f 100644 --- a/vespalib/src/tests/signalhandler/CMakeLists.txt +++ b/vespalib/src/tests/signalhandler/CMakeLists.txt @@ -5,6 +5,11 @@ vespa_add_library(vespalib_signalhandler_test_my_shared_library TEST DEPENDS vespalib ) + +# Don't convert call to jump when returning a value from a function with +# a compatible stack. +set_source_files_properties(my_shared_library.cpp PROPERTIES COMPILE_OPTIONS "-fno-optimize-sibling-calls") + vespa_add_executable(vespalib_signalhandler_test_app TEST SOURCES signalhandler_test.cpp diff --git a/vespalib/src/vespa/vespalib/btree/btree.h b/vespalib/src/vespa/vespalib/btree/btree.h index 32b538b65ec..c2f5aac01b7 100644 --- a/vespalib/src/vespa/vespalib/btree/btree.h +++ b/vespalib/src/vespa/vespalib/btree/btree.h @@ -61,9 +61,9 @@ public: } void - disableElemHoldList() + disable_entry_hold_list() { - _alloc.disableElemHoldList(); + _alloc.disable_entry_hold_list(); } // Inherit doc from BTreeRoot diff --git a/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h b/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h index 784e95e3817..b537602c703 100644 --- a/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h +++ b/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h @@ -60,8 +60,8 @@ public: _nodeStore.disableFreeLists(); } - void disableElemHoldList() { - _nodeStore.disableElemHoldList(); + void disable_entry_hold_list() { + _nodeStore.disable_entry_hold_list(); } /** diff --git a/vespalib/src/vespa/vespalib/btree/btreenodeallocator.hpp b/vespalib/src/vespa/vespalib/btree/btreenodeallocator.hpp index a38b68afe73..d23c8fc2054 100644 --- a/vespalib/src/vespa/vespalib/btree/btreenodeallocator.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreenodeallocator.hpp @@ -162,7 +162,7 @@ holdNode(BTreeNode::Ref nodeRef, InternalNodeType *node) { if (node->getFrozen()) { - _nodeStore.holdElem(nodeRef); + _nodeStore.hold_entry(nodeRef); } else { node->clean(); _internalHoldUntilFreeze.push_back(nodeRef); @@ -178,7 +178,7 @@ holdNode(BTreeNode::Ref nodeRef, LeafNodeType *node) { if (node->getFrozen()) { - _nodeStore.holdElem(nodeRef); + _nodeStore.hold_entry(nodeRef); } else { node->clean(); _leafHoldUntilFreeze.push_back(nodeRef); @@ -235,7 +235,7 @@ freeze() InternalNodeType *inode = mapInternalRef(i); (void) inode; assert(inode->getFrozen()); - _nodeStore.holdElem(i); + _nodeStore.hold_entry(i); } _internalHoldUntilFreeze.clear(); } @@ -245,7 +245,7 @@ freeze() LeafNodeType *lnode = mapLeafRef(i); (void) lnode; assert(lnode->getFrozen()); - _nodeStore.holdElem(i); + _nodeStore.hold_entry(i); } _leafHoldUntilFreeze.clear(); } diff --git a/vespalib/src/vespa/vespalib/btree/btreenodestore.h b/vespalib/src/vespa/vespalib/btree/btreenodestore.h index c59092bf75c..38bf4e5ed4e 100644 --- a/vespalib/src/vespa/vespalib/btree/btreenodestore.h +++ b/vespalib/src/vespa/vespalib/btree/btreenodestore.h @@ -29,16 +29,16 @@ class BTreeNodeBufferType : public datastore::BufferType<EntryType, FrozenBtreeN using ParentType = datastore::BufferType<EntryType, FrozenBtreeNode<EntryType>>; using ParentType::empty_entry; using ParentType::_arraySize; - using ElemCount = typename ParentType::ElemCount; + using EntryCount = typename ParentType::EntryCount; using CleanContext = typename ParentType::CleanContext; public: - BTreeNodeBufferType(uint32_t minArrays, uint32_t maxArrays) - : ParentType(1, minArrays, maxArrays) + BTreeNodeBufferType(uint32_t min_entries, uint32_t max_entries) + : ParentType(1, min_entries, max_entries) { } - void initializeReservedElements(void *buffer, ElemCount reservedElements) override; + void initialize_reserved_entries(void *buffer, EntryCount reserved_entries) override; - void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override; + void clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx) override; }; @@ -79,7 +79,7 @@ public: ~BTreeNodeStore(); void disableFreeLists() { _store.disableFreeLists(); } - void disableElemHoldList() { _store.disableElemHoldList(); } + void disable_entry_hold_list() { _store.disable_entry_hold_list(); } static bool isValidRef(EntryRef ref) { return ref.valid(); } @@ -152,8 +152,8 @@ public: return _store.freeListAllocator<InternalNodeType, BTreeNodeReclaimer>(NODETYPE_INTERNAL).alloc(rhs); } - void holdElem(EntryRef ref) { - _store.holdElem(ref, 1); + void hold_entry(EntryRef ref) { + _store.hold_entry(ref); } std::unique_ptr<vespalib::datastore::CompactingBuffers> start_compact_worst(const CompactionStrategy& compaction_strategy); diff --git a/vespalib/src/vespa/vespalib/btree/btreenodestore.hpp b/vespalib/src/vespa/vespalib/btree/btreenodestore.hpp index a1ffb4d445d..99054f35d61 100644 --- a/vespalib/src/vespa/vespalib/btree/btreenodestore.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreenodestore.hpp @@ -11,11 +11,11 @@ namespace vespalib::btree { template <typename EntryType> void -BTreeNodeBufferType<EntryType>::initializeReservedElements(void *buffer, ElemCount reservedElements) +BTreeNodeBufferType<EntryType>::initialize_reserved_entries(void *buffer, EntryCount reserved_entries) { - ParentType::initializeReservedElements(buffer, reservedElements); + ParentType::initialize_reserved_entries(buffer, reserved_entries); EntryType *e = static_cast<EntryType *>(buffer); - for (size_t j = reservedElements; j != 0; --j) { + for (size_t j = reserved_entries; j != 0; --j) { e->freeze(); ++e; } @@ -24,10 +24,10 @@ BTreeNodeBufferType<EntryType>::initializeReservedElements(void *buffer, ElemCou template <typename EntryType> void -BTreeNodeBufferType<EntryType>::cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext) +BTreeNodeBufferType<EntryType>::clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext) { EntryType *e = static_cast<EntryType *>(buffer) + offset; - for (size_t j = numElems; j != 0; --j) { + for (size_t j = num_entries; j != 0; --j) { e->cleanFrozen(); ++e; } diff --git a/vespalib/src/vespa/vespalib/btree/btreestore.h b/vespalib/src/vespa/vespalib/btree/btreestore.h index 9fdade850f1..7dd839f1529 100644 --- a/vespalib/src/vespa/vespalib/btree/btreestore.h +++ b/vespalib/src/vespa/vespalib/btree/btreestore.h @@ -50,8 +50,8 @@ public: using CompactionSpec = datastore::CompactionSpec; using CompactionStrategy = datastore::CompactionStrategy; using EntryRef = datastore::EntryRef; - template <typename EntryType> - using BufferType = datastore::BufferType<EntryType>; + template <typename ElemT> + using BufferType = datastore::BufferType<ElemT>; using BufferState = datastore::BufferState; static constexpr uint32_t clusterLimit = 8; @@ -105,9 +105,9 @@ public: _allocator.disableFreeLists(); } - void disableElemHoldList() { - _store.disableElemHoldList(); - _allocator.disableElemHoldList(); + void disable_entry_hold_list() { + _store.disable_entry_hold_list(); + _allocator.disable_entry_hold_list(); } BTreeTypeRefPair allocNewBTree() { diff --git a/vespalib/src/vespa/vespalib/btree/btreestore.hpp b/vespalib/src/vespa/vespalib/btree/btreestore.hpp index a19d0b34aa6..90c302af5e4 100644 --- a/vespalib/src/vespa/vespalib/btree/btreestore.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreestore.hpp @@ -74,7 +74,7 @@ allocNewKeyData(uint32_t clusterSize) { assert(clusterSize >= 1 && clusterSize <= clusterLimit); uint32_t typeId = clusterSize - 1; - return _store.allocator<KeyDataType>(typeId).allocArray(clusterSize); + return _store.allocator<KeyDataType>(typeId).allocArray(); } @@ -87,7 +87,7 @@ allocKeyData(uint32_t clusterSize) { assert(clusterSize >= 1 && clusterSize <= clusterLimit); uint32_t typeId = clusterSize - 1; - return _store.freeListAllocator<KeyDataType, datastore::DefaultReclaimer<KeyDataType>>(typeId).allocArray(clusterSize); + return _store.freeListAllocator<KeyDataType, datastore::DefaultReclaimer<KeyDataType>>(typeId).allocArray(); } @@ -156,7 +156,7 @@ makeTree(EntryRef &ref, lNode->freeze(); BTreeTypeRefPair tPair(allocBTree()); tPair.data->setRoots(lPair.ref); - _store.holdElem(ref, clusterSize); + _store.hold_entry(ref); ref = tPair.ref; } @@ -176,7 +176,7 @@ makeArray(EntryRef &ref, EntryRef root, LeafNodeType *leafNode) kd->setData(leafNode->getData(idx)); } assert(kd == kPair.data + clusterSize); - _store.holdElem(ref, 1); + _store.hold_entry(ref); if (!leafNode->getFrozen()) { leafNode->freeze(); } @@ -255,7 +255,7 @@ insert(EntryRef &ref, kd->setData(i->getData()); } assert(kd == kPair.data + clusterSize + 1); - _store.holdElem(ref, clusterSize); + _store.hold_entry(ref); ref = kPair.ref; return true; } @@ -284,7 +284,7 @@ insert(EntryRef &ref, lNode->freeze(); BTreeTypeRefPair tPair(allocBTree()); tPair.data->setRoots(lPair.ref); // allow immediate access to readers - _store.holdElem(ref, clusterSize); + _store.hold_entry(ref); ref = tPair.ref; return true; #endif @@ -339,7 +339,7 @@ remove(EntryRef &ref, if (oldi == olde || comp(key, oldi->_key)) return false; // not found if (clusterSize == 1) { - _store.holdElem(ref, 1); + _store.hold_entry(ref); ref = EntryRef(); return true; } @@ -357,7 +357,7 @@ remove(EntryRef &ref, kd->setData(i->getData()); } assert(kd == kPair.data + clusterSize - 1); - _store.holdElem(ref, clusterSize); + _store.hold_entry(ref); ref = kPair.ref; return true; } @@ -670,7 +670,7 @@ applyCluster(EntryRef &ref, if (newSizeMin <= clusterLimit) { uint32_t newSize = getNewClusterSize(ob, oe, a, ae, r, re, comp); if (newSize == 0) { - _store.holdElem(ref, clusterSize); + _store.hold_entry(ref); ref = EntryRef(); return true; } @@ -678,7 +678,7 @@ applyCluster(EntryRef &ref, KeyDataTypeRefPair kPair(allocKeyData(newSize)); applyCluster(ob, oe, kPair.data, kPair.data + newSize, a, ae, r, re, comp); - _store.holdElem(ref, clusterSize); + _store.hold_entry(ref); ref = kPair.ref; return true; } @@ -735,7 +735,7 @@ normalizeTree(EntryRef &ref, { EntryRef root = tree->getRoot(); if (!NodeAllocatorType::isValidRef(root)) { - _store.holdElem(ref, 1); + _store.hold_entry(ref); ref = EntryRef(); return; } @@ -798,10 +798,8 @@ clear(const EntryRef ref) if (clusterSize == 0) { BTreeType *tree = getWTreeEntry(iRef); tree->clear(_allocator); - _store.holdElem(ref, 1); - } else { - _store.holdElem(ref, clusterSize); } + _store.hold_entry(ref); } diff --git a/vespalib/src/vespa/vespalib/datastore/allocator.h b/vespalib/src/vespa/vespalib/datastore/allocator.h index 297270af0f5..30938bdc1c1 100644 --- a/vespalib/src/vespa/vespalib/datastore/allocator.h +++ b/vespalib/src/vespa/vespalib/datastore/allocator.h @@ -30,7 +30,7 @@ public: HandleType alloc(Args && ... args); HandleType allocArray(ConstArrayRef array); - HandleType allocArray(size_t size); + HandleType allocArray(); }; } diff --git a/vespalib/src/vespa/vespalib/datastore/allocator.hpp b/vespalib/src/vespa/vespalib/datastore/allocator.hpp index 85f0e842519..fa97ef9a5f5 100644 --- a/vespalib/src/vespa/vespalib/datastore/allocator.hpp +++ b/vespalib/src/vespa/vespalib/datastore/allocator.hpp @@ -20,7 +20,7 @@ template <typename ... Args> typename Allocator<EntryT, RefT>::HandleType Allocator<EntryT, RefT>::alloc(Args && ... args) { - _store.ensureBufferCapacity(_typeId, 1); + _store.ensure_buffer_capacity(_typeId, 1); uint32_t buffer_id = _store.primary_buffer_id(_typeId); BufferState &state = _store.getBufferState(buffer_id); assert(state.isActive()); @@ -36,39 +36,35 @@ template <typename EntryT, typename RefT> typename Allocator<EntryT, RefT>::HandleType Allocator<EntryT, RefT>::allocArray(ConstArrayRef array) { - _store.ensureBufferCapacity(_typeId, array.size()); + _store.ensure_buffer_capacity(_typeId, 1); uint32_t buffer_id = _store.primary_buffer_id(_typeId); BufferState &state = _store.getBufferState(buffer_id); assert(state.isActive()); assert(state.getArraySize() == array.size()); - size_t oldBufferSize = state.size(); - assert((oldBufferSize % array.size()) == 0); - RefT ref((oldBufferSize / array.size()), buffer_id); + RefT ref(state.size(), buffer_id); EntryT *buf = _store.template getEntryArray<EntryT>(ref, array.size()); for (size_t i = 0; i < array.size(); ++i) { new (static_cast<void *>(buf + i)) EntryT(array[i]); } - state.stats().pushed_back(array.size()); + state.stats().pushed_back(1); return HandleType(ref, buf); } template <typename EntryT, typename RefT> typename Allocator<EntryT, RefT>::HandleType -Allocator<EntryT, RefT>::allocArray(size_t size) +Allocator<EntryT, RefT>::allocArray() { - _store.ensureBufferCapacity(_typeId, size); + _store.ensure_buffer_capacity(_typeId, 1); uint32_t buffer_id = _store.primary_buffer_id(_typeId); BufferState &state = _store.getBufferState(buffer_id); assert(state.isActive()); - assert(state.getArraySize() == size); - size_t oldBufferSize = state.size(); - assert((oldBufferSize % size) == 0); - RefT ref((oldBufferSize / size), buffer_id); - EntryT *buf = _store.template getEntryArray<EntryT>(ref, size); - for (size_t i = 0; i < size; ++i) { + RefT ref(state.size(), buffer_id); + auto array_size = state.getArraySize(); + EntryT *buf = _store.template getEntryArray<EntryT>(ref, array_size); + for (size_t i = 0; i < array_size; ++i) { new (static_cast<void *>(buf + i)) EntryT(); } - state.stats().pushed_back(size); + state.stats().pushed_back(1); return HandleType(ref, buf); } diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.h b/vespalib/src/vespa/vespalib/datastore/array_store.h index c2b65d72f03..809ac10f6e3 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.h +++ b/vespalib/src/vespa/vespalib/datastore/array_store.h @@ -19,7 +19,7 @@ namespace vespalib::datastore { /** - * Datastore for storing arrays of type EntryT that is accessed via a 32-bit EntryRef. + * Datastore for storing arrays of type ElemT that is accessed via a 32-bit EntryRef. * * The default EntryRef type uses 19 bits for offset (524288 values) and 13 bits for buffer id (8192 buffers). * @@ -29,16 +29,18 @@ namespace vespalib::datastore { * * The max value of maxSmallArrayTypeId is (2^bufferBits - 1). */ -template <typename EntryT, typename RefT = EntryRefT<19>, typename TypeMapperT = ArrayStoreSimpleTypeMapper<EntryT> > +template <typename ElemT, typename RefT = EntryRefT<19>, typename TypeMapperT = ArrayStoreSimpleTypeMapper<ElemT> > class ArrayStore : public ICompactable { public: using AllocSpec = ArrayStoreConfig::AllocSpec; - using ArrayRef = vespalib::ArrayRef<EntryT>; - using ConstArrayRef = vespalib::ConstArrayRef<EntryT>; + using ArrayRef = vespalib::ArrayRef<ElemT>; + using ConstArrayRef = vespalib::ConstArrayRef<ElemT>; using DataStoreType = DataStoreT<RefT>; - using LargeArray = vespalib::Array<EntryT>; + using ElemType = ElemT; + using LargeArray = vespalib::Array<ElemT>; using LargeBufferType = typename TypeMapperT::LargeBufferType; + using RefType = RefT; using SmallBufferType = typename TypeMapperT::SmallBufferType; using TypeMapper = TypeMapperT; private: @@ -58,7 +60,7 @@ private: EntryRef addLargeArray(const ConstArrayRef &array); EntryRef allocate_large_array(size_t array_size); ConstArrayRef getSmallArray(RefT ref, size_t arraySize) const { - const EntryT *buf = _store.template getEntryArray<EntryT>(ref, arraySize); + const ElemT *buf = _store.template getEntryArray<ElemT>(ref, arraySize); return ConstArrayRef(buf, arraySize); } ConstArrayRef getLargeArray(RefT ref) const { @@ -85,7 +87,7 @@ public: } /** - * Allocate an array of the given size without any instantiation of EntryT elements. + * Allocate an array of the given size without any instantiation of ElemT elements. * * Use get_writable() to get a reference to the array for writing. * @@ -148,14 +150,14 @@ public: static ArrayStoreConfig optimizedConfigForHugePage(uint32_t maxSmallArrayTypeId, size_t hugePageSize, size_t smallPageSize, - size_t minNumArraysForNewBuffer, + size_t min_num_entries_for_new_buffer, float allocGrowFactor); static ArrayStoreConfig optimizedConfigForHugePage(uint32_t maxSmallArrayTypeId, const TypeMapper& mapper, size_t hugePageSize, size_t smallPageSize, - size_t minNumArraysForNewBuffer, + size_t min_num_entries_for_new_buffer, float allocGrowFactor); }; diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.hpp b/vespalib/src/vespa/vespalib/datastore/array_store.hpp index 301cff1e414..8e9fe779ba9 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.hpp +++ b/vespalib/src/vespa/vespalib/datastore/array_store.hpp @@ -15,9 +15,9 @@ namespace vespalib::datastore { -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> void -ArrayStore<EntryT, RefT, TypeMapperT>::initArrayTypes(const ArrayStoreConfig &cfg, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) +ArrayStore<ElemT, RefT, TypeMapperT>::initArrayTypes(const ArrayStoreConfig &cfg, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) { _largeArrayTypeId = _store.addType(&_largeArrayType); assert(_largeArrayTypeId == 0); @@ -31,14 +31,14 @@ ArrayStore<EntryT, RefT, TypeMapperT>::initArrayTypes(const ArrayStoreConfig &cf } } -template <typename EntryT, typename RefT, typename TypeMapperT> -ArrayStore<EntryT, RefT, TypeMapperT>::ArrayStore(const ArrayStoreConfig &cfg, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) +template <typename ElemT, typename RefT, typename TypeMapperT> +ArrayStore<ElemT, RefT, TypeMapperT>::ArrayStore(const ArrayStoreConfig &cfg, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) : ArrayStore(cfg, memory_allocator, TypeMapper()) { } -template <typename EntryT, typename RefT, typename TypeMapperT> -ArrayStore<EntryT, RefT, TypeMapperT>::ArrayStore(const ArrayStoreConfig &cfg, std::shared_ptr<alloc::MemoryAllocator> memory_allocator, +template <typename ElemT, typename RefT, typename TypeMapperT> +ArrayStore<ElemT, RefT, TypeMapperT>::ArrayStore(const ArrayStoreConfig &cfg, std::shared_ptr<alloc::MemoryAllocator> memory_allocator, TypeMapper&& mapper) : _largeArrayTypeId(0), _maxSmallArrayTypeId(cfg.maxSmallArrayTypeId()), @@ -56,25 +56,25 @@ ArrayStore<EntryT, RefT, TypeMapperT>::ArrayStore(const ArrayStoreConfig &cfg, s } } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> vespalib::MemoryUsage -ArrayStore<EntryT, RefT, TypeMapperT>::getMemoryUsage() const { +ArrayStore<ElemT, RefT, TypeMapperT>::getMemoryUsage() const { vespalib::MemoryUsage usage = _store.getMemoryUsage(); usage.incAllocatedBytes(_smallArrayTypes.capacity() * sizeof(SmallBufferType)); usage.incUsedBytes(_smallArrayTypes.size() * sizeof(SmallBufferType)); return usage; } -template <typename EntryT, typename RefT, typename TypeMapperT> -ArrayStore<EntryT, RefT, TypeMapperT>::~ArrayStore() +template <typename ElemT, typename RefT, typename TypeMapperT> +ArrayStore<ElemT, RefT, TypeMapperT>::~ArrayStore() { _store.reclaim_all_memory(); _store.dropBuffers(); } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> EntryRef -ArrayStore<EntryT, RefT, TypeMapperT>::add(const ConstArrayRef &array) +ArrayStore<ElemT, RefT, TypeMapperT>::add(const ConstArrayRef &array) { if (array.size() == 0) { return EntryRef(); @@ -86,9 +86,9 @@ ArrayStore<EntryT, RefT, TypeMapperT>::add(const ConstArrayRef &array) } } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> EntryRef -ArrayStore<EntryT, RefT, TypeMapperT>::allocate(size_t array_size) +ArrayStore<ElemT, RefT, TypeMapperT>::allocate(size_t array_size) { if (array_size == 0) { return EntryRef(); @@ -100,94 +100,93 @@ ArrayStore<EntryT, RefT, TypeMapperT>::allocate(size_t array_size) } } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> EntryRef -ArrayStore<EntryT, RefT, TypeMapperT>::addSmallArray(const ConstArrayRef &array) +ArrayStore<ElemT, RefT, TypeMapperT>::addSmallArray(const ConstArrayRef &array) { uint32_t typeId = _mapper.get_type_id(array.size()); - using NoOpReclaimer = DefaultReclaimer<EntryT>; - return _store.template freeListAllocator<EntryT, NoOpReclaimer>(typeId).allocArray(array).ref; + using NoOpReclaimer = DefaultReclaimer<ElemT>; + return _store.template freeListAllocator<ElemT, NoOpReclaimer>(typeId).allocArray(array).ref; } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> EntryRef -ArrayStore<EntryT, RefT, TypeMapperT>::allocate_small_array(size_t array_size) +ArrayStore<ElemT, RefT, TypeMapperT>::allocate_small_array(size_t array_size) { uint32_t type_id = _mapper.get_type_id(array_size); - return _store.template freeListRawAllocator<EntryT>(type_id).alloc(array_size).ref; + return _store.template freeListRawAllocator<ElemT>(type_id).alloc(1).ref; } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> EntryRef -ArrayStore<EntryT, RefT, TypeMapperT>::addLargeArray(const ConstArrayRef &array) +ArrayStore<ElemT, RefT, TypeMapperT>::addLargeArray(const ConstArrayRef &array) { using NoOpReclaimer = DefaultReclaimer<LargeArray>; auto handle = _store.template freeListAllocator<LargeArray, NoOpReclaimer>(_largeArrayTypeId) .alloc(array.cbegin(), array.cend()); auto& state = _store.getBufferState(RefT(handle.ref).bufferId()); - state.stats().inc_extra_used_bytes(sizeof(EntryT) * array.size()); + state.stats().inc_extra_used_bytes(sizeof(ElemT) * array.size()); return handle.ref; } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> EntryRef -ArrayStore<EntryT, RefT, TypeMapperT>::allocate_large_array(size_t array_size) +ArrayStore<ElemT, RefT, TypeMapperT>::allocate_large_array(size_t array_size) { using NoOpReclaimer = DefaultReclaimer<LargeArray>; auto handle = _store.template freeListAllocator<LargeArray, NoOpReclaimer>(_largeArrayTypeId).alloc(array_size); auto& state = _store.getBufferState(RefT(handle.ref).bufferId()); - state.stats().inc_extra_used_bytes(sizeof(EntryT) * array_size); + state.stats().inc_extra_used_bytes(sizeof(ElemT) * array_size); return handle.ref; } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> void -ArrayStore<EntryT, RefT, TypeMapperT>::remove(EntryRef ref) +ArrayStore<ElemT, RefT, TypeMapperT>::remove(EntryRef ref) { if (ref.valid()) { RefT internalRef(ref); uint32_t typeId = _store.getTypeId(internalRef.bufferId()); if (typeId != _largeArrayTypeId) { - size_t arraySize = _mapper.get_array_size(typeId); - _store.holdElem(ref, arraySize); + _store.hold_entry(ref); } else { - _store.holdElem(ref, 1, sizeof(EntryT) * get(ref).size()); + _store.hold_entry(ref, sizeof(ElemT) * get(ref).size()); } } } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> EntryRef -ArrayStore<EntryT, RefT, TypeMapperT>::move_on_compact(EntryRef ref) +ArrayStore<ElemT, RefT, TypeMapperT>::move_on_compact(EntryRef ref) { return add(get(ref)); } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> ICompactionContext::UP -ArrayStore<EntryT, RefT, TypeMapperT>::compact_worst(const CompactionStrategy &compaction_strategy) +ArrayStore<ElemT, RefT, TypeMapperT>::compact_worst(const CompactionStrategy &compaction_strategy) { auto compacting_buffers = _store.start_compact_worst_buffers(_compaction_spec, compaction_strategy); return std::make_unique<CompactionContext>(*this, std::move(compacting_buffers)); } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> std::unique_ptr<CompactingBuffers> -ArrayStore<EntryT, RefT, TypeMapperT>::start_compact_worst_buffers(const CompactionStrategy &compaction_strategy) +ArrayStore<ElemT, RefT, TypeMapperT>::start_compact_worst_buffers(const CompactionStrategy &compaction_strategy) { return _store.start_compact_worst_buffers(_compaction_spec, compaction_strategy); } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> vespalib::AddressSpace -ArrayStore<EntryT, RefT, TypeMapperT>::addressSpaceUsage() const +ArrayStore<ElemT, RefT, TypeMapperT>::addressSpaceUsage() const { return _store.getAddressSpaceUsage(); } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> vespalib::MemoryUsage -ArrayStore<EntryT, RefT, TypeMapperT>::update_stat(const CompactionStrategy& compaction_strategy) +ArrayStore<ElemT, RefT, TypeMapperT>::update_stat(const CompactionStrategy& compaction_strategy) { auto address_space_usage = _store.getAddressSpaceUsage(); auto memory_usage = getMemoryUsage(); @@ -195,20 +194,20 @@ ArrayStore<EntryT, RefT, TypeMapperT>::update_stat(const CompactionStrategy& com return memory_usage; } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> const BufferState & -ArrayStore<EntryT, RefT, TypeMapperT>::bufferState(EntryRef ref) +ArrayStore<ElemT, RefT, TypeMapperT>::bufferState(EntryRef ref) { RefT internalRef(ref); return _store.getBufferState(internalRef.bufferId()); } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> ArrayStoreConfig -ArrayStore<EntryT, RefT, TypeMapperT>::optimizedConfigForHugePage(uint32_t maxSmallArrayTypeId, +ArrayStore<ElemT, RefT, TypeMapperT>::optimizedConfigForHugePage(uint32_t maxSmallArrayTypeId, size_t hugePageSize, size_t smallPageSize, - size_t minNumArraysForNewBuffer, + size_t min_num_entries_for_new_buffer, float allocGrowFactor) { TypeMapper mapper; @@ -216,26 +215,26 @@ ArrayStore<EntryT, RefT, TypeMapperT>::optimizedConfigForHugePage(uint32_t maxSm mapper, hugePageSize, smallPageSize, - minNumArraysForNewBuffer, + min_num_entries_for_new_buffer, allocGrowFactor); } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> ArrayStoreConfig -ArrayStore<EntryT, RefT, TypeMapperT>::optimizedConfigForHugePage(uint32_t maxSmallArrayTypeId, +ArrayStore<ElemT, RefT, TypeMapperT>::optimizedConfigForHugePage(uint32_t maxSmallArrayTypeId, const TypeMapper& mapper, size_t hugePageSize, size_t smallPageSize, - size_t minNumArraysForNewBuffer, + size_t min_num_entries_for_new_buffer, float allocGrowFactor) { return ArrayStoreConfig::optimizeForHugePage(mapper.get_max_small_array_type_id(maxSmallArrayTypeId), [&](uint32_t type_id) noexcept { return mapper.get_array_size(type_id); }, hugePageSize, smallPageSize, - sizeof(EntryT), + sizeof(ElemT), RefT::offsetSize(), - minNumArraysForNewBuffer, + min_num_entries_for_new_buffer, allocGrowFactor); } diff --git a/vespalib/src/vespa/vespalib/datastore/array_store_config.cpp b/vespalib/src/vespa/vespalib/datastore/array_store_config.cpp index d5587841745..1df9354cd6c 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store_config.cpp +++ b/vespalib/src/vespa/vespalib/datastore/array_store_config.cpp @@ -49,19 +49,19 @@ ArrayStoreConfig::optimizeForHugePage(uint32_t maxSmallArrayTypeId, std::function<size_t(uint32_t)> type_id_to_array_size, size_t hugePageSize, size_t smallPageSize, - size_t entrySize, + size_t elem_size, size_t maxEntryRefOffset, - size_t minNumArraysForNewBuffer, + size_t min_num_entries_for_new_buffer, float allocGrowFactor) { AllocSpecVector allocSpecs; - allocSpecs.emplace_back(0, maxEntryRefOffset, minNumArraysForNewBuffer, allocGrowFactor); // large array spec; + allocSpecs.emplace_back(0, maxEntryRefOffset, min_num_entries_for_new_buffer, allocGrowFactor); // large array spec; for (uint32_t type_id = 1; type_id <= maxSmallArrayTypeId; ++type_id) { size_t arraySize = type_id_to_array_size(type_id); - size_t numArraysForNewBuffer = hugePageSize / (entrySize * arraySize); - numArraysForNewBuffer = capToLimits(numArraysForNewBuffer, minNumArraysForNewBuffer, maxEntryRefOffset); - numArraysForNewBuffer = alignToSmallPageSize(numArraysForNewBuffer, minNumArraysForNewBuffer, smallPageSize); - allocSpecs.emplace_back(0, maxEntryRefOffset, numArraysForNewBuffer, allocGrowFactor); + size_t num_entries_for_new_buffer = hugePageSize / (elem_size * arraySize); + num_entries_for_new_buffer = capToLimits(num_entries_for_new_buffer, min_num_entries_for_new_buffer, maxEntryRefOffset); + num_entries_for_new_buffer = alignToSmallPageSize(num_entries_for_new_buffer, min_num_entries_for_new_buffer, smallPageSize); + allocSpecs.emplace_back(0, maxEntryRefOffset, num_entries_for_new_buffer, allocGrowFactor); } return ArrayStoreConfig(allocSpecs); } diff --git a/vespalib/src/vespa/vespalib/datastore/array_store_config.h b/vespalib/src/vespa/vespalib/datastore/array_store_config.h index a326c00d042..cae241dba10 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store_config.h +++ b/vespalib/src/vespa/vespalib/datastore/array_store_config.h @@ -19,21 +19,21 @@ public: * Specification of buffer allocation strategy for arrays of a given size. */ struct AllocSpec { - // Minimum number of arrays to allocate in a buffer. - size_t minArraysInBuffer; - // Maximum number of arrays to allocate in a buffer. - size_t maxArraysInBuffer; - // Number of arrays needed before allocating a new buffer instead of just resizing the first one. - size_t numArraysForNewBuffer; + // Minimum number of entries to allocate in a buffer. + size_t min_entries_in_buffer; + // Maximum number of entries to allocate in a buffer. + size_t max_entries_in_buffer; + // Number of entries needed before allocating a new buffer instead of just resizing the first one. + size_t num_entries_for_new_buffer; // Grow factor used when allocating a new buffer. float allocGrowFactor; - AllocSpec(size_t minArraysInBuffer_, - size_t maxArraysInBuffer_, - size_t numArraysForNewBuffer_, + AllocSpec(size_t min_entries_in_buffer_, + size_t max_entries_in_buffer_, + size_t num_entries_for_new_buffer_, float allocGrowFactor_) noexcept - : minArraysInBuffer(minArraysInBuffer_), - maxArraysInBuffer(maxArraysInBuffer_), - numArraysForNewBuffer(numArraysForNewBuffer_), + : min_entries_in_buffer(min_entries_in_buffer_), + max_entries_in_buffer(max_entries_in_buffer_), + num_entries_for_new_buffer(num_entries_for_new_buffer_), allocGrowFactor(allocGrowFactor_) {} }; @@ -76,9 +76,9 @@ public: std::function<size_t(uint32_t)> type_id_to_array_size, size_t hugePageSize, size_t smallPageSize, - size_t entrySize, + size_t elem_size, size_t maxEntryRefOffset, - size_t minNumArraysForNewBuffer, + size_t min_num_entries_for_new_buffer, float allocGrowFactor); }; diff --git a/vespalib/src/vespa/vespalib/datastore/array_store_simple_type_mapper.h b/vespalib/src/vespa/vespalib/datastore/array_store_simple_type_mapper.h index a0cd7827b2d..5f9728a43cc 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store_simple_type_mapper.h +++ b/vespalib/src/vespa/vespalib/datastore/array_store_simple_type_mapper.h @@ -15,11 +15,11 @@ namespace vespalib::datastore { * * A more complex mapping can be used by creating a custom mapper and BufferType implementations. */ -template <typename EntryT> +template <typename ElemT> class ArrayStoreSimpleTypeMapper { public: - using SmallBufferType = SmallArrayBufferType<EntryT>; - using LargeBufferType = LargeArrayBufferType<EntryT>; + using SmallBufferType = SmallArrayBufferType<ElemT>; + using LargeBufferType = LargeArrayBufferType<ElemT>; uint32_t get_type_id(size_t array_size) const { return array_size; } size_t get_array_size(uint32_t type_id) const { return type_id; } diff --git a/vespalib/src/vespa/vespalib/datastore/array_store_type_mapper.h b/vespalib/src/vespa/vespalib/datastore/array_store_type_mapper.h index e707627de19..2a406a39bf9 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store_type_mapper.h +++ b/vespalib/src/vespa/vespalib/datastore/array_store_type_mapper.h @@ -12,7 +12,7 @@ namespace vespalib::datastore { * This class provides mapping between type ids and array sizes needed for * storing a value with size smaller than or equal to the array size. * - * The array sizes vector is a monotic increasing sequence that might end + * The array sizes vector is a monotonic strictly increasing sequence that might end * with exponential growth. */ class ArrayStoreTypeMapper diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_free_list.cpp b/vespalib/src/vespa/vespalib/datastore/buffer_free_list.cpp index 224ed4b0c8f..851db0222c2 100644 --- a/vespalib/src/vespa/vespalib/datastore/buffer_free_list.cpp +++ b/vespalib/src/vespa/vespalib/datastore/buffer_free_list.cpp @@ -20,9 +20,8 @@ BufferFreeList::detach() _free_list->detach(*this); } -BufferFreeList::BufferFreeList(std::atomic<ElemCount>& dead_elems) - : _dead_elems(dead_elems), - _array_size(0), +BufferFreeList::BufferFreeList(std::atomic<EntryCount>& dead_entries) + : _dead_entries(dead_entries), _free_list(), _free_refs() { @@ -66,7 +65,7 @@ BufferFreeList::pop_entry() { if (empty()) { detach(); } - _dead_elems.store(_dead_elems.load(std::memory_order_relaxed) - _array_size, std::memory_order_relaxed); + _dead_entries.store(_dead_entries.load(std::memory_order_relaxed) - 1, std::memory_order_relaxed); return ret; } diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_free_list.h b/vespalib/src/vespa/vespalib/datastore/buffer_free_list.h index 148ddd8db88..4348a41af04 100644 --- a/vespalib/src/vespa/vespalib/datastore/buffer_free_list.h +++ b/vespalib/src/vespa/vespalib/datastore/buffer_free_list.h @@ -19,8 +19,7 @@ class BufferFreeList { private: using EntryRefArray = vespalib::Array<EntryRef>; - std::atomic<ElemCount>& _dead_elems; - uint32_t _array_size; + std::atomic<EntryCount>& _dead_entries; FreeList* _free_list; EntryRefArray _free_refs; @@ -28,7 +27,7 @@ private: void detach(); public: - BufferFreeList(std::atomic<ElemCount>& dead_elems); + BufferFreeList(std::atomic<EntryCount>& dead_entrie); ~BufferFreeList(); BufferFreeList(BufferFreeList&&) = default; // Needed for emplace_back() during setup. BufferFreeList(const BufferFreeList&) = delete; @@ -37,10 +36,8 @@ public: void enable(FreeList& free_list); void disable(); - void set_array_size(uint32_t value) { _array_size = value; } bool enabled() const { return _free_list != nullptr; } bool empty() const { return _free_refs.empty(); } - uint32_t array_size() const { return _array_size; } void push_entry(EntryRef ref); EntryRef pop_entry(); }; diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_stats.cpp b/vespalib/src/vespa/vespalib/datastore/buffer_stats.cpp index 8d97414626e..0d96e3f6d47 100644 --- a/vespalib/src/vespa/vespalib/datastore/buffer_stats.cpp +++ b/vespalib/src/vespa/vespalib/datastore/buffer_stats.cpp @@ -6,27 +6,27 @@ namespace vespalib::datastore { BufferStats::BufferStats() - : _alloc_elems(0), - _used_elems(0), - _hold_elems(0), - _dead_elems(0), + : _alloc_entries(0), + _used_entries(0), + _hold_entries(0), + _dead_entries(0), _extra_used_bytes(0), _extra_hold_bytes(0) { } void -BufferStats::add_to_mem_stats(size_t element_size, MemoryStats& stats) const +BufferStats::add_to_mem_stats(size_t entry_size, MemoryStats& stats) const { size_t extra_used = extra_used_bytes(); - stats._allocElems += capacity(); - stats._usedElems += size(); - stats._deadElems += dead_elems(); - stats._holdElems += hold_elems(); - stats._allocBytes += (capacity() * element_size) + extra_used; - stats._usedBytes += (size() * element_size) + extra_used; - stats._deadBytes += dead_elems() * element_size; - stats._holdBytes += (hold_elems() * element_size) + extra_hold_bytes(); + stats._alloc_entries += capacity(); + stats._used_entries += size(); + stats._dead_entries += dead_entries(); + stats._hold_entries += hold_entries(); + stats._allocBytes += (capacity() * entry_size) + extra_used; + stats._usedBytes += (size() * entry_size) + extra_used; + stats._deadBytes += dead_entries() * entry_size; + stats._holdBytes += (hold_entries() * entry_size) + extra_hold_bytes(); } InternalBufferStats::InternalBufferStats() @@ -37,20 +37,20 @@ InternalBufferStats::InternalBufferStats() void InternalBufferStats::clear() { - _alloc_elems.store(0, std::memory_order_relaxed); - _used_elems.store(0, std::memory_order_relaxed); - _hold_elems.store(0, std::memory_order_relaxed); - _dead_elems.store(0, std::memory_order_relaxed); + _alloc_entries.store(0, std::memory_order_relaxed); + _used_entries.store(0, std::memory_order_relaxed); + _hold_entries.store(0, std::memory_order_relaxed); + _dead_entries.store(0, std::memory_order_relaxed); _extra_used_bytes.store(0, std::memory_order_relaxed); _extra_hold_bytes.store(0, std::memory_order_relaxed); } void -InternalBufferStats::dec_hold_elems(size_t value) +InternalBufferStats::dec_hold_entries(size_t value) { - ElemCount elems = hold_elems(); + EntryCount elems = hold_entries(); assert(elems >= value); - _hold_elems.store(elems - value, std::memory_order_relaxed); + _hold_entries.store(elems - value, std::memory_order_relaxed); } } diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_stats.h b/vespalib/src/vespa/vespalib/datastore/buffer_stats.h index 66f8b532c41..1974efa97ec 100644 --- a/vespalib/src/vespa/vespalib/datastore/buffer_stats.h +++ b/vespalib/src/vespa/vespalib/datastore/buffer_stats.h @@ -13,16 +13,16 @@ namespace vespalib::datastore { */ class BufferStats { protected: - // The number of elements that are allocated in the buffer. - std::atomic<ElemCount> _alloc_elems; - // The number of elements (of the allocated) that are used: _used_elems <= _alloc_elems. - std::atomic<ElemCount> _used_elems; - // The number of elements (of the used) that are on hold: _hold_elems <= _used_elems. - // "On hold" is a transitionary state used when removing elements. - std::atomic<ElemCount> _hold_elems; - // The number of elements (of the used) that are dead: _dead_elems <= _used_elems. - // A dead element was first on hold, and is now available for reuse in the free list (if enabled). - std::atomic<ElemCount> _dead_elems; + // The number of entries that are allocated in the buffer. + std::atomic<EntryCount> _alloc_entries; + // The number of entries (of the allocated) that are used: _used_entries <= _alloc_entries. + std::atomic<EntryCount> _used_entries; + // The number of entries (of the used) that are on hold: _hold_entries <= _used_entries. + // "On hold" is a transitionary state used when removing entries. + std::atomic<EntryCount> _hold_entries; + // The number of entries (of the used) that are dead: _dead_entries <= _used_entries. + // A dead entry was first on hold, and is now available for reuse in the free list (if enabled). + std::atomic<EntryCount> _dead_entries; // Number of bytes that are heap allocated (and used) by elements that are stored in this buffer. // For simple types this is always 0. @@ -34,22 +34,22 @@ protected: public: BufferStats(); - size_t size() const { return _used_elems.load(std::memory_order_relaxed); } - size_t capacity() const { return _alloc_elems.load(std::memory_order_relaxed); } + size_t size() const { return _used_entries.load(std::memory_order_relaxed); } + size_t capacity() const { return _alloc_entries.load(std::memory_order_relaxed); } size_t remaining() const { return capacity() - size(); } - void pushed_back(size_t num_elems) { - _used_elems.store(size() + num_elems, std::memory_order_relaxed); + void pushed_back(size_t num_entries) { + _used_entries.store(size() + num_entries, std::memory_order_relaxed); } - size_t dead_elems() const { return _dead_elems.load(std::memory_order_relaxed); } - size_t hold_elems() const { return _hold_elems.load(std::memory_order_relaxed); } + size_t dead_entries() const { return _dead_entries.load(std::memory_order_relaxed); } + size_t hold_entries() const { return _hold_entries.load(std::memory_order_relaxed); } size_t extra_used_bytes() const { return _extra_used_bytes.load(std::memory_order_relaxed); } size_t extra_hold_bytes() const { return _extra_hold_bytes.load(std::memory_order_relaxed); } void inc_extra_used_bytes(size_t value) { _extra_used_bytes.store(extra_used_bytes() + value, std::memory_order_relaxed); } - void add_to_mem_stats(size_t element_size, MemoryStats& stats) const; + void add_to_mem_stats(size_t entry_size, MemoryStats& stats) const; }; /** @@ -59,15 +59,15 @@ class InternalBufferStats : public BufferStats { public: InternalBufferStats(); void clear(); - void set_alloc_elems(size_t value) { _alloc_elems.store(value, std::memory_order_relaxed); } - void set_dead_elems(size_t value) { _dead_elems.store(value, std::memory_order_relaxed); } - void set_hold_elems(size_t value) { _hold_elems.store(value, std::memory_order_relaxed); } - void inc_dead_elems(size_t value) { _dead_elems.store(dead_elems() + value, std::memory_order_relaxed); } - void inc_hold_elems(size_t value) { _hold_elems.store(hold_elems() + value, std::memory_order_relaxed); } - void dec_hold_elems(size_t value); + void set_alloc_entries(size_t value) { _alloc_entries.store(value, std::memory_order_relaxed); } + void set_dead_entries(size_t value) { _dead_entries.store(value, std::memory_order_relaxed); } + void set_hold_entries(size_t value) { _hold_entries.store(value, std::memory_order_relaxed); } + void inc_dead_entries(size_t value) { _dead_entries.store(dead_entries() + value, std::memory_order_relaxed); } + void inc_hold_entries(size_t value) { _hold_entries.store(hold_entries() + value, std::memory_order_relaxed); } + void dec_hold_entries(size_t value); void inc_extra_hold_bytes(size_t value) { _extra_hold_bytes.store(extra_hold_bytes() + value, std::memory_order_relaxed); } - std::atomic<ElemCount>& used_elems_ref() { return _used_elems; } - std::atomic<ElemCount>& dead_elems_ref() { return _dead_elems; } + std::atomic<EntryCount>& used_entries_ref() { return _used_entries; } + std::atomic<EntryCount>& dead_entries_ref() { return _dead_entries; } std::atomic<size_t>& extra_used_bytes_ref() { return _extra_used_bytes; } std::atomic<size_t>& extra_hold_bytes_ref() { return _extra_hold_bytes; } }; diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp b/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp index 4a9ba2d33a8..0d43ede9e62 100644 --- a/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp +++ b/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp @@ -27,85 +27,85 @@ BufferTypeBase::CleanContext::extraBytesCleaned(size_t value) } BufferTypeBase::BufferTypeBase(uint32_t arraySize, - uint32_t minArrays, - uint32_t maxArrays, - uint32_t numArraysForNewBuffer, + uint32_t min_entries, + uint32_t max_entries, + uint32_t num_entries_for_new_buffer, float allocGrowFactor) noexcept : _arraySize(arraySize), - _minArrays(std::min(minArrays, maxArrays)), - _maxArrays(maxArrays), - _numArraysForNewBuffer(std::min(numArraysForNewBuffer, maxArrays)), + _min_entries(std::min(min_entries, max_entries)), + _max_entries(max_entries), + _num_entries_for_new_buffer(std::min(num_entries_for_new_buffer, max_entries)), _allocGrowFactor(allocGrowFactor), _holdBuffers(0), - _holdUsedElems(0), + _hold_used_entries(0), _aggr_counts(), _active_buffers() { } BufferTypeBase::BufferTypeBase(uint32_t arraySize, - uint32_t minArrays, - uint32_t maxArrays) noexcept - : BufferTypeBase(arraySize, minArrays, maxArrays, 0u, DEFAULT_ALLOC_GROW_FACTOR) + uint32_t min_entries, + uint32_t max_entries) noexcept + : BufferTypeBase(arraySize, min_entries, max_entries, 0u, DEFAULT_ALLOC_GROW_FACTOR) { } BufferTypeBase::~BufferTypeBase() { assert(_holdBuffers == 0); - assert(_holdUsedElems == 0); + assert(_hold_used_entries == 0); assert(_aggr_counts.empty()); assert(_active_buffers.empty()); } -ElemCount -BufferTypeBase::getReservedElements(uint32_t bufferId) const +EntryCount +BufferTypeBase::get_reserved_entries(uint32_t bufferId) const { - return bufferId == 0 ? _arraySize : 0u; + return bufferId == 0 ? 1u : 0u; } void -BufferTypeBase::onActive(uint32_t bufferId, std::atomic<ElemCount>* usedElems, std::atomic<ElemCount>* deadElems, void* buffer) +BufferTypeBase::on_active(uint32_t bufferId, std::atomic<EntryCount>* used_entries, std::atomic<EntryCount>* dead_entries, void* buffer) { - _aggr_counts.add_buffer(usedElems, deadElems); + _aggr_counts.add_buffer(used_entries, dead_entries); assert(std::find(_active_buffers.begin(), _active_buffers.end(), bufferId) == _active_buffers.end()); _active_buffers.emplace_back(bufferId); - size_t reservedElems = getReservedElements(bufferId); - if (reservedElems != 0u) { - initializeReservedElements(buffer, reservedElems); - *usedElems = reservedElems; - *deadElems = reservedElems; + auto reserved_entries = get_reserved_entries(bufferId); + if (reserved_entries != 0u) { + initialize_reserved_entries(buffer, reserved_entries); + *used_entries = reserved_entries; + *dead_entries = reserved_entries; } } void -BufferTypeBase::onHold(uint32_t buffer_id, const std::atomic<ElemCount>* usedElems, const std::atomic<ElemCount>* deadElems) +BufferTypeBase::on_hold(uint32_t buffer_id, const std::atomic<EntryCount>* used_entries, const std::atomic<EntryCount>* dead_entries) { ++_holdBuffers; auto itr = std::find(_active_buffers.begin(), _active_buffers.end(), buffer_id); assert(itr != _active_buffers.end()); _active_buffers.erase(itr); - _aggr_counts.remove_buffer(usedElems, deadElems); - _holdUsedElems += *usedElems; + _aggr_counts.remove_buffer(used_entries, dead_entries); + _hold_used_entries += *used_entries; } void -BufferTypeBase::onFree(ElemCount usedElems) +BufferTypeBase::on_free(EntryCount used_entries) { --_holdBuffers; - assert(_holdUsedElems >= usedElems); - _holdUsedElems -= usedElems; + assert(_hold_used_entries >= used_entries); + _hold_used_entries -= used_entries; } void -BufferTypeBase::resume_primary_buffer(uint32_t buffer_id, std::atomic<ElemCount>* used_elems, std::atomic<ElemCount>* dead_elems) +BufferTypeBase::resume_primary_buffer(uint32_t buffer_id, std::atomic<EntryCount>* used_entries, std::atomic<EntryCount>* dead_entries) { auto itr = std::find(_active_buffers.begin(), _active_buffers.end(), buffer_id); assert(itr != _active_buffers.end()); _active_buffers.erase(itr); _active_buffers.emplace_back(buffer_id); - _aggr_counts.remove_buffer(used_elems, dead_elems); - _aggr_counts.add_buffer(used_elems, dead_elems); + _aggr_counts.remove_buffer(used_entries, dead_entries); + _aggr_counts.add_buffer(used_entries, dead_entries); } const alloc::MemoryAllocator* @@ -115,17 +115,17 @@ BufferTypeBase::get_memory_allocator() const } void -BufferTypeBase::clampMaxArrays(uint32_t maxArrays) +BufferTypeBase::clamp_max_entries(uint32_t max_entries) { - _maxArrays = std::min(_maxArrays, maxArrays); - _minArrays = std::min(_minArrays, _maxArrays); - _numArraysForNewBuffer = std::min(_numArraysForNewBuffer, _maxArrays); + _max_entries = std::min(_max_entries, max_entries); + _min_entries = std::min(_min_entries, _max_entries); + _num_entries_for_new_buffer = std::min(_num_entries_for_new_buffer, _max_entries); } size_t -BufferTypeBase::calcArraysToAlloc(uint32_t bufferId, ElemCount elemsNeeded, bool resizing) const +BufferTypeBase::calc_entries_to_alloc(uint32_t bufferId, EntryCount free_entries_needed, bool resizing) const { - size_t reservedElems = getReservedElements(bufferId); + size_t reserved_entries = get_reserved_entries(bufferId); BufferCounts last_bc; BufferCounts bc; if (resizing) { @@ -134,56 +134,53 @@ BufferTypeBase::calcArraysToAlloc(uint32_t bufferId, ElemCount elemsNeeded, bool } } bc = _aggr_counts.all_buffers(); - assert((bc.used_elems % _arraySize) == 0); - assert((bc.dead_elems % _arraySize) == 0); - assert(bc.used_elems >= bc.dead_elems); - size_t neededArrays = (elemsNeeded + (resizing ? last_bc.used_elems : reservedElems) + _arraySize - 1) / _arraySize; - - size_t liveArrays = (bc.used_elems - bc.dead_elems) / _arraySize; - size_t growArrays = (liveArrays * _allocGrowFactor); - size_t usedArrays = last_bc.used_elems / _arraySize; - size_t wantedArrays = std::max((resizing ? usedArrays : 0u) + growArrays, - static_cast<size_t>(_minArrays)); - - size_t result = wantedArrays; - if (result < neededArrays) { - result = neededArrays; + assert(bc.used_entries >= bc.dead_entries); + size_t needed_entries = static_cast<size_t>(free_entries_needed) + (resizing ? last_bc.used_entries : reserved_entries); + size_t live_entries = (bc.used_entries - bc.dead_entries); + size_t grow_entries = (live_entries * _allocGrowFactor); + size_t used_entries = last_bc.used_entries; + size_t wanted_entries = std::max((resizing ? used_entries : 0u) + grow_entries, + static_cast<size_t>(_min_entries)); + + size_t result = wanted_entries; + if (result < needed_entries) { + result = needed_entries; } - if (result > _maxArrays) { - result = _maxArrays; + if (result > _max_entries) { + result = _max_entries; } - if (result < neededArrays) { + if (result < needed_entries) { vespalib::asciistream s; s << "BufferTypeBase::calcArraysToAlloc(" << "bufferId=" << bufferId << - ",elemsNeeeded=" << elemsNeeded << + ",free_entries_needed=" << free_entries_needed << ",resizing=" << (resizing ? "true" : "false") << ")" << - " wantedArrays=" << wantedArrays << + " wanted_entries=" << wanted_entries << ", _arraySize=" << _arraySize << - ", _maxArrays=" << _maxArrays << - ", reservedElems=" << reservedElems << - ", liveArrays=" << liveArrays << - ", growArrays=" << growArrays << - ", usedArrays=" << usedArrays << + ", _max_entries=" << _max_entries << + ", reserved_entries=" << reserved_entries << + ", live_entries=" << live_entries << + ", grow_entries=" << grow_entries << + ", used_entries=" << used_entries << ", typeid(*this).name=\"" << typeid(*this).name() << "\"" << - ", newArrays=" << result << - " < neededArrays=" << neededArrays;; + ", new_entries=" << result << + " < needed_entries=" << needed_entries; throw vespalib::OverflowException(s.c_str()); } return result; } uint32_t -BufferTypeBase::get_scaled_num_arrays_for_new_buffer() const +BufferTypeBase::get_scaled_num_entries_for_new_buffer() const { uint32_t active_buffers_count = get_active_buffers_count(); - if (active_buffers_count <= 1u || _numArraysForNewBuffer == 0u) { - return _numArraysForNewBuffer; + if (active_buffers_count <= 1u || _num_entries_for_new_buffer == 0u) { + return _num_entries_for_new_buffer; } double scale_factor = std::pow(1.0 + _allocGrowFactor, active_buffers_count - 1); - double scaled_result = _numArraysForNewBuffer * scale_factor; - if (scaled_result >= _maxArrays) { - return _maxArrays; + double scaled_result = _num_entries_for_new_buffer * scale_factor; + if (scaled_result >= _max_entries) { + return _max_entries; } return scaled_result; } @@ -194,22 +191,22 @@ BufferTypeBase::AggregatedBufferCounts::AggregatedBufferCounts() } void -BufferTypeBase::AggregatedBufferCounts::add_buffer(const std::atomic<ElemCount>* used_elems, const std::atomic<ElemCount>* dead_elems) +BufferTypeBase::AggregatedBufferCounts::add_buffer(const std::atomic<EntryCount>* used_entries, const std::atomic<EntryCount>* dead_entries) { for (const auto& elem : _counts) { - assert(elem.used_ptr != used_elems); - assert(elem.dead_ptr != dead_elems); + assert(elem.used_ptr != used_entries); + assert(elem.dead_ptr != dead_entries); } - _counts.emplace_back(used_elems, dead_elems); + _counts.emplace_back(used_entries, dead_entries); } void -BufferTypeBase::AggregatedBufferCounts::remove_buffer(const std::atomic<ElemCount>* used_elems, const std::atomic<ElemCount>* dead_elems) +BufferTypeBase::AggregatedBufferCounts::remove_buffer(const std::atomic<EntryCount>* used_entries, const std::atomic<EntryCount>* dead_entries) { auto itr = std::find_if(_counts.begin(), _counts.end(), - [=](const auto& elem){ return elem.used_ptr == used_elems; }); + [=](const auto& elem){ return elem.used_ptr == used_entries; }); assert(itr != _counts.end()); - assert(itr->dead_ptr == dead_elems); + assert(itr->dead_ptr == dead_entries); _counts.erase(itr); } @@ -219,8 +216,8 @@ BufferTypeBase::AggregatedBufferCounts::last_buffer() const BufferCounts result; assert(!_counts.empty()); const auto& last = _counts.back(); - result.used_elems += last.used_ptr->load(std::memory_order_relaxed); - result.dead_elems += last.dead_ptr->load(std::memory_order_relaxed); + result.used_entries += last.used_ptr->load(std::memory_order_relaxed); + result.dead_entries += last.dead_ptr->load(std::memory_order_relaxed); return result; } @@ -229,8 +226,8 @@ BufferTypeBase::AggregatedBufferCounts::all_buffers() const { BufferCounts result; for (const auto& elem : _counts) { - result.used_elems += elem.used_ptr->load(std::memory_order_relaxed); - result.dead_elems += elem.dead_ptr->load(std::memory_order_relaxed); + result.used_entries += elem.used_ptr->load(std::memory_order_relaxed); + result.dead_entries += elem.dead_ptr->load(std::memory_order_relaxed); } return result; } diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_type.h b/vespalib/src/vespa/vespalib/datastore/buffer_type.h index bedbb2c984e..ea52b026228 100644 --- a/vespalib/src/vespa/vespalib/datastore/buffer_type.h +++ b/vespalib/src/vespa/vespalib/datastore/buffer_type.h @@ -10,7 +10,7 @@ namespace vespalib::alloc { class MemoryAllocator; } namespace vespalib::datastore { -using ElemCount = uint64_t; +using EntryCount = uint32_t; /** * Abstract class used to manage allocation and de-allocation of a specific data type in underlying memory buffers in a data store. @@ -22,7 +22,7 @@ using ElemCount = uint64_t; class BufferTypeBase { public: - using ElemCount = vespalib::datastore::ElemCount; + using EntryCount = vespalib::datastore::EntryCount; class CleanContext { private: std::atomic<size_t> &_extraUsedBytes; @@ -39,53 +39,52 @@ public: BufferTypeBase & operator=(const BufferTypeBase &rhs) = delete; BufferTypeBase(BufferTypeBase &&rhs) noexcept = default; BufferTypeBase & operator=(BufferTypeBase &&rhs) noexcept = default; - BufferTypeBase(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays) noexcept; - BufferTypeBase(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays, - uint32_t numArraysForNewBuffer, float allocGrowFactor) noexcept; + BufferTypeBase(uint32_t arraySize, uint32_t min_entries, uint32_t max_entries) noexcept; + BufferTypeBase(uint32_t arraySize, uint32_t min_entries, uint32_t max_entries, + uint32_t num_entries_for_new_buffer, float allocGrowFactor) noexcept; virtual ~BufferTypeBase(); - virtual void destroyElements(void *buffer, ElemCount numElems) = 0; - virtual void fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems) = 0; + virtual void destroy_entries(void *buffer, EntryCount num_entries) = 0; + virtual void fallback_copy(void *newBuffer, const void *oldBuffer, EntryCount num_entries) = 0; /** - * Return number of reserved elements at start of buffer, to avoid - * invalid reference and handle data at negative offset (alignment - * hacks) as used by dense tensor store. + * Return number of reserved entries at start of buffer, to avoid + * invalid reference. */ - virtual ElemCount getReservedElements(uint32_t bufferId) const; + virtual EntryCount get_reserved_entries(uint32_t bufferId) const; /** * Initialize reserved elements at start of buffer. */ - virtual void initializeReservedElements(void *buffer, ElemCount reservedElements) = 0; - virtual size_t elementSize() const = 0; - virtual void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) = 0; + virtual void initialize_reserved_entries(void *buffer, EntryCount reserved_entries) = 0; + virtual size_t entry_size() const = 0; // Size of entry measured in bytes + virtual void clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx) = 0; size_t getArraySize() const { return _arraySize; } - virtual void onActive(uint32_t bufferId, std::atomic<ElemCount>* usedElems, std::atomic<ElemCount>* deadElems, void* buffer); - void onHold(uint32_t buffer_id, const std::atomic<ElemCount>* usedElems, const std::atomic<ElemCount>* deadElems); - virtual void onFree(ElemCount usedElems); - void resume_primary_buffer(uint32_t buffer_id, std::atomic<ElemCount>* used_elems, std::atomic<ElemCount>* dead_elems); + virtual void on_active(uint32_t bufferId, std::atomic<EntryCount>* used_entries, std::atomic<EntryCount>* dead_entries, void* buffer); + void on_hold(uint32_t buffer_id, const std::atomic<EntryCount>* used_entries, const std::atomic<EntryCount>* dead_entries); + virtual void on_free(EntryCount used_entries); + void resume_primary_buffer(uint32_t buffer_id, std::atomic<EntryCount>* used_entries, std::atomic<EntryCount>* dead_entries); virtual const alloc::MemoryAllocator* get_memory_allocator() const; /** - * Calculate number of arrays to allocate for new buffer given how many elements are needed. + * Calculate number of entries to allocate for new buffer given how many free entries are needed. */ - virtual size_t calcArraysToAlloc(uint32_t bufferId, ElemCount elementsNeeded, bool resizing) const; + virtual size_t calc_entries_to_alloc(uint32_t bufferId, EntryCount free_entries_needed, bool resizing) const; - void clampMaxArrays(uint32_t maxArrays); + void clamp_max_entries(uint32_t max_entries); uint32_t get_active_buffers_count() const { return _active_buffers.size(); } const std::vector<uint32_t>& get_active_buffers() const noexcept { return _active_buffers; } - size_t getMaxArrays() const { return _maxArrays; } - uint32_t get_scaled_num_arrays_for_new_buffer() const; - uint32_t get_num_arrays_for_new_buffer() const noexcept { return _numArraysForNewBuffer; } + size_t get_max_entries() const { return _max_entries; } + uint32_t get_scaled_num_entries_for_new_buffer() const; + uint32_t get_num_entries_for_new_buffer() const noexcept { return _num_entries_for_new_buffer; } protected: struct BufferCounts { - ElemCount used_elems; - ElemCount dead_elems; - BufferCounts() : used_elems(0), dead_elems(0) {} - BufferCounts(ElemCount used_elems_in, ElemCount dead_elems_in) - : used_elems(used_elems_in), dead_elems(dead_elems_in) + EntryCount used_entries; + EntryCount dead_entries; + BufferCounts() : used_entries(0), dead_entries(0) {} + BufferCounts(EntryCount used_entries_in, EntryCount dead_entries_in) + : used_entries(used_entries_in), dead_entries(dead_entries_in) {} }; @@ -94,45 +93,48 @@ protected: */ class AggregatedBufferCounts { private: - struct Element { - const std::atomic<ElemCount>* used_ptr; - const std::atomic<ElemCount>* dead_ptr; - Element() noexcept : used_ptr(nullptr), dead_ptr(nullptr) {} - Element(const std::atomic<ElemCount>* used_ptr_in, const std::atomic<ElemCount>* dead_ptr_in) noexcept + struct ActiveBufferCounts { + const std::atomic<EntryCount>* used_ptr; + const std::atomic<EntryCount>* dead_ptr; + ActiveBufferCounts() noexcept : used_ptr(nullptr), dead_ptr(nullptr) {} + ActiveBufferCounts(const std::atomic<EntryCount>* used_ptr_in, const std::atomic<EntryCount>* dead_ptr_in) noexcept : used_ptr(used_ptr_in), dead_ptr(dead_ptr_in) {} }; - std::vector<Element> _counts; + std::vector<ActiveBufferCounts> _counts; public: AggregatedBufferCounts(); - void add_buffer(const std::atomic<ElemCount>* used_elems, const std::atomic<ElemCount>* dead_elems); - void remove_buffer(const std::atomic<ElemCount>* used_elems, const std::atomic<ElemCount>* dead_elems); + void add_buffer(const std::atomic<EntryCount>* used_entries, const std::atomic<EntryCount>* dead_entries); + void remove_buffer(const std::atomic<EntryCount>* used_entries, const std::atomic<EntryCount>* dead_entries); BufferCounts last_buffer() const; BufferCounts all_buffers() const; bool empty() const { return _counts.empty(); } }; uint32_t _arraySize; // Number of elements in an allocation unit - uint32_t _minArrays; // Minimum number of arrays to allocate in a buffer - uint32_t _maxArrays; // Maximum number of arrays to allocate in a buffer - // Number of arrays needed before allocating a new buffer instead of just resizing the first one - uint32_t _numArraysForNewBuffer; + uint32_t _min_entries; // Minimum number of entries to allocate in a buffer + uint32_t _max_entries; // Maximum number of entries to allocate in a buffer + // Number of entries needed before allocating a new buffer instead of just resizing the first one + uint32_t _num_entries_for_new_buffer; float _allocGrowFactor; uint32_t _holdBuffers; - size_t _holdUsedElems; // Number of used elements in all held buffers for this type. + size_t _hold_used_entries; // Number of used entries in all held buffers for this type. AggregatedBufferCounts _aggr_counts; std::vector<uint32_t> _active_buffers; }; /** - * Concrete class used to manage allocation and de-allocation of elements of type EntryType in data store buffers. + * Concrete class used to manage allocation and de-allocation of elements of type ElemType in data store buffers. */ -template <typename EntryType, typename EmptyType = EntryType> +template <typename ElemT, typename EmptyT = ElemT> class BufferType : public BufferTypeBase { +public: + using ElemType = ElemT; + using EmptyType = EmptyT; protected: - static const EntryType& empty_entry() noexcept; + static const ElemType& empty_entry() noexcept; public: BufferType() noexcept : BufferType(1,1,1) {} @@ -140,15 +142,15 @@ public: BufferType & operator=(const BufferType &rhs) = delete; BufferType(BufferType && rhs) noexcept = default; BufferType & operator=(BufferType && rhs) noexcept = default; - BufferType(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays) noexcept; - BufferType(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays, - uint32_t numArraysForNewBuffer, float allocGrowFactor) noexcept; + BufferType(uint32_t arraySize, uint32_t min_entries, uint32_t max_entries) noexcept; + BufferType(uint32_t arraySize, uint32_t min_entries, uint32_t max_entries, + uint32_t num_entries_for_new_buffer, float allocGrowFactor) noexcept; ~BufferType() override; - void destroyElements(void *buffer, ElemCount numElems) override; - void fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems) override; - void initializeReservedElements(void *buffer, ElemCount reservedElements) override; - void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCxt) override; - size_t elementSize() const override { return sizeof(EntryType); } + void destroy_entries(void *buffer, EntryCount num_entries) override; + void fallback_copy(void *newBuffer, const void *oldBuffer, EntryCount num_entries) override; + void initialize_reserved_entries(void *buffer, EntryCount reserved_entries) override; + void clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext cleanCxt) override; + size_t entry_size() const override { return sizeof(ElemType) * _arraySize; } }; extern template class BufferType<char>; diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_type.hpp b/vespalib/src/vespa/vespalib/datastore/buffer_type.hpp index 72c8f574a70..60acca5ff39 100644 --- a/vespalib/src/vespa/vespalib/datastore/buffer_type.hpp +++ b/vespalib/src/vespa/vespalib/datastore/buffer_type.hpp @@ -6,78 +6,80 @@ namespace vespalib::datastore { -template <typename EntryType, typename EmptyType> -BufferType<EntryType, EmptyType>::BufferType(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays) noexcept - : BufferTypeBase(arraySize, minArrays, maxArrays) +template <typename ElemT, typename EmptyT> +BufferType<ElemT, EmptyT>::BufferType(uint32_t arraySize, uint32_t min_entries, uint32_t max_entries) noexcept + : BufferTypeBase(arraySize, min_entries, max_entries) { } -template <typename EntryType, typename EmptyType> -BufferType<EntryType, EmptyType>::BufferType(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays, - uint32_t numArraysForNewBuffer, float allocGrowFactor) noexcept - : BufferTypeBase(arraySize, minArrays, maxArrays, numArraysForNewBuffer, allocGrowFactor) +template <typename ElemT, typename EmptyT> +BufferType<ElemT, EmptyT>::BufferType(uint32_t arraySize, uint32_t min_entries, uint32_t max_entries, + uint32_t num_entries_for_new_buffer, float allocGrowFactor) noexcept + : BufferTypeBase(arraySize, min_entries, max_entries, num_entries_for_new_buffer, allocGrowFactor) { } -template <typename EntryType, typename EmptyType> -BufferType<EntryType, EmptyType>::~BufferType() = default; +template <typename ElemT, typename EmptyT> +BufferType<ElemT, EmptyT>::~BufferType() = default; -template <typename EntryType, typename EmptyType> +template <typename ElemT, typename EmptyT> void -BufferType<EntryType, EmptyType>::destroyElements(void *buffer, ElemCount numElems) +BufferType<ElemT, EmptyT>::destroy_entries(void *buffer, EntryCount num_entries) { - EntryType *e = static_cast<EntryType *>(buffer); - for (size_t j = numElems; j != 0; --j) { - e->~EntryType(); + auto num_elems = num_entries * getArraySize(); + ElemType *e = static_cast<ElemType *>(buffer); + for (size_t j = num_elems; j != 0; --j) { + e->~ElemType(); ++e; } } -template <typename EntryType, typename EmptyType> +template <typename ElemT, typename EmptyT> void -BufferType<EntryType, EmptyType>::fallbackCopy(void *newBuffer, - const void *oldBuffer, - ElemCount numElems) +BufferType<ElemT, EmptyT>::fallback_copy(void *newBuffer, const void *oldBuffer, EntryCount num_entries) { - EntryType *d = static_cast<EntryType *>(newBuffer); - const EntryType *s = static_cast<const EntryType *>(oldBuffer); - for (size_t j = numElems; j != 0; --j) { - new (static_cast<void *>(d)) EntryType(*s); + auto num_elems = num_entries * getArraySize(); + ElemType *d = static_cast<ElemType *>(newBuffer); + const ElemType *s = static_cast<const ElemType *>(oldBuffer); + for (size_t j = num_elems; j != 0; --j) { + new (static_cast<void *>(d)) ElemType(*s); ++s; ++d; } } -template <typename EntryType, typename EmptyType> +template <typename ElemT, typename EmptyT> void -BufferType<EntryType, EmptyType>::initializeReservedElements(void *buffer, ElemCount reservedElems) +BufferType<ElemT, EmptyT>::initialize_reserved_entries(void *buffer, EntryCount reserved_entries) { - EntryType *e = static_cast<EntryType *>(buffer); + auto reserved_elems = reserved_entries * getArraySize(); + ElemType *e = static_cast<ElemType *>(buffer); const auto& empty = empty_entry(); - for (size_t j = reservedElems; j != 0; --j) { - new (static_cast<void *>(e)) EntryType(empty); + for (size_t j = reserved_elems; j != 0; --j) { + new (static_cast<void *>(e)) ElemType(empty); ++e; } } -template <typename EntryType, typename EmptyType> +template <typename ElemT, typename EmptyT> void -BufferType<EntryType, EmptyType>::cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext) +BufferType<ElemT, EmptyT>::clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext) { - EntryType *e = static_cast<EntryType *>(buffer) + offset; + auto num_elems = num_entries * getArraySize(); + ElemType *e = static_cast<ElemType *>(buffer) + offset * getArraySize(); const auto& empty = empty_entry(); - for (size_t j = numElems; j != 0; --j) { + for (size_t j = num_elems; j != 0; --j) { *e = empty; ++e; } } -template <typename EntryType, typename EmptyType> -const EntryType& -BufferType<EntryType, EmptyType>::empty_entry() noexcept +template <typename ElemT, typename EmptyT> +const ElemT& +BufferType<ElemT, EmptyT>::empty_entry() noexcept { - // It's possible for EntryType to wrap e.g. an Alloc instance, which has a transitive + // It's possible for ElemType to wrap e.g. an Alloc instance, which has a transitive // dependency on globally constructed allocator object(s). To avoid issues with global // construction order, initialize the sentinel on the first access. - static EntryType empty = EmptyType(); + static ElemType empty = EmptyType(); return empty; } diff --git a/vespalib/src/vespa/vespalib/datastore/bufferstate.cpp b/vespalib/src/vespa/vespalib/datastore/bufferstate.cpp index 47fba1ef697..f312596d6f7 100644 --- a/vespalib/src/vespa/vespalib/datastore/bufferstate.cpp +++ b/vespalib/src/vespa/vespalib/datastore/bufferstate.cpp @@ -12,13 +12,13 @@ namespace vespalib::datastore { BufferState::BufferState() : _stats(), - _free_list(_stats.dead_elems_ref()), + _free_list(_stats.dead_entries_ref()), _typeHandler(nullptr), _buffer(Alloc::alloc(0, MemoryAllocator::HUGEPAGE_SIZE)), _arraySize(0), _typeId(0), _state(State::FREE), - _disableElemHoldList(false), + _disable_entry_hold_list(false), _compacting(false) { } @@ -28,15 +28,15 @@ BufferState::~BufferState() assert(getState() == State::FREE); assert(!_free_list.enabled()); assert(_free_list.empty()); - assert(_stats.hold_elems() == 0); + assert(_stats.hold_entries() == 0); } namespace { struct AllocResult { - size_t elements; + size_t entries; size_t bytes; - AllocResult(size_t elements_, size_t bytes_) : elements(elements_), bytes(bytes_) {} + AllocResult(size_t entries_, size_t bytes_) : entries(entries_), bytes(bytes_) {} }; size_t @@ -57,30 +57,30 @@ roundUpToMatchAllocator(size_t sz) } AllocResult -calcAllocation(uint32_t bufferId, - BufferTypeBase &typeHandler, - size_t elementsNeeded, - bool resizing) +calc_allocation(uint32_t bufferId, + BufferTypeBase &typeHandler, + size_t free_entries_needed, + bool resizing) { - size_t allocArrays = typeHandler.calcArraysToAlloc(bufferId, elementsNeeded, resizing); - size_t allocElements = allocArrays * typeHandler.getArraySize(); - size_t allocBytes = roundUpToMatchAllocator(allocElements * typeHandler.elementSize()); - size_t maxAllocBytes = typeHandler.getMaxArrays() * typeHandler.getArraySize() * typeHandler.elementSize(); + size_t alloc_entries = typeHandler.calc_entries_to_alloc(bufferId, free_entries_needed, resizing); + size_t entry_size = typeHandler.entry_size(); + size_t allocBytes = roundUpToMatchAllocator(alloc_entries * entry_size); + size_t maxAllocBytes = typeHandler.get_max_entries() * entry_size; if (allocBytes > maxAllocBytes) { // Ensure that allocated bytes does not exceed the maximum handled by this type. allocBytes = maxAllocBytes; } - size_t adjustedAllocElements = (allocBytes / typeHandler.elementSize()); - return AllocResult(adjustedAllocElements, allocBytes); + size_t adjusted_alloc_entries = allocBytes / entry_size; + return AllocResult(adjusted_alloc_entries, allocBytes); } } void -BufferState::onActive(uint32_t bufferId, uint32_t typeId, - BufferTypeBase *typeHandler, - size_t elementsNeeded, - std::atomic<void*>& buffer) +BufferState::on_active(uint32_t bufferId, uint32_t typeId, + BufferTypeBase *typeHandler, + size_t free_entries_needed, + std::atomic<void*>& buffer) { assert(buffer.load(std::memory_order_relaxed) == nullptr); assert(_buffer.get() == nullptr); @@ -88,30 +88,29 @@ BufferState::onActive(uint32_t bufferId, uint32_t typeId, assert(_typeHandler == nullptr); assert(capacity() == 0); assert(size() == 0); - assert(_stats.dead_elems() == 0u); - assert(_stats.hold_elems() == 0); + assert(_stats.dead_entries() == 0u); + assert(_stats.hold_entries() == 0); assert(_stats.extra_used_bytes() == 0); assert(_stats.extra_hold_bytes() == 0); assert(_free_list.empty()); - size_t reservedElements = typeHandler->getReservedElements(bufferId); - (void) reservedElements; - AllocResult alloc = calcAllocation(bufferId, *typeHandler, elementsNeeded, false); - assert(alloc.elements >= reservedElements + elementsNeeded); + size_t reserved_entries = typeHandler->get_reserved_entries(bufferId); + (void) reserved_entries; + AllocResult alloc = calc_allocation(bufferId, *typeHandler, free_entries_needed, false); + assert(alloc.entries >= reserved_entries + free_entries_needed); auto allocator = typeHandler->get_memory_allocator(); _buffer = (allocator != nullptr) ? Alloc::alloc_with_allocator(allocator) : Alloc::alloc(0, MemoryAllocator::HUGEPAGE_SIZE); _buffer.create(alloc.bytes).swap(_buffer); - assert(_buffer.get() != nullptr || alloc.elements == 0u); + assert(_buffer.get() != nullptr || alloc.entries == 0u); buffer.store(_buffer.get(), std::memory_order_release); - _stats.set_alloc_elems(alloc.elements); + _stats.set_alloc_entries(alloc.entries); _typeHandler.store(typeHandler, std::memory_order_release); assert(typeId <= std::numeric_limits<uint16_t>::max()); _typeId = typeId; _arraySize = typeHandler->getArraySize(); - _free_list.set_array_size(_arraySize); _state.store(State::ACTIVE, std::memory_order_release); - typeHandler->onActive(bufferId, &_stats.used_elems_ref(), &_stats.dead_elems_ref(), - buffer.load(std::memory_order::relaxed)); + typeHandler->on_active(bufferId, &_stats.used_entries_ref(), &_stats.dead_entries_ref(), + buffer.load(std::memory_order::relaxed)); } void @@ -121,11 +120,11 @@ BufferState::onHold(uint32_t buffer_id) assert(getTypeHandler() != nullptr); _state.store(State::HOLD, std::memory_order_release); _compacting = false; - assert(_stats.dead_elems() <= size()); - assert(_stats.hold_elems() <= (size() - _stats.dead_elems())); - _stats.set_dead_elems(0); - _stats.set_hold_elems(size()); - getTypeHandler()->onHold(buffer_id, &_stats.used_elems_ref(), &_stats.dead_elems_ref()); + assert(_stats.dead_entries() <= size()); + assert(_stats.hold_entries() <= (size() - _stats.dead_entries())); + _stats.set_dead_entries(0); + _stats.set_hold_entries(size()); + getTypeHandler()->on_hold(buffer_id, &_stats.used_entries_ref(), &_stats.dead_entries_ref()); _free_list.disable(); } @@ -135,20 +134,19 @@ BufferState::onFree(std::atomic<void*>& buffer) assert(buffer.load(std::memory_order_relaxed) == _buffer.get()); assert(getState() == State::HOLD); assert(_typeHandler != nullptr); - assert(_stats.dead_elems() <= size()); - assert(_stats.hold_elems() == (size() - _stats.dead_elems())); - getTypeHandler()->destroyElements(buffer, size()); + assert(_stats.dead_entries() <= size()); + assert(_stats.hold_entries() == (size() - _stats.dead_entries())); + getTypeHandler()->destroy_entries(buffer, size()); Alloc::alloc().swap(_buffer); - getTypeHandler()->onFree(size()); + getTypeHandler()->on_free(size()); buffer.store(nullptr, std::memory_order_release); _stats.clear(); _state.store(State::FREE, std::memory_order_release); _typeHandler = nullptr; _arraySize = 0; - _free_list.set_array_size(_arraySize); assert(!_free_list.enabled()); assert(_free_list.empty()); - _disableElemHoldList = false; + _disable_entry_hold_list = false; } @@ -171,67 +169,67 @@ BufferState::dropBuffer(uint32_t buffer_id, std::atomic<void*>& buffer) } void -BufferState::disable_elem_hold_list() +BufferState::disable_entry_hold_list() { - _disableElemHoldList = true; + _disable_entry_hold_list = true; } bool -BufferState::hold_elems(size_t num_elems, size_t extra_bytes) +BufferState::hold_entries(size_t num_entries, size_t extra_bytes) { assert(isActive()); - if (_disableElemHoldList) { + if (_disable_entry_hold_list) { // The elements are directly marked as dead as they are not put on hold. - _stats.inc_dead_elems(num_elems); + _stats.inc_dead_entries(num_entries); return true; } - _stats.inc_hold_elems(num_elems); + _stats.inc_hold_entries(num_entries); _stats.inc_extra_hold_bytes(extra_bytes); return false; } void -BufferState::free_elems(EntryRef ref, size_t num_elems, size_t ref_offset) +BufferState::free_entries(EntryRef ref, size_t num_entries, size_t ref_offset) { if (isActive()) { - if (_free_list.enabled() && (num_elems == getArraySize())) { + if (_free_list.enabled() && (num_entries == 1)) { _free_list.push_entry(ref); } } else { assert(isOnHold()); } - _stats.inc_dead_elems(num_elems); - _stats.dec_hold_elems(num_elems); - getTypeHandler()->cleanHold(_buffer.get(), (ref_offset * _arraySize), num_elems, - BufferTypeBase::CleanContext(_stats.extra_used_bytes_ref(), - _stats.extra_hold_bytes_ref())); + _stats.inc_dead_entries(num_entries); + _stats.dec_hold_entries(num_entries); + getTypeHandler()->clean_hold(_buffer.get(), ref_offset, num_entries, + BufferTypeBase::CleanContext(_stats.extra_used_bytes_ref(), + _stats.extra_hold_bytes_ref())); } void -BufferState::fallbackResize(uint32_t bufferId, - size_t elementsNeeded, +BufferState::fallback_resize(uint32_t bufferId, + size_t free_entries_needed, std::atomic<void*>& buffer, Alloc &holdBuffer) { assert(getState() == State::ACTIVE); assert(_typeHandler != nullptr); assert(holdBuffer.get() == nullptr); - AllocResult alloc = calcAllocation(bufferId, *_typeHandler, elementsNeeded, true); - assert(alloc.elements >= size() + elementsNeeded); - assert(alloc.elements > capacity()); + AllocResult alloc = calc_allocation(bufferId, *_typeHandler, free_entries_needed, true); + assert(alloc.entries >= size() + free_entries_needed); + assert(alloc.entries > capacity()); Alloc newBuffer = _buffer.create(alloc.bytes); - getTypeHandler()->fallbackCopy(newBuffer.get(), buffer.load(std::memory_order_relaxed), size()); + getTypeHandler()->fallback_copy(newBuffer.get(), buffer.load(std::memory_order_relaxed), size()); holdBuffer.swap(_buffer); std::atomic_thread_fence(std::memory_order_release); _buffer = std::move(newBuffer); buffer.store(_buffer.get(), std::memory_order_release); - _stats.set_alloc_elems(alloc.elements); + _stats.set_alloc_entries(alloc.entries); } void BufferState::resume_primary_buffer(uint32_t buffer_id) { - getTypeHandler()->resume_primary_buffer(buffer_id, &_stats.used_elems_ref(), &_stats.dead_elems_ref()); + getTypeHandler()->resume_primary_buffer(buffer_id, &_stats.used_entries_ref(), &_stats.dead_entries_ref()); } } diff --git a/vespalib/src/vespa/vespalib/datastore/bufferstate.h b/vespalib/src/vespa/vespalib/datastore/bufferstate.h index 5b98099ed69..f714f8e24d5 100644 --- a/vespalib/src/vespa/vespalib/datastore/bufferstate.h +++ b/vespalib/src/vespa/vespalib/datastore/bufferstate.h @@ -23,8 +23,8 @@ namespace vespalib::datastore { * It is kept in this state until all reader threads are no longer accessing the buffer. * Finally, it transitions back to FREE via onFree() and memory is de-allocated. * - * This class also supports use of free lists, where previously allocated elements in the buffer can be re-used. - * First the element is put on hold, then on the free list (counted as dead) to be re-used. + * This class also supports use of free lists, where previously allocated entries in the buffer can be re-used. + * First the entry is put on hold, then on the free list (counted as dead) to be re-used. */ class BufferState { @@ -45,7 +45,7 @@ private: uint32_t _arraySize; uint16_t _typeId; std::atomic<State> _state; - bool _disableElemHoldList : 1; + bool _disable_entry_hold_list : 1; bool _compacting : 1; public: @@ -62,14 +62,14 @@ public: /** * Transition from FREE to ACTIVE state. * - * @param bufferId Id of buffer to be active. - * @param typeId Registered data type id for buffer. - * @param typeHandler Type handler for registered data type. - * @param elementsNeeded Number of elements needed to be free in the memory allocated. - * @param buffer Start of allocated buffer return value. + * @param bufferId Id of buffer to be active. + * @param typeId Registered data type id for buffer. + * @param typeHandler Type handler for registered data type. + * @param free_entries_needed Number of entries needed to be free in the memory allocated. + * @param buffer Start of allocated buffer return value. */ - void onActive(uint32_t bufferId, uint32_t typeId, BufferTypeBase *typeHandler, - size_t elementsNeeded, std::atomic<void*>& buffer); + void on_active(uint32_t bufferId, uint32_t typeId, BufferTypeBase *typeHandler, + size_t free_entries_needed, std::atomic<void*>& buffer); /** * Transition from ACTIVE to HOLD state. @@ -82,24 +82,24 @@ public: void onFree(std::atomic<void*>& buffer); /** - * Disable hold of elements, just mark elements as dead without cleanup. + * Disable hold of entries, just mark entries as dead without cleanup. * Typically used when tearing down data structure in a controlled manner. */ - void disable_elem_hold_list(); + void disable_entry_hold_list(); /** - * Update stats to reflect that the given elements are put on hold. - * Returns true if element hold list is disabled for this buffer. + * Update stats to reflect that the given entries are put on hold. + * Returns true if entry hold list is disabled for this buffer. */ - bool hold_elems(size_t num_elems, size_t extra_bytes); + bool hold_entries(size_t num_entries, size_t extra_bytes); /** - * Free the given elements and update stats accordingly. + * Free the given entries and update stats accordingly. * * The given entry ref is put on the free list (if enabled). - * Hold cleaning of elements is executed on the buffer type. + * Hold cleaning of entries is executed on the buffer type. */ - void free_elems(EntryRef ref, size_t num_elems, size_t ref_offset); + void free_entries(EntryRef ref, size_t num_entries, size_t ref_offset); BufferStats& stats() { return _stats; } const BufferStats& stats() const { return _stats; } @@ -115,8 +115,7 @@ public: uint32_t getArraySize() const { return _arraySize; } bool getCompacting() const { return _compacting; } void setCompacting() { _compacting = true; } - uint32_t get_used_arrays() const noexcept { return size() / _arraySize; } - void fallbackResize(uint32_t bufferId, size_t elementsNeeded, std::atomic<void*>& buffer, Alloc &holdBuffer); + void fallback_resize(uint32_t bufferId, size_t free_entries_needed, std::atomic<void*>& buffer, Alloc &holdBuffer); bool isActive(uint32_t typeId) const { return (isActive() && (_typeId == typeId)); diff --git a/vespalib/src/vespa/vespalib/datastore/datastore.h b/vespalib/src/vespa/vespalib/datastore/datastore.h index 01b81d0fa58..f81348ce287 100644 --- a/vespalib/src/vespa/vespalib/datastore/datastore.h +++ b/vespalib/src/vespa/vespalib/datastore/datastore.h @@ -27,7 +27,7 @@ template <typename RefT = EntryRefT<22> > class DataStoreT : public DataStoreBase { private: - void free_elem_internal(EntryRef ref, size_t numElems); + void free_entry_internal(EntryRef ref, size_t num_entries); public: using RefType = RefT; @@ -38,12 +38,12 @@ public: ~DataStoreT() override; /** - * Hold element(s). + * Hold entries. */ - void holdElem(EntryRef ref, size_t numElems) { - holdElem(ref, numElems, 0); - } - void holdElem(EntryRef ref, size_t numElems, size_t extraBytes); + void hold_entry(EntryRef ref) { hold_entries(ref, 1, 0); } + void hold_entry(EntryRef ref, size_t extra_bytes) { hold_entries(ref, 1, extra_bytes); } + void hold_entries(EntryRef ref, size_t num_entries) { hold_entries(ref, num_entries, 0); } + void hold_entries(EntryRef ref, size_t num_entries, size_t extraBytes); void reclaim_entry_refs(generation_t oldest_used_gen) override; @@ -75,7 +75,7 @@ class DataStore : public DataStoreT<RefT> { protected: using ParentType = DataStoreT<RefT>; - using ParentType::ensureBufferCapacity; + using ParentType::ensure_buffer_capacity; using ParentType::getEntry; using ParentType::dropBuffers; using ParentType::init_primary_buffers; diff --git a/vespalib/src/vespa/vespalib/datastore/datastore.hpp b/vespalib/src/vespa/vespalib/datastore/datastore.hpp index bfb63954875..b21a5954eee 100644 --- a/vespalib/src/vespa/vespalib/datastore/datastore.hpp +++ b/vespalib/src/vespa/vespalib/datastore/datastore.hpp @@ -22,21 +22,21 @@ DataStoreT<RefT>::~DataStoreT() = default; template <typename RefT> void -DataStoreT<RefT>::free_elem_internal(EntryRef ref, size_t numElems) +DataStoreT<RefT>::free_entry_internal(EntryRef ref, size_t num_entries) { RefType intRef(ref); BufferState &state = getBufferState(intRef.bufferId()); - state.free_elems(ref, numElems, intRef.offset()); + state.free_entries(ref, num_entries, intRef.offset()); } template <typename RefT> void -DataStoreT<RefT>::holdElem(EntryRef ref, size_t numElems, size_t extraBytes) +DataStoreT<RefT>::hold_entries(EntryRef ref, size_t num_entries, size_t extraBytes) { RefType intRef(ref); BufferState &state = getBufferState(intRef.bufferId()); - if (!state.hold_elems(numElems, extraBytes)) { - _entry_ref_hold_list.insert({ref, numElems}); + if (!state.hold_entries(num_entries, extraBytes)) { + _entry_ref_hold_list.insert({ref, num_entries}); } } @@ -45,7 +45,7 @@ void DataStoreT<RefT>::reclaim_entry_refs(generation_t oldest_used_gen) { _entry_ref_hold_list.reclaim(oldest_used_gen, [this](const auto& elem) { - free_elem_internal(elem.ref, elem.num_elems); + free_entry_internal(elem.ref, elem.num_entries); }); } @@ -54,7 +54,7 @@ void DataStoreT<RefT>::reclaim_all_entry_refs() { _entry_ref_hold_list.reclaim_all([this](const auto& elem) { - free_elem_internal(elem.ref, elem.num_elems); + free_entry_internal(elem.ref, elem.num_entries); }); } diff --git a/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp b/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp index a40aa713bca..75ffe855a32 100644 --- a/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp +++ b/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp @@ -40,18 +40,18 @@ constexpr size_t TOO_DEAD_SLACK = 0x4000u; bool primary_buffer_too_dead(const BufferState &state) { - size_t deadElems = state.stats().dead_elems(); - size_t deadBytes = deadElems * state.getArraySize(); - return ((deadBytes >= TOO_DEAD_SLACK) && (deadElems * 2 >= state.size())); + size_t dead_entries = state.stats().dead_entries(); + size_t deadBytes = dead_entries * state.getTypeHandler()->entry_size(); + return ((deadBytes >= TOO_DEAD_SLACK) && (dead_entries * 2 >= state.size())); } } -DataStoreBase::FallbackHold::FallbackHold(size_t bytesSize, BufferState::Alloc &&buffer, size_t usedElems, +DataStoreBase::FallbackHold::FallbackHold(size_t bytesSize, BufferState::Alloc &&buffer, size_t used_entries, BufferTypeBase *typeHandler, uint32_t typeId) : GenerationHeldBase(bytesSize), _buffer(std::move(buffer)), - _usedElems(usedElems), + _used_entries(used_entries), _typeHandler(typeHandler), _typeId(typeId) { @@ -59,7 +59,7 @@ DataStoreBase::FallbackHold::FallbackHold(size_t bytesSize, BufferState::Alloc & DataStoreBase::FallbackHold::~FallbackHold() { - _typeHandler->destroyElements(_buffer.get(), _usedElems); + _typeHandler->destroy_entries(_buffer.get(), _used_entries); } class DataStoreBase::BufferHold : public GenerationHeldBase { @@ -80,7 +80,7 @@ public: } }; -DataStoreBase::DataStoreBase(uint32_t numBuffers, uint32_t offset_bits, size_t maxArrays) +DataStoreBase::DataStoreBase(uint32_t numBuffers, uint32_t offset_bits, size_t max_entries) : _entry_ref_hold_list(), _buffers(numBuffers), _primary_buffer_ids(), @@ -89,12 +89,12 @@ DataStoreBase::DataStoreBase(uint32_t numBuffers, uint32_t offset_bits, size_t m _free_lists(), _compaction_count(0u), _genHolder(), - _maxArrays(maxArrays), + _max_entries(max_entries), _bufferIdLimit(0u), _hold_buffer_count(0u), _offset_bits(offset_bits), _freeListsEnabled(false), - _disableElemHoldList(false), + _disable_entry_hold_list(false), _initializing(false) { } @@ -105,19 +105,19 @@ DataStoreBase::~DataStoreBase() } void -DataStoreBase::switch_primary_buffer(uint32_t typeId, size_t elemsNeeded) +DataStoreBase::switch_primary_buffer(uint32_t typeId, size_t entries_needed) { size_t buffer_id = getFirstFreeBufferId(); if (buffer_id >= getMaxNumBuffers()) { LOG_ABORT(vespalib::make_string("switch_primary_buffer(%u, %zu): did not find a free buffer", - typeId, elemsNeeded).c_str()); + typeId, entries_needed).c_str()); } - onActive(buffer_id, typeId, elemsNeeded); + on_active(buffer_id, typeId, entries_needed); _primary_buffer_ids[typeId] = buffer_id; } bool -DataStoreBase::consider_grow_active_buffer(uint32_t type_id, size_t elems_needed) +DataStoreBase::consider_grow_active_buffer(uint32_t type_id, size_t entries_needed) { auto type_handler = _typeHandlers[type_id]; uint32_t buffer_id = primary_buffer_id(type_id); @@ -126,7 +126,7 @@ DataStoreBase::consider_grow_active_buffer(uint32_t type_id, size_t elems_needed if (active_buffers_count < min_active_buffers) { return false; } - if (type_handler->get_num_arrays_for_new_buffer() == 0u) { + if (type_handler->get_num_entries_for_new_buffer() == 0u) { return false; } assert(!getBufferState(buffer_id).getCompacting()); @@ -146,8 +146,7 @@ DataStoreBase::consider_grow_active_buffer(uint32_t type_id, size_t elems_needed if (checked_active_buffers < min_active_buffers) { return false; } - auto array_size = type_handler->getArraySize(); - if (elems_needed + min_used > type_handler->getMaxArrays() * array_size) { + if (entries_needed + min_used > type_handler->get_max_entries()) { return false; } if (min_buffer_id != buffer_id) { @@ -181,24 +180,22 @@ DataStoreBase::getBufferState(uint32_t buffer_id) noexcept { } void -DataStoreBase::switch_or_grow_primary_buffer(uint32_t typeId, size_t elemsNeeded) +DataStoreBase::switch_or_grow_primary_buffer(uint32_t typeId, size_t entries_needed) { auto typeHandler = _typeHandlers[typeId]; - uint32_t arraySize = typeHandler->getArraySize(); - size_t numArraysForNewBuffer = typeHandler->get_scaled_num_arrays_for_new_buffer(); - size_t numEntriesForNewBuffer = numArraysForNewBuffer * arraySize; + size_t num_entries_for_new_buffer = typeHandler->get_scaled_num_entries_for_new_buffer(); uint32_t bufferId = primary_buffer_id(typeId); - if (elemsNeeded + getBufferState(bufferId).size() >= numEntriesForNewBuffer) { - if (consider_grow_active_buffer(typeId, elemsNeeded)) { + if (entries_needed + getBufferState(bufferId).size() >= num_entries_for_new_buffer) { + if (consider_grow_active_buffer(typeId, entries_needed)) { bufferId = primary_buffer_id(typeId); - if (elemsNeeded > getBufferState(bufferId).remaining()) { - fallbackResize(bufferId, elemsNeeded); + if (entries_needed > getBufferState(bufferId).remaining()) { + fallback_resize(bufferId, entries_needed); } } else { - switch_primary_buffer(typeId, elemsNeeded); + switch_primary_buffer(typeId, entries_needed); } } else { - fallbackResize(bufferId, elemsNeeded); + fallback_resize(bufferId, entries_needed); } } @@ -209,7 +206,7 @@ DataStoreBase::init_primary_buffers() for (uint32_t typeId = 0; typeId < numTypes; ++typeId) { size_t buffer_id = getFirstFreeBufferId(); assert(buffer_id <= get_bufferid_limit_relaxed()); - onActive(buffer_id, typeId, 0u); + on_active(buffer_id, typeId, 0u); _primary_buffer_ids[typeId] = buffer_id; } } @@ -219,7 +216,7 @@ DataStoreBase::addType(BufferTypeBase *typeHandler) { uint32_t typeId = _primary_buffer_ids.size(); assert(typeId == _typeHandlers.size()); - typeHandler->clampMaxArrays(_maxArrays); + typeHandler->clamp_max_entries(_max_entries); _primary_buffer_ids.push_back(0); _typeHandlers.push_back(typeHandler); _free_lists.emplace_back(); @@ -328,12 +325,12 @@ DataStoreBase::disableFreeLists() } void -DataStoreBase::disableElemHoldList() +DataStoreBase::disable_entry_hold_list() { for_each_buffer([](BufferState & state) { - if (!state.isFree()) state.disable_elem_hold_list(); + if (!state.isFree()) state.disable_entry_hold_list(); }); - _disableElemHoldList = true; + _disable_entry_hold_list = true; } MemoryStats @@ -351,13 +348,13 @@ DataStoreBase::getMemStats() const if ((state == BufferState::State::FREE) || (typeHandler == nullptr)) { ++stats._freeBuffers; } else if (state == BufferState::State::ACTIVE) { - size_t elementSize = typeHandler->elementSize(); + size_t entry_size = typeHandler->entry_size(); ++stats._activeBuffers; - bState->stats().add_to_mem_stats(elementSize, stats); + bState->stats().add_to_mem_stats(entry_size, stats); } else if (state == BufferState::State::HOLD) { - size_t elementSize = typeHandler->elementSize(); + size_t entry_size = typeHandler->entry_size(); ++stats._holdBuffers; - bState->stats().add_to_mem_stats(elementSize, stats); + bState->stats().add_to_mem_stats(entry_size, stats); } else { LOG_ABORT("should not be reached"); } @@ -373,32 +370,30 @@ vespalib::AddressSpace DataStoreBase::getAddressSpaceUsage() const { uint32_t buffer_id_limit = get_bufferid_limit_acquire(); - size_t usedArrays = 0; - size_t deadArrays = 0; - size_t limitArrays = size_t(_maxArrays) * (getMaxNumBuffers() - buffer_id_limit); + size_t used_entries = 0; + size_t dead_entries = 0; + size_t limit_entries = size_t(_max_entries) * (getMaxNumBuffers() - buffer_id_limit); for (uint32_t bufferId = 0; bufferId < buffer_id_limit; ++bufferId) { const BufferState * bState = _buffers[bufferId].get_state_acquire(); assert(bState != nullptr); if (bState->isFree()) { - limitArrays += _maxArrays; + limit_entries += _max_entries; } else if (bState->isActive()) { - uint32_t arraySize = bState->getArraySize(); - usedArrays += bState->size() / arraySize; - deadArrays += bState->stats().dead_elems() / arraySize; - limitArrays += bState->capacity() / arraySize; + used_entries += bState->size(); + dead_entries += bState->stats().dead_entries(); + limit_entries += bState->capacity(); } else if (bState->isOnHold()) { - uint32_t arraySize = bState->getArraySize(); - usedArrays += bState->size() / arraySize; - limitArrays += bState->capacity() / arraySize; + used_entries += bState->size(); + limit_entries += bState->capacity(); } else { LOG_ABORT("should not be reached"); } } - return {usedArrays, deadArrays, limitArrays}; + return {used_entries, dead_entries, limit_entries}; } void -DataStoreBase::onActive(uint32_t bufferId, uint32_t typeId, size_t elemsNeeded) +DataStoreBase::on_active(uint32_t bufferId, uint32_t typeId, size_t entries_needed) { assert(typeId < _typeHandlers.size()); assert(bufferId <= _bufferIdLimit); @@ -407,8 +402,8 @@ DataStoreBase::onActive(uint32_t bufferId, uint32_t typeId, size_t elemsNeeded) BufferState *state = bufferMeta.get_state_relaxed(); if (state == nullptr) { BufferState & newState = _stash.create<BufferState>(); - if (_disableElemHoldList) { - newState.disable_elem_hold_list(); + if (_disable_entry_hold_list) { + newState.disable_entry_hold_list(); } if ( ! _freeListsEnabled) { newState.disable_free_list(); @@ -418,7 +413,7 @@ DataStoreBase::onActive(uint32_t bufferId, uint32_t typeId, size_t elemsNeeded) _bufferIdLimit.store(bufferId + 1, std::memory_order_release); } assert(state->isFree()); - state->onActive(bufferId, typeId, _typeHandlers[typeId], elemsNeeded, bufferMeta.get_atomic_buffer()); + state->on_active(bufferId, typeId, _typeHandlers[typeId], entries_needed, bufferMeta.get_atomic_buffer()); bufferMeta.setTypeId(typeId); bufferMeta.setArraySize(state->getArraySize()); if (_freeListsEnabled && state->isActive() && !state->getCompacting()) { @@ -436,17 +431,17 @@ DataStoreBase::finishCompact(const std::vector<uint32_t> &toHold) } void -DataStoreBase::fallbackResize(uint32_t bufferId, size_t elemsNeeded) +DataStoreBase::fallback_resize(uint32_t bufferId, size_t entries_needed) { BufferState &state = getBufferState(bufferId); BufferState::Alloc toHoldBuffer; - size_t oldUsedElems = state.size(); - size_t oldAllocElems = state.capacity(); - size_t elementSize = state.getTypeHandler()->elementSize(); - state.fallbackResize(bufferId, elemsNeeded, _buffers[bufferId].get_atomic_buffer(), toHoldBuffer); - auto hold = std::make_unique<FallbackHold>(oldAllocElems * elementSize, + size_t old_used_entries = state.size(); + size_t old_alloc_entries = state.capacity(); + size_t entry_size = state.getTypeHandler()->entry_size(); + state.fallback_resize(bufferId, entries_needed, _buffers[bufferId].get_atomic_buffer(), toHoldBuffer); + auto hold = std::make_unique<FallbackHold>(old_alloc_entries * entry_size, std::move(toHoldBuffer), - oldUsedElems, + old_used_entries, state.getTypeHandler(), state.getTypeId()); if (!_initializing) { @@ -465,7 +460,7 @@ DataStoreBase::markCompacting(uint32_t bufferId) } assert(!state.getCompacting()); state.setCompacting(); - state.disable_elem_hold_list(); + state.disable_entry_hold_list(); state.disable_free_list(); inc_compaction_count(); } @@ -492,15 +487,15 @@ DataStoreBase::start_compact_worst_buffers(CompactionSpec compaction_spec, const free_buffers++; } else if (state->isActive()) { auto typeHandler = state->getTypeHandler(); - uint32_t arraySize = typeHandler->getArraySize(); - uint32_t reservedElements = typeHandler->getReservedElements(bufferId); - size_t used_elems = state->size(); - size_t deadElems = state->stats().dead_elems() - reservedElements; + uint32_t reserved_entries = typeHandler->get_reserved_entries(bufferId); + size_t used_entries = state->size(); + size_t dead_entries = state->stats().dead_entries() - reserved_entries; + size_t entry_size = typeHandler->entry_size(); if (compaction_spec.compact_memory()) { - elem_buffers.add(bufferId, used_elems, deadElems); + elem_buffers.add(bufferId, used_entries * entry_size, dead_entries * entry_size); } if (compaction_spec.compact_address_space()) { - array_buffers.add(bufferId, used_elems / arraySize, deadElems / arraySize); + array_buffers.add(bufferId, used_entries, dead_entries); } } } diff --git a/vespalib/src/vespa/vespalib/datastore/datastorebase.h b/vespalib/src/vespa/vespalib/datastore/datastorebase.h index 9cab7a2e375..e5a38e3fd41 100644 --- a/vespalib/src/vespa/vespalib/datastore/datastorebase.h +++ b/vespalib/src/vespa/vespalib/datastore/datastorebase.h @@ -35,15 +35,16 @@ public: void init_primary_buffers(); /** - * Ensure that the primary buffer for the given type has a given number of elements free at end. + * Ensure that the primary buffer for the given type has a given number of entries free at end. * Switch to new buffer if current buffer is too full. * - * @param typeId Registered data type for buffer. - * @param elemsNeeded Number of elements needed to be free. + * @param typeId Registered data type for buffer. + * @param entries_needed Number of entries needed to be free. */ - void ensureBufferCapacity(uint32_t typeId, size_t elemsNeeded) { - if (elemsNeeded > getBufferState(primary_buffer_id(typeId)).remaining()) [[unlikely]] { - switch_or_grow_primary_buffer(typeId, elemsNeeded); + void ensure_buffer_capacity(uint32_t typeId, size_t entries_needed) { + auto &state = getBufferState(primary_buffer_id(typeId)); + if (entries_needed > state.remaining()) [[unlikely]] { + switch_or_grow_primary_buffer(typeId, entries_needed); } } @@ -58,10 +59,10 @@ public: * Switch to a new primary buffer, typically in preparation for compaction * or when the current primary buffer no longer has free space. * - * @param typeId Registered data type for buffer. - * @param elemsNeeded Number of elements needed to be free. + * @param typeId Registered data type for buffer. + * @param entries_needed Number of entries needed to be free. */ - void switch_primary_buffer(uint32_t typeId, size_t elemsNeeded); + void switch_primary_buffer(uint32_t typeId, size_t entries_needed); vespalib::MemoryUsage getMemoryUsage() const; vespalib::MemoryUsage getDynamicMemoryUsage() const; @@ -127,7 +128,7 @@ public: /** * Enable free list management. - * This only works for fixed size elements. + * This only works for fixed size entries. */ void enableFreeLists(); @@ -135,7 +136,7 @@ public: * Disable free list management. */ void disableFreeLists(); - void disableElemHoldList(); + void disable_entry_hold_list(); bool has_free_lists_enabled() const { return _freeListsEnabled; } @@ -177,25 +178,25 @@ public: bool has_held_buffers() const noexcept { return _hold_buffer_count != 0u; } /** - * Trim elem hold list, freeing elements that no longer needs to be held. + * Trim entry hold list, freeing entries that no longer needs to be held. * * @param oldest_used_gen the oldest generation that is still used. */ virtual void reclaim_entry_refs(generation_t oldest_used_gen) = 0; protected: - DataStoreBase(uint32_t numBuffers, uint32_t offset_bits, size_t maxArrays); + DataStoreBase(uint32_t numBuffers, uint32_t offset_bits, size_t max_entries); virtual ~DataStoreBase(); void* getBuffer(uint32_t bufferId) { return _buffers[bufferId].get_buffer_relaxed(); } struct EntryRefHoldElem { EntryRef ref; - size_t num_elems; + size_t num_entries; - EntryRefHoldElem(EntryRef ref_in, size_t num_elems_in) + EntryRefHoldElem(EntryRef ref_in, size_t num_entries_in) : ref(ref_in), - num_elems(num_elems_in) + num_entries(num_entries_in) {} }; @@ -215,11 +216,11 @@ private: { public: BufferState::Alloc _buffer; - size_t _usedElems; + size_t _used_entries; BufferTypeBase *_typeHandler; uint32_t _typeId; - FallbackHold(size_t bytesSize, BufferState::Alloc &&buffer, size_t usedElems, + FallbackHold(size_t bytesSize, BufferState::Alloc &&buffer, size_t used_entries, BufferTypeBase *typeHandler, uint32_t typeId); ~FallbackHold() override; @@ -227,8 +228,8 @@ private: class BufferHold; - bool consider_grow_active_buffer(uint32_t type_id, size_t elems_needed); - void switch_or_grow_primary_buffer(uint32_t typeId, size_t elemsNeeded); + bool consider_grow_active_buffer(uint32_t type_id, size_t entries_needed); + void switch_or_grow_primary_buffer(uint32_t typeId, size_t entries_needed); void markCompacting(uint32_t bufferId); /** * Hold of buffer has ended. @@ -238,14 +239,14 @@ private: /** * Switch buffer state to active for the given buffer. * - * @param bufferId Id of buffer to be active. - * @param typeId Registered data type for buffer. - * @param elemsNeeded Number of elements needed to be free. + * @param bufferId Id of buffer to be active. + * @param typeId Registered data type for buffer. + * @param entries_needed Number of entries needed to be free. */ - void onActive(uint32_t bufferId, uint32_t typeId, size_t elemsNeeded); + void on_active(uint32_t bufferId, uint32_t typeId, size_t entries_needed); void inc_hold_buffer_count(); - void fallbackResize(uint32_t bufferId, size_t elementsNeeded); + void fallback_resize(uint32_t bufferId, size_t entries_needed); uint32_t getFirstFreeBufferId(); template<typename FuncType> @@ -261,7 +262,7 @@ private: std::vector<BufferAndMeta> _buffers; // For fast mapping with known types // Provides a mapping from typeId -> primary buffer for that type. - // The primary buffer is used for allocations of new element(s) if no available slots are found in free lists. + // The primary buffer is used for allocations of new entries if no available slots are found in free lists. std::vector<uint32_t> _primary_buffer_ids; Stash _stash; @@ -269,12 +270,12 @@ private: std::vector<FreeList> _free_lists; mutable std::atomic<uint64_t> _compaction_count; vespalib::GenerationHolder _genHolder; - const uint32_t _maxArrays; + const uint32_t _max_entries; std::atomic<uint32_t> _bufferIdLimit; uint32_t _hold_buffer_count; const uint8_t _offset_bits; bool _freeListsEnabled; - bool _disableElemHoldList; + bool _disable_entry_hold_list; bool _initializing; }; diff --git a/vespalib/src/vespa/vespalib/datastore/free_list.h b/vespalib/src/vespa/vespalib/datastore/free_list.h index 20d4a6b96df..cd475af3104 100644 --- a/vespalib/src/vespa/vespalib/datastore/free_list.h +++ b/vespalib/src/vespa/vespalib/datastore/free_list.h @@ -30,7 +30,6 @@ public: bool empty() const { return _free_lists.empty(); } size_t size() const { return _free_lists.size(); } - uint32_t array_size() const { return _free_lists.back()->array_size(); } EntryRef pop_entry() { return _free_lists.back()->pop_entry(); } diff --git a/vespalib/src/vespa/vespalib/datastore/free_list_allocator.h b/vespalib/src/vespa/vespalib/datastore/free_list_allocator.h index cf899a76712..dc2d1ea3c34 100644 --- a/vespalib/src/vespa/vespalib/datastore/free_list_allocator.h +++ b/vespalib/src/vespa/vespalib/datastore/free_list_allocator.h @@ -29,7 +29,7 @@ public: HandleType alloc(Args && ... args); HandleType allocArray(ConstArrayRef array); - HandleType allocArray(size_t size); + HandleType allocArray(); }; } diff --git a/vespalib/src/vespa/vespalib/datastore/free_list_allocator.hpp b/vespalib/src/vespa/vespalib/datastore/free_list_allocator.hpp index b793e4f77a2..4e69db08a3c 100644 --- a/vespalib/src/vespa/vespalib/datastore/free_list_allocator.hpp +++ b/vespalib/src/vespa/vespalib/datastore/free_list_allocator.hpp @@ -71,8 +71,9 @@ FreeListAllocator<EntryT, RefT, ReclaimerT>::allocArray(ConstArrayRef array) if (free_list.empty()) { return ParentType::allocArray(array); } - assert(free_list.array_size() == array.size()); RefT ref = free_list.pop_entry(); + auto& state = _store.getBufferState(ref.bufferId()); + assert(state.getArraySize() == array.size()); EntryT *buf = _store.template getEntryArray<EntryT>(ref, array.size()); for (size_t i = 0; i < array.size(); ++i) { *(buf + i) = array[i]; @@ -82,14 +83,15 @@ FreeListAllocator<EntryT, RefT, ReclaimerT>::allocArray(ConstArrayRef array) template <typename EntryT, typename RefT, typename ReclaimerT> typename Allocator<EntryT, RefT>::HandleType -FreeListAllocator<EntryT, RefT, ReclaimerT>::allocArray(size_t size) +FreeListAllocator<EntryT, RefT, ReclaimerT>::allocArray() { auto& free_list = _store.getFreeList(_typeId); if (free_list.empty()) { - return ParentType::allocArray(size); + return ParentType::allocArray(); } - assert(free_list.array_size() == size); RefT ref = free_list.pop_entry(); + auto& state = _store.getBufferState(ref.bufferId()); + auto size = state.getArraySize(); EntryT *buf = _store.template getEntryArray<EntryT>(ref, size); return HandleType(ref, buf); } diff --git a/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.h b/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.h index 1b71c22f0ce..29684267546 100644 --- a/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.h +++ b/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.h @@ -27,7 +27,7 @@ private: public: FreeListRawAllocator(DataStoreBase &store, uint32_t typeId); - HandleType alloc(size_t numElems); + HandleType alloc(size_t num_entries); }; } diff --git a/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.hpp b/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.hpp index af832955cb7..7680cd8a9a5 100644 --- a/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.hpp +++ b/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.hpp @@ -14,16 +14,18 @@ FreeListRawAllocator<EntryT, RefT>::FreeListRawAllocator(DataStoreBase &store, u template <typename EntryT, typename RefT> typename FreeListRawAllocator<EntryT, RefT>::HandleType -FreeListRawAllocator<EntryT, RefT>::alloc(size_t numElems) +FreeListRawAllocator<EntryT, RefT>::alloc(size_t num_entries) { auto& free_list = _store.getFreeList(_typeId); if (free_list.empty()) { - return ParentType::alloc(numElems); + return ParentType::alloc(num_entries); } - assert(free_list.array_size() == numElems); + assert(num_entries == 1); RefT ref = free_list.pop_entry(); + auto& state = _store.getBufferState(ref.bufferId()); + auto array_size = state.getArraySize(); // We must scale the offset according to array size as it was divided when the entry ref was created. - EntryT *entry = _store.template getEntryArray<EntryT>(ref, numElems); + EntryT *entry = _store.template getEntryArray<EntryT>(ref, array_size); return HandleType(ref, entry); } diff --git a/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.h b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.h index 600925969a3..e2718b94cd2 100644 --- a/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.h +++ b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.h @@ -14,11 +14,11 @@ namespace vespalib::datastore { /* * Class representing buffer type for large arrays in ArrayStore */ -template <typename EntryT> -class LargeArrayBufferType : public BufferType<Array<EntryT>> +template <typename ElemT> +class LargeArrayBufferType : public BufferType<Array<ElemT>> { using AllocSpec = ArrayStoreConfig::AllocSpec; - using ArrayType = Array<EntryT>; + using ArrayType = Array<ElemT>; using ParentType = BufferType<ArrayType>; using ParentType::empty_entry; using CleanContext = typename ParentType::CleanContext; @@ -31,7 +31,7 @@ public: { } ~LargeArrayBufferType() override; - void cleanHold(void* buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override; + void clean_hold(void* buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx) override; const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override; }; diff --git a/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.hpp b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.hpp index 3042bbff73f..72a2662991b 100644 --- a/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.hpp +++ b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.hpp @@ -7,32 +7,32 @@ namespace vespalib::datastore { -template <typename EntryT> -LargeArrayBufferType<EntryT>::LargeArrayBufferType(const AllocSpec& spec, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) noexcept - : BufferType<Array<EntryT>>(1u, spec.minArraysInBuffer, spec.maxArraysInBuffer, spec.numArraysForNewBuffer, spec.allocGrowFactor), +template <typename ElemT> +LargeArrayBufferType<ElemT>::LargeArrayBufferType(const AllocSpec& spec, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) noexcept + : BufferType<Array<ElemT>>(1u, spec.min_entries_in_buffer, spec.max_entries_in_buffer, spec.num_entries_for_new_buffer, spec.allocGrowFactor), _memory_allocator(std::move(memory_allocator)) { } -template <typename EntryT> -LargeArrayBufferType<EntryT>::~LargeArrayBufferType() = default; +template <typename ElemT> +LargeArrayBufferType<ElemT>::~LargeArrayBufferType() = default; -template <typename EntryT> +template <typename ElemT> void -LargeArrayBufferType<EntryT>::cleanHold(void* buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) +LargeArrayBufferType<ElemT>::clean_hold(void* buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx) { ArrayType* elem = static_cast<ArrayType*>(buffer) + offset; const auto& empty = empty_entry(); - for (size_t i = 0; i < numElems; ++i) { - cleanCtx.extraBytesCleaned(sizeof(EntryT) * elem->size()); + for (size_t i = 0; i < num_entries; ++i) { + cleanCtx.extraBytesCleaned(sizeof(ElemT) * elem->size()); *elem = empty; ++elem; } } -template <typename EntryT> +template <typename ElemT> const vespalib::alloc::MemoryAllocator* -LargeArrayBufferType<EntryT>::get_memory_allocator() const +LargeArrayBufferType<ElemT>::get_memory_allocator() const { return _memory_allocator.get(); } diff --git a/vespalib/src/vespa/vespalib/datastore/memory_stats.cpp b/vespalib/src/vespa/vespalib/datastore/memory_stats.cpp index 8e060b4cfb4..5cb04796c5b 100644 --- a/vespalib/src/vespa/vespalib/datastore/memory_stats.cpp +++ b/vespalib/src/vespa/vespalib/datastore/memory_stats.cpp @@ -5,10 +5,10 @@ namespace vespalib::datastore { MemoryStats::MemoryStats() - : _allocElems(0), - _usedElems(0), - _deadElems(0), - _holdElems(0), + : _alloc_entries(0), + _used_entries(0), + _dead_entries(0), + _hold_entries(0), _allocBytes(0), _usedBytes(0), _deadBytes(0), @@ -22,10 +22,10 @@ MemoryStats::MemoryStats() MemoryStats& MemoryStats::operator+=(const MemoryStats& rhs) { - _allocElems += rhs._allocElems; - _usedElems += rhs._usedElems; - _deadElems += rhs._deadElems; - _holdElems += rhs._holdElems; + _alloc_entries += rhs._alloc_entries; + _used_entries += rhs._used_entries; + _dead_entries += rhs._dead_entries; + _hold_entries += rhs._hold_entries; _allocBytes += rhs._allocBytes; _usedBytes += rhs._usedBytes; _deadBytes += rhs._deadBytes; diff --git a/vespalib/src/vespa/vespalib/datastore/memory_stats.h b/vespalib/src/vespa/vespalib/datastore/memory_stats.h index 18d7dd77559..72a570dd625 100644 --- a/vespalib/src/vespa/vespalib/datastore/memory_stats.h +++ b/vespalib/src/vespa/vespalib/datastore/memory_stats.h @@ -13,10 +13,10 @@ namespace vespalib::datastore { class MemoryStats { public: - size_t _allocElems; - size_t _usedElems; - size_t _deadElems; - size_t _holdElems; + size_t _alloc_entries; + size_t _used_entries; + size_t _dead_entries; + size_t _hold_entries; size_t _allocBytes; size_t _usedBytes; size_t _deadBytes; diff --git a/vespalib/src/vespa/vespalib/datastore/raw_allocator.h b/vespalib/src/vespa/vespalib/datastore/raw_allocator.h index c10c8152e72..e7a59fadcf8 100644 --- a/vespalib/src/vespa/vespalib/datastore/raw_allocator.h +++ b/vespalib/src/vespa/vespalib/datastore/raw_allocator.h @@ -25,10 +25,10 @@ protected: public: RawAllocator(DataStoreBase &store, uint32_t typeId); - HandleType alloc(size_t numElems) { - return alloc(numElems, 0); + HandleType alloc(size_t num_entries) { + return alloc(num_entries, 0); } - HandleType alloc(size_t numElems, size_t extraElems); + HandleType alloc(size_t num_entries, size_t extra_entries); }; } diff --git a/vespalib/src/vespa/vespalib/datastore/raw_allocator.hpp b/vespalib/src/vespa/vespalib/datastore/raw_allocator.hpp index 04d99588218..9de361a8b19 100644 --- a/vespalib/src/vespa/vespalib/datastore/raw_allocator.hpp +++ b/vespalib/src/vespa/vespalib/datastore/raw_allocator.hpp @@ -16,19 +16,15 @@ RawAllocator<EntryT, RefT>::RawAllocator(DataStoreBase &store, uint32_t typeId) template <typename EntryT, typename RefT> typename RawAllocator<EntryT, RefT>::HandleType -RawAllocator<EntryT, RefT>::alloc(size_t numElems, size_t extraElems) +RawAllocator<EntryT, RefT>::alloc(size_t num_entries, size_t extra_entries) { - _store.ensureBufferCapacity(_typeId, numElems + extraElems); + _store.ensure_buffer_capacity(_typeId, num_entries + extra_entries); uint32_t buffer_id = _store.primary_buffer_id(_typeId); BufferState &state = _store.getBufferState(buffer_id); assert(state.isActive()); - size_t oldBufferSize = state.size(); - // Must perform scaling ourselves, according to array size - size_t arraySize = state.getArraySize(); - assert((numElems % arraySize) == 0u); - RefT ref((oldBufferSize / arraySize), buffer_id); - EntryT *buffer = _store.getEntryArray<EntryT>(ref, arraySize); - state.stats().pushed_back(numElems); + RefT ref(state.size(), buffer_id); + EntryT *buffer = _store.getEntryArray<EntryT>(ref, state.getArraySize()); + state.stats().pushed_back(num_entries); return HandleType(ref, buffer); } diff --git a/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.h b/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.h index 676a9d3790f..6bb6601839d 100644 --- a/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.h +++ b/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.h @@ -13,8 +13,8 @@ namespace vespalib::datastore { /* * Class representing buffer type for small arrays in ArrayStore */ -template <typename EntryT> -class SmallArrayBufferType : public BufferType<EntryT> +template <typename ElemT> +class SmallArrayBufferType : public BufferType<ElemT> { using AllocSpec = ArrayStoreConfig::AllocSpec; std::shared_ptr<alloc::MemoryAllocator> _memory_allocator; diff --git a/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.hpp b/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.hpp index 414804417eb..c9033936bd6 100644 --- a/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.hpp +++ b/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.hpp @@ -6,19 +6,19 @@ namespace vespalib::datastore { -template <typename EntryT> -SmallArrayBufferType<EntryT>::SmallArrayBufferType(uint32_t array_size, const AllocSpec& spec, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) noexcept - : BufferType<EntryT>(array_size, spec.minArraysInBuffer, spec.maxArraysInBuffer, spec.numArraysForNewBuffer, spec.allocGrowFactor), +template <typename ElemT> +SmallArrayBufferType<ElemT>::SmallArrayBufferType(uint32_t array_size, const AllocSpec& spec, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) noexcept + : BufferType<ElemT>(array_size, spec.min_entries_in_buffer, spec.max_entries_in_buffer, spec.num_entries_for_new_buffer, spec.allocGrowFactor), _memory_allocator(std::move(memory_allocator)) { } -template <typename EntryT> -SmallArrayBufferType<EntryT>::~SmallArrayBufferType() = default; +template <typename ElemT> +SmallArrayBufferType<ElemT>::~SmallArrayBufferType() = default; -template <typename EntryT> +template <typename ElemT> const vespalib::alloc::MemoryAllocator* -SmallArrayBufferType<EntryT>::get_memory_allocator() const +SmallArrayBufferType<ElemT>::get_memory_allocator() const { return _memory_allocator.get(); } diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp index 52b0798543f..0efaf04b26e 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp @@ -106,7 +106,7 @@ private: _mapping.resize(data_store.get_bufferid_limit_relaxed()); for (const auto bufferId : _compacting_buffers->get_buffer_ids()) { BufferState &state = data_store.getBufferState(bufferId); - _mapping[bufferId].resize(state.get_used_arrays()); + _mapping[bufferId].resize(state.size()); } } diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.hpp index 8ad11b18218..49d2631e73f 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.hpp @@ -43,7 +43,7 @@ template <typename EntryT, typename RefT> void UniqueStoreAllocator<EntryT, RefT>::hold(EntryRef ref) { - _store.holdElem(ref, 1); + _store.hold_entry(ref); } template <typename EntryT, typename RefT> diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp index 32513d09c72..4a517521d77 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp @@ -44,7 +44,7 @@ UniqueStoreEnumerator<RefT>::allocate_enum_values(DataStoreBase & store) { _enumValues.resize(store.get_bufferid_limit_relaxed()); store.for_each_active_buffer([this](uint32_t buffer_id, const BufferState & state) { - _enumValues[buffer_id].resize(state.get_used_arrays()); + _enumValues[buffer_id].resize(state.size()); }); } diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.cpp b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.cpp index 1d3ba27d6bf..9f2105b7f08 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.cpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.cpp @@ -42,28 +42,28 @@ UniqueStoreSmallStringBufferType::UniqueStoreSmallStringBufferType(uint32_t arra UniqueStoreSmallStringBufferType::~UniqueStoreSmallStringBufferType() = default; void -UniqueStoreSmallStringBufferType::destroyElements(void *, ElemCount) +UniqueStoreSmallStringBufferType::destroy_entries(void *, EntryCount) { static_assert(std::is_trivially_destructible<UniqueStoreSmallStringEntry>::value, "UniqueStoreSmallStringEntry must be trivially destructable"); } void -UniqueStoreSmallStringBufferType::fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems) +UniqueStoreSmallStringBufferType::fallback_copy(void *newBuffer, const void *oldBuffer, EntryCount num_entries) { static_assert(std::is_trivially_copyable<UniqueStoreSmallStringEntry>::value, "UniqueStoreSmallStringEntry must be trivially copyable"); - if (numElems > 0) { - memcpy(newBuffer, oldBuffer, numElems); + if (num_entries > 0) { + memcpy(newBuffer, oldBuffer, num_entries * getArraySize()); } } void -UniqueStoreSmallStringBufferType::cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext) +UniqueStoreSmallStringBufferType::clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext) { - void *e = static_cast<char *>(buffer) + offset; - void *e_end = static_cast<char *>(e) + numElems; size_t array_size = getArraySize(); + void *e = static_cast<char *>(buffer) + offset * array_size; + void *e_end = static_cast<char *>(e) + num_entries * array_size; while (e < e_end) { static_cast<UniqueStoreSmallStringEntry *>(e)->clean_hold(array_size); e = static_cast<char *>(e) + array_size; @@ -86,10 +86,10 @@ UniqueStoreExternalStringBufferType::UniqueStoreExternalStringBufferType(uint32_ UniqueStoreExternalStringBufferType::~UniqueStoreExternalStringBufferType() = default; void -UniqueStoreExternalStringBufferType::cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) +UniqueStoreExternalStringBufferType::clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx) { UniqueStoreEntry<std::string> *elem = static_cast<UniqueStoreEntry<std::string> *>(buffer) + offset; - for (size_t i = 0; i < numElems; ++i) { + for (size_t i = 0; i < num_entries; ++i) { cleanCtx.extraBytesCleaned(elem->value().size() + 1); std::string().swap(elem->value()); ++elem; diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.h b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.h index a85b73f423d..d3348950891 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.h @@ -62,9 +62,9 @@ class UniqueStoreSmallStringBufferType : public BufferType<char> { public: UniqueStoreSmallStringBufferType(uint32_t array_size, uint32_t max_arrays, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator); ~UniqueStoreSmallStringBufferType() override; - void destroyElements(void *, ElemCount) override; - void fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems) override; - void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext) override; + void destroy_entries(void *, EntryCount) override; + void fallback_copy(void *newBuffer, const void *oldBuffer, EntryCount numElems) override; + void clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext) override; const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override; }; @@ -76,7 +76,7 @@ class UniqueStoreExternalStringBufferType : public BufferType<UniqueStoreEntry<s public: UniqueStoreExternalStringBufferType(uint32_t array_size, uint32_t max_arrays, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator); ~UniqueStoreExternalStringBufferType() override; - void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override; + void clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx) override; const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override; }; diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.hpp index 65cab4850ba..4ff8e1e1ab4 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.hpp @@ -42,7 +42,7 @@ UniqueStoreStringAllocator<RefT>::allocate(const char *value) uint32_t type_id = string_allocator::get_type_id(value_len); if (type_id != 0) { size_t array_size = string_allocator::array_sizes[type_id - 1]; - auto handle = _store.template freeListRawAllocator<char>(type_id).alloc(array_size); + auto handle = _store.template freeListRawAllocator<char>(type_id).alloc(1); new (static_cast<void *>(handle.data)) UniqueStoreSmallStringEntry(value, value_len, array_size); return handle.ref; } else { @@ -61,11 +61,10 @@ UniqueStoreStringAllocator<RefT>::hold(EntryRef ref) RefT iRef(ref); uint32_t type_id = _store.getTypeId(iRef.bufferId()); if (type_id != 0) { - size_t array_size = string_allocator::array_sizes[type_id - 1]; - _store.holdElem(ref, array_size); + _store.hold_entry(ref); } else { auto &value = _store.template getEntry<WrappedExternalEntryType>(iRef)->value(); - _store.holdElem(ref, 1, value.size() + 1); + _store.hold_entry(ref, value.size() + 1); } } @@ -79,7 +78,7 @@ UniqueStoreStringAllocator<RefT>::move_on_compact(EntryRef ref) static_assert(std::is_trivially_copyable<UniqueStoreSmallStringEntry>::value, "UniqueStoreSmallStringEntry must be trivially copyable"); size_t array_size = string_allocator::array_sizes[type_id - 1]; - auto handle = _store.template rawAllocator<char>(type_id).alloc(array_size); + auto handle = _store.template rawAllocator<char>(type_id).alloc(1); auto orig = _store.template getEntryArray<char>(iRef, array_size); memcpy(handle.data, orig, array_size); return handle.ref; diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index 663b3b65638..8ee3957af32 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -100,3 +100,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT zstdcompressor.cpp DEPENDS ) + +# Don't convert call to jump when returning a value from a function with +# a compatible stack. +set_source_files_properties(signalhandler.cpp PROPERTIES COMPILE_OPTIONS "-fno-optimize-sibling-calls") diff --git a/vespalib/src/vespa/vespalib/util/time.cpp b/vespalib/src/vespa/vespalib/util/time.cpp index cba26f24059..dd4972b1c21 100644 --- a/vespalib/src/vespa/vespalib/util/time.cpp +++ b/vespalib/src/vespa/vespalib/util/time.cpp @@ -93,7 +93,7 @@ Timer::waitAtLeast(duration dur, bool busyWait) { } -#if (defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 160000) || (!defined(_LIBCPP_VERSION) && defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 12) +#if (defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 170000) || (!defined(_LIBCPP_VERSION) && defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 12) // Temporary workaround until libc++ supports stream operators for duration // Temporary workaround while using libstdc++ 11 diff --git a/vespalib/src/vespa/vespalib/util/time.h b/vespalib/src/vespa/vespalib/util/time.h index b893661832f..10077a0b49a 100644 --- a/vespalib/src/vespa/vespalib/util/time.h +++ b/vespalib/src/vespa/vespalib/util/time.h @@ -101,7 +101,7 @@ duration adjustTimeoutByHz(duration timeout, long hz); } -#if (defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 160000) || (!defined(_LIBCPP_VERSION) && defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 12) +#if (defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 170000) || (!defined(_LIBCPP_VERSION) && defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 12) // Temporary workaround until libc++ supports stream operators for duration // Temporary workaround while using libstdc++ 11 |