diff options
author | Harald Musum <musum@yahooinc.com> | 2023-04-14 09:18:05 +0200 |
---|---|---|
committer | Harald Musum <musum@yahooinc.com> | 2023-04-14 09:18:05 +0200 |
commit | 16176985401e8fef7bd3d95e081c333cd652fe30 (patch) | |
tree | f8b2b3663c09b77f349d364f4fe3854b5ac5e39f | |
parent | 9a71eb340d65db6b8fb8a815e3a913708d57e2c1 (diff) | |
parent | 2279c18214465fd88ce2cc8aa22d8856342e2d85 (diff) |
Merge branch 'master' into hmusum/allow-groups-to-be-down
395 files changed, 4957 insertions, 3730 deletions
diff --git a/client/go/internal/admin/prog/valgrind.go b/client/go/internal/admin/prog/valgrind.go index 7d3fb059f8f..2d7f0a597d9 100644 --- a/client/go/internal/admin/prog/valgrind.go +++ b/client/go/internal/admin/prog/valgrind.go @@ -22,9 +22,10 @@ func (p *Spec) ConfigureValgrind() { p.shouldUseValgrind = false p.shouldUseCallgrind = false env := p.Getenv(envvars.VESPA_USE_VALGRIND) + allValgrind := env == "all" parts := strings.Split(env, " ") for _, part := range parts { - if p.BaseName == part { + if p.BaseName == part || allValgrind { trace.Trace("using valgrind as", p.Program, "has basename in", envvars.VESPA_USE_VALGRIND, "=>", env) backticks := util.BackTicksWithStderr out, err := backticks.Run(VALGRIND_PROG, "--help") diff --git a/client/go/internal/admin/prog/valgrind_test.go b/client/go/internal/admin/prog/valgrind_test.go index 6ec622277c6..11d9424405f 100644 --- a/client/go/internal/admin/prog/valgrind_test.go +++ b/client/go/internal/admin/prog/valgrind_test.go @@ -52,6 +52,11 @@ func TestValgrindDetection(t *testing.T) { assert.Equal(t, false, spec.shouldUseValgrind) assert.Equal(t, false, spec.shouldUseCallgrind) + t.Setenv("VESPA_USE_VALGRIND", "all") + spec.ConfigureValgrind() + assert.Equal(t, true, spec.shouldUseValgrind) + assert.Equal(t, false, spec.shouldUseCallgrind) + t.Setenv("VESPA_USE_VALGRIND", "foo bar") spec.ConfigureValgrind() assert.Equal(t, false, spec.shouldUseValgrind) diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind.go b/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind.go index 43a1ed602bd..cccb37df8e5 100644 --- a/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind.go +++ b/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind.go @@ -18,9 +18,10 @@ func (p *ProgSpec) configureValgrind() { p.shouldUseValgrind = false p.shouldUseCallgrind = false env := p.getenv(envvars.VESPA_USE_VALGRIND) + allValgrind := env == "all" parts := strings.Split(env, " ") for _, part := range parts { - if p.BaseName == part { + if p.BaseName == part || allValgrind { trace.Trace("using valgrind as", p.Program, "has basename in", envvars.VESPA_USE_VALGRIND, "=>", env) backticks := util.BackTicksWithStderr out, err := backticks.Run("which", "valgrind") diff --git a/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind_test.go b/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind_test.go index 48cc78474ed..1a105d66c4a 100644 --- a/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind_test.go +++ b/client/go/internal/admin/vespa-wrapper/startcbinary/valgrind_test.go @@ -52,6 +52,11 @@ func TestValgrindDetection(t *testing.T) { assert.Equal(t, false, spec.shouldUseValgrind) assert.Equal(t, false, spec.shouldUseCallgrind) + t.Setenv("VESPA_USE_VALGRIND", "all") + spec.configureValgrind() + assert.Equal(t, true, spec.shouldUseValgrind) + assert.Equal(t, false, spec.shouldUseCallgrind) + t.Setenv("VESPA_USE_VALGRIND", "foo bar") spec.configureValgrind() assert.Equal(t, false, spec.shouldUseValgrind) diff --git a/client/go/internal/cli/auth/zts/zts.go b/client/go/internal/cli/auth/zts/zts.go index 1e84912a271..caa2d03367d 100644 --- a/client/go/internal/cli/auth/zts/zts.go +++ b/client/go/internal/cli/auth/zts/zts.go @@ -37,7 +37,7 @@ func (c *Client) AccessToken(domain string, certificate tls.Certificate) (string if err != nil { return "", err } - util.SetCertificate(c.client, []tls.Certificate{certificate}) + util.SetCertificates(c.client, []tls.Certificate{certificate}) response, err := c.client.Do(req, 10*time.Second) if err != nil { return "", err diff --git a/client/go/internal/cli/cmd/config.go b/client/go/internal/cli/cmd/config.go index fd049864096..2d32c454842 100644 --- a/client/go/internal/cli/cmd/config.go +++ b/client/go/internal/cli/cmd/config.go @@ -544,12 +544,12 @@ func (c *Config) list(includeUnset bool) []string { } // flagValue returns the set value and default value of the named flag. -func (c *Config) flagValue(name string) (string, string) { +func (c *Config) flagValue(name string) (string, string, bool) { f, ok := c.flags[name] if !ok { - return "", "" + return "", "", ok } - return f.Value.String(), f.DefValue + return f.Value.String(), f.DefValue, f.Changed } // getNonEmpty returns value of given option, if that value is non-empty @@ -564,9 +564,9 @@ func (c *Config) getNonEmpty(option string) (string, bool) { // get returns the value associated with option, from the most preferred source in the following order: flag > local // config > global config. func (c *Config) get(option string) (string, bool) { - flagValue, flagDefault := c.flagValue(option) + flagValue, flagDefault, changed := c.flagValue(option) // explicit flag value always takes precedence over everything else - if flagValue != flagDefault { + if changed { return flagValue, true } // ... then local config, if option is explicitly defined there diff --git a/client/go/internal/cli/cmd/config_test.go b/client/go/internal/cli/cmd/config_test.go index 612904061de..458878b4356 100644 --- a/client/go/internal/cli/cmd/config_test.go +++ b/client/go/internal/cli/cmd/config_test.go @@ -28,6 +28,7 @@ func TestConfig(t *testing.T) { assertConfigCommand(t, configHome, "", "config", "set", "target", "http://127.0.0.1:8080") assertConfigCommand(t, configHome, "", "config", "set", "target", "https://127.0.0.1") assertConfigCommand(t, configHome, "target = https://127.0.0.1\n", "config", "get", "target") + assertConfigCommand(t, configHome, "target = local\n", "config", "get", "-t", "local", "target") // application assertConfigCommandErr(t, configHome, "Error: invalid application: \"foo\"\n", "config", "set", "application", "foo") diff --git a/client/go/internal/cli/cmd/feed.go b/client/go/internal/cli/cmd/feed.go index f273c5aa826..895a22d2be5 100644 --- a/client/go/internal/cli/cmd/feed.go +++ b/client/go/internal/cli/cmd/feed.go @@ -9,16 +9,20 @@ import ( "time" "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" "github.com/vespa-engine/vespa/client/go/internal/vespa/document" ) -func addFeedFlags(cmd *cobra.Command, concurrency *int) { - cmd.PersistentFlags().IntVarP(concurrency, "concurrency", "T", 64, "Number of goroutines to use for dispatching") +func addFeedFlags(cmd *cobra.Command, verbose *bool, connections *int) { + cmd.PersistentFlags().IntVarP(connections, "connections", "N", 8, "The number of connections to use") + cmd.PersistentFlags().BoolVarP(verbose, "verbose", "v", false, "Verbose mode. Print errors as they happen") } func newFeedCmd(cli *CLI) *cobra.Command { var ( - concurrency int + verbose bool + connections int ) cmd := &cobra.Command{ Use: "feed FILE", @@ -26,39 +30,66 @@ func newFeedCmd(cli *CLI) *cobra.Command { Long: `Feed documents to a Vespa cluster. A high performance feeding client. This can be used to feed large amounts of -documents to Vespa cluster efficiently. +documents to a Vespa cluster efficiently. The contents of FILE must be either a JSON array or JSON objects separated by newline (JSONL). + +If FILE is a single dash ('-'), documents will be read from standard input. `, Example: `$ vespa feed documents.jsonl +$ cat documents.jsonl | vespa feed - `, Args: cobra.ExactArgs(1), DisableAutoGenTag: true, SilenceUsage: true, Hidden: true, // TODO(mpolden): Remove when ready for public use RunE: func(cmd *cobra.Command, args []string) error { - f, err := os.Open(args[0]) - if err != nil { - return err + var r io.Reader + if args[0] == "-" { + r = cli.Stdin + } else { + f, err := os.Open(args[0]) + if err != nil { + return err + } + defer f.Close() + r = f } - defer f.Close() - return feed(f, cli, concurrency) + return feed(r, cli, verbose, connections) }, } - addFeedFlags(cmd, &concurrency) + addFeedFlags(cmd, &verbose, &connections) return cmd } -func feed(r io.Reader, cli *CLI, concurrency int) error { +func createServiceClients(service *vespa.Service, n int) []util.HTTPClient { + clients := make([]util.HTTPClient, 0, n) + for i := 0; i < n; i++ { + client := service.Client().Clone() + util.ForceHTTP2(client, service.TLSOptions.KeyPair) // Feeding should always use HTTP/2 + clients = append(clients, client) + } + return clients +} + +func feed(r io.Reader, cli *CLI, verbose bool, connections int) error { service, err := documentService(cli) if err != nil { return err } + clients := createServiceClients(service, connections) client := document.NewClient(document.ClientOptions{ BaseURL: service.BaseURL, - }, service) - dispatcher := document.NewDispatcher(client, concurrency) + }, clients) + throttler := document.NewThrottler(connections) + // TODO(mpolden): Make doom duration configurable + circuitBreaker := document.NewCircuitBreaker(10*time.Second, 0) + errWriter := io.Discard + if verbose { + errWriter = cli.Stderr + } + dispatcher := document.NewDispatcher(client, throttler, circuitBreaker, errWriter) dec := document.NewDecoder(r) start := cli.now() @@ -78,7 +109,7 @@ func feed(r io.Reader, cli *CLI, concurrency int) error { return err } elapsed := cli.now().Sub(start) - return writeSummaryJSON(cli.Stdout, client.Stats(), elapsed) + return writeSummaryJSON(cli.Stdout, dispatcher.Stats(), elapsed) } type number float32 diff --git a/client/go/internal/cli/cmd/feed_test.go b/client/go/internal/cli/cmd/feed_test.go index 1bf1ef6ab9b..521d2b2abd0 100644 --- a/client/go/internal/cli/cmd/feed_test.go +++ b/client/go/internal/cli/cmd/feed_test.go @@ -1,6 +1,7 @@ package cmd import ( + "bytes" "os" "path/filepath" "testing" @@ -42,7 +43,7 @@ func TestFeed(t *testing.T) { require.Nil(t, cli.Run("feed", jsonFile)) assert.Equal(t, "", stderr.String()) - assert.Equal(t, `{ + want := `{ "feeder.seconds": 1.000, "feeder.ok.count": 1, "feeder.ok.rate": 1.000, @@ -63,5 +64,15 @@ func TestFeed(t *testing.T) { "200": 1 } } -`, stdout.String()) +` + assert.Equal(t, want, stdout.String()) + + stdout.Reset() + cli.Stdin = bytes.NewBuffer([]byte(`{ + "put": "id:ns:type::doc1", + "fields": {"foo": "123"} +}`)) + httpClient.NextResponseString(200, `{"message":"OK"}`) + require.Nil(t, cli.Run("feed", "-")) + assert.Equal(t, want, stdout.String()) } diff --git a/client/go/internal/cli/cmd/prod.go b/client/go/internal/cli/cmd/prod.go index 0e00597221a..f322bd667df 100644 --- a/client/go/internal/cli/cmd/prod.go +++ b/client/go/internal/cli/cmd/prod.go @@ -105,21 +105,17 @@ https://cloud.vespa.ai/en/reference/deployment`, func newProdSubmitCmd(cli *CLI) *cobra.Command { return &cobra.Command{ Use: "submit", - Short: "Submit your application for production deployment", - Long: `Submit your application for production deployment. + Short: "Submit an application for production deployment", + Long: `Submit an application for production deployment. -This commands uploads your application package to Vespa Cloud and deploys it to +This commands uploads an application package to Vespa Cloud and deploys it to the production zones specified in deployment.xml. -Nodes are allocated to your application according to resources specified in +Nodes are allocated to the application according to resources specified in services.xml. -While submitting an application from a local development environment is -supported, it's strongly recommended that production deployments are performed -by a continuous build system. - For more information about production deployments in Vespa Cloud see: -https://cloud.vespa.ai/en/getting-to-production +https://cloud.vespa.ai/en/production-deployment https://cloud.vespa.ai/en/automated-deployments`, DisableAutoGenTag: true, SilenceUsage: true, diff --git a/client/go/internal/cli/cmd/root.go b/client/go/internal/cli/cmd/root.go index 58e940d59ef..360af9d0dcf 100644 --- a/client/go/internal/cli/cmd/root.go +++ b/client/go/internal/cli/cmd/root.go @@ -366,7 +366,7 @@ func (c *CLI) createCloudTarget(targetType string, opts targetOptions) (vespa.Ta return nil, errHint(err, "Deployment to cloud requires a certificate. Try 'vespa auth cert'") } deploymentTLSOptions = vespa.TLSOptions{ - KeyPair: &kp.KeyPair, + KeyPair: []tls.Certificate{kp.KeyPair}, CertificateFile: kp.CertificateFile, PrivateKeyFile: kp.PrivateKeyFile, } @@ -377,7 +377,7 @@ func (c *CLI) createCloudTarget(targetType string, opts targetOptions) (vespa.Ta return nil, errHint(err, "Deployment to hosted requires an Athenz certificate", "Try renewing certificate with 'athenz-user-cert'") } apiTLSOptions = vespa.TLSOptions{ - KeyPair: &kp.KeyPair, + KeyPair: []tls.Certificate{kp.KeyPair}, CertificateFile: kp.CertificateFile, PrivateKeyFile: kp.PrivateKeyFile, } diff --git a/client/go/internal/cli/cmd/test.go b/client/go/internal/cli/cmd/test.go index 4a53fe6bed3..05633b1135e 100644 --- a/client/go/internal/cli/cmd/test.go +++ b/client/go/internal/cli/cmd/test.go @@ -263,7 +263,7 @@ func verify(step step, defaultCluster string, defaultParameters map[string]strin var response *http.Response if externalEndpoint { - util.SetCertificate(context.cli.httpClient, []tls.Certificate{}) + util.SetCertificates(context.cli.httpClient, []tls.Certificate{}) response, err = context.cli.httpClient.Do(request, 60*time.Second) } else { response, err = service.Do(request, 600*time.Second) // Vespa should provide a response within the given request timeout diff --git a/client/go/internal/cli/cmd/visit_test.go b/client/go/internal/cli/cmd/visit_test.go index 4302680b9d9..b6e5b893e0b 100644 --- a/client/go/internal/cli/cmd/visit_test.go +++ b/client/go/internal/cli/cmd/visit_test.go @@ -47,7 +47,7 @@ func TestQuoteFunc(t *testing.T) { res := quoteArgForUrl(s) if i < 32 || i > 127 { assert.Equal(t, "a+z", res) - } else { + } else if testing.Verbose() { // go test -v fmt.Printf("res %3d => '%s'\n", i, res) } } diff --git a/client/go/internal/mock/http.go b/client/go/internal/mock/http.go index 9c55f2e79bf..58614d7e5bd 100644 --- a/client/go/internal/mock/http.go +++ b/client/go/internal/mock/http.go @@ -6,6 +6,8 @@ import ( "net/http" "strconv" "time" + + "github.com/vespa-engine/vespa/client/go/internal/util" ) type HTTPClient struct { @@ -58,3 +60,5 @@ func (c *HTTPClient) Do(request *http.Request, timeout time.Duration) (*http.Res }, nil } + +func (c *HTTPClient) Clone() util.HTTPClient { return c } diff --git a/client/go/internal/util/http.go b/client/go/internal/util/http.go index cb35932c8e7..dcf05ed3a14 100644 --- a/client/go/internal/util/http.go +++ b/client/go/internal/util/http.go @@ -2,9 +2,10 @@ package util import ( - "bytes" + "context" "crypto/tls" "fmt" + "net" "net/http" "time" @@ -14,6 +15,7 @@ import ( type HTTPClient interface { Do(request *http.Request, timeout time.Duration) (response *http.Response, error error) + Clone() HTTPClient } type defaultHTTPClient struct { @@ -31,50 +33,57 @@ func (c *defaultHTTPClient) Do(request *http.Request, timeout time.Duration) (re return c.client.Do(request) } -func SetCertificate(client HTTPClient, certificates []tls.Certificate) { +func (c *defaultHTTPClient) Clone() HTTPClient { return CreateClient(c.client.Timeout) } + +func SetCertificates(client HTTPClient, certificates []tls.Certificate) { c, ok := client.(*defaultHTTPClient) if !ok { return } - // Use HTTP/2 transport explicitly. Connection reuse does not work properly when using regular http.Transport, even - // though it upgrades to HTTP/2 automatically - // https://github.com/golang/go/issues/16582 - // https://github.com/golang/go/issues/22091 - var transport *http2.Transport - if _, ok := c.client.Transport.(*http.Transport); ok { - transport = &http2.Transport{} - c.client.Transport = transport - } else if t, ok := c.client.Transport.(*http2.Transport); ok { - transport = t - } else { - panic(fmt.Sprintf("unknown transport type: %T", c.client.Transport)) - } - if ok && !c.hasCertificates(transport.TLSClientConfig, certificates) { - transport.TLSClientConfig = &tls.Config{ + var tlsConfig *tls.Config = nil + if certificates != nil { + tlsConfig = &tls.Config{ Certificates: certificates, MinVersion: tls.VersionTLS12, } } + if tr, ok := c.client.Transport.(*http.Transport); ok { + tr.TLSClientConfig = tlsConfig + } else if tr, ok := c.client.Transport.(*http2.Transport); ok { + tr.TLSClientConfig = tlsConfig + } else { + panic(fmt.Sprintf("unknown transport type: %T", c.client.Transport)) + } } -func (c *defaultHTTPClient) hasCertificates(tlsConfig *tls.Config, certs []tls.Certificate) bool { - if tlsConfig == nil { - return false - } - if len(tlsConfig.Certificates) != len(certs) { - return false +func ForceHTTP2(client HTTPClient, certificates []tls.Certificate) { + c, ok := client.(*defaultHTTPClient) + if !ok { + return } - for i := 0; i < len(certs); i++ { - if len(tlsConfig.Certificates[i].Certificate) != len(certs[i].Certificate) { - return false + var tlsConfig *tls.Config = nil + var dialFunc func(ctx context.Context, network, addr string, cfg *tls.Config) (net.Conn, error) + if certificates != nil { + tlsConfig = &tls.Config{ + Certificates: certificates, + MinVersion: tls.VersionTLS12, } - for j := 0; j < len(certs[i].Certificate); j++ { - if !bytes.Equal(tlsConfig.Certificates[i].Certificate[j], certs[i].Certificate[j]) { - return false - } + } else { + // No certificate, so force H2C (HTTP/2 over clear-text) by using a non-TLS Dialer + dialer := net.Dialer{} + dialFunc = func(ctx context.Context, network, addr string, cfg *tls.Config) (net.Conn, error) { + return dialer.DialContext(ctx, network, addr) } } - return true + // Use HTTP/2 transport explicitly. Connection reuse does not work properly when using regular http.Transport, even + // though it upgrades to HTTP/2 automatically + // https://github.com/golang/go/issues/16582 + // https://github.com/golang/go/issues/22091 + c.client.Transport = &http2.Transport{ + AllowHTTP: true, + TLSClientConfig: tlsConfig, + DialTLSContext: dialFunc, + } } func CreateClient(timeout time.Duration) HTTPClient { diff --git a/client/go/internal/vespa/document/circuit_breaker.go b/client/go/internal/vespa/document/circuit_breaker.go new file mode 100644 index 00000000000..17fc595d58f --- /dev/null +++ b/client/go/internal/vespa/document/circuit_breaker.go @@ -0,0 +1,77 @@ +package document + +import ( + "math" + "sync/atomic" + "time" +) + +type CircuitState int + +const ( + // CircuitClosed represents a closed circuit. Documents are processed successfully + CircuitClosed CircuitState = iota + // CircuitHalfOpen represents a half-open circuit. Some errors have happend, but processing may still recover + CircuitHalfOpen + // CircuitOpen represents a open circuit. Something is broken. We should no longer process documents + CircuitOpen +) + +type CircuitBreaker interface { + Success() + Error(error) + State() CircuitState +} + +type timeCircuitBreaker struct { + graceDuration time.Duration + doomDuration time.Duration + + failingSinceMillis int64 + lastError atomic.Value + halfOpen atomic.Value + open atomic.Value + + now func() time.Time +} + +func (b *timeCircuitBreaker) Success() { + atomic.StoreInt64(&b.failingSinceMillis, math.MaxInt64) + if !b.open.Load().(bool) { + b.halfOpen.CompareAndSwap(true, false) + } +} + +func (b *timeCircuitBreaker) Error(err error) { + if atomic.CompareAndSwapInt64(&b.failingSinceMillis, math.MaxInt64, b.now().UnixMilli()) { + b.lastError.Store(err) + } +} + +func (b *timeCircuitBreaker) State() CircuitState { + failingDuration := b.now().Sub(time.UnixMilli(atomic.LoadInt64(&b.failingSinceMillis))) + if failingDuration > b.graceDuration { + b.halfOpen.CompareAndSwap(false, true) + } + if b.doomDuration > 0 && failingDuration > b.doomDuration { + b.open.CompareAndSwap(false, true) + } + if b.open.Load().(bool) { + return CircuitOpen + } else if b.halfOpen.Load().(bool) { + return CircuitHalfOpen + } + return CircuitClosed +} + +func NewCircuitBreaker(graceDuration, doomDuration time.Duration) *timeCircuitBreaker { + b := &timeCircuitBreaker{ + graceDuration: graceDuration, + doomDuration: doomDuration, + now: time.Now, + failingSinceMillis: math.MaxInt64, + } + b.open.Store(false) + b.halfOpen.Store(false) + return b +} diff --git a/client/go/internal/vespa/document/circuit_breaker_test.go b/client/go/internal/vespa/document/circuit_breaker_test.go new file mode 100644 index 00000000000..7a4fffaae27 --- /dev/null +++ b/client/go/internal/vespa/document/circuit_breaker_test.go @@ -0,0 +1,49 @@ +package document + +import ( + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestCircuitBreaker(t *testing.T) { + clock := &manualClock{} + breaker := NewCircuitBreaker(time.Second, time.Minute) + breaker.now = clock.now + err := errors.New("error") + + assert.Equal(t, CircuitClosed, breaker.State(), "Initial state is closed") + + clock.advance(100 * time.Second) + assert.Equal(t, CircuitClosed, breaker.State(), "State is closed after some time without activity") + + breaker.Success() + assert.Equal(t, CircuitClosed, breaker.State(), "State is closed after a success") + + clock.advance(100 * time.Second) + assert.Equal(t, CircuitClosed, breaker.State(), "State is closed some time after a success") + + breaker.Error(err) + assert.Equal(t, CircuitClosed, breaker.State(), "State is closed right after a failure") + + clock.advance(time.Second) + assert.Equal(t, CircuitClosed, breaker.State(), "State is closed until grace duration has passed") + + clock.advance(time.Millisecond) + assert.Equal(t, CircuitHalfOpen, breaker.State(), "State is half-open when grace duration has passed") + + breaker.Success() + assert.Equal(t, CircuitClosed, breaker.State(), "State is closed after a new success") + + breaker.Error(err) + clock.advance(time.Minute) + assert.Equal(t, CircuitHalfOpen, breaker.State(), "State is half-open until doom duration has passed") + + clock.advance(time.Millisecond) + assert.Equal(t, CircuitOpen, breaker.State(), "State is open when doom duration has passed") + + breaker.Success() + assert.Equal(t, CircuitOpen, breaker.State(), "State remains open in spite of new successes") +} diff --git a/client/go/internal/vespa/document/dispatcher.go b/client/go/internal/vespa/document/dispatcher.go index feb562a241a..838a7bc45ee 100644 --- a/client/go/internal/vespa/document/dispatcher.go +++ b/client/go/internal/vespa/document/dispatcher.go @@ -1,129 +1,212 @@ package document import ( + "container/list" "fmt" + "io" "sync" + "sync/atomic" + "time" ) const maxAttempts = 10 // Dispatcher dispatches documents from a queue to a Feeder. type Dispatcher struct { - workers int - feeder Feeder - ready chan Id - inflight map[string]*documentGroup + feeder Feeder + throttler Throttler + circuitBreaker CircuitBreaker + stats Stats + + started bool + ready chan Id + results chan Result + inflight map[string]*documentGroup + inflightCount int64 + errWriter io.Writer + mu sync.RWMutex wg sync.WaitGroup - closed bool -} - -// documentGroup holds document operations which share their ID, and must be dispatched in order. -type documentGroup struct { - id Id - failed bool - operations []documentOp - mu sync.Mutex + resultWg sync.WaitGroup } +// documentOp represents a document operation and the number of times it has been attempted. type documentOp struct { document Document attempts int } -func (g *documentGroup) append(op documentOp) { +// documentGroup holds document operations which share an ID, and must be dispatched in order. +type documentGroup struct { + ops *list.List + mu sync.Mutex +} + +func (g *documentGroup) add(op documentOp, first bool) { g.mu.Lock() defer g.mu.Unlock() - g.operations = append(g.operations, op) + if g.ops == nil { + g.ops = list.New() + } + if first { + g.ops.PushFront(op) + } else { + g.ops.PushBack(op) + } } -func NewDispatcher(feeder Feeder, workers int) *Dispatcher { - if workers < 1 { - workers = 1 - } +func NewDispatcher(feeder Feeder, throttler Throttler, breaker CircuitBreaker, errWriter io.Writer) *Dispatcher { d := &Dispatcher{ - workers: workers, - feeder: feeder, - inflight: make(map[string]*documentGroup), + feeder: feeder, + throttler: throttler, + circuitBreaker: breaker, + inflight: make(map[string]*documentGroup), + errWriter: errWriter, } d.start() return d } -func (d *Dispatcher) dispatchAll(g *documentGroup) int { - g.mu.Lock() - defer g.mu.Unlock() - failCount := len(g.operations) - for i := 0; !g.failed && i < len(g.operations); i++ { - op := g.operations[i] - ok := false - for op.attempts <= maxAttempts && !ok { - op.attempts += 1 - // TODO(mpolden): Extract function which does throttling/circuit-breaking - result := d.feeder.Send(op.document) - ok = result.Status.Success() +func (d *Dispatcher) sendDocumentIn(group *documentGroup) { + group.mu.Lock() + defer group.mu.Unlock() + defer d.releaseSlot() + first := group.ops.Front() + if first == nil { + panic("sending from empty document group, this should not happen") + } + op := group.ops.Remove(first).(documentOp) + op.attempts++ + result := d.feeder.Send(op.document) + d.results <- result + if d.shouldRetry(op, result) { + d.enqueue(op) + } +} + +func (d *Dispatcher) shouldRetry(op documentOp, result Result) bool { + if result.HTTPStatus/100 == 2 || result.HTTPStatus == 404 || result.HTTPStatus == 412 { + d.throttler.Success() + d.circuitBreaker.Success() + return false + } + if result.HTTPStatus == 429 || result.HTTPStatus == 503 { + fmt.Fprintf(d.errWriter, "feed: %s was throttled with status %d: retrying\n", op.document, result.HTTPStatus) + d.throttler.Throttled(atomic.LoadInt64(&d.inflightCount)) + return true + } + if result.Err != nil || result.HTTPStatus == 500 || result.HTTPStatus == 502 || result.HTTPStatus == 504 { + retry := op.attempts <= maxAttempts + msg := "feed: " + op.document.String() + " failed with " + if result.Err != nil { + msg += "error " + result.Err.Error() + } else { + msg += fmt.Sprintf("status %d", result.HTTPStatus) } - if ok { - failCount-- + if retry { + msg += ": retrying" } else { - g.failed = true - failCount = len(g.operations) - i + msg += fmt.Sprintf(": giving up after %d attempts", maxAttempts) + } + fmt.Fprintln(d.errWriter, msg) + d.circuitBreaker.Error(fmt.Errorf("request failed with status %d", result.HTTPStatus)) + if retry { + return true } } - g.operations = nil - return failCount + return false } func (d *Dispatcher) start() { d.mu.Lock() defer d.mu.Unlock() - d.closed = false - d.ready = make(chan Id, 4*d.workers) - for i := 0; i < d.workers; i++ { + if d.started { + return + } + d.ready = make(chan Id, 4096) + d.results = make(chan Result, 4096) + d.started = true + d.wg.Add(1) + go func() { + defer d.wg.Done() + d.readDocuments() + }() + d.resultWg.Add(1) + go func() { + defer d.resultWg.Done() + d.readResults() + }() +} + +func (d *Dispatcher) readDocuments() { + for id := range d.ready { + d.mu.RLock() + group := d.inflight[id.String()] + d.mu.RUnlock() d.wg.Add(1) go func() { defer d.wg.Done() - for id := range d.ready { - d.mu.RLock() - group := d.inflight[id.String()] - d.mu.RUnlock() - if group != nil { - failedDocs := d.dispatchAll(group) - d.feeder.AddStats(Stats{Errors: int64(failedDocs)}) - } - } + d.sendDocumentIn(group) }() } } -func (d *Dispatcher) Enqueue(doc Document) error { +func (d *Dispatcher) readResults() { + for result := range d.results { + d.stats.Add(result.Stats) + } +} + +func (d *Dispatcher) enqueue(op documentOp) error { d.mu.Lock() - defer d.mu.Unlock() - if d.closed { + if !d.started { return fmt.Errorf("dispatcher is closed") } - group, ok := d.inflight[doc.Id.String()] - if ok { - group.append(documentOp{document: doc}) - } else { - group = &documentGroup{ - id: doc.Id, - operations: []documentOp{{document: doc}}, - } - d.inflight[doc.Id.String()] = group + group, ok := d.inflight[op.document.Id.String()] + if !ok { + group = &documentGroup{} + d.inflight[op.document.Id.String()] = group } - d.ready <- doc.Id + d.mu.Unlock() + group.add(op, op.attempts > 0) + d.enqueueWithSlot(op.document.Id) return nil } -// Close closes the dispatcher and waits for all inflight operations to complete. -func (d *Dispatcher) Close() error { +func (d *Dispatcher) enqueueWithSlot(id Id) { + d.acquireSlot() + d.ready <- id + d.throttler.Sent() +} + +func (d *Dispatcher) acquireSlot() { + for atomic.LoadInt64(&d.inflightCount) >= d.throttler.TargetInflight() { + time.Sleep(time.Millisecond) + } + atomic.AddInt64(&d.inflightCount, 1) +} + +func (d *Dispatcher) releaseSlot() { atomic.AddInt64(&d.inflightCount, -1) } + +func closeAndWait[T any](ch chan T, wg *sync.WaitGroup, d *Dispatcher, markClosed bool) { d.mu.Lock() - if !d.closed { - close(d.ready) - d.closed = true + if d.started { + close(ch) + if markClosed { + d.started = false + } } d.mu.Unlock() - d.wg.Wait() + wg.Wait() +} + +func (d *Dispatcher) Enqueue(doc Document) error { return d.enqueue(documentOp{document: doc}) } + +func (d *Dispatcher) Stats() Stats { return d.stats } + +// Close closes the dispatcher and waits for all inflight operations to complete. +func (d *Dispatcher) Close() error { + closeAndWait(d.ready, &d.wg, d, false) + closeAndWait(d.results, &d.resultWg, d, true) return nil } diff --git a/client/go/internal/vespa/document/dispatcher_test.go b/client/go/internal/vespa/document/dispatcher_test.go index 04e0928f2a3..80bc5f603ae 100644 --- a/client/go/internal/vespa/document/dispatcher_test.go +++ b/client/go/internal/vespa/document/dispatcher_test.go @@ -1,8 +1,10 @@ package document import ( + "io" "sync" "testing" + "time" "github.com/stretchr/testify/assert" ) @@ -10,7 +12,6 @@ import ( type mockFeeder struct { failAfterNDocs int documents []Document - stats Stats mu sync.Mutex } @@ -23,24 +24,24 @@ func (f *mockFeeder) failAfterN(docs int) { func (f *mockFeeder) Send(doc Document) Result { f.mu.Lock() defer f.mu.Unlock() + result := Result{Id: doc.Id} if f.failAfterNDocs > 0 && len(f.documents) >= f.failAfterNDocs { - return Result{Id: doc.Id, Status: StatusVespaFailure} + result.Status = StatusVespaFailure + } else { + f.documents = append(f.documents, doc) } - f.documents = append(f.documents, doc) - return Result{Id: doc.Id} -} - -func (f *mockFeeder) Stats() Stats { return f.stats } - -func (f *mockFeeder) AddStats(stats Stats) { - f.mu.Lock() - defer f.mu.Unlock() - f.stats.Add(stats) + if !result.Success() { + result.Stats.Errors = 1 + } + return result } func TestDispatcher(t *testing.T) { feeder := &mockFeeder{} - dispatcher := NewDispatcher(feeder, 2) + clock := &manualClock{tick: time.Second} + throttler := newThrottler(8, clock.now) + breaker := NewCircuitBreaker(time.Second, 0) + dispatcher := NewDispatcher(feeder, throttler, breaker, io.Discard) docs := []Document{ {Id: mustParseId("id:ns:type::doc1"), Operation: OperationPut, Body: []byte(`{"fields":{"foo": "123"}}`)}, {Id: mustParseId("id:ns:type::doc2"), Operation: OperationPut, Body: []byte(`{"fields":{"bar": "456"}}`)}, @@ -70,7 +71,10 @@ func TestDispatcherOrdering(t *testing.T) { {Id: mustParseId("id:ns:type::doc8"), Operation: OperationPut}, {Id: mustParseId("id:ns:type::doc9"), Operation: OperationPut}, } - dispatcher := NewDispatcher(feeder, len(docs)) + clock := &manualClock{tick: time.Second} + throttler := newThrottler(8, clock.now) + breaker := NewCircuitBreaker(time.Second, 0) + dispatcher := NewDispatcher(feeder, throttler, breaker, io.Discard) for _, d := range docs { dispatcher.Enqueue(d) } @@ -90,7 +94,7 @@ func TestDispatcherOrdering(t *testing.T) { } assert.Equal(t, len(docs), len(feeder.documents)) assert.Equal(t, wantDocs, gotDocs) - assert.Equal(t, int64(0), feeder.Stats().Errors) + assert.Equal(t, int64(0), dispatcher.Stats().Errors) } func TestDispatcherOrderingWithFailures(t *testing.T) { @@ -103,26 +107,26 @@ func TestDispatcherOrderingWithFailures(t *testing.T) { {Id: commonId, Operation: OperationRemove}, // fails } feeder.failAfterN(2) - dispatcher := NewDispatcher(feeder, len(docs)) + clock := &manualClock{tick: time.Second} + throttler := newThrottler(8, clock.now) + breaker := NewCircuitBreaker(time.Second, 0) + dispatcher := NewDispatcher(feeder, throttler, breaker, io.Discard) for _, d := range docs { dispatcher.Enqueue(d) } dispatcher.Close() wantDocs := docs[:2] assert.Equal(t, wantDocs, feeder.documents) - assert.Equal(t, int64(2), feeder.Stats().Errors) + assert.Equal(t, int64(2), dispatcher.Stats().Errors) - // Dispatching more documents for same ID fails implicitly + // Dispatching more documents for same ID succeed feeder.failAfterN(0) dispatcher.start() dispatcher.Enqueue(Document{Id: commonId, Operation: OperationPut}) dispatcher.Enqueue(Document{Id: commonId, Operation: OperationRemove}) - // Other IDs are fine - doc2 := Document{Id: mustParseId("id:ns:type::doc2"), Operation: OperationPut} - doc3 := Document{Id: mustParseId("id:ns:type::doc3"), Operation: OperationPut} - dispatcher.Enqueue(doc2) - dispatcher.Enqueue(doc3) + dispatcher.Enqueue(Document{Id: mustParseId("id:ns:type::doc2"), Operation: OperationPut}) + dispatcher.Enqueue(Document{Id: mustParseId("id:ns:type::doc3"), Operation: OperationPut}) dispatcher.Close() - assert.Equal(t, int64(4), feeder.Stats().Errors) - assert.Equal(t, 4, len(feeder.documents)) + assert.Equal(t, int64(2), dispatcher.Stats().Errors) + assert.Equal(t, 6, len(feeder.documents)) } diff --git a/client/go/internal/vespa/document/document.go b/client/go/internal/vespa/document/document.go index 98cb2d1b6c6..efb60ad8c0a 100644 --- a/client/go/internal/vespa/document/document.go +++ b/client/go/internal/vespa/document/document.go @@ -130,6 +130,27 @@ type Decoder struct { jsonl bool } +func (d Document) String() string { + var sb strings.Builder + switch d.Operation { + case OperationPut: + sb.WriteString("put ") + case OperationUpdate: + sb.WriteString("update ") + case OperationRemove: + sb.WriteString("remove ") + } + sb.WriteString(d.Id.String()) + if d.Condition != "" { + sb.WriteString(", condition=") + sb.WriteString(d.Condition) + } + if d.Create { + sb.WriteString(", create=true") + } + return sb.String() +} + func (d *Decoder) guessMode() error { for !d.array && !d.jsonl { b, err := d.buf.ReadByte() diff --git a/client/go/internal/vespa/document/feeder.go b/client/go/internal/vespa/document/feeder.go index 6996e649d24..9e6768d0eb4 100644 --- a/client/go/internal/vespa/document/feeder.go +++ b/client/go/internal/vespa/document/feeder.go @@ -17,23 +17,24 @@ const ( // StatusTransportFailure indicates that there was failure in the transport layer error while sending the document // operation to Vespa. StatusTransportFailure - // StatusError is a catch-all status for any other error that might occur. - StatusError ) // Result represents the result of a feeding operation. type Result struct { - Id Id - Status Status - Message string - Trace string - Err error + Id Id + Status Status + HTTPStatus int + Message string + Trace string + Err error + Stats Stats } -// Success returns whether status s is considered a success. -func (s Status) Success() bool { return s == StatusSuccess || s == StatusConditionNotMet } +func (r Result) Success() bool { + return r.Err == nil && (r.Status == StatusSuccess || r.Status == StatusConditionNotMet) +} -// Stats represents the summed statistics of a feeder. +// Stats represents feeding operation statistics. type Stats struct { Requests int64 Responses int64 @@ -47,8 +48,6 @@ type Stats struct { BytesRecv int64 } -func NewStats() Stats { return Stats{ResponsesByCode: make(map[int]int64)} } - // AvgLatency returns the average latency for a request. func (s Stats) AvgLatency() time.Duration { requests := s.Requests @@ -69,6 +68,9 @@ func (s Stats) Successes() int64 { func (s *Stats) Add(other Stats) { s.Requests += other.Requests s.Responses += other.Responses + if s.ResponsesByCode == nil && other.ResponsesByCode != nil { + s.ResponsesByCode = make(map[int]int64) + } for code, count := range other.ResponsesByCode { _, ok := s.ResponsesByCode[code] if ok { @@ -91,8 +93,4 @@ func (s *Stats) Add(other Stats) { } // Feeder is the interface for a consumer of documents. -type Feeder interface { - Send(Document) Result - Stats() Stats - AddStats(Stats) -} +type Feeder interface{ Send(Document) Result } diff --git a/client/go/internal/vespa/document/feeder_test.go b/client/go/internal/vespa/document/feeder_test.go index 1368d871436..a7d92495889 100644 --- a/client/go/internal/vespa/document/feeder_test.go +++ b/client/go/internal/vespa/document/feeder_test.go @@ -7,7 +7,7 @@ import ( ) func TestStatsAdd(t *testing.T) { - got := NewStats() + var got Stats got.Add(Stats{Requests: 1}) got.Add(Stats{Requests: 1}) got.Add(Stats{Responses: 1}) diff --git a/client/go/internal/vespa/document/http.go b/client/go/internal/vespa/document/http.go index 2e01d4564ab..588330a0574 100644 --- a/client/go/internal/vespa/document/http.go +++ b/client/go/internal/vespa/document/http.go @@ -5,11 +5,12 @@ import ( "encoding/json" "fmt" "io" + "math" "net/http" "net/url" "strconv" "strings" - "sync" + "sync/atomic" "time" "github.com/vespa-engine/vespa/client/go/internal/util" @@ -17,11 +18,10 @@ import ( // Client represents a HTTP client for the /document/v1/ API. type Client struct { - options ClientOptions - httpClient util.HTTPClient - stats Stats - mu sync.Mutex - now func() time.Time + options ClientOptions + httpClients []countingHTTPClient + now func() time.Time + sendCount int32 } // ClientOptions specifices the configuration options of a feed client. @@ -32,6 +32,18 @@ type ClientOptions struct { TraceLevel *int } +type countingHTTPClient struct { + client util.HTTPClient + inflight int64 +} + +func (c *countingHTTPClient) addInflight(n int64) { atomic.AddInt64(&c.inflight, n) } + +func (c *countingHTTPClient) Do(req *http.Request, timeout time.Duration) (*http.Response, error) { + defer c.addInflight(-1) + return c.client.Do(req, timeout) +} + type countingReader struct { reader io.Reader bytesRead int64 @@ -43,14 +55,19 @@ func (r *countingReader) Read(p []byte) (int, error) { return n, err } -func NewClient(options ClientOptions, httpClient util.HTTPClient) *Client { - c := &Client{ - options: options, - httpClient: httpClient, - stats: NewStats(), - now: time.Now, +func NewClient(options ClientOptions, httpClients []util.HTTPClient) *Client { + if len(httpClients) < 1 { + panic("need at least one HTTP client") + } + countingClients := make([]countingHTTPClient, 0, len(httpClients)) + for _, client := range httpClients { + countingClients = append(countingClients, countingHTTPClient{client: client}) + } + return &Client{ + options: options, + httpClients: countingClients, + now: time.Now, } - return c } func (c *Client) queryParams() url.Values { @@ -113,52 +130,56 @@ func (c *Client) feedURL(d Document, queryParams url.Values) (string, *url.URL, return httpMethod, u, nil } -// Send given document the URL configured in this client. +func (c *Client) leastBusyClient() *countingHTTPClient { + leastBusy := c.httpClients[0] + min := int64(math.MaxInt64) + next := atomic.AddInt32(&c.sendCount, 1) + start := int(next) % len(c.httpClients) + for i := range c.httpClients { + j := (i + start) % len(c.httpClients) + client := c.httpClients[j] + inflight := atomic.LoadInt64(&client.inflight) + if inflight < min { + leastBusy = client + min = inflight + } + } + leastBusy.addInflight(1) + return &leastBusy +} + +// Send given document to the endpoint configured in this client. func (c *Client) Send(document Document) Result { start := c.now() - stats := NewStats() - stats.Requests = 1 - defer func() { - latency := c.now().Sub(start) - stats.TotalLatency = latency - stats.MinLatency = latency - stats.MaxLatency = latency - c.AddStats(stats) - }() + result := Result{Id: document.Id, Stats: Stats{Requests: 1}} method, url, err := c.feedURL(document, c.queryParams()) if err != nil { - stats.Errors = 1 - return Result{Status: StatusError, Err: err} + return resultWithErr(result, err) } req, err := http.NewRequest(method, url.String(), bytes.NewReader(document.Body)) if err != nil { - stats.Errors = 1 - return Result{Status: StatusError, Err: err} + return resultWithErr(result, err) } - resp, err := c.httpClient.Do(req, c.options.Timeout) + resp, err := c.leastBusyClient().Do(req, 190*time.Second) if err != nil { - stats.Errors = 1 - return Result{Status: StatusTransportFailure, Err: err} + return resultWithErr(result, err) } defer resp.Body.Close() - stats.Responses = 1 - stats.ResponsesByCode = map[int]int64{ - resp.StatusCode: 1, - } - stats.BytesSent = int64(len(document.Body)) - return c.createResult(document.Id, &stats, resp) + elapsed := c.now().Sub(start) + return c.resultWithResponse(resp, result, document, elapsed) } -func (c *Client) Stats() Stats { return c.stats } - -func (c *Client) AddStats(stats Stats) { - c.mu.Lock() - defer c.mu.Unlock() - c.stats.Add(stats) +func resultWithErr(result Result, err error) Result { + result.Stats.Errors++ + result.Status = StatusTransportFailure + result.Err = err + return result } -func (c *Client) createResult(id Id, stats *Stats, resp *http.Response) Result { - result := Result{Id: id} +func (c *Client) resultWithResponse(resp *http.Response, result Result, document Document, elapsed time.Duration) Result { + result.HTTPStatus = resp.StatusCode + result.Stats.Responses++ + result.Stats.ResponsesByCode = map[int]int64{resp.StatusCode: 1} switch resp.StatusCode { case 200: result.Status = StatusSuccess @@ -176,14 +197,18 @@ func (c *Client) createResult(id Id, stats *Stats, resp *http.Response) Result { cr := countingReader{reader: resp.Body} jsonDec := json.NewDecoder(&cr) if err := jsonDec.Decode(&body); err != nil { - result.Status = StatusError + result.Status = StatusVespaFailure result.Err = fmt.Errorf("failed to decode json response: %w", err) } result.Message = body.Message result.Trace = string(body.Trace) - stats.BytesRecv = cr.bytesRead - if !result.Status.Success() { - stats.Errors = 1 - } + result.Stats.BytesSent = int64(len(document.Body)) + result.Stats.BytesRecv = cr.bytesRead + if !result.Success() { + result.Stats.Errors++ + } + result.Stats.TotalLatency = elapsed + result.Stats.MinLatency = elapsed + result.Stats.MaxLatency = elapsed return result } diff --git a/client/go/internal/vespa/document/http_test.go b/client/go/internal/vespa/document/http_test.go index f02c87730d5..43eaf1bfdf9 100644 --- a/client/go/internal/vespa/document/http_test.go +++ b/client/go/internal/vespa/document/http_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/vespa-engine/vespa/client/go/internal/mock" + "github.com/vespa-engine/vespa/client/go/internal/util" ) type manualClock struct { @@ -19,9 +20,41 @@ type manualClock struct { } func (c *manualClock) now() time.Time { - t := c.t - c.t = c.t.Add(c.tick) - return t + c.advance(c.tick) + return c.t +} + +func (c *manualClock) advance(d time.Duration) { c.t = c.t.Add(d) } + +type mockHTTPClient struct { + id int + *mock.HTTPClient +} + +func TestLeastBusyClient(t *testing.T) { + httpClient := mock.HTTPClient{} + var httpClients []util.HTTPClient + for i := 0; i < 4; i++ { + httpClients = append(httpClients, &mockHTTPClient{i, &httpClient}) + } + client := NewClient(ClientOptions{}, httpClients) + client.httpClients[0].addInflight(1) + client.httpClients[1].addInflight(1) + assertLeastBusy(t, 2, client) + assertLeastBusy(t, 2, client) + assertLeastBusy(t, 3, client) + client.httpClients[3].addInflight(1) + client.httpClients[1].addInflight(-1) + assertLeastBusy(t, 1, client) +} + +func assertLeastBusy(t *testing.T, id int, client *Client) { + t.Helper() + leastBusy := client.leastBusyClient() + got := leastBusy.client.(*mockHTTPClient).id + if got != id { + t.Errorf("got client.id=%d, want %d", got, id) + } } func TestClientSend(t *testing.T) { @@ -34,19 +67,43 @@ func TestClientSend(t *testing.T) { client := NewClient(ClientOptions{ BaseURL: "https://example.com:1337", Timeout: time.Duration(5 * time.Second), - }, &httpClient) + }, []util.HTTPClient{&httpClient}) clock := manualClock{t: time.Now(), tick: time.Second} client.now = clock.now + var stats Stats for i, doc := range docs { + wantRes := Result{ + Id: doc.Id, + Stats: Stats{ + Requests: 1, + Responses: 1, + TotalLatency: time.Second, + MinLatency: time.Second, + MaxLatency: time.Second, + BytesSent: 25, + }, + } if i < 2 { httpClient.NextResponseString(200, `{"message":"All good!"}`) + wantRes.Status = StatusSuccess + wantRes.HTTPStatus = 200 + wantRes.Message = "All good!" + wantRes.Stats.ResponsesByCode = map[int]int64{200: 1} + wantRes.Stats.BytesRecv = 23 } else { httpClient.NextResponseString(502, `{"message":"Good bye, cruel world!"}`) + wantRes.Status = StatusVespaFailure + wantRes.HTTPStatus = 502 + wantRes.Message = "Good bye, cruel world!" + wantRes.Stats.ResponsesByCode = map[int]int64{502: 1} + wantRes.Stats.Errors = 1 + wantRes.Stats.BytesRecv = 36 } res := client.Send(doc) - if res.Err != nil { - t.Fatalf("got unexpected error %q", res.Err) + if !reflect.DeepEqual(res, wantRes) { + t.Fatalf("got result %+v, want %+v", res, wantRes) } + stats.Add(res.Stats) r := httpClient.LastRequest if r.Method != http.MethodPut { t.Errorf("got r.Method = %q, want %q", r.Method, http.MethodPut) @@ -64,7 +121,6 @@ func TestClientSend(t *testing.T) { t.Errorf("got r.Body = %q, want %q", string(body), string(wantBody)) } } - stats := client.Stats() want := Stats{ Requests: 3, Responses: 3, @@ -174,7 +230,7 @@ func TestClientFeedURL(t *testing.T) { httpClient := mock.HTTPClient{} client := NewClient(ClientOptions{ BaseURL: "https://example.com", - }, &httpClient) + }, []util.HTTPClient{&httpClient}) for i, tt := range tests { moreParams := url.Values{} moreParams.Set("foo", "ba/r") diff --git a/client/go/internal/vespa/document/throttler.go b/client/go/internal/vespa/document/throttler.go new file mode 100644 index 00000000000..5b0aab6174e --- /dev/null +++ b/client/go/internal/vespa/document/throttler.go @@ -0,0 +1,120 @@ +package document + +import ( + "math" + "math/rand" + "sync/atomic" + "time" +) + +const throttlerWeight = 0.7 + +type Throttler interface { + // Sent notifies the the throttler that a document has been sent. + Sent() + // Success notifies the throttler that document operation succeeded. + Success() + // Throttled notifies the throttler that a throttling event occured while count documents were in-flight. + Throttled(count int64) + // TargetInflight returns the ideal number of documents to have in-flight now. + TargetInflight() int64 +} + +type dynamicThrottler struct { + minInflight int64 + maxInflight int64 + targetInflight int64 + targetTimesTen int64 + + throughputs []float64 + ok int64 + sent int64 + + start time.Time + now func() time.Time +} + +func newThrottler(connections int, nowFunc func() time.Time) *dynamicThrottler { + var ( + minInflight = 16 * int64(connections) + maxInflight = 256 * minInflight // 4096 max streams per connection on the server side + ) + return &dynamicThrottler{ + minInflight: minInflight, + maxInflight: maxInflight, + targetInflight: 8 * minInflight, + targetTimesTen: 10 * maxInflight, + + throughputs: make([]float64, 128), + + start: nowFunc(), + now: nowFunc, + } +} + +func NewThrottler(connections int) Throttler { return newThrottler(connections, time.Now) } + +func (t *dynamicThrottler) Sent() { + currentInflight := atomic.LoadInt64(&t.targetInflight) + t.sent++ + if t.sent*t.sent*t.sent < 100*currentInflight*currentInflight { + return + } + t.sent = 0 + now := t.now() + elapsed := now.Sub(t.start) + t.start = now + currentThroughput := float64(atomic.SwapInt64(&t.ok, 0)) / float64(elapsed) + + // Use buckets for throughput over inflight, along the log-scale, in [minInflight, maxInflight). + index := int(float64(len(t.throughputs)) * math.Log(max(1, min(255, float64(currentInflight)/float64(t.minInflight)))) / math.Log(256)) + t.throughputs[index] = currentThroughput + + // Loop over throughput measurements and pick the one which optimises throughput and latency. + choice := float64(currentInflight) + maxObjective := float64(-1) + for i := len(t.throughputs) - 1; i >= 0; i-- { + if t.throughputs[i] == 0 { + continue // Skip unknown values + } + inflight := float64(t.minInflight) * math.Pow(256, (float64(i)+0.5)/float64(len(t.throughputs))) + objective := t.throughputs[i] * math.Pow(inflight, throttlerWeight-1) // Optimise throughput (weight), but also latency (1 - weight) + if objective > maxObjective { + maxObjective = objective + choice = inflight + } + } + target := int64((rand.Float64()*0.20 + 0.92) * choice) // Random walk, skewed towards increase + atomic.StoreInt64(&t.targetInflight, max(t.minInflight, min(t.maxInflight, target))) +} + +func (t *dynamicThrottler) Success() { + atomic.AddInt64(&t.targetTimesTen, 1) + atomic.AddInt64(&t.ok, 1) +} + +func (t *dynamicThrottler) Throttled(inflight int64) { + atomic.StoreInt64(&t.targetTimesTen, max(inflight*5, t.minInflight*10)) +} + +func (t *dynamicThrottler) TargetInflight() int64 { + staticTargetInflight := min(t.maxInflight, atomic.LoadInt64(&t.targetTimesTen)/10) + targetInflight := atomic.LoadInt64(&t.targetInflight) + return min(staticTargetInflight, targetInflight) +} + +type number interface{ float64 | int64 } + +func min[T number](x, y T) T { + if x < y { + return x + } + return y +} + +func max[T number](x, y T) T { + if x > y { + return x + } + return y +} diff --git a/client/go/internal/vespa/document/throttler_test.go b/client/go/internal/vespa/document/throttler_test.go new file mode 100644 index 00000000000..a22f059207f --- /dev/null +++ b/client/go/internal/vespa/document/throttler_test.go @@ -0,0 +1,21 @@ +package document + +import ( + "testing" + "time" +) + +func TestThrottler(t *testing.T) { + clock := &manualClock{tick: time.Second} + tr := newThrottler(8, clock.now) + for i := 0; i < 100; i++ { + tr.Sent() + } + if got, want := tr.TargetInflight(), int64(1024); got != want { + t.Errorf("got TargetInflight() = %d, but want %d", got, want) + } + tr.Throttled(5) + if got, want := tr.TargetInflight(), int64(128); got != want { + t.Errorf("got TargetInflight() = %d, but want %d", got, want) + } +} diff --git a/client/go/internal/vespa/target.go b/client/go/internal/vespa/target.go index 51861eb12ab..bc936623bcb 100644 --- a/client/go/internal/vespa/target.go +++ b/client/go/internal/vespa/target.go @@ -74,7 +74,7 @@ type Target interface { // TLSOptions configures the client certificate to use for cloud API or service requests. type TLSOptions struct { - KeyPair *tls.Certificate + KeyPair []tls.Certificate CertificateFile string PrivateKeyFile string AthenzDomain string @@ -93,7 +93,7 @@ type LogOptions struct { // Do sends request to this service. Any required authentication happens automatically. func (s *Service) Do(request *http.Request, timeout time.Duration) (*http.Response, error) { if s.TLSOptions.AthenzDomain != "" && s.TLSOptions.KeyPair != nil { - accessToken, err := s.zts.AccessToken(s.TLSOptions.AthenzDomain, *s.TLSOptions.KeyPair) + accessToken, err := s.zts.AccessToken(s.TLSOptions.AthenzDomain, s.TLSOptions.KeyPair[0]) if err != nil { return nil, err } @@ -105,6 +105,8 @@ func (s *Service) Do(request *http.Request, timeout time.Duration) (*http.Respon return s.httpClient.Do(request, timeout) } +func (s *Service) Client() util.HTTPClient { return s.httpClient } + // Wait polls the health check of this service until it succeeds or timeout passes. func (s *Service) Wait(timeout time.Duration) (int, error) { url := s.BaseURL @@ -139,18 +141,18 @@ type requestFunc func() *http.Request // waitForOK queries url and returns its status code. If the url returns a non-200 status code, it is repeatedly queried // until timeout elapses. -func waitForOK(client util.HTTPClient, url string, certificate *tls.Certificate, timeout time.Duration) (int, error) { +func waitForOK(client util.HTTPClient, url string, certificates []tls.Certificate, timeout time.Duration) (int, error) { req, err := http.NewRequest("GET", url, nil) if err != nil { return 0, err } okFunc := func(status int, response []byte) (bool, error) { return isOK(status), nil } - return wait(client, okFunc, func() *http.Request { return req }, certificate, timeout) + return wait(client, okFunc, func() *http.Request { return req }, certificates, timeout) } -func wait(client util.HTTPClient, fn responseFunc, reqFn requestFunc, certificate *tls.Certificate, timeout time.Duration) (int, error) { - if certificate != nil { - util.SetCertificate(client, []tls.Certificate{*certificate}) +func wait(client util.HTTPClient, fn responseFunc, reqFn requestFunc, certificates []tls.Certificate, timeout time.Duration) (int, error) { + if certificates != nil { + util.SetCertificates(client, certificates) } var ( httpErr error diff --git a/client/go/internal/vespa/target_cloud.go b/client/go/internal/vespa/target_cloud.go index 2335d4f3432..1fb3edd78c5 100644 --- a/client/go/internal/vespa/target_cloud.go +++ b/client/go/internal/vespa/target_cloud.go @@ -161,7 +161,7 @@ func (t *cloudTarget) Service(name string, timeout time.Duration, runID int64, c } if service.TLSOptions.KeyPair != nil { - util.SetCertificate(service.httpClient, []tls.Certificate{*service.TLSOptions.KeyPair}) + util.SetCertificates(service.httpClient, service.TLSOptions.KeyPair) } return service, nil } @@ -175,7 +175,7 @@ func (t *cloudTarget) SignRequest(req *http.Request, keyID string) error { return t.addAuth0AccessToken(req) } } else { - if t.apiOptions.TLSOptions.KeyPair.Certificate == nil { + if t.apiOptions.TLSOptions.KeyPair == nil { return fmt.Errorf("system %s requires a certificate for authentication", t.apiOptions.System.Name) } return nil diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeCheckerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeCheckerTest.java index 1d28a90352b..08265248959 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeCheckerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/NodeStateChangeCheckerTest.java @@ -507,7 +507,7 @@ public class NodeStateChangeCheckerTest { } private Result transitionToSameState(String oldDescription, String newDescription) { - return transitionToSameState(State.MAINTENANCE, oldDescription, newDescription); + return transitionToSameState(MAINTENANCE, oldDescription, newDescription); } @Test diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 21797f0f469..fef2354c452 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -72,7 +72,7 @@ public interface ModelContext { * - Remove all flag data files from hosted-feature-flag repository */ interface FeatureFlags { - @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Revisit in May or June 2021") default double defaultTermwiseLimit() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Revisit in May or June 2023") default double defaultTermwiseLimit() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Select sequencer type use while feeding") default String feedSequencerType() { return "THROUGHPUT"; } @ModelFeatureFlag(owners = {"baldersheim"}) default String responseSequencerType() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default String queryDispatchPolicy() { return "adaptive"; } @@ -112,6 +112,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"arnej","baldersheim"}, removeAfter = "8.110") default boolean useOldJdiscContainerStartup() { return false; } @ModelFeatureFlag(owners = {"tokle, bjorncs"}, removeAfter = "8.108") default boolean enableDataPlaneFilter() { return true; } @ModelFeatureFlag(owners = {"arnej, bjorncs"}) default boolean enableGlobalPhase() { return true; } + @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Select summary decode type") default String summaryDecodePolicy() { return "eager"; } //Below are all flags that must be kept until 7 is out of the door @ModelFeatureFlag(owners = {"arnej"}, removeAfter="7.last") default boolean ignoreThreadStackSizes() { return false; } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index ecbb990f096..c72aa23a836 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -43,6 +43,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private double defaultTermwiseLimit = 1.0; private String jvmGCOptions = null; private String queryDispatchPolicy = "adaptive"; + private String summaryDecodePolicy = "eager"; private String sequencerType = "THROUGHPUT"; private boolean firstTimeDeployment = false; private String responseSequencerType = "ADAPTIVE"; @@ -134,6 +135,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public int heapSizePercentage() { return heapSizePercentage; } @Override public int rpcEventsBeforeWakeup() { return rpc_events_before_wakeup; } @Override public String queryDispatchPolicy() { return queryDispatchPolicy; } + @Override public String summaryDecodePolicy() { return summaryDecodePolicy; } @Override public boolean useRestrictedDataPlaneBindings() { return useRestrictedDataPlaneBindings; } @Override public Optional<CloudAccount> cloudAccount() { return cloudAccount; } @Override public boolean allowUserFilters() { return allowUserFilters; } @@ -191,6 +193,10 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea queryDispatchPolicy = policy; return this; } + public TestProperties setSummaryDecodePolicy(String type) { + summaryDecodePolicy = type; + return this; + } public TestProperties setFeedSequencerType(String type) { sequencerType = type; return this; diff --git a/config-model/src/main/java/com/yahoo/schema/OnnxModel.java b/config-model/src/main/java/com/yahoo/schema/OnnxModel.java index 272b668b5fb..90a27d1f036 100644 --- a/config-model/src/main/java/com/yahoo/schema/OnnxModel.java +++ b/config-model/src/main/java/com/yahoo/schema/OnnxModel.java @@ -1,15 +1,17 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.schema; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.model.ml.OnnxModelInfo; -import com.yahoo.searchlib.rankingexpression.Reference; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; /** * A global ONNX model distributed using file distribution, similar to ranking constants. @@ -21,6 +23,7 @@ public class OnnxModel extends DistributableResource { private OnnxModelInfo modelInfo = null; private final Map<String, String> inputMap = new HashMap<>(); private final Map<String, String> outputMap = new HashMap<>(); + private final Set<String> initializers = new HashSet<>(); private String statelessExecutionMode = null; private Integer statelessInterOpThreads = null; @@ -101,11 +104,13 @@ public class OnnxModel extends DistributableResource { for (String onnxName : modelInfo.getOutputs()) { addOutputNameMapping(onnxName, OnnxModelInfo.asValidIdentifier(onnxName), false); } + initializers.addAll(modelInfo.getInitializers()); this.modelInfo = modelInfo; } public Map<String, String> getInputMap() { return Collections.unmodifiableMap(inputMap); } public Map<String, String> getOutputMap() { return Collections.unmodifiableMap(outputMap); } + public Set<String> getInitializers() { return Set.copyOf(initializers); } public String getDefaultOutput() { return modelInfo != null ? modelInfo.getDefaultOutput() : ""; diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java index af072c5b59a..7f578f07fe3 100644 --- a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java @@ -2,7 +2,6 @@ package com.yahoo.schema.expressiontransforms; import com.yahoo.schema.FeatureNames; -import com.yahoo.schema.RankProfile; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.parser.ParseException; @@ -12,13 +11,12 @@ import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.searchlib.rankingexpression.rule.TensorFunctionNode; import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; -import com.yahoo.tensor.functions.DynamicTensor; import com.yahoo.tensor.functions.Generate; -import com.yahoo.tensor.functions.Slice; import java.io.StringReader; import java.util.HashSet; import java.util.Set; +import java.util.logging.Logger; /** * Analyzes expression to figure out what inputs it needs @@ -27,6 +25,8 @@ import java.util.Set; */ public class InputRecorder extends ExpressionTransformer<InputRecorderContext> { + private static final Logger log = Logger.getLogger(InputRecorder.class.getName()); + private final Set<String> neededInputs; private final Set<String> handled = new HashSet<>(); @@ -120,7 +120,11 @@ public class InputRecorder extends ExpressionTransformer<InputRecorderContext> { if (model == null) { throw new IllegalArgumentException("missing onnx model: " + arg); } - for (String onnxInput : model.getInputMap().values()) { + model.getInputMap().forEach((onnxName, onnxInput) -> { + if (model.getInitializers().contains(onnxName)) { + log.fine(() -> "For input '%s': skipping name '%s' as it's an initializer".formatted(onnxInput, onnxName)); + return; + } var reader = new StringReader(onnxInput); try { var asExpression = new RankingExpression(reader); @@ -128,7 +132,7 @@ public class InputRecorder extends ExpressionTransformer<InputRecorderContext> { } catch (ParseException e) { throw new IllegalArgumentException("illegal onnx input '" + onnxInput + "': " + e.getMessage()); } - } + }); return; } neededInputs.add(feature.toString()); diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertSchemaCollection.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertSchemaCollection.java index 5509d11885c..b414d3757e2 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertSchemaCollection.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertSchemaCollection.java @@ -20,7 +20,7 @@ import java.util.List; * Class converting a collection of schemas from the intermediate format. * * @author arnej27959 - **/ + */ public class ConvertSchemaCollection { private final IntermediateCollection input; diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java index d8c1fb3125f..3c7e9b4066f 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java @@ -62,7 +62,7 @@ public class IndexingValidation extends Processor { final Set<String> prevNames = new HashSet<>(); @Override - protected ExpressionConverter branch() { + public ExpressionConverter branch() { MyConverter ret = new MyConverter(); ret.outputs.addAll(outputs); ret.prevNames.addAll(prevNames); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java deleted file mode 100644 index 9d4e732d3f8..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/CloudWatch.java +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.admin.monitoring; - -import java.util.Optional; - -/** - * Helper object for CloudWatch configuration. - * - * @author gjoranv - */ -public class CloudWatch { - - private final String region; - private final String namespace; - private final MetricsConsumer consumer; - - private HostedAuth hostedAuth; - private SharedCredentials sharedCredentials; - - public CloudWatch(String region, String namespace, MetricsConsumer consumer) { - this.region = region; - this.namespace = namespace; - this.consumer = consumer; - } - - public String region() { return region; } - public String namespace() { return namespace; } - public String consumer() { return consumer.id(); } - - public Optional<HostedAuth> hostedAuth() {return Optional.ofNullable(hostedAuth); } - public Optional<SharedCredentials> sharedCredentials() {return Optional.ofNullable(sharedCredentials); } - - public void setHostedAuth(String accessKeyName, String secretKeyName) { - hostedAuth = new HostedAuth(accessKeyName, secretKeyName); - } - - public void setSharedCredentials(String file, Optional<String> profile) { - sharedCredentials = new SharedCredentials(file, profile); - } - - public static class HostedAuth { - public final String accessKeyName; - public final String secretKeyName; - - HostedAuth(String accessKeyName, String secretKeyName) { - this.accessKeyName = accessKeyName; - this.secretKeyName = secretKeyName; - } - } - - public static class SharedCredentials { - public final String file; - public final Optional<String> profile; - - SharedCredentials(String file, Optional<String> profile) { - this.file = file; - this.profile = profile; - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java index 671a90a0fee..c6bd4c6e295 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/MetricsConsumer.java @@ -41,9 +41,6 @@ public class MetricsConsumer { private final String id; private final MetricSet metricSet; - // TODO: This shouldn't be here - private final List<CloudWatch> cloudWatches = new ArrayList<>(); - /** * @param id the consumer * @param metricSet the metrics for this consumer @@ -66,14 +63,6 @@ public class MetricsConsumer { return metricSet.getMetrics(); } - public void addCloudWatch(CloudWatch cloudWatch) { - cloudWatches.add(cloudWatch); - } - - public List<CloudWatch> cloudWatches() { - return unmodifiableList(cloudWatches); - } - private static MetricsConsumer consumer(String id, MetricSet ... metricSets) { return new MetricsConsumer(id, new MetricSet(id + "-consumer-metrics", List.of(), Arrays.asList(metricSets))); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 9c0a6b1b224..ac354d66206 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -142,26 +142,26 @@ public class VespaMetricSet { addMetric(metrics, ContainerMetrics.HANDLED_REQUESTS.count()); addMetric(metrics, ContainerMetrics.HANDLED_LATENCY, EnumSet.of(sum, count, max)); - addMetric(metrics, ContainerMetrics.SERVER_NUM_OPEN_CONNECTIONS, EnumSet.of(max, last, average)); - addMetric(metrics, ContainerMetrics.SERVER_NUM_CONNECTIONS, EnumSet.of(max, last, average)); + addMetric(metrics, ContainerMetrics.SERVER_NUM_OPEN_CONNECTIONS, EnumSet.of(max, last, average)); // TODO: Vespa 9: Remove last + addMetric(metrics, ContainerMetrics.SERVER_NUM_CONNECTIONS, EnumSet.of(max, last, average)); // TODO: Vespa 9: Remove last addMetric(metrics, ContainerMetrics.SERVER_BYTES_RECEIVED, EnumSet.of(sum, count)); addMetric(metrics, ContainerMetrics.SERVER_BYTES_SENT, EnumSet.of(sum, count)); - addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_UNHANDLED_EXCEPTIONS, EnumSet.of(sum, count, last, min, max)); - addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_WORK_QUEUE_CAPACITY, EnumSet.of(sum, count, last, min, max)); - addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_WORK_QUEUE_SIZE, EnumSet.of(sum, count, last, min, max)); - addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_REJECTED_TASKS, EnumSet.of(sum, count, last, min, max)); - addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_SIZE, EnumSet.of(sum, count, last, min, max)); - addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_MAX_ALLOWED_SIZE, EnumSet.of(sum, count, last, min, max)); - addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_ACTIVE_THREADS, EnumSet.of(sum, count, last, min, max)); - - addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_MAX_THREADS, EnumSet.of(sum, count, last, min, max)); - addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_MIN_THREADS, EnumSet.of(sum, count, last, min, max)); - addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_RESERVED_THREADS, EnumSet.of(sum, count, last, min, max)); - addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_BUSY_THREADS, EnumSet.of(sum, count, last, min, max)); - addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_TOTAL_THREADS, EnumSet.of(sum, count, last, min, max)); - addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_QUEUE_SIZE, EnumSet.of(sum, count, last, min, max)); + addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_UNHANDLED_EXCEPTIONS, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove last, min, max + addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_WORK_QUEUE_CAPACITY, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove sum, count, last, min + addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_WORK_QUEUE_SIZE, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove last + addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_REJECTED_TASKS, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove last, min, max + addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_SIZE, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove sum, count, last, min + addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_MAX_ALLOWED_SIZE, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove sum, count, last, min + addMetric(metrics, ContainerMetrics.JDISC_THREAD_POOL_ACTIVE_THREADS, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove last + + addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_MAX_THREADS, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove. + addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_MIN_THREADS, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove. + addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_RESERVED_THREADS, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove. + addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_BUSY_THREADS, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove last, min + addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_TOTAL_THREADS, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove sum, count, last, min + addMetric(metrics, ContainerMetrics.JETTY_THREADPOOL_QUEUE_SIZE, EnumSet.of(sum, count, last, min, max)); // TODO: Vespa 9: Remove sum, count, last, min addMetric(metrics, ContainerMetrics.HTTPAPI_LATENCY, EnumSet.of(max, sum, count)); addMetric(metrics, ContainerMetrics.HTTPAPI_PENDING, EnumSet.of(max, sum, count)); @@ -192,8 +192,8 @@ public class VespaMetricSet { addMetric(metrics, ContainerMetrics.JDISC_MEMORY_MAPPINGS.max()); addMetric(metrics, ContainerMetrics.JDISC_OPEN_FILE_DESCRIPTORS.max()); - addMetric(metrics, ContainerMetrics.JDISC_GC_COUNT, EnumSet.of(average, max, last)); - addMetric(metrics, ContainerMetrics.JDISC_GC_MS, EnumSet.of(average, max, last)); + addMetric(metrics, ContainerMetrics.JDISC_GC_COUNT, EnumSet.of(average, max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, ContainerMetrics.JDISC_GC_MS, EnumSet.of(average, max, last)); // TODO: Vespa 9: Remove last addMetric(metrics, ContainerMetrics.JDISC_DEACTIVATED_CONTAINERS.last()); addMetric(metrics, ContainerMetrics.JDISC_DEACTIVATED_CONTAINERS_WITH_RETAINED_REFS.last()); @@ -263,19 +263,19 @@ public class VespaMetricSet { addMetric(metrics, ClusterControllerMetrics.STOPPING_COUNT.last()); addMetric(metrics, ClusterControllerMetrics.UP_COUNT.last()); addMetric(metrics, ClusterControllerMetrics.CLUSTER_STATE_CHANGE_COUNT.baseName()); - addMetric(metrics, ClusterControllerMetrics.BUSY_TICK_TIME_MS, EnumSet.of(last, max, sum, count)); - addMetric(metrics, ClusterControllerMetrics.IDLE_TICK_TIME_MS, EnumSet.of(last, max, sum, count)); + addMetric(metrics, ClusterControllerMetrics.BUSY_TICK_TIME_MS, EnumSet.of(last, max, sum, count)); // TODO: Vespa 9: Remove last + addMetric(metrics, ClusterControllerMetrics.IDLE_TICK_TIME_MS, EnumSet.of(last, max, sum, count)); // TODO: Vespa 9: Remove last - addMetric(metrics, ClusterControllerMetrics.WORK_MS, EnumSet.of(last, sum, count)); + addMetric(metrics, ClusterControllerMetrics.WORK_MS, EnumSet.of(last, sum, count)); // TODO: Vespa 9: Remove last addMetric(metrics, ClusterControllerMetrics.IS_MASTER.last()); addMetric(metrics, ClusterControllerMetrics.REMOTE_TASK_QUEUE_SIZE.last()); // TODO(hakonhall): Update this name once persistent "count" metrics has been implemented. // DO NOT RELY ON THIS METRIC YET. addMetric(metrics, ClusterControllerMetrics.NODE_EVENT_COUNT.baseName()); - addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_NODES_ABOVE_LIMIT, EnumSet.of(last, max)); - addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_MAX_MEMORY_UTILIZATION, EnumSet.of(last, max)); - addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_MAX_DISK_UTILIZATION, EnumSet.of(last, max)); + addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_NODES_ABOVE_LIMIT, EnumSet.of(last, max)); // TODO: Vespa 9: Remove last + addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_MAX_MEMORY_UTILIZATION, EnumSet.of(last, max)); // TODO: Vespa 9: Remove last + addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_MAX_DISK_UTILIZATION, EnumSet.of(last, max)); // TODO: Vespa 9: Remove last addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_MEMORY_LIMIT.last()); addMetric(metrics, ClusterControllerMetrics.RESOURCE_USAGE_DISK_LIMIT.last()); addMetric(metrics, ClusterControllerMetrics.REINDEXING_PROGRESS.last()); @@ -311,7 +311,7 @@ public class VespaMetricSet { addMetric(metrics, ContainerMetrics.DOCUMENTS_COVERED.count()); addMetric(metrics, ContainerMetrics.DOCUMENTS_TOTAL.count()); addMetric(metrics, ContainerMetrics.DOCUMENTS_TARGET_TOTAL.count()); - addMetric(metrics, ContainerMetrics.JDISC_RENDER_LATENCY, EnumSet.of(min, max, count, sum, last, average)); + addMetric(metrics, ContainerMetrics.JDISC_RENDER_LATENCY, EnumSet.of(min, max, count, sum, last, average)); // TODO: Vespa 9: Remove last, average addMetric(metrics, ContainerMetrics.QUERY_ITEM_COUNT, EnumSet.of(max, sum, count)); addMetric(metrics, ContainerMetrics.TOTAL_HITS_PER_QUERY, EnumSet.of(sum, count, max, ninety_five_percentile, ninety_nine_percentile)); addMetric(metrics, ContainerMetrics.EMPTY_RESULTS.rate()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java index 5e76311dce3..aa61d2ffb74 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/Metrics.java @@ -29,12 +29,4 @@ public class Metrics { .anyMatch(existing -> existing.equalsIgnoreCase(id)); } - /** - * Returns true if any of the consumers have specified external metric systems. - */ - public boolean usesExternalMetricSystems() { - return consumers.values().stream() - .anyMatch(consumer -> ! consumer.cloudWatches().isEmpty()); - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java deleted file mode 100644 index ada4cb76d8e..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/CloudWatchBuilder.java +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.admin.monitoring.builder.xml; - -import com.yahoo.vespa.model.admin.monitoring.CloudWatch; -import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; -import org.w3c.dom.Element; - -import static com.yahoo.config.model.builder.xml.XmlHelper.getOptionalAttribute; -import static com.yahoo.config.model.builder.xml.XmlHelper.getOptionalChild; - -/** - * @author gjoranv - */ -public class CloudWatchBuilder { - - private static final String REGION_ATTRIBUTE = "region"; - private static final String NAMESPACE_ATTRIBUTE = "namespace"; - private static final String CREDENTIALS_ELEMENT = "credentials"; - private static final String ACCESS_KEY_ATTRIBUTE = "access-key-name"; - private static final String SECRET_KEY_ATTRIBUTE = "secret-key-name"; - private static final String SHARED_CREDENTIALS_ELEMENT = "shared-credentials"; - private static final String PROFILE_ATTRIBUTE = "profile"; - private static final String FILE_ATTRIBUTE = "file"; - - public static CloudWatch buildCloudWatch(Element cloudwatchElement, MetricsConsumer consumer) { - CloudWatch cloudWatch = new CloudWatch(cloudwatchElement.getAttribute(REGION_ATTRIBUTE), - cloudwatchElement.getAttribute(NAMESPACE_ATTRIBUTE), - consumer); - - getOptionalChild(cloudwatchElement, CREDENTIALS_ELEMENT) - .ifPresent(elem -> cloudWatch.setHostedAuth(elem.getAttribute(ACCESS_KEY_ATTRIBUTE), - elem.getAttribute(SECRET_KEY_ATTRIBUTE))); - - getOptionalChild(cloudwatchElement, SHARED_CREDENTIALS_ELEMENT) - .ifPresent(elem -> cloudWatch.setSharedCredentials(elem.getAttribute(FILE_ATTRIBUTE), - getOptionalAttribute(elem, PROFILE_ATTRIBUTE))); - - return cloudWatch; - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java index cfd2ebb2fd1..f75b2a864f9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/builder/xml/MetricsBuilder.java @@ -41,9 +41,6 @@ public class MetricsBuilder { MetricSet metricSet = buildMetricSet(consumerId, consumerElement); var consumer = new MetricsConsumer(consumerId, metricSet); - for (Element cloudwatchElement : XML.getChildren(consumerElement, "cloudwatch")) { - consumer.addCloudWatch(CloudWatchBuilder.buildCloudWatch(cloudwatchElement, consumer)); - } metrics.addConsumer(consumer); } return metrics; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java deleted file mode 100644 index 847a3402dc5..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/CloudWatchValidator.java +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.application.validation; - -import com.yahoo.config.model.ConfigModelContext; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; - -import java.util.List; - -/** - * @author gjoranv - */ -public class CloudWatchValidator extends Validator { - - @Override - public void validate(VespaModel model, DeployState deployState) { - if (!deployState.isHosted()) return; - if (deployState.zone().system().isPublic()) return; - if (model.getAdmin().getApplicationType() != ConfigModelContext.ApplicationType.DEFAULT) return; - - var offendingConsumers = model.getAdmin().getUserMetrics().getConsumers().values().stream() - .filter(consumer -> !consumer.cloudWatches().isEmpty()) - .toList(); - - if (! offendingConsumers.isEmpty()) { - throw new IllegalArgumentException("CloudWatch cannot be set up for non-public hosted Vespa and must " + - "be removed for consumers: " + consumerIds(offendingConsumers)); - } - } - - private List<String> consumerIds(List<MetricsConsumer> offendingConsumers) { - return offendingConsumers.stream().map(MetricsConsumer::id).toList(); - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index c7a363010b7..2576d9cb392 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -82,7 +82,6 @@ public class Validation { new SecretStoreValidator().validate(model, deployState); new EndpointCertificateSecretsValidator().validate(model, deployState); new AccessControlFilterValidator().validate(model, deployState); - new CloudWatchValidator().validate(model, deployState); new QuotaValidator().validate(model, deployState); new UriBindingsValidator().validate(model, deployState); new CloudDataPlaneFilterValidator().validate(model, deployState); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java index 0cc52edf3cc..b1eace947cc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidator.java @@ -2,11 +2,15 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.vespa.model.VespaModel; import com.yahoo.config.application.api.ValidationId; +import com.yahoo.vespa.model.container.ApplicationContainer; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.content.cluster.ContentCluster; +import java.util.ArrayList; import java.util.List; /** @@ -19,16 +23,23 @@ public class ContentClusterRemovalValidator implements ChangeValidator { @Override public List<ConfigChangeAction> validate(VespaModel current, VespaModel next, DeployState deployState) { + List<ConfigChangeAction> actions = new ArrayList<>(); for (String currentClusterId : current.getContentClusters().keySet()) { ContentCluster nextCluster = next.getContentClusters().get(currentClusterId); - if (nextCluster == null) + if (nextCluster == null) { deployState.validationOverrides().invalid(ValidationId.contentClusterRemoval, - "Content cluster '" + currentClusterId + "' is removed. " + - "This will cause loss of all data in this cluster", - deployState.now()); - } + "Content cluster '" + currentClusterId + "' is removed. " + + "This will cause loss of all data in this cluster", + deployState.now()); - return List.of(); + // If we allow the removal, we must restart all containers to ensure mbus is OK. + for (ApplicationContainerCluster cluster : next.getContainerClusters().values()) { + actions.add(new VespaRestartAction(cluster.id(), + "Content cluster '" + currentClusterId + "' has been removed", + cluster.getContainers().stream().map(ApplicationContainer::getServiceInfo).toList())); + } + } + } + return actions; } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java index 5e8bb85c29d..2b5dadf5512 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/IdentityProvider.java @@ -51,15 +51,18 @@ public class IdentityProvider extends SimpleComponent implements IdentityConfig. builder.loadBalancerAddress(loadBalancerName.value()); builder.ztsUrl(ztsUrl != null ? ztsUrl.toString() : ""); builder.athenzDnsSuffix(athenzDnsSuffix != null ? athenzDnsSuffix : ""); - builder.nodeIdentityName("vespa.vespa.tenant"); // TODO Move to Oath configmodel amender + builder.nodeIdentityName(configServerDomain() + ".tenant"); // TODO Move to Oath configmodel amender builder.configserverIdentityName(getConfigserverIdentityName()); } // TODO Move to Oath configmodel amender private String getConfigserverIdentityName() { return String.format("%s.provider_%s_%s", - zone.system() == SystemName.main ? "vespa.vespa" : "vespa.vespa.cd", + configServerDomain(), zone.environment().value(), zone.region().value()); } + private String configServerDomain() { + return zone.system() == SystemName.main ? "vespa.vespa" : "vespa.vespa.cd"; + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java index e2bc9897c85..5184d4ef07a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java @@ -112,7 +112,7 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer private void addFieldChildren(QueryProfilesConfig.Queryprofile.Builder qpB, QueryProfile profile, String namePrefix) { List<Map.Entry<String, Object>> content = new ArrayList<>(profile.declaredContent().entrySet()); - Collections.sort(content, new MapEntryKeyComparator()); + content.sort(new MapEntryKeyComparator()); if (profile.getValue() != null) { // Add "prefix with dot removed"=value: QueryProfilesConfig.Queryprofile.Property.Builder propB = new QueryProfilesConfig.Queryprofile.Property.Builder(); String fullName = namePrefix.substring(0, namePrefix.length() - 1); @@ -132,7 +132,7 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer QueryProfile profile, String namePrefix) { List<Map.Entry<String, Object>> content = new ArrayList<>(profile.declaredContent().entrySet()); - Collections.sort(content,new MapEntryKeyComparator()); + content.sort(new MapEntryKeyComparator()); if (profile.getValue() != null) { // Add "prefix with dot removed"=value: QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder propB = new QueryProfilesConfig.Queryprofile.Queryprofilevariant.Property.Builder(); String fullName = namePrefix.substring(0, namePrefix.length() - 1); @@ -152,8 +152,7 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer private void addField(QueryProfilesConfig.Queryprofile.Builder qpB, QueryProfile profile, Entry<String, Object> field, String namePrefix) { String fullName=namePrefix + field.getKey(); - if (field.getValue() instanceof QueryProfile) { - QueryProfile subProfile=(QueryProfile)field.getValue(); + if (field.getValue() instanceof QueryProfile subProfile) { if ( ! subProfile.isExplicit()) { // Implicitly defined profile - add content addFieldChildren(qpB, subProfile,fullName + "."); } @@ -172,8 +171,7 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer private void addVariantField(QueryProfilesConfig.Queryprofile.Queryprofilevariant.Builder qpB, Entry<String, Object> field, Boolean overridable, String namePrefix) { String fullName = namePrefix + field.getKey(); - if (field.getValue() instanceof QueryProfile) { - QueryProfile subProfile = (QueryProfile)field.getValue(); + if (field.getValue() instanceof QueryProfile subProfile) { if ( ! subProfile.isExplicit()) { // Implicitly defined profile - add content addVariantFieldChildren(qpB, subProfile,fullName + "."); } @@ -203,7 +201,7 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer varB.inherit(inherited.getId().stringValue()); List<Map.Entry<String,Object>> content = new ArrayList<>(variant.getValues().entrySet()); - Collections.sort(content, new MapEntryKeyComparator()); + content.sort(new MapEntryKeyComparator()); for (Map.Entry<String, Object> entry : content) { addVariantField(varB, entry, variant.getOverriable().get(entry.getKey()), ""); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java index 2742dc59fcd..7c89a349d7d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/OnnxModelInfo.java @@ -42,13 +42,16 @@ public class OnnxModelInfo { private final Map<String, OnnxTypeInfo> inputs; private final Map<String, OnnxTypeInfo> outputs; private final Map<String, TensorType> vespaTypes = new HashMap<>(); + private final Set<String> initializers; - private OnnxModelInfo(ApplicationPackage app, String path, Map<String, OnnxTypeInfo> inputs, Map<String, OnnxTypeInfo> outputs, String defaultOutput) { + private OnnxModelInfo(ApplicationPackage app, String path, Map<String, OnnxTypeInfo> inputs, + Map<String, OnnxTypeInfo> outputs, Set<String> initializers, String defaultOutput) { this.app = app; this.modelPath = path; this.inputs = Collections.unmodifiableMap(inputs); this.outputs = Collections.unmodifiableMap(outputs); this.defaultOutput = defaultOutput; + this.initializers = Set.copyOf(initializers); } public String getModelPath() { @@ -63,6 +66,8 @@ public class OnnxModelInfo { return outputs.keySet(); } + public Set<String> getInitializers() { return initializers; } + public String getDefaultOutput() { return defaultOutput; } @@ -208,6 +213,14 @@ public class OnnxModelInfo { } g.writeEndArray(); + g.writeArrayFieldStart("initializers"); + for (Onnx.TensorProto initializers : model.getGraph().getInitializerList()) { + g.writeStartObject(); + g.writeStringField("name", initializers.getName()); + g.writeEndObject(); + } + g.writeEndArray(); + g.writeEndObject(); g.close(); return out.toString(); @@ -218,6 +231,7 @@ public class OnnxModelInfo { JsonNode root = m.readTree(json); Map<String, OnnxTypeInfo> inputs = new HashMap<>(); Map<String, OnnxTypeInfo> outputs = new HashMap<>(); + Set<String> initializers = new HashSet<>(); String defaultOutput = ""; String path = null; @@ -233,7 +247,13 @@ public class OnnxModelInfo { if (root.get("outputs").has(0)) { defaultOutput = root.get("outputs").get(0).get("name").textValue(); } - return new OnnxModelInfo(app, path, inputs, outputs, defaultOutput); + var initializerRoot = root.get("initializers"); + if (initializerRoot != null) { + for (JsonNode initializer : initializerRoot) { + initializers.add(initializer.get("name").textValue()); + } + } + return new OnnxModelInfo(app, path, inputs, outputs, initializers, defaultOutput); } static private void onnxTypeToJson(JsonGenerator g, Onnx.ValueInfoProto valueInfo) throws IOException { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 4ed8c5ab2e8..670460a9f9f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -64,6 +64,7 @@ public class IndexedSearchCluster extends SearchCluster private final List<SearchNode> searchNodes = new ArrayList<>(); private final DispatchTuning.DispatchPolicy defaultDispatchPolicy; private final double dispatchWarmup; + private final String summaryDecodePolicy; /** * Returns the document selector that is able to resolve what documents are to be routed to this search cluster. * This string uses the document selector language as defined in the "document" module. @@ -80,6 +81,7 @@ public class IndexedSearchCluster extends SearchCluster rootDispatch = new DispatchGroup(this); defaultDispatchPolicy = DispatchTuning.Builder.toDispatchPolicy(featureFlags.queryDispatchPolicy()); dispatchWarmup = featureFlags.queryDispatchWarmup(); + summaryDecodePolicy = featureFlags.summaryDecodePolicy(); } @Override @@ -337,6 +339,15 @@ public class IndexedSearchCluster extends SearchCluster builder.maxWaitAfterCoverageFactor(searchCoverage.getMaxWaitAfterCoverageFactor()); } builder.warmuptime(dispatchWarmup); + builder.summaryDecodePolicy(toSummaryDecoding(summaryDecodePolicy)); + } + + private DispatchConfig.SummaryDecodePolicy.Enum toSummaryDecoding(String summaryDecodeType) { + return switch (summaryDecodeType.toLowerCase()) { + case "eager" -> DispatchConfig.SummaryDecodePolicy.EAGER; + case "ondemand","on-demand" -> DispatchConfig.SummaryDecodePolicy.Enum.ONDEMAND; + default -> DispatchConfig.SummaryDecodePolicy.Enum.EAGER; + }; } @Override diff --git a/config-model/src/main/javacc/SchemaParser.jj b/config-model/src/main/javacc/SchemaParser.jj index 61e8574bc87..9d6e16b3f67 100644 --- a/config-model/src/main/javacc/SchemaParser.jj +++ b/config-model/src/main/javacc/SchemaParser.jj @@ -164,13 +164,13 @@ TOKEN : | < FIELD: "field" > | < FIELDS: "fields" > | < FIELDSET: "fieldset" > -| < STRUCTFIELD: "struct-field" > +| < STRUCT_FIELD: "struct-field" > | < IMPORT: "import" > | < AS: "as" > | < INDEXING: "indexing" > -| < SUMMARYTO: "summary-to" > -| < DOCUMENTSUMMARY: "document-summary" > -| < RANKTYPE: "rank-type" > +| < SUMMARY_TO: "summary-to" > +| < DOCUMENT_SUMMARY: "document-summary" > +| < RANK_TYPE: "rank-type" > | < WEIGHT: "weight" > | < TYPE: "type" > | < INDEX: "index" > @@ -179,26 +179,29 @@ TOKEN : | < TEXT: "text" > | < WORD: "word" > | < GRAM: "gram" > -| < GRAMSIZE: "gram-size" > -| < MAXLENGTH: "max-length" > +| < GRAM_SIZE: "gram-size" > +| < MAX_LENGTH: "max-length" > | < PREFIX: "prefix" > | < SUBSTRING: "substring" > | < SUFFIX: "suffix" > | < CONSTANT: "constant"> -| < ONNXMODEL: "onnx-model"> -| < INTRAOPTHREADS: "intraop-threads"> -| < INTEROPTHREADS: "interop-threads"> -| < GPUDEVICE: "gpu-device"> +| < ONNX_MODEL: "onnx-model"> +| < INTRAOP_THREADS: "intraop-threads"> +| < INTEROP_THREADS: "interop-threads"> +| < GPU_DEVICE: "gpu-device"> +| < EXECUTION_MODE: "execution-mode"> +| < PARALLEL: "parallel"> +| < SEQUENTIAL: "sequential"> | < MODEL: "model" > | < MUTATE: "mutate" > | < QUERY: "query" > -| < RANKPROFILE: "rank-profile" > +| < RANK_PROFILE: "rank-profile" > | < RAW_AS_BASE64_IN_SUMMARY: "raw-as-base64-in-summary" > | < SUMMARY: "summary" > | < FULL: "full" > | < STATIC: "static" > | < DYNAMIC: "dynamic" > -| < MATCHEDELEMENTSONLY: "matched-elements-only" > +| < MATCHED_ELEMENTS_ONLY: "matched-elements-only" > | < SSCONTEXTUAL: "contextual" > | < SSOVERRIDE: "override" > | < SSTITLE: "title" > @@ -233,7 +236,7 @@ TOKEN : | < TRUE: "true" > | < FALSE: "false" > | < SYMMETRIC: "symmetric" > -| < QUERYCOMMAND: "query-command" > +| < QUERY_COMMAND: "query-command" > | < ALIAS: "alias" > | < MATCH: "match" > | < RANK: "rank" > @@ -241,24 +244,24 @@ TOKEN : | < EXACT: "exact" > | < FILTER: "filter" > | < NORMAL: "normal" > -| < EXACTTERMINATOR: "exact-terminator" > -| < IGNOREDEFAULTRANKFEATURES: "ignore-default-rank-features" > +| < EXACT_TERMINATOR: "exact-terminator" > +| < IGNORE_DEFAULT_RANK_FEATURES: "ignore-default-rank-features" > | < ID: "id" > | < SOURCE: "source" > | < TO: "to" > | < DIRECT: "direct" > -| < FROMDISK: "from-disk" > -| < OMITSUMMARYFEATURES: "omit-summary-features" > +| < FROM_DISK: "from-disk" > +| < OMIT_SUMMARY_FEATURES: "omit-summary-features" > | < ALWAYS: "always" > -| < ONDEMAND: "on-demand" > +| < ON_DEMAND: "on-demand" > | < NEVER: "never" > -| < ENABLEBITVECTORS: "enable-bit-vectors" > -| < ENABLEONLYBITVECTOR: "enable-only-bit-vector" > -| < FASTACCESS: "fast-access" > +| < ENABLE_BIT_VECTORS: "enable-bit-vectors" > +| < ENABLE_ONLY_BIT_VECTOR: "enable-only-bit-vector" > +| < FAST_ACCESS: "fast-access" > | < MUTABLE: "mutable" > | < PAGED: "paged" > -| < FASTRANK: "fast-rank" > -| < FASTSEARCH: "fast-search" > +| < FAST_RANK: "fast-rank" > +| < FAST_SEARCH: "fast-search" > | < TENSOR_TYPE: "tensor" ("<" (~["<",">"])+ ">")? "(" (~["(",")"])* ")" > | < TENSOR_VALUE_SL: "value" (" ")* ":" (" ")* ("{"<BRACE_SL_LEVEL_1>) ("\n")? > | < TENSOR_VALUE_ML: "value" (<SEARCHLIB_SKIP>)? "{" (["\n"," "])* ("{"<BRACE_ML_LEVEL_1>) (["\n"," "])* "}" ("\n")? > @@ -272,30 +275,30 @@ TOKEN : | < MAP: "map" > | < REFERENCE: "reference" > | < QUESTIONMARK: "?" > -| < CREATEIFNONEXISTENT: "create-if-nonexistent" > -| < REMOVEIFZERO: "remove-if-zero" > -| < MATCHPHASE: "match-phase" > +| < CREATE_IF_NONEXISTENT: "create-if-nonexistent" > +| < REMOVE_IF_ZERO: "remove-if-zero" > +| < MATCH_PHASE: "match-phase" > | < EVALUATION_POINT: "evaluation-point" > | < PRE_POST_FILTER_TIPPING_POINT: "pre-post-filter-tipping-point" > | < ORDER: "order" > -| < MAXFILTERCOVERAGE: "max-filter-coverage" > -| < MAXHITS: "max-hits" > -| < FIRSTPHASE: "first-phase" > -| < SECONDPHASE: "second-phase" > -| < GLOBALPHASE: "global-phase" > +| < MAX_FILTER_COVERAGE: "max-filter-coverage" > +| < MAX_HITS: "max-hits" > +| < FIRST_PHASE: "first-phase" > +| < SECOND_PHASE: "second-phase" > +| < GLOBAL_PHASE: "global-phase" > | < MACRO: "macro" > | < INLINE: "inline" > | < ARITY: "arity" > -| < LOWERBOUND: "lower-bound" > -| < UPPERBOUND: "upper-bound" > -| < DENSEPOSTINGLISTTHRESHOLD: "dense-posting-list-threshold" > +| < LOWER_BOUND: "lower-bound" > +| < UPPER_BOUND: "upper-bound" > +| < DENSE_POSTING_LIST_THRESHOLD: "dense-posting-list-threshold" > | < ENABLE_BM25: "enable-bm25" > | < HNSW: "hnsw" > -| < MAXLINKSPERNODE: "max-links-per-node" > +| < MAX_LINKS_PER_NODE: "max-links-per-node" > | < DOUBLE_KEYWORD: "double" > -| < DISTANCEMETRIC: "distance-metric" > -| < NEIGHBORSTOEXPLOREATINSERT: "neighbors-to-explore-at-insert" > -| < MULTITHREADEDINDEXING: "multi-threaded-indexing" > +| < DISTANCE_METRIC: "distance-metric" > +| < NEIGHBORS_TO_EXPLORE_AT_INSERT: "neighbors-to-explore-at-insert" > +| < MULTI_THREADED_INDEXING: "multi-threaded-indexing" > | < MATCHFEATURES_SL: "match-features" (" ")* ":" (~["}","\n"])* ("\n")? > | < MATCHFEATURES_ML: "match-features" (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" > | < MATCHFEATURES_ML_INHERITS: "match-features inherits " (<IDENTIFIER_WITH_DASH>) (<SEARCHLIB_SKIP>)? "{" (~["}"])* "}" > @@ -315,16 +318,16 @@ TOKEN : | < #BRACE_ML_LEVEL_3: <BRACE_ML_CONTENT> "}" > | < #BRACE_ML_CONTENT: (~["{","}"])* > | < #SEARCHLIB_SKIP: ([" ","\f","\n","\r","\t"])+ > -| < RANKPROPERTIES: "rank-properties" > -| < RERANKCOUNT: "rerank-count" > -| < NUMTHREADSPERSEARCH: "num-threads-per-search" > -| < MINHITSPERTHREAD: "min-hits-per-thread" > -| < NUMSEARCHPARTITIONS: "num-search-partitions" > -| < TERMWISELIMIT: "termwise-limit" > -| < POSTFILTERTHRESHOLD: "post-filter-threshold" > -| < APPROXIMATETHRESHOLD: "approximate-threshold" > -| < KEEPRANKCOUNT: "keep-rank-count" > -| < RANKSCOREDROPLIMIT: "rank-score-drop-limit" > +| < RANK_PROPERTIES: "rank-properties" > +| < RERANK_COUNT: "rerank-count" > +| < NUM_THREADS_PER_SEARCH: "num-threads-per-search" > +| < MIN_HITS_PER_THREAD: "min-hits-per-thread" > +| < NUM_SEARCH_PARTITIONS: "num-search-partitions" > +| < TERMWISE_LIMIT: "termwise-limit" > +| < POST_FILTER_THRESHOLD: "post-filter-threshold" > +| < APPROXIMATE_THRESHOLD: "approximate-threshold" > +| < KEEP_RANK_COUNT: "keep-rank-count" > +| < RANK_SCORE_DROP_LIMIT: "rank-score-drop-limit" > | < CONSTANTS: "constants" > | < FILE: "file" > | < URI: "uri" > @@ -608,7 +611,7 @@ void fieldSet(ParsedSchema schema) : (( ( <FIELDS><COLON> name = identifier() { fieldSet.addField(name); } ( <COMMA> name = identifier() { fieldSet.addField(name); } )* ) - | ( <QUERYCOMMAND> <COLON> (queryCommand = identifierWithDash() | queryCommand = quotedString())) { fieldSet.addQueryCommand(queryCommand); } + | ( <QUERY_COMMAND> <COLON> (queryCommand = identifierWithDash() | queryCommand = quotedString())) { fieldSet.addQueryCommand(queryCommand); } | matchSettings(fieldSet.matchSettings()) ) (<NL>)* )+ <RBRACE> @@ -825,7 +828,7 @@ void structField(ParsedField field) : ParsedField structField; } { - <STRUCTFIELD> name = identifier() { + <STRUCT_FIELD> name = identifier() { if (name != null && Schema.isReservedName(name.toLowerCase())) { throw new IllegalArgumentException("Reserved name '" + name + "' can not be used as a field name."); } @@ -917,7 +920,7 @@ void summaryTo(ParsedField field) : ParsedSummaryField psf; } { - <SUMMARYTO> [name = identifier()] <COLON> destination = identifier() + <SUMMARY_TO> [name = identifier()] <COLON> destination = identifier() { psf = field.summaryFieldFor(name); psf.addDestination(destination); @@ -956,8 +959,8 @@ void weightedset(ParsedType fieldType) : { } */ void weightedsetBody(ParsedType type) : { } { - ( <CREATEIFNONEXISTENT> { type.setCreateIfNonExistent(true); } - | <REMOVEIFZERO> { type.setRemoveIfZero(true); } ) + ( <CREATE_IF_NONEXISTENT> { type.setCreateIfNonExistent(true); } + | <REMOVE_IF_ZERO> { type.setRemoveIfZero(true); } ) } /** @@ -971,7 +974,7 @@ void rankType(ParsedField field) : String indexName = ""; } { - <RANKTYPE> [indexName = identifier()] <COLON> typeName = identifier() + <RANK_TYPE> [indexName = identifier()] <COLON> typeName = identifier() { field.addRankType(indexName, typeName); } @@ -1049,18 +1052,18 @@ void attributeSetting(ParsedAttribute attribute) : } { ( - <FASTRANK> { attribute.setFastRank(true); } - | <FASTSEARCH> { attribute.setFastSearch(true); } - | <FASTACCESS> { attribute.setFastAccess(true); } + <FAST_RANK> { attribute.setFastRank(true); } + | <FAST_SEARCH> { attribute.setFastSearch(true); } + | <FAST_ACCESS> { attribute.setFastAccess(true); } | <MUTABLE> { attribute.setMutable(true); } | <PAGED> { attribute.setPaged(true); } - | <ENABLEBITVECTORS> { deployLogger.logApplicationPackage(Level.WARNING, "'enable-bit-vectors' is deprecated and void -> remove it. Will be removed in vespa-9"); } - | <ENABLEONLYBITVECTOR> { attribute.setEnableOnlyBitVector(true); } + | <ENABLE_BIT_VECTORS> { deployLogger.logApplicationPackage(Level.WARNING, "'enable-bit-vectors' is deprecated and void -> remove it. Will be removed in vespa-9"); } + | <ENABLE_ONLY_BIT_VECTOR> { attribute.setEnableOnlyBitVector(true); } | attributeSorting(attribute) | <ALIAS> { String alias; String aliasedName=attribute.name(); } [aliasedName = identifier()] <COLON> alias = identifierWithDash() { attribute.addAlias(aliasedName, alias); } - | <DISTANCEMETRIC> <COLON> str = identifierWithDash() { attribute.setDistanceMetric(str); } + | <DISTANCE_METRIC> <COLON> str = identifierWithDash() { attribute.setDistanceMetric(str); } ) } @@ -1116,7 +1119,7 @@ void summaryInFieldShort(ParsedField field) : psf = field.summaryFieldFor(name); } <COLON> ( <DYNAMIC> { psf.setDynamic(); } - | <MATCHEDELEMENTSONLY> { psf.setMatchedElementsOnly(); } + | <MATCHED_ELEMENTS_ONLY> { psf.setMatchedElementsOnly(); } | (<FULL> | <STATIC>) { psf.setFull(); } ) } @@ -1161,7 +1164,7 @@ void summaryItem(ParsedSummaryField field) : { } void summaryTransform(ParsedSummaryField field) : { } { ( <DYNAMIC> { field.setDynamic(); } - | <MATCHEDELEMENTSONLY> { field.setMatchedElementsOnly(); } + | <MATCHED_ELEMENTS_ONLY> { field.setMatchedElementsOnly(); } | (<FULL> | <STATIC>) { field.setFull(); } ) } @@ -1295,7 +1298,7 @@ void queryCommand(ParsedField field) : String command; } { - <QUERYCOMMAND> <COLON> ( command = identifierWithDash() | command = quotedString() ) + <QUERY_COMMAND> <COLON> ( command = identifierWithDash() | command = quotedString() ) { field.addQueryCommand(command); } @@ -1349,7 +1352,7 @@ void exactTerminator(ParsedMatchSettings matchInfo) : String terminator; } { - <EXACTTERMINATOR> <COLON> terminator = quotedString() + <EXACT_TERMINATOR> <COLON> terminator = quotedString() { matchInfo.setExactTerminator(terminator); } @@ -1360,7 +1363,7 @@ void gramSize(ParsedMatchSettings matchInfo) : int gramSize; } { - <GRAMSIZE> <COLON> gramSize = integer() + <GRAM_SIZE> <COLON> gramSize = integer() { matchInfo.setGramSize(gramSize); } @@ -1371,7 +1374,7 @@ void matchSize(ParsedMatchSettings matchInfo) : int matchSize; } { - <MAXLENGTH> <COLON> matchSize = integer() { + <MAX_LENGTH> <COLON> matchSize = integer() { matchInfo.setMaxLength(matchSize); } } @@ -1426,13 +1429,13 @@ void documentSummary(ParsedSchema schema) : ParsedDocumentSummary summary; } { - ( <DOCUMENTSUMMARY> + ( <DOCUMENT_SUMMARY> name = identifierWithDash() { summary = new ParsedDocumentSummary(name); } [inheritsDocumentSummary(summary)] lbrace() ( - <FROMDISK> { summary.setFromDisk(true); } | - <OMITSUMMARYFEATURES> { summary.setOmitSummaryFeatures(true); } | + <FROM_DISK> { summary.setFromDisk(true); } | + <OMIT_SUMMARY_FEATURES> { summary.setOmitSummaryFeatures(true); } | documentSummaryItem(summary) | <NL> )* @@ -1529,9 +1532,9 @@ void indexBody(ParsedIndex index) : | <ALIAS> <COLON> str = identifierWithDash() { index.addAlias(str); } | <STEMMING> <COLON> str = identifierWithDash() { index.setStemming(Stemming.get(str)); } | <ARITY> <COLON> arity = integer() { index.setArity(arity); } - | <LOWERBOUND> <COLON> num = longValue() { index.setLowerBound(num); } - | <UPPERBOUND> <COLON> num = longValue() { index.setUpperBound(num); } - | <DENSEPOSTINGLISTTHRESHOLD> <COLON> threshold = floatValue() { index.setDensePostingListThreshold(threshold); } + | <LOWER_BOUND> <COLON> num = longValue() { index.setLowerBound(num); } + | <UPPER_BOUND> <COLON> num = longValue() { index.setUpperBound(num); } + | <DENSE_POSTING_LIST_THRESHOLD> <COLON> threshold = floatValue() { index.setDensePostingListThreshold(threshold); } | <ENABLE_BM25> { index.setEnableBm25(true); } | hnswIndex(index) { } ) @@ -1556,9 +1559,9 @@ void hnswIndexBody(HnswIndexParams.Builder params) : boolean bool; } { - ( <MAXLINKSPERNODE> <COLON> num = integer() { params.setMaxLinksPerNode(num); } - | <NEIGHBORSTOEXPLOREATINSERT> <COLON> num = integer() { params.setNeighborsToExploreAtInsert(num); } - | <MULTITHREADEDINDEXING> <COLON> bool = bool() { params.setMultiThreadedIndexing(bool); } ) + ( <MAX_LINKS_PER_NODE> <COLON> num = integer() { params.setMaxLinksPerNode(num); } + | <NEIGHBORS_TO_EXPLORE_AT_INSERT> <COLON> num = integer() { params.setNeighborsToExploreAtInsert(num); } + | <MULTI_THREADED_INDEXING> <COLON> bool = bool() { params.setMultiThreadedIndexing(bool); } ) } void onnxModelInSchema(ParsedSchema schema) : @@ -1584,7 +1587,7 @@ OnnxModel onnxModel() : OnnxModel onnxModel; } { - ( <ONNXMODEL> name = identifier() { onnxModel = new OnnxModel(name); } + ( <ONNX_MODEL> name = identifier() { onnxModel = new OnnxModel(name); } lbrace() (onnxModelItem(onnxModel) (<NL>)*)+ <RBRACE> ) { return onnxModel; } } @@ -1603,9 +1606,11 @@ void onnxModelItem(OnnxModel onnxModel) : ( (path = fileItem()) { onnxModel.setFileName(path); } | (path = uriItem()) { onnxModel.setUri(path); } | - <GPUDEVICE> <COLON> num = integer() { onnxModel.setGpuDevice(num, false); } | - <INTRAOPTHREADS> <COLON> num = integer() { onnxModel.setStatelessIntraOpThreads(num); } | - <INTEROPTHREADS> <COLON> num = integer() { onnxModel.setStatelessInterOpThreads(num); } | + <GPU_DEVICE> <COLON> num = integer() { onnxModel.setGpuDevice(num, false); } | + <INTRAOP_THREADS> <COLON> num = integer() { onnxModel.setStatelessIntraOpThreads(num); } | + <INTEROP_THREADS> <COLON> num = integer() { onnxModel.setStatelessInterOpThreads(num); } | + <EXECUTION_MODE> <COLON> ( <PARALLEL> { onnxModel.setStatelessExecutionMode("parallel"); } + | <SEQUENTIAL> { onnxModel.setStatelessExecutionMode("sequential"); } ) | (<ONNX_INPUT_SL>) { String name = token.image.substring(5, token.image.lastIndexOf(":")).trim(); if (name.startsWith("\"")) { name = name.substring(1, name.length() - 1); } @@ -1680,7 +1685,7 @@ void rankProfile(ParsedSchema schema) : ParsedRankProfile profile; } { - ( <MODEL> | <RANKPROFILE> ) name = identifierWithDash() + ( <MODEL> | <RANK_PROFILE> ) name = identifierWithDash() { profile = new ParsedRankProfile(name); } [inheritsRankProfile(profile)] lbrace() (rankProfileItem(schema, profile) (<NL>)*)* <RBRACE> @@ -1819,7 +1824,7 @@ void matchPhase(ParsedRankProfile profile) : MatchPhaseSettings settings = new MatchPhaseSettings(); } { - <MATCHPHASE> lbrace() (matchPhaseItem(settings) (<NL>)*)* <RBRACE> + <MATCH_PHASE> lbrace() (matchPhaseItem(settings) (<NL>)*)* <RBRACE> { settings.checkValid(); profile.setMatchPhaseSettings(settings); @@ -1838,8 +1843,8 @@ void matchPhaseItem(MatchPhaseSettings settings) : | diversity(settings) | <ORDER> <COLON> ( <ASCENDING> { settings.setAscending(true); } | <DESCENDING> { settings.setAscending(false); } ) - | <MAXHITS> <COLON> num = integer() { settings.setMaxHits(num); } - | <MAXFILTERCOVERAGE> <COLON> coverage = floatValue() { settings.setMaxFilterCoverage(coverage); } + | <MAX_HITS> <COLON> num = integer() { settings.setMaxHits(num); } + | <MAX_FILTER_COVERAGE> <COLON> coverage = floatValue() { settings.setMaxFilterCoverage(coverage); } | <EVALUATION_POINT> <COLON> multiplier = floatValue() { settings.setEvaluationPoint(multiplier); } | <PRE_POST_FILTER_TIPPING_POINT> <COLON> multiplier = floatValue() { settings.setPrePostFilterTippingPoint(multiplier); } ) @@ -1890,7 +1895,7 @@ void firstPhase(ParsedRankProfile profile) : String exp; } { - <FIRSTPHASE> lbrace() (firstPhaseItem(profile) (<NL>)*)* <RBRACE> + <FIRST_PHASE> lbrace() (firstPhaseItem(profile) (<NL>)*)* <RBRACE> } void firstPhaseItem(ParsedRankProfile profile) : @@ -1901,8 +1906,8 @@ void firstPhaseItem(ParsedRankProfile profile) : } { ( expression = expression() { profile.setFirstPhaseRanking(expression); } - | (<KEEPRANKCOUNT> <COLON> keepRankCount = integer()) { profile.setKeepRankCount(keepRankCount); } - | (<RANKSCOREDROPLIMIT> <COLON> dropLimit = floatValue()) { profile.setRankScoreDropLimit(dropLimit); } + | (<KEEP_RANK_COUNT> <COLON> keepRankCount = integer()) { profile.setKeepRankCount(keepRankCount); } + | (<RANK_SCORE_DROP_LIMIT> <COLON> dropLimit = floatValue()) { profile.setRankScoreDropLimit(dropLimit); } ) } @@ -1913,7 +1918,7 @@ void firstPhaseItem(ParsedRankProfile profile) : */ void secondPhase(ParsedRankProfile profile) : { } { - <SECONDPHASE> lbrace() (secondPhaseItem(profile) (<NL>)*)* <RBRACE> + <SECOND_PHASE> lbrace() (secondPhaseItem(profile) (<NL>)*)* <RBRACE> } /** @@ -1928,7 +1933,7 @@ void secondPhaseItem(ParsedRankProfile profile) : } { ( expression = expression() { profile.setSecondPhaseRanking(expression); } - | (<RERANKCOUNT> <COLON> rerankCount = integer()) { profile.setRerankCount(rerankCount); } + | (<RERANK_COUNT> <COLON> rerankCount = integer()) { profile.setRerankCount(rerankCount); } ) } @@ -1939,7 +1944,7 @@ void secondPhaseItem(ParsedRankProfile profile) : */ void globalPhase(ParsedRankProfile profile) : { } { - <GLOBALPHASE> lbrace() (globalPhaseItem(profile) (<NL>)*)* <RBRACE> + <GLOBAL_PHASE> lbrace() (globalPhaseItem(profile) (<NL>)*)* <RBRACE> } /** @@ -1954,7 +1959,7 @@ void globalPhaseItem(ParsedRankProfile profile) : } { ( expression = expression() { profile.setGlobalPhaseExpression(expression); } - | (<RERANKCOUNT> <COLON> rerankCount = integer()) { profile.setGlobalPhaseRerankCount(rerankCount); } + | (<RERANK_COUNT> <COLON> rerankCount = integer()) { profile.setGlobalPhaseRerankCount(rerankCount); } ) } @@ -2092,7 +2097,7 @@ void rankFeatures(ParsedRankProfile profile) : */ void ignoreRankFeatures(ParsedRankProfile profile) : { } { - <IGNOREDEFAULTRANKFEATURES> { profile.setIgnoreDefaultRankFeatures(true); } + <IGNORE_DEFAULT_RANK_FEATURES> { profile.setIgnoreDefaultRankFeatures(true); } } /** @@ -2105,7 +2110,7 @@ void numThreadsPerSearch(ParsedRankProfile profile) : int num; } { - (<NUMTHREADSPERSEARCH> <COLON> num = integer()) { profile.setNumThreadsPerSearch(num); } + (<NUM_THREADS_PER_SEARCH> <COLON> num = integer()) { profile.setNumThreadsPerSearch(num); } } /** @@ -2118,7 +2123,7 @@ void minHitsPerThread(ParsedRankProfile profile) : int num; } { - (<MINHITSPERTHREAD> <COLON> num = integer()) { profile.setMinHitsPerThread(num); } + (<MIN_HITS_PER_THREAD> <COLON> num = integer()) { profile.setMinHitsPerThread(num); } } /** @@ -2131,7 +2136,7 @@ void numSearchPartitions(ParsedRankProfile profile) : int num; } { - (<NUMSEARCHPARTITIONS> <COLON> num = integer()) { profile.setNumSearchPartitions(num); } + (<NUM_SEARCH_PARTITIONS> <COLON> num = integer()) { profile.setNumSearchPartitions(num); } } /** @@ -2144,7 +2149,7 @@ void termwiseLimit(ParsedRankProfile profile) : double num; } { - (<TERMWISELIMIT> <COLON> num = floatValue()) { profile.setTermwiseLimit(num); } + (<TERMWISE_LIMIT> <COLON> num = floatValue()) { profile.setTermwiseLimit(num); } } /** @@ -2157,7 +2162,7 @@ void postFilterThreshold(ParsedRankProfile profile) : double threshold; } { - (<POSTFILTERTHRESHOLD> <COLON> threshold = floatValue()) { profile.setPostFilterThreshold(threshold); } + (<POST_FILTER_THRESHOLD> <COLON> threshold = floatValue()) { profile.setPostFilterThreshold(threshold); } } /** @@ -2170,7 +2175,7 @@ void approximateThreshold(ParsedRankProfile profile) : double threshold; } { - (<APPROXIMATETHRESHOLD> <COLON> threshold = floatValue()) { profile.setApproximateThreshold(threshold); } + (<APPROXIMATE_THRESHOLD> <COLON> threshold = floatValue()) { profile.setApproximateThreshold(threshold); } } /** @@ -2182,7 +2187,7 @@ void approximateThreshold(ParsedRankProfile profile) : */ void rankProperties(ParsedRankProfile profile) : { } { - <RANKPROPERTIES> lbrace() (LOOKAHEAD(rankPropertyItem() <COLON> rankPropertyItem() <NL>) + <RANK_PROPERTIES> lbrace() (LOOKAHEAD(rankPropertyItem() <COLON> rankPropertyItem() <NL>) rankProperty(profile) (<NL>)+)* [rankProperty(profile)] <RBRACE> } @@ -2245,7 +2250,7 @@ void fieldRankType(ParsedRankProfile profile) : String type; } { - <RANKTYPE> name = identifier() <COLON> type = identifier() + <RANK_TYPE> name = identifier() <COLON> type = identifier() { profile.addFieldRankType(name, type); } } @@ -2566,7 +2571,68 @@ String identifierWithDash() : { ( identifier = identifier() { return identifier; } ) | - ( <IDENTIFIER_WITH_DASH> { return token.image; } ) + ( <IDENTIFIER_WITH_DASH> + | <APPROXIMATE_THRESHOLD> + | <CREATE_IF_NONEXISTENT> + | <CUTOFF_FACTOR> + | <CUTOFF_STRATEGY> + | <DENSE_POSTING_LIST_THRESHOLD> + | <DISTANCE_METRIC> + | <DOCUMENT_SUMMARY> + | <ENABLE_BIT_VECTORS> + | <ENABLE_BM25> + | <ENABLE_ONLY_BIT_VECTOR> + | <EVALUATION_POINT> + | <EXACT_TERMINATOR> + | <EXECUTION_MODE> + | <FAST_ACCESS> + | <FAST_RANK> + | <FAST_SEARCH> + | <FIRST_PHASE> + | <FROM_DISK> + | <GLOBAL_PHASE> + | <GPU_DEVICE> + | <GRAM_SIZE> + | <IGNORE_DEFAULT_RANK_FEATURES> + | <INTEROP_THREADS> + | <INTRAOP_THREADS> + | <KEEP_RANK_COUNT> + | <LOWER_BOUND> + | <MATCHED_ELEMENTS_ONLY> + | <MATCH_PHASE> + | <MAX_FILTER_COVERAGE> + | <MAX_HITS> + | <MAX_LENGTH> + | <MAX_LINKS_PER_NODE> + | <MIN_GROUPS> + | <MIN_HITS_PER_THREAD> + | <MULTI_THREADED_INDEXING> + | <NEIGHBORS_TO_EXPLORE_AT_INSERT> + | <NUM_SEARCH_PARTITIONS> + | <NUM_THREADS_PER_SEARCH> + | <OMIT_SUMMARY_FEATURES> + | <ON_DEMAND> + | <ON_FIRST_PHASE> + | <ON_MATCH> + | <ONNX_MODEL> + | <ON_SECOND_PHASE> + | <ON_SUMMARY> + | <POST_FILTER_THRESHOLD> + | <PRE_POST_FILTER_TIPPING_POINT> + | <QUERY_COMMAND> + | <RANK_PROFILE> + | <RANK_PROPERTIES> + | <RANK_SCORE_DROP_LIMIT> + | <RANK_TYPE> + | <RAW_AS_BASE64_IN_SUMMARY> + | <REMOVE_IF_ZERO> + | <RERANK_COUNT> + | <SECOND_PHASE> + | <STRUCT_FIELD> + | <SUMMARY_TO> + | <TERMWISE_LIMIT> + | <UPPER_BOUND> + ) { return token.image; } } /** @@ -2592,38 +2658,28 @@ String identifier() : { } | <CONSTANT> | <CONSTANTS> | <CONTEXT> - | <GLOBALPHASE> - | <CREATEIFNONEXISTENT> - | <DENSEPOSTINGLISTTHRESHOLD> | <DESCENDING> | <DICTIONARY> | <DIRECT> + | <DIVERSITY> | <DOCUMENT> - | <DOCUMENTSUMMARY> | <DOUBLE_KEYWORD> | <DYNAMIC> - | <ENABLEBITVECTORS> - | <ENABLEONLYBITVECTOR> | <EXACT> - | <EXACTTERMINATOR> | <FALSE> - | <FASTACCESS> - | <FASTRANK> - | <FASTSEARCH> | <FIELD> | <FIELDS> | <FIELDSET> | <FILE> | <FILTER> - | <FIRSTPHASE> | <FULL> | <FUNCTION> | <GRAM> | <HASH> + | <HNSW> | <ID> | <IDENTICAL> | <IDENTIFIER> - | <IGNOREDEFAULTRANKFEATURES> | <IMPORT> | <INDEX> | <INDEXING> @@ -2631,52 +2687,42 @@ String identifier() : { } | <INLINE> | <INPUTS> | <INTEGER> - | <KEEPRANKCOUNT> | <LITERAL> | <LOCALE> | <LONG> | <LOOSE> - | <LOWERBOUND> | <LOWERCASE> | <MACRO> | <MAP> | <MATCH> - | <MATCHPHASE> - | <MAXFILTERCOVERAGE> - | <MAXHITS> + | <MODEL> | <MTOKEN> | <MUTABLE> + | <MUTATE> | <NEVER> | <NONE> | <NORMAL> | <NORMALIZING> | <OFF> | <ON> - | <ONDEMAND> + | <OPERATION> | <ORDER> + | <PAGED> + | <PARALLEL> | <PREFIX> | <PRIMARY> | <PROPERTIES> | <QUATERNARY> | <QUERY> - | <QUERYCOMMAND> | <RANK> - | <MODEL> - | <RANKPROFILE> - | <RANKPROPERTIES> - | <RANKSCOREDROPLIMIT> - | <RANKTYPE> | <RAW> | <REFERENCE> - | <REMOVEIFZERO> - | <RERANKCOUNT> | <SCHEMA> | <SEARCH> | <SECONDARY> - | <SECONDPHASE> + | <SEQUENTIAL> | <SORTING> | <SOURCE> - | <PAGED> | <SSCONTEXTUAL> | <SSOVERRIDE> | <SSTITLE> @@ -2690,7 +2736,6 @@ String identifier() : { } | <SUBSTRING> | <SUFFIX> | <SUMMARY> - | <SUMMARYTO> | <SYMMETRIC> | <TERTIARY> | <TEXT> @@ -2700,7 +2745,6 @@ String identifier() : { } | <UCA> | <UNCASED> | <URI> - | <UPPERBOUND> | <VARIABLE> | <WEIGHT> | <WEIGHTEDSET> diff --git a/config-model/src/test/derived/globalphase_onnx_inside/onnx-models.cfg b/config-model/src/test/derived/globalphase_onnx_inside/onnx-models.cfg index f705f896540..d63e85e2f19 100644 --- a/config-model/src/test/derived/globalphase_onnx_inside/onnx-models.cfg +++ b/config-model/src/test/derived/globalphase_onnx_inside/onnx-models.cfg @@ -25,7 +25,7 @@ model[].input[].source "rankingExpression(indirect_x)" model[].output[].name "vector_Y" model[].output[].as "foobar" model[].dry_run_on_setup true -model[].stateless_execution_mode "" +model[].stateless_execution_mode "parallel" model[].stateless_interop_threads 5 model[].stateless_intraop_threads 3 model[].gpu_device 2 diff --git a/config-model/src/test/derived/globalphase_onnx_inside/test.sd b/config-model/src/test/derived/globalphase_onnx_inside/test.sd index cd3dfcea3d0..f7749e648e0 100644 --- a/config-model/src/test/derived/globalphase_onnx_inside/test.sd +++ b/config-model/src/test/derived/globalphase_onnx_inside/test.sd @@ -52,6 +52,7 @@ schema test { intraop-threads: 3 interop-threads: 5 gpu-device: 2 + execution-mode: parallel } first-phase { expression: sum(attribute(aa)) diff --git a/config-model/src/test/derived/indexswitches/ilscripts.cfg b/config-model/src/test/derived/indexswitches/ilscripts.cfg index 77ac18e3261..5cda0a9fdc7 100644 --- a/config-model/src/test/derived/indexswitches/ilscripts.cfg +++ b/config-model/src/test/derived/indexswitches/ilscripts.cfg @@ -4,7 +4,7 @@ ilscript[].doctype "indexswitches" ilscript[].docfield[] "title" ilscript[].docfield[] "descr" ilscript[].docfield[] "source_src" -ilscript[].content[] "clear_state | guard { input source_src | switch { case \"theweb\": input source_src | tokenize normalize | summary source | index source; case \"amg\": input source_src | tokenize normalize | summary source; default: input source_src . \" partner\" | tokenize normalize | summary source | index source; }; }" +ilscript[].content[] "clear_state | guard { input source_src | switch { case \"amg\": input source_src | tokenize normalize | summary source; case \"theweb\": input source_src | tokenize normalize | summary source | index source; default: input source_src . \" partner\" | tokenize normalize | summary source | index source; }; }" ilscript[].content[] "clear_state | guard { input title | tokenize normalize stem:\"BEST\" | summary title | index title; }" ilscript[].content[] "clear_state | guard { input descr | tokenize normalize stem:\"BEST\" | summary descr | index descr; }" ilscript[].content[] "input source_src | passthrough source_src" diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudWatchValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudWatchValidatorTest.java deleted file mode 100644 index f600900d3cd..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/CloudWatchValidatorTest.java +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.application.validation; - -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.model.NullConfigModelRegistry; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.model.VespaModel; -import org.junit.jupiter.api.Test; -import org.xml.sax.SAXException; - -import java.io.IOException; - -import static com.yahoo.config.provision.Environment.prod; -import static org.junit.jupiter.api.Assertions.*; - - -/** - * @author gjoranv - */ -public class CloudWatchValidatorTest { - - @Test - void cloudwatch_in_public_zones_passes_validation() throws IOException, SAXException { - DeployState deployState = deployState(servicesWithCloudwatch(), true, true); - VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - - new CloudWatchValidator().validate(model, deployState); - } - - @Test - void cloudwatch_passes_validation_for_self_hosted_vespa() throws IOException, SAXException { - DeployState deployState = deployState(servicesWithCloudwatch(), false, false); - VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - - new CloudWatchValidator().validate(model, deployState); - } - - @Test - void cloudwatch_in_non_public_zones_fails_validation() throws IOException, SAXException { - DeployState deployState = deployState(servicesWithCloudwatch(), true, false); - VespaModel model = new VespaModel(new NullConfigModelRegistry(), deployState); - - try { - new CloudWatchValidator().validate(model, deployState); - fail(); - } catch (IllegalArgumentException e) { - assertEquals("CloudWatch cannot be set up for non-public hosted Vespa and must be removed for consumers: [cloudwatch-consumer]", - e.getMessage()); - } - } - - private static DeployState deployState(String servicesXml, boolean isHosted, boolean isPublic) { - ApplicationPackage app = new MockApplicationPackage.Builder() - .withServices(servicesXml) - .build(); - - DeployState.Builder builder = new DeployState.Builder() - .applicationPackage(app) - .properties(new TestProperties().setHostedVespa(isHosted)); - if (isHosted) { - var system = isPublic ? SystemName.Public : SystemName.main; - builder.zone(new Zone(system, Environment.prod, RegionName.from("foo"))); - } - final DeployState deployState = builder.build(); - - if (isHosted) { - assertTrue(deployState.isHosted(), "Test must emulate a hosted deployment."); - assertEquals(prod, deployState.zone().environment(), "Test must emulate a prod environment."); - } - return deployState; - } - - private String servicesWithCloudwatch() { - return String.join("\n", - "<services>", - " <admin version='2.0'>", - " <adminserver hostalias='node1'/>", - " <metrics>", - " <consumer id='cloudwatch-consumer'>", - " <metric id='my-metric'/>", - " <cloudwatch region='us-east-1' namespace='my-namespace' >", - " <credentials access-key-name='my-access-key' ", - " secret-key-name='my-secret-key' />", - " </cloudwatch>", - " </consumer>", - " </metrics>", - " </admin>", - "</services>" - ); - } - -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java index 5c360a9343f..65dfce8ff6c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ContentClusterRemovalValidatorTest.java @@ -1,14 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change; +import com.yahoo.collections.Pair; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.provision.Environment; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.ValidationTester; import com.yahoo.yolean.Exceptions; import org.junit.jupiter.api.Test; +import java.util.List; + import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.fail; @@ -36,7 +40,12 @@ public class ContentClusterRemovalValidatorTest { @Test void testOverridingContentRemovalValidation() { VespaModel previous = tester.deploy(null, getServices("contentClusterId"), Environment.prod, null).getFirst(); - tester.deploy(previous, getServices("newContentClusterId"), Environment.prod, removalOverride); // Allowed due to override + var result = tester.deploy(previous, getServices("newContentClusterId"), Environment.prod, removalOverride); // Allowed due to override + assertEquals(result.getFirst().getContainerClusters().values().stream() + .flatMap(cluster -> cluster.getContainers().stream()) + .map(container -> container.getServiceInfo()) + .toList(), + result.getSecond().stream().flatMap(action -> action.getServices().stream()).toList()); } private static String getServices(String contentClusterId) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index b2175014cd4..488ad9f8484 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1099,6 +1099,30 @@ public class ContentClusterTest extends ContentBaseTest { } } + private void verifySummaryDecodeType(String policy, DispatchConfig.SummaryDecodePolicy.Enum expected) { + TestProperties properties = new TestProperties(); + if (policy != null) { + properties.setSummaryDecodePolicy(policy); + } + VespaModel model = createEnd2EndOneNode(properties); + + ContentCluster cc = model.getContentClusters().get("storage"); + DispatchConfig.Builder builder = new DispatchConfig.Builder(); + cc.getSearch().getConfig(builder); + + DispatchConfig cfg = new DispatchConfig(builder); + assertEquals(expected, cfg.summaryDecodePolicy()); + } + + @Test + public void verify_summary_decoding_controlled_by_properties() { + verifySummaryDecodeType(null, DispatchConfig.SummaryDecodePolicy.EAGER); + verifySummaryDecodeType("illegal-config", DispatchConfig.SummaryDecodePolicy.EAGER); + verifySummaryDecodeType("eager", DispatchConfig.SummaryDecodePolicy.EAGER); + verifySummaryDecodeType("ondemand", DispatchConfig.SummaryDecodePolicy.ONDEMAND); + verifySummaryDecodeType("on-demand", DispatchConfig.SummaryDecodePolicy.ONDEMAND); + } + private int resolveMaxCompactBuffers(OptionalInt maxCompactBuffers) { TestProperties testProperties = new TestProperties(); if (maxCompactBuffers.isPresent()) { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java index 5d5775275c3..17025b10568 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/filedistribution/FileReferencesAndDownloadsMaintainer.java @@ -23,7 +23,9 @@ import java.util.stream.Collectors; import static java.nio.file.Files.readAttributes; /** - * Deletes file references and url downloads that have not been used for some time + * Deletes file references and url downloads that have not been used for some time. + * See {@link com.yahoo.vespa.config.proxy.filedistribution.RequestTracker} for how we track + * when a file reference or download was last used. * * @author hmusum */ @@ -32,7 +34,7 @@ class FileReferencesAndDownloadsMaintainer implements Runnable { private static final Logger log = Logger.getLogger(FileReferencesAndDownloadsMaintainer.class.getName()); private static final File defaultUrlDownloadDir = UrlDownloadRpcServer.downloadDir; private static final File defaultFileReferencesDownloadDir = FileDownloader.defaultDownloadDirectory; - private static final Duration defaultDurationToKeepFiles = Duration.ofDays(14); + private static final Duration defaultDurationToKeepFiles = Duration.ofDays(21); private static final Duration interval = Duration.ofMinutes(1); private final ScheduledExecutorService executor = diff --git a/configdefinitions/src/vespa/dispatch.def b/configdefinitions/src/vespa/dispatch.def index 936b1400053..01fc5d48dfa 100644 --- a/configdefinitions/src/vespa/dispatch.def +++ b/configdefinitions/src/vespa/dispatch.def @@ -3,11 +3,11 @@ namespace=vespa.config.search -# The active docs a group must have as a % of the average active docs of all other groups, -# for that group to be included in queries +## The active docs a group must have as a % of the average active docs of all other groups, +## for that group to be included in queries minActivedocsPercentage double default=97.0 -# Distribution policy for group selection +## Distribution policy for group selection distributionPolicy enum { ROUNDROBIN, BEST_OF_RANDOM_2, LATENCY_AMORTIZED_OVER_REQUESTS, LATENCY_AMORTIZED_OVER_TIME, ADAPTIVE } default=ADAPTIVE ## Maximum number of hits that will be requested from a single node @@ -26,24 +26,28 @@ maxHitsPerNode int default=2147483647 ## a setting of 1.0. This is a significant optimisation with with very little loss in presicion. topKProbability double default=0.9999 -# Number of document replicas _per group_ that will be present in a stable cluster. -# Should always be >= searchableCopies. +## Number of document replicas _per group_ that will be present in a stable cluster. +## Should always be >= searchableCopies. redundancy long default=1 -# Minimum search coverage required before returning the results of a query +## Minimum search coverage required before returning the results of a query minSearchCoverage double default=100 -# Minimum wait time for full coverage after minimum coverage is achieved, factored based on time left at minimum coverage +## Minimum wait time for full coverage after minimum coverage is achieved, factored based on time left at minimum coverage minWaitAfterCoverageFactor double default=0 -# Maximum wait time for full coverage after minimum coverage is achieved, factored based on time left at minimum coverage +## Maximum wait time for full coverage after minimum coverage is achieved, factored based on time left at minimum coverage maxWaitAfterCoverageFactor double default=1 -# Number of JRT transport threads +## Number of JRT transport threads numJrtTransportThreads int default=8 -# Number of JRT connections per backend node +## Number of JRT connections per backend node numJrtConnectionsPerNode int default=8 -# Number of seconds to spend warming up code to prevent JIT cold start issues. +## Number of seconds to spend warming up code to prevent JIT cold start issues. warmuptime double default=0.1 + +## Specifies how summary data are decoded +## Eager will build java objects immediately, while ONDEMAND will do so when it is needed +summaryDecodePolicy enum {EAGER, ONDEMAND} default=EAGER diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index f6988a6b566..955b1bc8f4f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -534,7 +534,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye NestedTransaction transaction = new NestedTransaction(); Optional<ApplicationTransaction> applicationTransaction = hostProvisioner.map(provisioner -> provisioner.lock(applicationId)) .map(lock -> new ApplicationTransaction(lock, transaction)); - try (var applicationLock = tenantApplications.lock(applicationId)) { + try (@SuppressWarnings("unused") var applicationLock = tenantApplications.lock(applicationId)) { Optional<Long> activeSession = tenantApplications.activeSessionOf(applicationId); CompletionWaiter waiter; if (activeSession.isPresent()) { @@ -796,7 +796,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye NestedTransaction transaction = new NestedTransaction(); Optional<ApplicationTransaction> applicationTransaction = hostProvisioner.map(provisioner -> provisioner.lock(applicationId)) .map(lock -> new ApplicationTransaction(lock, transaction)); - try (var sessionLock = tenant.getApplicationRepo().lock(applicationId)) { + try (@SuppressWarnings("unused") var sessionLock = tenant.getApplicationRepo().lock(applicationId)) { Optional<Session> activeSession = getActiveSession(applicationId); var sessionZooKeeperClient = tenant.getSessionRepository().createSessionZooKeeperClient(session.getSessionId()); CompletionWaiter waiter = sessionZooKeeperClient.createActiveWaiter(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 88e3134ccad..cddcb0f316d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -74,7 +74,7 @@ public class TenantApplications implements RequestHandler, HostValidator { private final MetricUpdater tenantMetricUpdater; private final Clock clock; private final TenantFileSystemDirs tenantFileSystemDirs; - private final ConfigserverConfig configserverConfig; + private final String serverId; private final ListFlag<String> incompatibleVersions; public TenantApplications(TenantName tenant, Curator curator, StripedExecutor<TenantName> zkWatcherExecutor, @@ -95,7 +95,7 @@ public class TenantApplications implements RequestHandler, HostValidator { this.hostRegistry = hostRegistry; this.tenantFileSystemDirs = tenantFileSystemDirs; this.clock = clock; - this.configserverConfig = configserverConfig; + this.serverId = configserverConfig.serverId(); this.incompatibleVersions = PermanentFlags.INCOMPATIBLE_VERSIONS.bindTo(flagSource); } @@ -230,7 +230,7 @@ public class TenantApplications implements RequestHandler, HostValidator { */ public void activateApplication(ApplicationSet applicationSet, long activeSessionId) { ApplicationId id = applicationSet.getId(); - try (Lock lock = lock(id)) { + try (@SuppressWarnings("unused") Lock lock = lock(id)) { if ( ! exists(id)) return; // Application was deleted before activation. if (applicationSet.getApplicationGeneration() != activeSessionId) @@ -269,7 +269,7 @@ public class TenantApplications implements RequestHandler, HostValidator { public void removeApplicationsExcept(Set<ApplicationId> applications) { for (ApplicationId activeApplication : applicationMapper.listApplicationIds()) { if ( ! applications.contains(activeApplication)) { - try (var applicationLock = lock(activeApplication)){ + try (@SuppressWarnings("unused") var applicationLock = lock(activeApplication)){ removeApplication(activeApplication); } } @@ -404,11 +404,11 @@ public class TenantApplications implements RequestHandler, HostValidator { public TenantFileSystemDirs getTenantFileSystemDirs() { return tenantFileSystemDirs; } public CompletionWaiter createRemoveApplicationWaiter(ApplicationId applicationId) { - return RemoveApplicationWaiter.createAndInitialize(curator, applicationId, configserverConfig.serverId()); + return RemoveApplicationWaiter.createAndInitialize(curator, applicationId, serverId); } public CompletionWaiter getRemoveApplicationWaiter(ApplicationId applicationId) { - return RemoveApplicationWaiter.create(curator, applicationId, configserverConfig.serverId()); + return RemoveApplicationWaiter.create(curator, applicationId, serverId); } /** diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index ad5423f0a94..7a2377594a1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -203,6 +203,7 @@ public class ModelContextImpl implements ModelContext { private final boolean useRestrictedDataPlaneBindings; private final int heapPercentage; private final boolean enableGlobalPhase; + private final String summaryDecodePolicy; public FeatureFlags(FlagSource source, ApplicationId appId, Version version) { this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -248,11 +249,13 @@ public class ModelContextImpl implements ModelContext { this.useRestrictedDataPlaneBindings = flagValue(source, appId, version, Flags.RESTRICT_DATA_PLANE_BINDINGS); this.heapPercentage = flagValue(source, appId, version, PermanentFlags.HEAP_SIZE_PERCENTAGE); this.enableGlobalPhase = flagValue(source, appId, version, Flags.ENABLE_GLOBAL_PHASE); + this.summaryDecodePolicy = flagValue(source, appId, version, Flags.SUMMARY_DECODE_POLICY); } @Override public int heapSizePercentage() { return heapPercentage; } @Override public String queryDispatchPolicy() { return queryDispatchPolicy; } @Override public double queryDispatchWarmup() { return queryDispatchWarmup; } + @Override public String summaryDecodePolicy() { return summaryDecodePolicy; } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @Override public String feedSequencerType() { return feedSequencer; } @Override public String responseSequencerType() { return responseSequencer; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index 86c0c90ca12..62a1704b350 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -97,8 +97,6 @@ public class ApplicationHandler extends HttpHandler { if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/filedistributionstatus")) return filedistributionStatus(applicationId(path), request); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/logs")) return logs(applicationId(path), request); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/metrics/deployment")) return deploymentMetrics(applicationId(path)); - // TODO: Remove when all usage has migrated to .../metrics/searchnode - if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/metrics/proton")) return searchNodeMetrics(applicationId(path)); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/metrics/searchnode")) return searchNodeMetrics(applicationId(path)); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindexing")) return getReindexingStatus(applicationId(path)); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/service/{service}/{hostname}/status/{*}")) return serviceStatusPage(applicationId(path), path.get("service"), path.get("hostname"), path.getRest(), request); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java index 2948b82dd96..22ef6cc2547 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ApplicationPackageMaintainer.java @@ -88,7 +88,7 @@ public class ApplicationPackageMaintainer extends ConfigServerMaintainer { createLocalSessionIfMissing(applicationId, sessionId); } } - return asSuccessFactor(attempts, failures); + return asSuccessFactorDeviation(attempts, failures); } private static FileDownloader createFileDownloader(List<String> otherConfigServersInCluster, diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java index dad687aae67..ef48c3bdb98 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintainer.java @@ -46,9 +46,9 @@ public abstract class ConfigServerMaintainer extends Maintainer { } @Override - public void completed(String job, double successFactor, long durationMs) { + public void completed(String job, double successFactorDeviation, long durationMs) { var context = metric.createContext(Map.of("job", job)); - metric.set("maintenance.successFactor", successFactor, context); + metric.set("maintenance.successFactorDeviation", successFactorDeviation, context); metric.set("maintenance.duration", durationMs, context); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java index a9672455d09..e81c9b6bcbd 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ReindexingMaintainer.java @@ -77,7 +77,7 @@ public class ReindexingMaintainer extends ConfigServerMaintainer { } }); } - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactorDeviation(attempts.get(), failures.get()); } private Supplier<Long> lazyGeneration(Application application) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java index 728f3e8510f..2ad04fdd572 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java @@ -12,8 +12,8 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.ConfigKey; -import com.yahoo.vespa.config.server.ConfigServerDB; import com.yahoo.vespa.config.server.ConfigActivationListener; +import com.yahoo.vespa.config.server.ConfigServerDB; import com.yahoo.vespa.config.server.ServerCache; import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; import com.yahoo.vespa.config.server.host.HostRegistry; @@ -26,7 +26,6 @@ import com.yahoo.vespa.config.server.tenant.TestTenantRepository; import com.yahoo.vespa.curator.CompletionTimeoutException; import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; -import com.yahoo.vespa.curator.mock.MockCuratorFramework; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.model.VespaModel; @@ -37,22 +36,17 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.xml.sax.SAXException; - import java.io.File; import java.io.IOException; import java.time.Clock; import java.time.Duration; import java.util.Arrays; -import java.util.Collection; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.IntStream; -import static com.yahoo.vespa.config.server.application.TenantApplications.RemoveApplicationWaiter; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -195,7 +189,6 @@ public class TenantApplicationsTest { public static class MockConfigActivationListener implements ConfigActivationListener { public final AtomicInteger activated = new AtomicInteger(0); final AtomicInteger removed = new AtomicInteger(0); - final Map<String, Collection<String>> tenantHosts = new LinkedHashMap<>(); @Override public void configActivated(ApplicationSet application) { @@ -247,22 +240,22 @@ public class TenantApplicationsTest { @Test public void testRemoveApplication2of3Respond() throws InterruptedException { - Curator curator = new MockCurator3ConfigServers(); - Thread t1 = setupWaiter(curator); - notifyCompletion(curator, 2); + TenantApplications applications = createZKAppRepo(new InMemoryFlagSource()); + Thread t1 = setupWaiter(applications); + notifyCompletion(applications, 2); t1.join(); } @Test public void testRemoveApplicationAllRespond() throws InterruptedException { - Curator curator = new MockCurator3ConfigServers(); - Thread t1 = setupWaiter(curator); - notifyCompletion(curator, 3); + TenantApplications applications = createZKAppRepo(new InMemoryFlagSource()); + Thread t1 = setupWaiter(applications); + notifyCompletion(applications, 3); t1.join(); } - private Thread setupWaiter(Curator curator) { - Curator.CompletionWaiter waiter = RemoveApplicationWaiter.createAndInitialize(curator, createApplicationId(), "cfg1", Duration.ofSeconds(1)); + private Thread setupWaiter(TenantApplications applications) { + Curator.CompletionWaiter waiter = applications.getRemoveApplicationWaiter(createApplicationId()); Thread t1 = new Thread(() -> { try { waiter.awaitCompletion(Duration.ofSeconds(120)); @@ -274,10 +267,10 @@ public class TenantApplicationsTest { return t1; } - private void notifyCompletion(Curator curator, int respondentCount) { + private void notifyCompletion(TenantApplications applications, int respondentCount) { IntStream.range(0, respondentCount) - .forEach(i -> RemoveApplicationWaiter.create(curator, createApplicationId(), "cfg" + i, Duration.ofSeconds(1)) - .notifyCompletion()); + .forEach(i -> applications.createRemoveApplicationWaiter(createApplicationId()) + .notifyCompletion()); } private TenantApplications createZKAppRepo() { @@ -332,12 +325,4 @@ public class TenantApplicationsTest { flagSource); } - private static class MockCurator3ConfigServers extends Curator { - - public MockCurator3ConfigServers() { - super("host1:2181,host2:2181,host3:2181", "host1:2181,host2:2181,host3:2181", (retryPolicy) -> new MockCuratorFramework(true, false)); - } - - } - } diff --git a/container-core/src/main/java/com/yahoo/processing/Request.java b/container-core/src/main/java/com/yahoo/processing/Request.java index cff546f1fd4..313df855cd4 100644 --- a/container-core/src/main/java/com/yahoo/processing/Request.java +++ b/container-core/src/main/java/com/yahoo/processing/Request.java @@ -30,13 +30,13 @@ public class Request extends FreezableClass implements Cloneable { * The name of the chain of Processor instances which will be invoked when * executing a request. */ - public static final CompoundName CHAIN = new CompoundName("chain"); + public static final CompoundName CHAIN = CompoundName.from("chain"); /** * The name of the request property used in the processing framework to * store the incoming JDisc request. */ - public static final CompoundName JDISC_REQUEST = new CompoundName("jdisc.request"); + public static final CompoundName JDISC_REQUEST = CompoundName.from("jdisc.request"); /** * Creates a request with no properties diff --git a/container-core/src/main/java/com/yahoo/processing/Response.java b/container-core/src/main/java/com/yahoo/processing/Response.java index 59533900a0c..9f87f42ba65 100644 --- a/container-core/src/main/java/com/yahoo/processing/Response.java +++ b/container-core/src/main/java/com/yahoo/processing/Response.java @@ -35,7 +35,7 @@ import java.util.concurrent.TimeoutException; */ public class Response extends ListenableFreezableClass { - private final static CompoundName freezeListenerKey =new CompoundName("processing.freezeListener"); + private final static CompoundName freezeListenerKey = CompoundName.from("processing.freezeListener"); private final DataList<?> data; diff --git a/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java b/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java index f7aea8abbd1..837f356d4d9 100644 --- a/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java +++ b/container-core/src/main/java/com/yahoo/processing/handler/AbstractProcessingHandler.java @@ -46,7 +46,7 @@ import static com.yahoo.component.chain.ChainsConfigurer.prepareChainRegistry; @SuppressWarnings("deprecation") // super class is deprecated public abstract class AbstractProcessingHandler<COMPONENT extends Processor> extends LoggingRequestHandler { - private final static CompoundName freezeListenerKey =new CompoundName("processing.freezeListener"); + private final static CompoundName freezeListenerKey = CompoundName.from("processing.freezeListener"); public final static String DEFAULT_RENDERER_ID = "default"; @@ -112,7 +112,7 @@ public abstract class AbstractProcessingHandler<COMPONENT extends Processor> ext public HttpResponse handle(HttpRequest request, ContentChannel channel) { com.yahoo.processing.Request processingRequest = new com.yahoo.processing.Request(); populate("", request.propertyMap(), processingRequest.properties()); - populate("context", request.getJDiscRequest().context(), processingRequest.properties()); + populate("context.", request.getJDiscRequest().context(), processingRequest.properties()); processingRequest.properties().set(Request.JDISC_REQUEST, request); FreezeListener freezeListener = new FreezeListener(processingRequest, renderers, defaultRenderer, channel, renderingExecutor); @@ -183,10 +183,9 @@ public abstract class AbstractProcessingHandler<COMPONENT extends Processor> ext return properties.getString(Request.CHAIN,"default"); } - private void populate(String prefixName,Map<String,?> parameters,Properties properties) { - CompoundName prefix = new CompoundName(prefixName); + private void populate(String prefixName, Map<String,?> parameters,Properties properties) { for (Map.Entry<String,?> entry : parameters.entrySet()) - properties.set(prefix.append(entry.getKey()), entry.getValue()); + properties.set(CompoundName.from(prefixName + entry.getKey()), entry.getValue()); } private static class FreezeListener implements Runnable, ResponseReceiver { diff --git a/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java b/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java index ac7b3d24d08..0edff9162b5 100644 --- a/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java +++ b/container-core/src/main/java/com/yahoo/processing/request/CompoundName.java @@ -21,9 +21,12 @@ import static com.yahoo.text.Lowercase.toLowerCase; */ public final class CompoundName { - /** - * The string name of this compound. - */ + private static final int MAX_CACHE_SIZE = 10_000; + private static final Map<String, CompoundName> cache = new CopyOnWriteHashMap<>(); + /** The empty compound */ + public static final CompoundName empty = CompoundName.from(""); + + /* The string name of this compound. */ private final String name; private final String lowerCasedName; @@ -37,17 +40,16 @@ public final class CompoundName { /** This name with the last component removed */ private final CompoundName first; - /** The empty compound */ - public static final CompoundName empty = new CompoundName(""); - private static final Map<String, CompoundName> cache = new CopyOnWriteHashMap<>(); - private static final int MAX_CACHE_SIZE = 10_000; /** * Constructs this from a string which may contains dot-separated components * * @throws NullPointerException if name is null */ public CompoundName(String name) { - this(name, parse(name).toArray(new String[1])); + this(name, false); + } + private CompoundName(String name, boolean useCache) { + this(name, parse(name).toArray(new String[0]), useCache); } /** Constructs this from an array of name components which are assumed not to contain dots */ @@ -61,7 +63,7 @@ public final class CompoundName { } private CompoundName(String [] compounds) { - this(toCompoundString(compounds), compounds); + this(toCompoundString(compounds), compounds, false); } /** @@ -71,7 +73,7 @@ public final class CompoundName { * @param name the string representation of the compounds * @param compounds the compounds of this name */ - private CompoundName(String name, String [] compounds) { + private CompoundName(String name, String [] compounds, boolean useCache) { if (name == null) throw new NullPointerException("Name can not be null"); this.name = name; @@ -86,12 +88,27 @@ public final class CompoundName { this.compounds = new ImmutableArrayList(compounds); this.hashCode = this.compounds.hashCode(); - rest = (compounds.length > 1) - ? new CompoundName(name.substring(compounds[0].length()+1), Arrays.copyOfRange(compounds, 1, compounds.length)) - : empty; - first = (compounds.length > 1) - ? new CompoundName(name.substring(0, name.length() - (compounds[compounds.length-1].length()+1)), Arrays.copyOfRange(compounds, 0, compounds.length-1)) - : empty; + if (compounds.length > 1) { + String restName = name.substring(compounds[0].length()+1); + if (useCache) { + rest = cache.computeIfAbsent(restName, (key) -> new CompoundName(key, Arrays.copyOfRange(compounds, 1, compounds.length), useCache)); + } else { + rest = new CompoundName(restName, Arrays.copyOfRange(compounds, 1, compounds.length), useCache); + } + } else { + rest = empty; + } + + if (compounds.length > 1) { + String firstName = name.substring(0, name.length() - (compounds[compounds.length-1].length()+1)); + if (useCache) { + first = cache.computeIfAbsent(firstName, (key) -> new CompoundName(key, Arrays.copyOfRange(compounds, 0, compounds.length-1), useCache)); + } else { + first = new CompoundName(firstName, Arrays.copyOfRange(compounds, 0, compounds.length-1), useCache); + } + } else { + first = empty; + } } private static List<String> parse(String s) { @@ -138,7 +155,7 @@ public final class CompoundName { int count = 0; for (String s : compounds) { newCompounds[count++] = s; } for (String s : name.compounds) { newCompounds[count++] = s; } - return new CompoundName(concat(this.name, name.name), newCompounds); + return new CompoundName(concat(this.name, name.name), newCompounds, false); } private static String concat(String name1, String name2) { @@ -310,11 +327,12 @@ public final class CompoundName { CompoundName found = cache.get(name); if (found != null) return found; - CompoundName compound = new CompoundName(name); if (cache.size() < MAX_CACHE_SIZE) { + CompoundName compound = new CompoundName(name, true); cache.put(name, compound); + return compound; } - return compound; + return new CompoundName(name, false); } private static class ImmutableArrayList extends AbstractList<String> { diff --git a/container-core/src/test/java/com/yahoo/processing/request/CompoundNameTestCase.java b/container-core/src/test/java/com/yahoo/processing/request/CompoundNameTestCase.java index 4bbece0af29..eccc4dd8842 100644 --- a/container-core/src/test/java/com/yahoo/processing/request/CompoundNameTestCase.java +++ b/container-core/src/test/java/com/yahoo/processing/request/CompoundNameTestCase.java @@ -18,59 +18,61 @@ import static org.junit.jupiter.api.Assertions.*; public class CompoundNameTestCase { private static final String NAME = "com.yahoo.processing.request.CompoundNameTestCase"; - private final CompoundName cn = new CompoundName(NAME); + private static final CompoundName C_NAME = CompoundName.from(NAME); + private static final CompoundName C_abcde = CompoundName.from("a.b.c.d.e"); void verifyStrict(CompoundName expected, CompoundName actual) { assertEquals(expected, actual); assertEquals(expected.asList(), actual.asList()); } void verifyStrict(String expected, CompoundName actual) { - verifyStrict(new CompoundName(expected), actual); + verifyStrict(CompoundName.from(expected), actual); } @Test final void testLast() { - assertEquals(NAME.substring(NAME.lastIndexOf('.') + 1), cn.last()); + assertEquals(NAME.substring(NAME.lastIndexOf('.') + 1), C_NAME.last()); } @Test final void testFirst() { - assertEquals(NAME.substring(0, NAME.indexOf('.')), cn.first()); + assertEquals(NAME.substring(0, NAME.indexOf('.')), C_NAME.first()); } @Test final void testRest() { - verifyStrict(NAME.substring(NAME.indexOf('.') + 1), cn.rest()); + verifyStrict(NAME.substring(NAME.indexOf('.') + 1), C_NAME.rest()); } @Test final void testRestN() { - verifyStrict("a.b.c.d.e", new CompoundName("a.b.c.d.e").rest(0)); - verifyStrict("b.c.d.e", new CompoundName("a.b.c.d.e").rest(1)); - verifyStrict("c.d.e", new CompoundName("a.b.c.d.e").rest(2)); - verifyStrict("d.e", new CompoundName("a.b.c.d.e").rest(3)); - verifyStrict("e", new CompoundName("a.b.c.d.e").rest(4)); - verifyStrict(CompoundName.empty, new CompoundName("a.b.c.d.e").rest(5)); + verifyStrict("a.b.c.d.e", C_abcde.rest(0)); + verifyStrict("b.c.d.e", C_abcde.rest(1)); + verifyStrict("c.d.e", C_abcde.rest(2)); + verifyStrict("d.e", C_abcde.rest(3)); + verifyStrict("e", C_abcde.rest(4)); + verifyStrict(CompoundName.empty, C_abcde.rest(5)); } @Test final void testFirstN() { - verifyStrict("a.b.c.d.e", new CompoundName("a.b.c.d.e").first(5)); - verifyStrict("a.b.c.d", new CompoundName("a.b.c.d.e").first(4)); - verifyStrict("a.b.c", new CompoundName("a.b.c.d.e").first(3)); - verifyStrict("a.b", new CompoundName("a.b.c.d.e").first(2)); - verifyStrict("a", new CompoundName("a.b.c.d.e").first(1)); - verifyStrict(CompoundName.empty, new CompoundName("a.b.c.d.e").first(0)); + verifyStrict("a.b.c.d.e", C_abcde.first(5)); + verifyStrict("a.b.c.d", C_abcde.first(4)); + verifyStrict("a.b.c", C_abcde.first(3)); + verifyStrict("a.b", C_abcde.first(2)); + verifyStrict("a", C_abcde.first(1)); + verifyStrict(CompoundName.empty, C_abcde.first(0)); } @Test final void testPrefix() { - assertTrue(new CompoundName("a.b.c").hasPrefix(new CompoundName(""))); - assertTrue(new CompoundName("a.b.c").hasPrefix(new CompoundName("a"))); - assertTrue(new CompoundName("a.b.c").hasPrefix(new CompoundName("a.b"))); - assertTrue(new CompoundName("a.b.c").hasPrefix(new CompoundName("a.b.c"))); + CompoundName abc = CompoundName.from("a.b.c"); + assertTrue(abc.hasPrefix(CompoundName.empty)); + assertTrue(abc.hasPrefix(CompoundName.from("a"))); + assertTrue(abc.hasPrefix(CompoundName.from("a.b"))); + assertTrue(abc.hasPrefix(CompoundName.from("a.b.c"))); - assertFalse(new CompoundName("a.b.c").hasPrefix(new CompoundName("a.b.c.d"))); - assertFalse(new CompoundName("a.b.c").hasPrefix(new CompoundName("a.b.d"))); + assertFalse(abc.hasPrefix(CompoundName.from("a.b.c.d"))); + assertFalse(abc.hasPrefix(CompoundName.from("a.b.d"))); } @Test @@ -87,28 +89,28 @@ public class CompoundNameTestCase { for (@SuppressWarnings("unused") String x : i) { ++n; } - assertEquals(n, cn.size()); + assertEquals(n, C_NAME.size()); } @Test final void testGet() { - String s = cn.get(0); + String s = C_NAME.get(0); assertEquals(NAME.substring(0, NAME.indexOf('.')), s); } @Test final void testIsCompound() { - assertTrue(cn.isCompound()); + assertTrue(C_NAME.isCompound()); } @Test final void testIsEmpty() { - assertFalse(cn.isEmpty()); + assertFalse(C_NAME.isEmpty()); } @Test final void testAsList() { - List<String> l = cn.asList(); + List<String> l = C_NAME.asList(); Splitter peoplesFront = Splitter.on('.'); Iterable<String> answer = peoplesFront.split(NAME); Iterator<String> expected = answer.iterator(); @@ -120,35 +122,35 @@ public class CompoundNameTestCase { @Test final void testEqualsObject() { - assertNotEquals(cn, NAME); - assertNotEquals(cn, null); - verifyStrict(cn, cn); - verifyStrict(cn, new CompoundName(NAME)); + assertNotEquals(C_NAME, NAME); + assertNotEquals(C_NAME, null); + verifyStrict(C_NAME, C_NAME); + verifyStrict(C_NAME, CompoundName.from(NAME)); } @Test final void testEmptyNonEmpty() { - assertTrue(new CompoundName("").isEmpty()); - assertEquals(0, new CompoundName("").size()); - assertFalse(new CompoundName("a").isEmpty()); - assertEquals(1, new CompoundName("a").size()); - CompoundName empty = new CompoundName("a.b.c"); + assertTrue(CompoundName.empty.isEmpty()); + assertEquals(0, CompoundName.empty.size()); + assertFalse(CompoundName.from("a").isEmpty()); + assertEquals(1, CompoundName.from("a").size()); + CompoundName empty = CompoundName.from("a.b.c"); verifyStrict(empty, empty.rest(0)); assertNotEquals(empty, empty.rest(1)); } @Test final void testGetLowerCasedName() { - assertEquals(Lowercase.toLowerCase(NAME), cn.getLowerCasedName()); + assertEquals(Lowercase.toLowerCase(NAME), C_NAME.getLowerCasedName()); } @Test void testAppendCompound() { - verifyStrict("a.b.c.d", new CompoundName("").append(new CompoundName("a.b.c.d"))); - verifyStrict("a.b.c.d", new CompoundName("a").append(new CompoundName("b.c.d"))); - verifyStrict("a.b.c.d", new CompoundName("a.b").append(new CompoundName("c.d"))); - verifyStrict("a.b.c.d", new CompoundName("a.b.c").append(new CompoundName("d"))); - verifyStrict("a.b.c.d", new CompoundName("a.b.c.d").append(new CompoundName(""))); + verifyStrict("a.b.c.d", CompoundName.empty.append(CompoundName.from("a.b.c.d"))); + verifyStrict("a.b.c.d", CompoundName.from("a").append(CompoundName.from("b.c.d"))); + verifyStrict("a.b.c.d", CompoundName.from("a.b").append(CompoundName.from("c.d"))); + verifyStrict("a.b.c.d", CompoundName.from("a.b.c").append(CompoundName.from("d"))); + verifyStrict("a.b.c.d", CompoundName.from("a.b.c.d").append(CompoundName.empty)); } @Test @@ -156,13 +158,13 @@ public class CompoundNameTestCase { CompoundName empty = new CompoundName(""); assertTrue(empty.hasPrefix(empty)); - assertTrue(new CompoundName("a").hasPrefix(empty)); + assertTrue(CompoundName.from("a").hasPrefix(empty)); } @Test void whole_components_must_match_to_be_prefix() { - CompoundName stringPrefix = new CompoundName("a"); - CompoundName name = new CompoundName("aa"); + CompoundName stringPrefix = CompoundName.from("a"); + CompoundName name = CompoundName.from("aa"); assertFalse(name.hasPrefix(stringPrefix)); } @@ -171,7 +173,7 @@ public class CompoundNameTestCase { void testFirstRest() { verifyStrict(CompoundName.empty, CompoundName.empty.rest()); - CompoundName n = new CompoundName("on.two.three"); + CompoundName n = CompoundName.from("on.two.three"); assertEquals("on", n.first()); verifyStrict("two.three", n.rest()); n = n.rest(); @@ -190,7 +192,7 @@ public class CompoundNameTestCase { @Test void testHashCodeAndEquals() { - CompoundName n1 = new CompoundName("venn.d.a"); + CompoundName n1 = CompoundName.from("venn.d.a"); CompoundName n2 = new CompoundName(n1.asList()); assertEquals(n1.hashCode(), n2.hashCode()); verifyStrict(n1, n2); @@ -198,12 +200,12 @@ public class CompoundNameTestCase { @Test void testAppendString() { - verifyStrict("a", new CompoundName("a").append("")); - verifyStrict("a", new CompoundName("").append("a")); - verifyStrict("a.b", new CompoundName("a").append("b")); - verifyStrict("a.b.c.d", new CompoundName("a.b").append("c.d")); + verifyStrict("a", CompoundName.from("a").append("")); + verifyStrict("a", CompoundName.empty.append("a")); + verifyStrict("a.b", CompoundName.from("a").append("b")); + verifyStrict("a.b.c.d", CompoundName.from("a.b").append("c.d")); - CompoundName name = new CompoundName("a.b"); + CompoundName name = CompoundName.from("a.b"); verifyStrict("a.b.c", name.append("c")); verifyStrict("a.b.d", name.append("d")); verifyStrict("a.b.d.e", name.append("d.e")); @@ -218,7 +220,7 @@ public class CompoundNameTestCase { @Test void testAsList2() { - assertEquals("[one]", new CompoundName("one").asList().toString()); - assertEquals("[one, two, three]", new CompoundName("one.two.three").asList().toString()); + assertEquals("[one]", CompoundName.from("one").asList().toString()); + assertEquals("[one, two, three]", CompoundName.from("one.two.three").asList().toString()); } } diff --git a/container-core/src/test/java/com/yahoo/processing/request/test/RequestTestCase.java b/container-core/src/test/java/com/yahoo/processing/request/test/RequestTestCase.java index c82b85d84d9..8e960bd1305 100644 --- a/container-core/src/test/java/com/yahoo/processing/request/test/RequestTestCase.java +++ b/container-core/src/test/java/com/yahoo/processing/request/test/RequestTestCase.java @@ -9,7 +9,9 @@ import com.yahoo.processing.request.properties.PropertyMap; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * Tests using requests @@ -19,9 +21,26 @@ import static org.junit.jupiter.api.Assertions.assertNull; public class RequestTestCase { private static final double delta = 0.0000000001; + private static final CompoundName C_a = CompoundName.from("a"); + + private static final CompoundName C_B = CompoundName.from("b"); + private static final CompoundName C_C = CompoundName.from("c"); + private static final CompoundName C_D = CompoundName.from("d"); + private static final CompoundName C_f = CompoundName.from("f"); + private static final CompoundName C_g = CompoundName.from("g"); + private static final CompoundName C_I = CompoundName.from("i"); + private static final CompoundName C_L = CompoundName.from("l"); + private static final CompoundName C_M = CompoundName.from("m"); + private static final CompoundName C_N = CompoundName.from("n"); + private static final CompoundName C_o = CompoundName.from("o"); + private static final CompoundName C_x = CompoundName.from("x"); + private static final CompoundName C_none = CompoundName.from("none"); + + @Test void testProperties() { + Properties p = new PropertyMap(); p.set("a", "a1"); Request r = new Request(p); @@ -33,38 +52,38 @@ public class RequestTestCase { assertEquals("b1", r.properties().get("b", "default")); assertEquals("default", r.properties().get("c", "default")); assertNull(r.properties().get("c")); - assertEquals("b1", r.properties().get(new CompoundName("b"))); - assertEquals("b1", r.properties().get(new CompoundName("b"), "default")); - assertEquals("default", r.properties().get(new CompoundName("c"), "default")); - assertNull(r.properties().get(new CompoundName("c"))); + assertEquals("b1", r.properties().get(C_B)); + assertEquals("b1", r.properties().get(C_B, "default")); + assertEquals("default", r.properties().get(C_C, "default")); + assertNull(r.properties().get(C_C)); assertEquals("b1", r.properties().getString("b")); assertEquals("b1", r.properties().getString("b", "default")); assertEquals("default", r.properties().getString("c", "default")); assertNull(r.properties().getString("c")); - assertEquals("b1", r.properties().getString(new CompoundName("b"))); - assertEquals("b1", r.properties().getString(new CompoundName("b"), "default")); - assertEquals("default", r.properties().getString(new CompoundName("c"), "default")); - assertNull(r.properties().getString(new CompoundName("c"))); + assertEquals("b1", r.properties().getString(C_B)); + assertEquals("b1", r.properties().getString(C_B, "default")); + assertEquals("default", r.properties().getString(C_C, "default")); + assertNull(r.properties().getString(C_C)); r.properties().set("i", 7); assertEquals(7, (int) r.properties().getInteger("i")); assertEquals(7, (int) r.properties().getInteger("i", 3)); assertEquals(3, (int) r.properties().getInteger("n", 3)); assertNull(r.properties().getInteger("n")); - assertEquals(7, (int) r.properties().getInteger(new CompoundName("i"))); - assertEquals(7, (int) r.properties().getInteger(new CompoundName("i"), 3)); - assertEquals(3, (int) r.properties().getInteger(new CompoundName("n"), 3)); + assertEquals(7, (int) r.properties().getInteger(C_I)); + assertEquals(7, (int) r.properties().getInteger(C_I, 3)); + assertEquals(3, (int) r.properties().getInteger(C_N, 3)); assertNull(r.properties().getInteger("n")); - r.properties().set(new CompoundName("l"), 7); + r.properties().set(C_L, 7); assertEquals(7, (long) r.properties().getLong("l")); - assertEquals(7, (long) r.properties().getLong("l", 3l)); - assertEquals(3, (long) r.properties().getLong("m", 3l)); + assertEquals(7, (long) r.properties().getLong("l", 3L)); + assertEquals(3, (long) r.properties().getLong("m", 3L)); assertNull(r.properties().getInteger("m")); - assertEquals(7, (long) r.properties().getLong(new CompoundName("l"))); - assertEquals(7, (long) r.properties().getLong(new CompoundName("l"), 3l)); - assertEquals(3, (long) r.properties().getLong(new CompoundName("m"), 3l)); + assertEquals(7, (long) r.properties().getLong(C_L)); + assertEquals(7, (long) r.properties().getLong(C_L, 3L)); + assertEquals(3, (long) r.properties().getLong(C_M, 3L)); assertNull(r.properties().getInteger("m")); r.properties().set("d", 7.3); @@ -72,29 +91,29 @@ public class RequestTestCase { assertEquals(7.3, r.properties().getDouble("d", 3.4d), delta); assertEquals(3.4, r.properties().getDouble("f", 3.4d), delta); assertNull(r.properties().getDouble("f")); - assertEquals(7.3, r.properties().getDouble(new CompoundName("d")), delta); - assertEquals(7.3, r.properties().getDouble(new CompoundName("d"), 3.4d), delta); - assertEquals(3.4, r.properties().getDouble(new CompoundName("f"), 3.4d), delta); + assertEquals(7.3, r.properties().getDouble(C_D), delta); + assertEquals(7.3, r.properties().getDouble(C_D, 3.4d), delta); + assertEquals(3.4, r.properties().getDouble(C_f, 3.4d), delta); assertNull(r.properties().getDouble("f")); r.properties().set("o", true); - assertEquals(true, r.properties().getBoolean("o")); - assertEquals(true, r.properties().getBoolean("o", true)); - assertEquals(true, r.properties().getBoolean("g", true)); - assertEquals(false, r.properties().getBoolean("g")); - assertEquals(true, r.properties().getBoolean(new CompoundName("o"))); - assertEquals(true, r.properties().getBoolean(new CompoundName("o"), true)); - assertEquals(true, r.properties().getBoolean(new CompoundName("g"), true)); - assertEquals(false, r.properties().getBoolean("g")); - - r.properties().set(new CompoundName("x.y"), "x1.y1"); + assertTrue(r.properties().getBoolean("o")); + assertTrue(r.properties().getBoolean("o", true)); + assertTrue(r.properties().getBoolean("g", true)); + assertFalse(r.properties().getBoolean("g")); + assertTrue(r.properties().getBoolean(C_o)); + assertTrue(r.properties().getBoolean(C_o, true)); + assertTrue(r.properties().getBoolean(C_g, true)); + assertFalse(r.properties().getBoolean("g")); + + r.properties().set(CompoundName.from("x.y"), "x1.y1"); r.properties().set("x.z", "x1.z1"); assertEquals(8, r.properties().listProperties().size()); assertEquals(0, r.properties().listProperties("a").size()); - assertEquals(0, r.properties().listProperties(new CompoundName("a")).size()); - assertEquals(0, r.properties().listProperties(new CompoundName("none")).size()); - assertEquals(2, r.properties().listProperties(new CompoundName("x")).size()); + assertEquals(0, r.properties().listProperties(C_a).size()); + assertEquals(0, r.properties().listProperties(C_none).size()); + assertEquals(2, r.properties().listProperties(C_x).size()); assertEquals(2, r.properties().listProperties("x").size()); } diff --git a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java index 88a37ea5a02..92ce6abb319 100644 --- a/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java +++ b/container-search/src/main/java/com/yahoo/prelude/IndexFacts.java @@ -6,11 +6,11 @@ import com.yahoo.search.Query; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import static com.yahoo.text.Lowercase.toLowerCase; @@ -32,6 +32,16 @@ public class IndexFacts { private Map<String, List<String>> clusterByDocument; + private static class DocumentTypeListOffset { + public final int offset; + public final SearchDefinition searchDefinition; + + public DocumentTypeListOffset(int offset, SearchDefinition searchDefinition) { + this.offset = offset; + this.searchDefinition = searchDefinition; + } + } + /** A Map of all known search definitions indexed by name */ private Map<String, SearchDefinition> searchDefinitions = new LinkedHashMap<>(); @@ -100,32 +110,34 @@ public class IndexFacts { private boolean isIndexFromDocumentTypes(String indexName, List<String> documentTypes) { if ( ! isInitialized()) return true; - if (documentTypes.isEmpty()) return unionSearchDefinition.getIndex(indexName) != null; + if (documentTypes.isEmpty()) { + return unionSearchDefinition.getIndex(indexName) != null; + } - for (String docName : documentTypes) { - SearchDefinition sd = searchDefinitions.get(docName); - if (sd != null) { - Index index = sd.getIndex(indexName); - if (index != null) return true; + DocumentTypeListOffset sd = chooseSearchDefinition(documentTypes, 0); + while (sd != null) { + Index index = sd.searchDefinition.getIndex(indexName); + if (index != null) { + return true; } + sd = chooseSearchDefinition(documentTypes, sd.offset); } + return false; } private String getCanonicNameFromDocumentTypes(String indexName, List<String> documentTypes) { if (!isInitialized()) return indexName; - String lowerCased = toLowerCase(indexName); if (documentTypes.isEmpty()) { - Index index = unionSearchDefinition.getIndexByLowerCase(lowerCased); + Index index = unionSearchDefinition.getIndexByLowerCase(toLowerCase(indexName)); return index == null ? indexName : index.getName(); } - for (String docName : documentTypes) { - SearchDefinition sd = searchDefinitions.get(docName); - if (sd != null) { - Index index = sd.getIndexByLowerCase(lowerCased); - if (index != null) return index.getName(); - } + DocumentTypeListOffset sd = chooseSearchDefinition(documentTypes, 0); + while (sd != null) { + Index index = sd.searchDefinition.getIndexByLowerCase(toLowerCase(indexName)); + if (index != null) return index.getName(); + sd = chooseSearchDefinition(documentTypes, sd.offset); } return indexName; } @@ -146,12 +158,13 @@ public class IndexFacts { return index; } - for (String docName : documentTypes) { - SearchDefinition sd = searchDefinitions.get(docName); - if (sd != null) { - Index index = sd.getIndex(canonicName); - if (index != null) return index; - } + DocumentTypeListOffset sd = chooseSearchDefinition(documentTypes, 0); + + while (sd != null) { + Index index = sd.searchDefinition.getIndex(canonicName); + + if (index != null) return index; + sd = chooseSearchDefinition(documentTypes, sd.offset); } return Index.nullIndex; } @@ -174,7 +187,7 @@ public class IndexFacts { * Given a search list which is a mixture of document types and cluster * names, and a restrict list which is a list of document types, return a * set of all valid document types for this combination. Most use-cases for - * fetching index settings will involve calling this method with the + * fetching index settings will involve calling this method with the the * incoming query's {@link com.yahoo.search.query.Model#getSources()} and * {@link com.yahoo.search.query.Model#getRestrict()} as input parameters * before calling any other method of this class. @@ -183,20 +196,20 @@ public class IndexFacts { * @param restrict the restrict list for a query * @return a (possibly empty) set of valid document types */ - private Set<String> resolveDocumentTypes(Collection<String> sources, Set<String> restrict, + private Set<String> resolveDocumentTypes(Collection<String> sources, Collection<String> restrict, Set<String> candidateDocumentTypes) { sources = emptyCollectionIfNull(sources); - restrict = emptySetIfNull(restrict); + restrict = emptyCollectionIfNull(restrict); if (sources.isEmpty()) { if ( ! restrict.isEmpty()) { - return Set.copyOf(restrict); + return new TreeSet<>(restrict); } else { return candidateDocumentTypes; } } - Set<String> toSearch = new HashSet<>(); + Set<String> toSearch = new TreeSet<>(); for (String source : sources) { // source: a document type or a cluster containing them List<String> clusterDocTypes = clusters.get(source); if (clusterDocTypes == null) { // source was a document type @@ -222,8 +235,21 @@ public class IndexFacts { private Collection<String> emptyCollectionIfNull(Collection<String> collection) { return collection == null ? List.of() : collection; } - private Set<String> emptySetIfNull(Set<String> collection) { - return collection == null ? Set.of() : collection; + + /** + * Chooses the correct search definition, default if in doubt. + * + * @return the search definition to use + */ + private DocumentTypeListOffset chooseSearchDefinition(List<String> documentTypes, int index) { + while (index < documentTypes.size()) { + String docName = documentTypes.get(index++); + SearchDefinition sd = searchDefinitions.get(docName); + if (sd != null) { + return new DocumentTypeListOffset(index, sd); + } + } + return null; } /** @@ -253,6 +279,10 @@ public class IndexFacts { return frozen; } + private void ensureNotFrozen() { + if (frozen) throw new IllegalStateException("Tried to modify frozen IndexFacts instance."); + } + public String getDefaultPosition(String sdName) { SearchDefinition sd; if (sdName == null) { @@ -270,16 +300,12 @@ public class IndexFacts { return new Session(query); } - public Session newSession() { - return new Session(Set.of(), Set.of()); - } - - public Session newSession(Collection<String> sources, Set<String> restrict) { + public Session newSession(Collection<String> sources, Collection<String> restrict) { return new Session(sources, restrict); } public Session newSession(Collection<String> sources, - Set<String> restrict, + Collection<String> restrict, Set<String> candidateDocumentTypes) { return new Session(sources, restrict, candidateDocumentTypes); } @@ -297,12 +323,12 @@ public class IndexFacts { documentTypes = List.copyOf(resolveDocumentTypes(query)); } - private Session(Collection<String> sources, Set<String> restrict) { + private Session(Collection<String> sources, Collection<String> restrict) { // Assumption: Search definition name equals document name. documentTypes = List.copyOf(resolveDocumentTypes(sources, restrict, searchDefinitions.keySet())); } - private Session(Collection<String> sources, Set<String> restrict, Set<String> candidateDocumentTypes) { + private Session(Collection<String> sources, Collection<String> restrict, Set<String> candidateDocumentTypes) { documentTypes = List.copyOf(resolveDocumentTypes(sources, restrict, candidateDocumentTypes)); } diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index f0e3e3f3e44..46332d632fe 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.cluster; -import com.yahoo.component.ComponentId; import com.yahoo.component.annotation.Inject; +import com.yahoo.component.ComponentId; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.QrSearchersConfig; @@ -28,6 +28,10 @@ import com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher; import com.yahoo.yolean.Exceptions; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Set; import java.util.UUID; @@ -53,7 +57,8 @@ public class ClusterSearcher extends Searcher { private final String searchClusterName; - private final SchemaResolver schemaResolver; + // The set of document types contained in this search cluster + private final Set<String> schemas; private final long maxQueryTimeout; // in milliseconds private final long maxQueryCacheTimeout; // in milliseconds @@ -79,7 +84,7 @@ public class ClusterSearcher extends Searcher { searchClusterName = clusterConfig.clusterName(); QrSearchersConfig.Searchcluster searchClusterConfig = getSearchClusterConfigFromClusterName(qrsConfig, searchClusterName); this.globalPhaseRanker = searchClusterConfig.globalphase() ? globalPhaseRanker : null; - this.schemaResolver = new SchemaResolver(documentDbConfig); + schemas = new LinkedHashSet<>(); maxQueryTimeout = ParameterParser.asMilliSeconds(clusterConfig.maxQueryTimeout(), DEFAULT_MAX_QUERY_TIMEOUT); maxQueryCacheTimeout = ParameterParser.asMilliSeconds(clusterConfig.maxQueryCacheTimeout(), DEFAULT_MAX_QUERY_CACHE_TIMEOUT); @@ -88,6 +93,9 @@ public class ClusterSearcher extends Searcher { .com().yahoo().prelude().fastsearch().FastSearcher().docsum() .defaultclass()); + for (DocumentdbInfoConfig.Documentdb docDb : documentDbConfig.documentdb()) + schemas.add(docDb.name()); + String uniqueServerId = UUID.randomUUID().toString(); if (searchClusterConfig.indexingmode() == STREAMING) { server = vdsCluster(uniqueServerId, searchClusterIndex, @@ -149,7 +157,7 @@ public class ClusterSearcher extends Searcher { /** Do not use, for internal testing purposes only. **/ ClusterSearcher(Set<String> schemas, VespaBackEndSearcher searcher, Executor executor) { - this.schemaResolver = new SchemaResolver(schemas); + this.schemas = schemas; searchClusterName = "testScenario"; maxQueryTimeout = DEFAULT_MAX_QUERY_TIMEOUT; maxQueryCacheTimeout = DEFAULT_MAX_QUERY_CACHE_TIMEOUT; @@ -221,9 +229,8 @@ public class ClusterSearcher extends Searcher { } private Result doSearch(Searcher searcher, Query query, Execution execution) { - var schemas = schemaResolver.resolve(query, execution); if (schemas.size() > 1) { - return searchMultipleDocumentTypes(searcher, query, execution, schemas); + return searchMultipleDocumentTypes(searcher, query, execution); } else { String docType = schemas.iterator().next(); query.getModel().setRestrict(docType); @@ -259,7 +266,8 @@ public class ClusterSearcher extends Searcher { } } - private Result searchMultipleDocumentTypes(Searcher searcher, Query query, Execution execution, Set<String> schemas) { + private Result searchMultipleDocumentTypes(Searcher searcher, Query query, Execution execution) { + Set<String> schemas = resolveSchemas(query, execution.context().getIndexFacts()); List<Query> queries = createQueries(query, schemas); if (queries.size() == 1) { return perSchemaSearch(searcher, queries.get(0), execution); @@ -293,7 +301,25 @@ public class ClusterSearcher extends Searcher { } Set<String> resolveSchemas(Query query, IndexFacts indexFacts) { - return schemaResolver.resolve(query, indexFacts); + Set<String> restrict = query.getModel().getRestrict(); + if (restrict == null || restrict.isEmpty()) { + Set<String> sources = query.getModel().getSources(); + return (sources == null || sources.isEmpty()) + ? schemas + : new HashSet<>(indexFacts.newSession(sources, Collections.emptyList(), schemas).documentTypes()); + } else { + return filterValidDocumentTypes(restrict); + } + } + + private Set<String> filterValidDocumentTypes(Collection<String> restrict) { + Set<String> retval = new LinkedHashSet<>(); + for (String docType : restrict) { + if (docType != null && schemas.contains(docType)) { + retval.add(docType); + } + } + return retval; } private List<Query> createQueries(Query query, Set<String> docTypes) { diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/SchemaResolver.java b/container-search/src/main/java/com/yahoo/prelude/cluster/SchemaResolver.java deleted file mode 100644 index 3a2125d1d38..00000000000 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/SchemaResolver.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.prelude.cluster; - -import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; -import com.yahoo.search.Query; -import com.yahoo.search.searchchain.Execution; - -import java.util.Collection; -import java.util.LinkedHashSet; -import java.util.Set; - -/** - * Resolves schemas from query and execution context - * - * @author bjorncs - */ -class SchemaResolver { - - private final Set<String> schemas; - - SchemaResolver(DocumentdbInfoConfig cfg) { - this(cfg.documentdb().stream().map(DocumentdbInfoConfig.Documentdb::name).toList()); - } - - SchemaResolver(Collection<String> schemas) { - this.schemas = new LinkedHashSet<>(schemas); - } - - Set<String> resolve(Query query, Execution execution) { - return resolve(query, execution.context().getIndexFacts()); - } - - Set<String> resolve(Query query, IndexFacts indexFacts) { - if (schemas.size() == 1) return Set.of(schemas.iterator().next()); - var restrict = query.getModel().getRestrict(); - if (restrict == null || restrict.isEmpty()) { - Set<String> sources = query.getModel().getSources(); - return (sources == null || sources.isEmpty()) - ? schemas - : new LinkedHashSet<>(indexFacts.newSession(sources, Set.of(), schemas).documentTypes()); - } else { - return filterValidDocumentTypes(restrict); - } - } - - private Set<String> filterValidDocumentTypes(Collection<String> restrict) { - Set<String> retval = new LinkedHashSet<>(); - for (String docType : restrict) { - if (docType != null && schemas.contains(docType)) { - retval.add(docType); - } - } - return retval; - } - -} diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/CustomParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/CustomParser.java index 2bd408220cd..e3b2278475b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/CustomParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/CustomParser.java @@ -6,6 +6,7 @@ import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.query.Item; import com.yahoo.search.query.parser.Parser; +import java.util.Collections; import java.util.Set; /** @@ -22,7 +23,7 @@ public interface CustomParser extends Parser { Set<String> toSearch, IndexFacts indexFacts, String defaultIndexName) { if (indexFacts == null) indexFacts = new IndexFacts(); - return parse(queryToParse, filterToParse, parsingLanguage, indexFacts.newSession(toSearch, Set.of()), defaultIndexName); + return parse(queryToParse, filterToParse, parsingLanguage, indexFacts.newSession(toSearch, Collections.emptySet()), defaultIndexName); } Item parse(String queryToParse, String filterToParse, Language parsingLanguage, diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java index 9952ec64d13..c1d415b8e27 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java @@ -8,6 +8,7 @@ import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.query.Substring; +import java.util.Collections; import java.util.List; import static com.yahoo.prelude.query.parser.Token.Kind.*; @@ -62,7 +63,7 @@ public final class Tokenizer { * @return a read-only list of tokens. This list can only be used by this thread */ public List<Token> tokenize(String string) { - return tokenize(string, new IndexFacts().newSession()); + return tokenize(string, new IndexFacts().newSession(Collections.emptySet(), Collections.emptySet())); } /** @@ -170,10 +171,13 @@ public final class Tokenizer { // this is a heuristic to check whether we probably have reached the end of an URL element for (int i = tokens.size() - 1; i >= 0; --i) { switch (tokens.get(i).kind) { - case COLON -> { if (i == indexLastExplicitlyChangedAt) return false; } - case SPACE -> { return true; } - default -> { } - // do nothing + case COLON: + if (i == indexLastExplicitlyChangedAt) return false; + break; + case SPACE: + return true; + default: + // do nothing } } // really not sure whether we should choose false instead, on cause of the guard at diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/NonPhrasingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/NonPhrasingSearcher.java index 6bae5175d84..1eb7eb8abe7 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/NonPhrasingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/NonPhrasingSearcher.java @@ -22,7 +22,7 @@ import java.util.List; @Before("transformedQuery") public class NonPhrasingSearcher extends Searcher { - private static final CompoundName suggestonly=new CompoundName("suggestonly"); + private static final CompoundName suggestonly = CompoundName.from("suggestonly"); private PhraseMatcher phraseMatcher; diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/PhrasingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/PhrasingSearcher.java index b67ddade7ee..4da61f5eeca 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/PhrasingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/PhrasingSearcher.java @@ -29,7 +29,7 @@ import java.util.List; @Provides(PhrasingSearcher.PHRASE_REPLACEMENT) public class PhrasingSearcher extends Searcher { - private static final CompoundName suggestonly = new CompoundName("suggestonly"); + private static final CompoundName suggestonly = CompoundName.from("suggestonly"); public static final String PHRASE_REPLACEMENT = "PhraseReplacement"; diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java index 2c7d78c0671..9f8facd2b28 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/RecallSearcher.java @@ -39,7 +39,7 @@ import static com.yahoo.prelude.querytransform.StemmingSearcher.STEMMING; @Before({STEMMING, ACCENT_REMOVAL}) public class RecallSearcher extends Searcher { - public static final CompoundName recallName=new CompoundName("recall"); + public static final CompoundName recallName = CompoundName.from("recall"); @Override public Result search(Query query, Execution execution) { diff --git a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java index 6aa262ca49e..7c4bcb38c41 100644 --- a/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java @@ -9,11 +9,33 @@ import com.yahoo.language.Language; import com.yahoo.language.Linguistics; import com.yahoo.language.process.StemMode; import com.yahoo.language.process.StemList; + +import java.util.ArrayList; +import java.util.IdentityHashMap; +import java.util.Iterator; +import java.util.List; +import java.util.ListIterator; +import java.util.Map; +import java.util.Set; import java.util.logging.Level; import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.query.*; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.AndSegmentItem; +import com.yahoo.prelude.query.BlockItem; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.Highlight; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.PhraseItem; +import com.yahoo.prelude.query.PhraseSegmentItem; +import com.yahoo.prelude.query.PrefixItem; +import com.yahoo.prelude.query.SegmentingRule; +import com.yahoo.prelude.query.Substring; +import com.yahoo.prelude.query.TaggableItem; +import com.yahoo.prelude.query.TermItem; +import com.yahoo.prelude.query.WordAlternativesItem; import com.yahoo.prelude.query.WordAlternativesItem.Alternative; +import com.yahoo.prelude.query.WordItem; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -21,7 +43,6 @@ import com.yahoo.search.Searcher; import com.yahoo.search.searchchain.Execution; import com.yahoo.search.searchchain.PhaseNames; -import java.util.*; import static com.yahoo.prelude.querytransform.CJKSearcher.TERM_ORDER_RELAXATION; @@ -46,7 +67,7 @@ public class StemmingSearcher extends Searcher { } public static final String STEMMING = "Stemming"; - public static final CompoundName DISABLE = new CompoundName("nostemming"); + public static final CompoundName DISABLE = CompoundName.from("nostemming"); private final Linguistics linguistics; public StemmingSearcher(Linguistics linguistics) { @@ -107,15 +128,13 @@ public class StemmingSearcher extends Searcher { } private Map<Item, TaggableItem> populateReverseConnectivityMap(Item root, Map<Item, TaggableItem> reverseConnectivity) { - if (root instanceof TaggableItem) { - TaggableItem asTaggable = (TaggableItem) root; + if (root instanceof TaggableItem asTaggable) { Item connectsTo = asTaggable.getConnectedItem(); if (connectsTo != null) { reverseConnectivity.put(connectsTo, asTaggable); } } - if (root instanceof CompositeItem && !(root instanceof BlockItem)) { - CompositeItem c = (CompositeItem) root; + if (root instanceof CompositeItem c && !(root instanceof BlockItem)) { for (Iterator<Item> i = c.getItemIterator(); i.hasNext();) { Item item = i.next(); populateReverseConnectivityMap(item, reverseConnectivity); @@ -134,8 +153,7 @@ public class StemmingSearcher extends Searcher { } if (item instanceof BlockItem) { item = checkBlock((BlockItem) item, context); - } else if (item instanceof CompositeItem) { - CompositeItem comp = (CompositeItem) item; + } else if (item instanceof CompositeItem comp) { ListIterator<Item> i = comp.getItemIterator(); while (i.hasNext()) { @@ -220,8 +238,7 @@ public class StemmingSearcher extends Searcher { copyAttributes(blockAsItem, composite); composite.lock(); - if (composite instanceof PhraseSegmentItem) { - PhraseSegmentItem replacement = (PhraseSegmentItem) composite; + if (composite instanceof PhraseSegmentItem replacement) { setSignificance(replacement, current); phraseSegmentConnectivity(current, context.reverseConnectivity, replacement); } @@ -258,10 +275,9 @@ public class StemmingSearcher extends Searcher { } private Connectivity getConnectivity(BlockItem current) { - if (!(current instanceof TaggableItem)) { + if (!(current instanceof TaggableItem t)) { return null; } - TaggableItem t = (TaggableItem) current; if (t.getConnectedItem() == null) { return null; } @@ -294,7 +310,7 @@ public class StemmingSearcher extends Searcher { Substring substring, boolean insidePhrase) { String indexName = current.getIndexName(); - if (insidePhrase == false && ((index.getLiteralBoost() || index.getStemMode() == StemMode.ALL))) { + if (!insidePhrase && ((index.getLiteralBoost() || index.getStemMode() == StemMode.ALL))) { List<Alternative> terms = new ArrayList<>(segment.size() + 1); terms.add(new Alternative(current.stringValue(), 1.0d)); for (String term : segment) { @@ -305,8 +321,7 @@ public class StemmingSearcher extends Searcher { return alternatives; } } - WordItem first = singleStemSegment((Item) current, segment.get(0), indexName, substring); - return first; + return singleStemSegment((Item) current, segment.get(0), indexName, substring); } private void setMetaData(BlockItem current, Map<Item, TaggableItem> reverseConnectivity, TaggableItem replacement) { @@ -353,14 +368,13 @@ public class StemmingSearcher extends Searcher { if (current.getSegmentingRule() == SegmentingRule.LANGUAGE_DEFAULT) return chooseComposite(current, parent, indexName); - switch (current.getSegmentingRule()) { // TODO: Why for CJK only? The segmentingRule says nothing about being for CJK only - case PHRASE: return createPhraseSegment(current, indexName); - case BOOLEAN_AND: return createAndSegment(current); - default: - throw new IllegalArgumentException("Unknown segmenting rule: " + current.getSegmentingRule() + - ". This is a bug in Vespa, as the implementation has gotten out of sync." + - " Please create an issue."); - } + return switch (current.getSegmentingRule()) { // TODO: Why for CJK only? The segmentingRule says nothing about being for CJK only + case PHRASE -> createPhraseSegment(current, indexName); + case BOOLEAN_AND -> createAndSegment(current); + default -> throw new IllegalArgumentException("Unknown segmenting rule: " + current.getSegmentingRule() + + ". This is a bug in Vespa, as the implementation has gotten out of sync." + + " Please create an issue."); + }; } private AndSegmentItem createAndSegment(BlockItem current) { diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java index ead6ad53715..9927880c476 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/FieldCollapsingSearcher.java @@ -25,10 +25,10 @@ import java.util.Map; @Before(PhaseNames.TRANSFORMED_QUERY) public class FieldCollapsingSearcher extends Searcher { - private static final CompoundName collapse = new CompoundName("collapse"); - private static final CompoundName collapsefield = new CompoundName("collapsefield"); - private static final CompoundName collapsesize = new CompoundName("collapsesize"); - private static final CompoundName collapseSummaryName = new CompoundName("collapse.summary"); + private static final CompoundName collapse = CompoundName.from("collapse"); + private static final CompoundName collapsefield = CompoundName.from("collapsefield"); + private static final CompoundName collapsesize = CompoundName.from("collapsesize"); + private static final CompoundName collapseSummaryName = CompoundName.from("collapse.summary"); /** Maximum number of queries to send next searcher */ private static final int maxQueries = 4; diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/JSONDebugSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/JSONDebugSearcher.java index 409b502f086..7750bd17108 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/JSONDebugSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/JSONDebugSearcher.java @@ -25,7 +25,7 @@ public class JSONDebugSearcher extends Searcher { public static final String STRUCT_FIELD = "Structured data field (as json): "; public static final String FEATURE_FIELD = "Feature data field (as json): "; - private static final CompoundName PROPERTYNAME = new CompoundName("dumpjson"); + private static final CompoundName PROPERTYNAME = CompoundName.from("dumpjson"); @Override public Result search(Query query, Execution execution) { diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/MultipleResultsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/MultipleResultsSearcher.java index 3c61a361cbb..dd2c4a1da7f 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/MultipleResultsSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/MultipleResultsSearcher.java @@ -10,7 +10,12 @@ import com.yahoo.search.result.Hit; import com.yahoo.search.result.HitGroup; import com.yahoo.search.searchchain.Execution; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; /** * Groups hits according to document type. @@ -21,9 +26,9 @@ import java.util.*; public class MultipleResultsSearcher extends Searcher { private final static String propertyPrefix = "multipleresultsets."; - private static final CompoundName additionalHitsFactorName=new CompoundName(propertyPrefix + "additionalHitsFactor"); - private static final CompoundName maxTimesRetrieveHeterogeneousHitsName=new CompoundName(propertyPrefix + "maxTimesRetrieveHeterogeneousHits"); - private static final CompoundName numHits=new CompoundName(propertyPrefix + "numHits"); + private static final CompoundName additionalHitsFactorName = CompoundName.from(propertyPrefix + "additionalHitsFactor"); + private static final CompoundName maxTimesRetrieveHeterogeneousHitsName = CompoundName.from(propertyPrefix + "maxTimesRetrieveHeterogeneousHits"); + private static final CompoundName numHits = CompoundName.from(propertyPrefix + "numHits"); @Override public Result search(Query query, Execution e) { @@ -162,9 +167,9 @@ public class MultipleResultsSearcher extends Searcher { // Assumes that field sddocname is available private static class PartitionedResult { - private Map<String, HitGroup> resultSets = new HashMap<>(); + private final Map<String, HitGroup> resultSets = new HashMap<>(); - private List<Hit> otherHits = new ArrayList<>(); + private final List<Hit> otherHits = new ArrayList<>(); PartitionedResult(List<DocumentGroup> documentGroups,Result result) throws ParameterException { for (DocumentGroup group : documentGroups) @@ -174,9 +179,8 @@ public class MultipleResultsSearcher extends Searcher { } void addHits(Result result, boolean addOtherHits) { - Iterator<Hit> i = result.hits().iterator(); - while (i.hasNext()) { - add(i.next(), addOtherHits); + for (Hit hit : result.hits()) { + add(hit, addOtherHits); } } diff --git a/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java b/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java index 0df21df8842..fd4c9e89f3b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/searcher/PosSearcher.java @@ -37,12 +37,11 @@ public class PosSearcher extends Searcher { public static final String POSITION_PARSING = "PositionParsing"; - private static final CompoundName posBb = new CompoundName("pos.bb"); - private static final CompoundName posLl = new CompoundName("pos.ll"); - private static final CompoundName posXy = new CompoundName("pos.xy"); - private static final CompoundName posAttributeName = new CompoundName("pos.attribute"); - private static final CompoundName posRadius = new CompoundName("pos.radius"); - private static final CompoundName posUnits = new CompoundName("pos.units"); + private static final CompoundName posBb = CompoundName.from("pos.bb"); + private static final CompoundName posLl = CompoundName.from("pos.ll"); + private static final CompoundName posXy = CompoundName.from("pos.xy"); + private static final CompoundName posAttributeName = CompoundName.from("pos.attribute"); + private static final CompoundName posRadius = CompoundName.from("pos.radius"); // according to wikipedia: // Earth's equatorial radius = 6378137 meter - not used @@ -117,13 +116,13 @@ public class PosSearcher extends Searcher { if (radius == null) { radiusdegrees = 50.0 * km2deg; } else if (radius.endsWith("km")) { - double radiuskm = Double.valueOf(radius.substring(0, radius.length()-2)); + double radiuskm = Double.parseDouble(radius.substring(0, radius.length()-2)); radiusdegrees = radiuskm * km2deg; } else if (radius.endsWith("m")) { - double radiusm = Double.valueOf(radius.substring(0, radius.length()-1)); + double radiusm = Double.parseDouble(radius.substring(0, radius.length()-1)); radiusdegrees = radiusm * km2deg / 1000.0; } else if (radius.endsWith("mi")) { - double radiusmiles = Double.valueOf(radius.substring(0, radius.length()-2)); + double radiusmiles = Double.parseDouble(radius.substring(0, radius.length()-2)); radiusdegrees = radiusmiles * mi2deg; } else { radiusdegrees = Integer.parseInt(radius) * 0.000001; @@ -151,15 +150,15 @@ public class PosSearcher extends Searcher { double radiusdegrees = radiuskm * km2deg; radiusUnits = (int)(radiusdegrees * 1000000); } else if (radius.endsWith("km")) { - double radiuskm = Double.valueOf(radius.substring(0, radius.length()-2)); + double radiuskm = Double.parseDouble(radius.substring(0, radius.length()-2)); double radiusdegrees = radiuskm * km2deg; radiusUnits = (int)(radiusdegrees * 1000000); } else if (radius.endsWith("m")) { - double radiusm = Double.valueOf(radius.substring(0, radius.length()-1)); + double radiusm = Double.parseDouble(radius.substring(0, radius.length()-1)); double radiusdegrees = radiusm * km2deg / 1000.0; radiusUnits = (int)(radiusdegrees * 1000000); } else if (radius.endsWith("mi")) { - double radiusmiles = Double.valueOf(radius.substring(0, radius.length()-2)); + double radiusmiles = Double.parseDouble(radius.substring(0, radius.length()-2)); double radiusdegrees = radiusmiles * mi2deg; radiusUnits = (int)(radiusdegrees * 1000000); } else { diff --git a/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java b/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java index 42a2b4f4e9b..5b3806485b5 100644 --- a/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/semantics/SemanticSearcher.java @@ -29,9 +29,9 @@ import static com.yahoo.prelude.querytransform.StemmingSearcher.STEMMING; @Before({PhaseNames.TRANSFORMED_QUERY, STEMMING}) public class SemanticSearcher extends Searcher { - private static final CompoundName rulesRulebase = new CompoundName("rules.rulebase"); - private static final CompoundName rulesOff = new CompoundName("rules.off"); - private static final CompoundName tracelevelRules = new CompoundName("tracelevel.rules"); + private static final CompoundName rulesRulebase = CompoundName.from("rules.rulebase"); + private static final CompoundName rulesOff = CompoundName.from("rules.off"); + private static final CompoundName tracelevelRules = CompoundName.from("tracelevel.rules"); /** The default rule base of this */ private RuleBase defaultRuleBase; diff --git a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java index c13bdff1f50..de24196327e 100644 --- a/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/statistics/StatisticsSearcher.java @@ -56,7 +56,7 @@ import static com.yahoo.container.protect.Error.UNSPECIFIED; @Before(PhaseNames.RAW_QUERY) public class StatisticsSearcher extends Searcher { - private static final CompoundName IGNORE_QUERY = new CompoundName("metrics.ignore"); + private static final CompoundName IGNORE_QUERY = CompoundName.from("metrics.ignore"); private static final String MAX_QUERY_LATENCY_METRIC = ContainerMetrics.MAX_QUERY_LATENCY.baseName(); private static final String EMPTY_RESULTS_METRIC = ContainerMetrics.EMPTY_RESULTS.baseName(); private static final String HITS_PER_QUERY_METRIC = ContainerMetrics.HITS_PER_QUERY.baseName(); diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java index 39fd372d2a7..73ed89687fe 100644 --- a/container-search/src/main/java/com/yahoo/search/Query.java +++ b/container-search/src/main/java/com/yahoo/search/Query.java @@ -48,7 +48,6 @@ import com.yahoo.yolean.Exceptions; import java.net.URI; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -180,23 +179,23 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { //---------------- Static property handling ------------------------------------ - public static final CompoundName OFFSET = new CompoundName("offset"); - public static final CompoundName HITS = new CompoundName("hits"); + public static final CompoundName OFFSET = CompoundName.from("offset"); + public static final CompoundName HITS = CompoundName.from("hits"); - public static final CompoundName QUERY_PROFILE = new CompoundName("queryProfile"); - public static final CompoundName SEARCH_CHAIN = new CompoundName("searchChain"); + public static final CompoundName QUERY_PROFILE = CompoundName.from("queryProfile"); + public static final CompoundName SEARCH_CHAIN = CompoundName.from("searchChain"); - public static final CompoundName NO_CACHE = new CompoundName("noCache"); - public static final CompoundName GROUPING_SESSION_CACHE = new CompoundName("groupingSessionCache"); - public static final CompoundName TIMEOUT = new CompoundName("timeout"); + public static final CompoundName NO_CACHE = CompoundName.from("noCache"); + public static final CompoundName GROUPING_SESSION_CACHE = CompoundName.from("groupingSessionCache"); + public static final CompoundName TIMEOUT = CompoundName.from("timeout"); /** @deprecated use Trace.LEVEL */ @Deprecated // TODO: Remove on Vespa 9 - public static final CompoundName TRACE_LEVEL = new CompoundName("traceLevel"); + public static final CompoundName TRACE_LEVEL = CompoundName.from("traceLevel"); /** @deprecated use Trace.EXPLAIN_LEVEL */ @Deprecated // TODO: Remove on Vespa 9 - public static final CompoundName EXPLAIN_LEVEL = new CompoundName("explainLevel"); + public static final CompoundName EXPLAIN_LEVEL = CompoundName.from("explainLevel"); private static final QueryProfileType argumentType; static { @@ -228,17 +227,17 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { private static final Map<String, CompoundName> propertyAliases; static { Map<String, CompoundName> propertyAliasesBuilder = new HashMap<>(); - addAliases(Query.getArgumentType(), CompoundName.empty, propertyAliasesBuilder); + addAliases(Query.getArgumentType(), "", propertyAliasesBuilder); propertyAliases = ImmutableMap.copyOf(propertyAliasesBuilder); } - private static void addAliases(QueryProfileType arguments, CompoundName prefix, Map<String, CompoundName> aliases) { + private static void addAliases(QueryProfileType arguments, String prefix, Map<String, CompoundName> aliases) { for (FieldDescription field : arguments.fields().values()) { for (String alias : field.getAliases()) - aliases.put(alias, prefix.append(field.getName())); + aliases.put(alias, CompoundName.from(append(prefix, field.getName()))); if (field.getType() instanceof QueryProfileFieldType) { var type = ((QueryProfileFieldType) field.getType()).getQueryProfileType(); if (type != null) - addAliases(type, prefix.append(type.getComponentIdAsCompoundName()), aliases); + addAliases(type, append(prefix, type.getComponentIdAsCompoundName().toString()), aliases); } } } @@ -261,18 +260,18 @@ public class Query extends com.yahoo.processing.Request implements Cloneable { /** Returns an unmodifiable list of all the native properties under a Query */ public static final List<CompoundName> nativeProperties = - List.copyOf(namesUnder(CompoundName.empty, Query.getArgumentType())); + List.copyOf(namesUnder("", Query.getArgumentType())); - private static List<CompoundName> namesUnder(CompoundName prefix, QueryProfileType type) { - if (type == null) return Collections.emptyList(); // Names not known statically + private static List<CompoundName> namesUnder(String prefix, QueryProfileType type) { + if (type == null) return List.of(); // Names not known statically List<CompoundName> names = new ArrayList<>(); for (Map.Entry<String, FieldDescription> field : type.fields().entrySet()) { + String name = append(prefix, field.getKey()); if (field.getValue().getType() instanceof QueryProfileFieldType) { - names.addAll(namesUnder(prefix.append(field.getKey()), - ((QueryProfileFieldType) field.getValue().getType()).getQueryProfileType())); + names.addAll(namesUnder(name, ((QueryProfileFieldType) field.getValue().getType()).getQueryProfileType())); } else { - names.add(prefix.append(field.getKey())); + names.add(CompoundName.from(name)); } } return names; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 09db1be4732..4e4b77422c1 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -52,7 +52,7 @@ public class Dispatcher extends AbstractComponent { private static final int MAX_GROUP_SELECTION_ATTEMPTS = 3; /** If set will control computation of how many hits will be fetched from each partition.*/ - public static final CompoundName topKProbability = CompoundName.fromComponents(DISPATCH, TOP_K_PROBABILITY); + public static final CompoundName topKProbability = CompoundName.from(DISPATCH + "." + TOP_K_PROBABILITY); private final DispatchConfig dispatchConfig; private final RpcResourcePool rpcResourcePool; diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/CompressService.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/CompressService.java index 9e7fc9b5b29..c10375a3014 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/CompressService.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/CompressService.java @@ -13,7 +13,7 @@ import com.yahoo.search.Query; */ public class CompressService implements CompressPayload { /** The compression method which will be used with rpc dispatch. "lz4" (default) and "none" is supported. */ - public final static CompoundName dispatchCompression = new CompoundName("dispatch.compression"); + public static final CompoundName dispatchCompression = CompoundName.from("dispatch.compression"); private final Compressor compressor = new Compressor(CompressionType.LZ4, 5, 0.95, 256); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java index 4466b03a713..154002c4f77 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcInvokerFactory.java @@ -20,11 +20,20 @@ public class RpcInvokerFactory extends InvokerFactory { private final RpcConnectionPool rpcResourcePool; private final CompressPayload compressor; + private final RpcProtobufFillInvoker.DecodePolicy decodeType; + + private static RpcProtobufFillInvoker.DecodePolicy convert(DispatchConfig.SummaryDecodePolicy.Enum decoding) { + return switch (decoding) { + case EAGER -> RpcProtobufFillInvoker.DecodePolicy.EAGER; + case ONDEMAND -> RpcProtobufFillInvoker.DecodePolicy.ONDEMAND; + }; + } public RpcInvokerFactory(RpcConnectionPool rpcResourcePool, SearchGroups cluster, DispatchConfig dispatchConfig) { super(cluster, dispatchConfig); this.rpcResourcePool = rpcResourcePool; this.compressor = new CompressService(); + decodeType = convert(dispatchConfig.summaryDecodePolicy()); } @Override @@ -37,6 +46,7 @@ public class RpcInvokerFactory extends InvokerFactory { Query query = result.getQuery(); boolean summaryNeedsQuery = searcher.summaryNeedsQuery(query); - return new RpcProtobufFillInvoker(rpcResourcePool, compressor, searcher.getDocumentDatabase(query), searcher.getServerId(), summaryNeedsQuery); + return new RpcProtobufFillInvoker(rpcResourcePool, compressor, searcher.getDocumentDatabase(query), + searcher.getServerId(), decodeType, summaryNeedsQuery); } } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java index 4e538fb54dc..2bdafecfaba 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcProtobufFillInvoker.java @@ -21,6 +21,7 @@ import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.result.Hit; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.BinaryFormat; +import com.yahoo.slime.BinaryView; import java.util.Iterator; import java.util.List; @@ -42,11 +43,14 @@ public class RpcProtobufFillInvoker extends FillInvoker { private static final Logger log = Logger.getLogger(RpcProtobufFillInvoker.class.getName()); + enum DecodePolicy {EAGER, ONDEMAND} + private final DocumentDatabase documentDb; private final RpcConnectionPool resourcePool; private final boolean summaryNeedsQuery; private final String serverId; private final CompressPayload compressor; + private final DecodePolicy decodePolicy; private BlockingQueue<Pair<Client.ResponseOrError<ProtobufResponse>, List<FastHit>>> responses; @@ -56,12 +60,14 @@ public class RpcProtobufFillInvoker extends FillInvoker { /** The number of responses we should receive (and process) before this is complete */ private int outstandingResponses; - RpcProtobufFillInvoker(RpcConnectionPool resourcePool, CompressPayload compressor, DocumentDatabase documentDb, String serverId, boolean summaryNeedsQuery) { + RpcProtobufFillInvoker(RpcConnectionPool resourcePool, CompressPayload compressor, DocumentDatabase documentDb, + String serverId, DecodePolicy decodePolicy, boolean summaryNeedsQuery) { this.documentDb = documentDb; this.resourcePool = resourcePool; this.serverId = serverId; this.summaryNeedsQuery = summaryNeedsQuery; this.compressor = compressor; + this.decodePolicy = decodePolicy; } @Override @@ -211,7 +217,9 @@ public class RpcProtobufFillInvoker extends FillInvoker { private int fill(Result result, List<FastHit> hits, String summaryClass, byte[] payload) { try { var protobuf = SearchProtocol.DocsumReply.parseFrom(payload); - var root = BinaryFormat.decode(protobuf.getSlimeSummaries().toByteArray()).get(); + var root = (decodePolicy == DecodePolicy.ONDEMAND) + ? BinaryView.inspect(protobuf.getSlimeSummaries().toByteArray()) + : BinaryFormat.decode(protobuf.getSlimeSummaries().toByteArray()).get(); var errors = root.field("errors"); boolean hasErrors = errors.valid() && (errors.entries() > 0); if (hasErrors) { diff --git a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java index 80a41ffdf22..b8be1dc210b 100644 --- a/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/federation/FederationSearcher.java @@ -70,8 +70,8 @@ public class FederationSearcher extends ForkingSearcher { private static final Logger log = Logger.getLogger(FederationSearcher.class.getName()); /** The name of the query property containing the source name added to the query to each source by this */ - public final static CompoundName SOURCENAME = new CompoundName("sourceName"); - public final static CompoundName PROVIDERNAME = new CompoundName("providerName"); + public static final CompoundName SOURCENAME = CompoundName.from("sourceName"); + public static final CompoundName PROVIDERNAME = CompoundName.from("providerName"); public static final String FEDERATION = "Federation"; public static final String LOG_COUNT_PREFIX = "count_"; @@ -686,33 +686,6 @@ public class FederationSearcher extends ForkingSearcher { } - private static class CompoundKey { - - private final String sourceName; - private final String propertyName; - - CompoundKey(String sourceName, String propertyName) { - this.sourceName = sourceName; - this.propertyName = propertyName; - } - - @Override - public int hashCode() { - return sourceName.hashCode() ^ propertyName.hashCode(); - } - - @Override - public boolean equals(Object o) { - CompoundKey rhs = (CompoundKey) o; - return sourceName.equals(rhs.sourceName) && propertyName.equals(rhs.propertyName); - } - - @Override - public String toString() { - return sourceName + '.' + propertyName; - } - } - private static class Window { private final int hits; diff --git a/container-search/src/main/java/com/yahoo/search/grouping/GroupingQueryParser.java b/container-search/src/main/java/com/yahoo/search/grouping/GroupingQueryParser.java index 54194221958..180687d5274 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/GroupingQueryParser.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/GroupingQueryParser.java @@ -38,13 +38,13 @@ import java.util.TimeZone; public class GroupingQueryParser extends Searcher { public static final String SELECT_PARAMETER_PARSING = "SelectParameterParsing"; - public static final CompoundName PARAM_CONTINUE = new CompoundName("continue"); - public static final CompoundName PARAM_REQUEST = new CompoundName(Select.SELECT); - public static final CompoundName PARAM_TIMEZONE = new CompoundName("timezone"); - @Beta public static final CompoundName PARAM_DEFAULT_MAX_HITS = new CompoundName("grouping.defaultMaxHits"); - @Beta public static final CompoundName PARAM_DEFAULT_MAX_GROUPS = new CompoundName("grouping.defaultMaxGroups"); - @Beta public static final CompoundName PARAM_DEFAULT_PRECISION_FACTOR = new CompoundName("grouping.defaultPrecisionFactor"); - @Beta public static final CompoundName GROUPING_GLOBAL_MAX_GROUPS = new CompoundName("grouping.globalMaxGroups"); + public static final CompoundName PARAM_CONTINUE = CompoundName.from("continue"); + public static final CompoundName PARAM_REQUEST = CompoundName.from(Select.SELECT); + public static final CompoundName PARAM_TIMEZONE = CompoundName.from("timezone"); + @Beta public static final CompoundName PARAM_DEFAULT_MAX_HITS = CompoundName.from("grouping.defaultMaxHits"); + @Beta public static final CompoundName PARAM_DEFAULT_MAX_GROUPS = CompoundName.from("grouping.defaultMaxGroups"); + @Beta public static final CompoundName PARAM_DEFAULT_PRECISION_FACTOR = CompoundName.from("grouping.defaultPrecisionFactor"); + @Beta public static final CompoundName GROUPING_GLOBAL_MAX_GROUPS = CompoundName.from("grouping.globalMaxGroups"); private static final ThreadLocal<ZoneCache> zoneCache = new ThreadLocal<>(); @Override @@ -121,7 +121,6 @@ public class GroupingQueryParser extends Searcher { return val != null ? OptionalDouble.of(val) : OptionalDouble.empty(); } - @SuppressWarnings("serial") private static class ZoneCache extends LinkedHashMap<String, TimeZone> { ZoneCache() { diff --git a/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java b/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java index 35cf72d6124..878fbbb733d 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/GroupingValidator.java @@ -37,7 +37,7 @@ import static com.yahoo.search.grouping.GroupingQueryParser.SELECT_PARAMETER_PAR public class GroupingValidator extends Searcher { public static final String GROUPING_VALIDATED = "GroupingValidated"; - public static final CompoundName PARAM_ENABLED = new CompoundName("validate_" + GroupingQueryParser.PARAM_REQUEST); + public static final CompoundName PARAM_ENABLED = CompoundName.from("validate_" + GroupingQueryParser.PARAM_REQUEST); private final HashMap<String, AttributesConfig.Attribute> attributes = new HashMap<>(); private final String clusterName; private final boolean enabled; diff --git a/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java b/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java index a1446367f79..27d8a76511d 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/UniqueGroupingSearcher.java @@ -42,7 +42,7 @@ import java.util.logging.Logger; @Before(PhaseNames.TRANSFORMED_QUERY) public class UniqueGroupingSearcher extends Searcher { - public static final CompoundName PARAM_UNIQUE = new CompoundName("unique"); + public static final CompoundName PARAM_UNIQUE = CompoundName.from("unique"); private static final Logger log = Logger.getLogger(UniqueGroupingSearcher.class.getName()); private static final HitOrderer NOP_ORDERER = new HitOrderer() { @@ -144,17 +144,13 @@ public class UniqueGroupingSearcher extends Searcher { for (Sorting.FieldOrder fieldOrder : sortingSpec.fieldOrders()) { Sorting.Order sortOrder = fieldOrder.getSortOrder(); switch (sortOrder) { - case ASCENDING: - case UNDEFINED: - // When we want ascending order, the hit with the smallest value should come first (and be surfaced). - orderingClause.add(new MinAggregator(new AttributeValue(fieldOrder.getFieldName()))); - break; - case DESCENDING: - // When we sort in descending order, the hit with the largest value should come first (and be surfaced). - orderingClause.add(new NegFunction(new MaxAggregator(new AttributeValue(fieldOrder.getFieldName())))); - break; - default: - throw new UnsupportedOperationException("Can not handle sort order " + sortOrder + "."); + case ASCENDING, UNDEFINED -> + // When we want ascending order, the hit with the smallest value should come first (and be surfaced). + orderingClause.add(new MinAggregator(new AttributeValue(fieldOrder.getFieldName()))); + case DESCENDING -> + // When we sort in descending order, the hit with the largest value should come first (and be surfaced). + orderingClause.add(new NegFunction(new MaxAggregator(new AttributeValue(fieldOrder.getFieldName())))); + default -> throw new UnsupportedOperationException("Can not handle sort order " + sortOrder + "."); } } return orderingClause; @@ -170,18 +166,13 @@ public class UniqueGroupingSearcher extends Searcher { GroupingExpression groupingClause = null; for (Sorting.FieldOrder fieldOrder : sortingSpec.fieldOrders()) { Sorting.Order sortOrder = fieldOrder.getSortOrder(); - switch (sortOrder) { - case ASCENDING: - case UNDEFINED: - groupingClause = new AttributeValue(fieldOrder.getFieldName()); - break; - case DESCENDING: - // To sort descending, just take the negative. This is the most common case - groupingClause = new NegFunction(new AttributeValue(fieldOrder.getFieldName())); - break; - default: - throw new UnsupportedOperationException("Can not handle sort order " + sortOrder + "."); - } + groupingClause = switch (sortOrder) { + case ASCENDING, UNDEFINED -> new AttributeValue(fieldOrder.getFieldName()); + case DESCENDING -> + // To sort descending, just take the negative. This is the most common case + new NegFunction(new AttributeValue(fieldOrder.getFieldName())); + default -> throw new UnsupportedOperationException("Can not handle sort order " + sortOrder + "."); + }; } return groupingClause; } diff --git a/container-search/src/main/java/com/yahoo/search/grouping/result/FlatteningSearcher.java b/container-search/src/main/java/com/yahoo/search/grouping/result/FlatteningSearcher.java index e56b88c3d3e..027ba92f587 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/result/FlatteningSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/result/FlatteningSearcher.java @@ -22,7 +22,7 @@ import java.util.Iterator; @Before(GroupingExecutor.COMPONENT_NAME) public class FlatteningSearcher extends Searcher { - private final CompoundName flatten = CompoundName.fromComponents("grouping", "flatten"); + private final CompoundName flatten = CompoundName.from("grouping.flatten"); @Override public Result search(Query query, Execution execution) { diff --git a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java index 2ba33f60ea1..c85dbed83c8 100644 --- a/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java +++ b/container-search/src/main/java/com/yahoo/search/grouping/vespa/GroupingExecutor.java @@ -379,7 +379,7 @@ public class GroupingExecutor extends Searcher { } private static CompoundName newCompoundName(String name) { - return new CompoundName(GroupingExecutor.class.getName() + "." + name); + return CompoundName.from(GroupingExecutor.class.getName() + "." + name); } private static class RequestContext { diff --git a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java index fac6c598e4b..3b4d15f788f 100644 --- a/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java +++ b/container-search/src/main/java/com/yahoo/search/handler/SearchHandler.java @@ -75,8 +75,8 @@ public class SearchHandler extends LoggingRequestHandler { // max number of threads for the executor for this handler private final int maxThreads; - private static final CompoundName DETAILED_TIMING_LOGGING = new CompoundName("trace.timingDetails"); - private static final CompoundName FORCE_TIMESTAMPS = new CompoundName("trace.timestamps"); + private static final CompoundName DETAILED_TIMING_LOGGING = CompoundName.from("trace.timingDetails"); + private static final CompoundName FORCE_TIMESTAMPS = CompoundName.from("trace.timestamps"); /** Event name for number of connections to the search subsystem */ private static final String SEARCH_CONNECTIONS = "search_connections"; diff --git a/container-search/src/main/java/com/yahoo/search/intent/model/IntentModel.java b/container-search/src/main/java/com/yahoo/search/intent/model/IntentModel.java index 9aafd2c4693..0c4b22fbbf2 100644 --- a/container-search/src/main/java/com/yahoo/search/intent/model/IntentModel.java +++ b/container-search/src/main/java/com/yahoo/search/intent/model/IntentModel.java @@ -5,7 +5,12 @@ import com.yahoo.search.Query; import com.yahoo.processing.request.CompoundName; import com.yahoo.text.interpretation.Interpretation; -import java.util.*; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; /** * This is the root node of an intent model. @@ -19,9 +24,9 @@ import java.util.*; public class IntentModel extends ParentNode<InterpretationNode> { /** The name of the property carrying the intent model string: intentModel */ - public static final CompoundName intentModelStringName=new CompoundName("intentModel"); + public static final CompoundName intentModelStringName = CompoundName.from("intentModel"); /** The name of the property carrying the intent model object: IntentModel */ - public static final CompoundName intentModelObjectName=new CompoundName("IntentModel"); + public static final CompoundName intentModelObjectName = CompoundName.from("IntentModel"); private static final InterpretationNodeComparator inodeComp = new InterpretationNodeComparator(); @@ -45,7 +50,7 @@ public class IntentModel extends ParentNode<InterpretationNode> { /** Sort interpretations by descending score order */ public void sortChildren() { - Collections.sort(children(), inodeComp); + children().sort(inodeComp); } /** diff --git a/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateSearcher.java b/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateSearcher.java index c23627accf4..5fef1821de2 100644 --- a/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/pagetemplates/PageTemplateSearcher.java @@ -72,13 +72,13 @@ import java.util.Map; public class PageTemplateSearcher extends Searcher { /** The name of the query property containing the resolved candidate page template list */ - public static final CompoundName pagePageTemplateListName=new CompoundName("page.PageTemplateList"); + public static final CompoundName pagePageTemplateListName = CompoundName.from("page.PageTemplateList"); /** The name of the query property containing a list of candidate pages to consider */ - public static final CompoundName pageIdListName=new CompoundName("page.idList"); + public static final CompoundName pageIdListName = CompoundName.from("page.idList"); /** The name of the query property containing the page id to use */ - public static final CompoundName pageIdName=new CompoundName("page.id"); + public static final CompoundName pageIdName = CompoundName.from("page.id"); /** The name of the query property containing the resolver id to use */ - public static final CompoundName pageResolverName=new CompoundName("page.resolver"); + public static final CompoundName pageResolverName = CompoundName.from("page.resolver"); private final ResolverRegistry resolverRegistry; diff --git a/container-search/src/main/java/com/yahoo/search/query/Model.java b/container-search/src/main/java/com/yahoo/search/query/Model.java index 190ad675015..09b2f394f20 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Model.java +++ b/container-search/src/main/java/com/yahoo/search/query/Model.java @@ -71,13 +71,13 @@ public class Model implements Cloneable { argumentType.addField(new FieldDescription(SEARCH_PATH, "string", "searchpath")); argumentType.addField(new FieldDescription(RESTRICT, "string", "restrict")); argumentType.freeze(); - argumentTypeName = new CompoundName(argumentType.getId().getName()); + argumentTypeName = CompoundName.from(argumentType.getId().getName()); } public static QueryProfileType getArgumentType() { return argumentType; } /** The name of the query property used for generating hit count estimate queries. */ - public static final CompoundName ESTIMATE = new CompoundName("hitcountestimate"); // TODO: Cleanup + public static final CompoundName ESTIMATE = CompoundName.from("hitcountestimate"); // TODO: Cleanup private String encoding = null; private String queryString = ""; diff --git a/container-search/src/main/java/com/yahoo/search/query/Ranking.java b/container-search/src/main/java/com/yahoo/search/query/Ranking.java index e8738a19412..5426268d173 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Ranking.java +++ b/container-search/src/main/java/com/yahoo/search/query/Ranking.java @@ -27,7 +27,7 @@ import java.util.Objects; public class Ranking implements Cloneable { /** An alias for listing features */ - public static final CompoundName RANKFEATURES = new CompoundName("rankfeatures"); + public static final CompoundName RANKFEATURES = CompoundName.from("rankfeatures"); /** The type representing the property arguments consumed by this */ private static final QueryProfileType argumentType; @@ -70,7 +70,7 @@ public class Ranking implements Cloneable { argumentType.addField(new FieldDescription(FEATURES, "query-profile", "rankfeature input")); // Repeated at the end of RankFeatures argumentType.addField(new FieldDescription(PROPERTIES, "query-profile", "rankproperty")); argumentType.freeze(); - argumentTypeName = new CompoundName(argumentType.getId().getName()); + argumentTypeName = CompoundName.from(argumentType.getId().getName()); } public static QueryProfileType getArgumentType() { return argumentType; } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java index ad9d3f4c1a5..ae531c67dd1 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/compiled/CompiledQueryProfile.java @@ -169,7 +169,7 @@ public class CompiledQueryProfile extends AbstractComponent implements Cloneable } public final Object get(String name) { - return get(name, Collections.emptyMap()); + return get(name, Map.of()); } public final Object get(String name, Map<String, String> context) { return get(name, context, new QueryProfileProperties(this)); diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java index 8fdbf8b2281..719a5a2c281 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/FieldDescription.java @@ -64,7 +64,7 @@ public class FieldDescription implements Comparable<FieldDescription> { } public FieldDescription(String name, FieldType type, String aliases, boolean mandatory, boolean overridable) { - this(new CompoundName(name), type, aliases, mandatory, overridable); + this(CompoundName.from(name), type, aliases, mandatory, overridable); } /** diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java index 3da2ad53f9a..9cbfe5b7112 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/types/QueryProfileType.java @@ -52,7 +52,7 @@ public class QueryProfileType extends FreezableSimpleComponent { private QueryProfileType(ComponentId id, Map<String, FieldDescription> fields, List<QueryProfileType> inherited) { super(id); QueryProfile.validateName(id.getName()); - componentIdAsCompoundName = new CompoundName(getId().getName()); + componentIdAsCompoundName = CompoundName.from(getId().getName()); this.fields = fields; this.inherited = inherited; } @@ -318,10 +318,9 @@ public class QueryProfileType extends FreezableSimpleComponent { QueryProfileType type = null; FieldDescription fieldDescription = getField(name); if (fieldDescription != null) { - if ( ! (fieldDescription.getType() instanceof QueryProfileFieldType)) + if ( ! (fieldDescription.getType() instanceof QueryProfileFieldType fieldType)) throw new IllegalArgumentException("Cannot use name '" + name + "' as a prefix because it is " + "already a " + fieldDescription.getType()); - QueryProfileFieldType fieldType = (QueryProfileFieldType) fieldDescription.getType(); type = fieldType.getQueryProfileType(); } @@ -399,8 +398,7 @@ public class QueryProfileType extends FreezableSimpleComponent { @Override public boolean equals(Object o) { if (o == this) return true; - if ( ! (o instanceof QueryProfileType)) return false; - QueryProfileType other = (QueryProfileType)o; + if ( ! (o instanceof QueryProfileType other)) return false; return other.getId().equals(this.getId()); } diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/DefaultProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/DefaultProperties.java index 221368afeb6..6ccf4792efb 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/DefaultProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/DefaultProperties.java @@ -16,9 +16,9 @@ import java.util.Map; */ public final class DefaultProperties extends Properties { - public static final CompoundName MAX_OFFSET = new CompoundName("maxOffset"); - public static final CompoundName MAX_HITS = new CompoundName("maxHits"); - public static final CompoundName MAX_QUERY_ITEMS = new CompoundName("maxQueryItems"); + public static final CompoundName MAX_OFFSET = CompoundName.from("maxOffset"); + public static final CompoundName MAX_HITS = CompoundName.from("maxHits"); + public static final CompoundName MAX_QUERY_ITEMS = CompoundName.from("maxQueryItems"); public static final QueryProfileType argumentType = new QueryProfileType("DefaultProperties"); diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/SubProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/SubProperties.java index 42fb4e44040..4fc9dfa03cd 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/SubProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/SubProperties.java @@ -10,14 +10,16 @@ import java.util.Map; * A wrapper around a chain of property objects that prefixes all gets/sets with a given path * * @author Arne Bergene Fossaa + * @deprecated Unused and will go away on vespa 9 */ +@Deprecated (forRemoval = true) public class SubProperties extends com.yahoo.search.query.Properties { final private CompoundName pathPrefix; final private Properties parent; public SubProperties(String pathPrefix, Properties properties) { - this(new CompoundName(pathPrefix),properties); + this(CompoundName.from(pathPrefix), properties); } public SubProperties(CompoundName pathPrefix, Properties properties) { diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java b/container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java index 30ee2f473b7..c49b14f0978 100644 --- a/container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java +++ b/container-search/src/main/java/com/yahoo/search/query/ranking/SoftTimeout.java @@ -24,8 +24,7 @@ public class SoftTimeout implements Cloneable { public static final String TAILCOST = "tailcost"; /** The full property name for turning softtimeout on or off */ - public static final CompoundName enableProperty = - CompoundName.fromComponents(Ranking.RANKING, Ranking.SOFTTIMEOUT, ENABLE); + public static final CompoundName enableProperty = CompoundName.from(Ranking.RANKING + "." + Ranking.SOFTTIMEOUT + "." + ENABLE); static { argumentType = new QueryProfileType(Ranking.SOFTTIMEOUT); diff --git a/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterConstants.java b/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterConstants.java index b0fce45de46..eeb8b675ae1 100644 --- a/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterConstants.java +++ b/container-search/src/main/java/com/yahoo/search/query/rewrite/RewriterConstants.java @@ -42,7 +42,7 @@ public class RewriterConstants { public static final String REWRITER_CHAIN = "QRWChain"; /** Name for rewrite metadata retrieval from query properties */ - public static final CompoundName REWRITE_META = new CompoundName("RewriteMeta"); + public static final CompoundName REWRITE_META = CompoundName.from("RewriteMeta"); /** Name for rewritten field retrieval from query properties */ public static final String REWRITTEN = "Rewritten"; diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java index f43be20e0ac..91315fe1bb3 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/BooleanSearcher.java @@ -28,9 +28,9 @@ import static com.yahoo.yolean.Exceptions.toMessageString; @Provides(BooleanSearcher.PREDICATE) public class BooleanSearcher extends Searcher { - private static final CompoundName FIELD = new CompoundName("boolean.field"); - private static final CompoundName ATTRIBUTES = new CompoundName("boolean.attributes"); - private static final CompoundName RANGE_ATTRIBUTES = new CompoundName("boolean.rangeAttributes"); + private static final CompoundName FIELD = CompoundName.from("boolean.field"); + private static final CompoundName ATTRIBUTES = CompoundName.from("boolean.attributes"); + private static final CompoundName RANGE_ATTRIBUTES = CompoundName.from("boolean.rangeAttributes"); public static final String PREDICATE = "predicate"; @Override diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/SortingDegrader.java b/container-search/src/main/java/com/yahoo/search/querytransform/SortingDegrader.java index 75b06c29193..81881685a1d 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/SortingDegrader.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/SortingDegrader.java @@ -41,9 +41,9 @@ public class SortingDegrader extends Searcher { /** Set this to false in query.properties to turn off degrading. Default: on */ // (this is not called ranking.sorting.degrading because it should not be part of the query object model - public static final CompoundName DEGRADING = new CompoundName("sorting.degrading"); + public static final CompoundName DEGRADING = CompoundName.from("sorting.degrading"); - public static final CompoundName PAGINATION = new CompoundName("to_be_removed_pagination"); + public static final CompoundName PAGINATION = CompoundName.from("to_be_removed_pagination"); @Override public Result search(Query query, Execution execution) { diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java index 89245eaf137..bfac668b6c4 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/WandSearcher.java @@ -63,12 +63,12 @@ public class WandSearcher extends Searcher { */ private static class InputResolver { - private static final CompoundName WAND_FIELD = new CompoundName("wand.field"); - private static final CompoundName WAND_TOKENS = new CompoundName("wand.tokens"); - private static final CompoundName WAND_HEAP_SIZE = new CompoundName("wand.heapSize"); - private static final CompoundName WAND_TYPE = new CompoundName("wand.type"); - private static final CompoundName WAND_SCORE_THRESHOLD = new CompoundName("wand.scoreThreshold"); - private static final CompoundName WAND_THRESHOLD_BOOST_FACTOR = new CompoundName("wand.thresholdBoostFactor"); + private static final CompoundName WAND_FIELD = CompoundName.from("wand.field"); + private static final CompoundName WAND_TOKENS = CompoundName.from("wand.tokens"); + private static final CompoundName WAND_HEAP_SIZE = CompoundName.from("wand.heapSize"); + private static final CompoundName WAND_TYPE = CompoundName.from("wand.type"); + private static final CompoundName WAND_SCORE_THRESHOLD = CompoundName.from("wand.scoreThreshold"); + private static final CompoundName WAND_THRESHOLD_BOOST_FACTOR = CompoundName.from("wand.thresholdBoostFactor"); private final String fieldName; private final WandType wandType; private final Map<Object, Integer> tokens; diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java index 2d6e059342e..9aa7a9d998d 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java @@ -21,8 +21,8 @@ import com.yahoo.yolean.chain.After; */ @After(MinimalQueryInserter.EXTERNAL_YQL) public class WeakAndReplacementSearcher extends Searcher { - static final CompoundName WEAKAND_REPLACE = new CompoundName("weakAnd.replace"); - static final CompoundName WAND_HITS = new CompoundName("wand.hits"); + static final CompoundName WEAKAND_REPLACE = CompoundName.from("weakAnd.replace"); + static final CompoundName WAND_HITS = CompoundName.from("wand.hits"); @Override public Result search(Query query, Execution execution) { if (!query.properties().getBoolean(WEAKAND_REPLACE)) { diff --git a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java index 44620179c1d..b36c8788877 100644 --- a/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java +++ b/container-search/src/main/java/com/yahoo/search/rendering/JsonRenderer.java @@ -76,10 +76,10 @@ import static com.fasterxml.jackson.databind.SerializationFeature.FLUSH_AFTER_WR // NOTE: The JSON format is a public API. If new elements are added be sure to update the reference doc. public class JsonRenderer extends AsynchronousSectionedRenderer<Result> { - private static final CompoundName WRAP_DEEP_MAPS = new CompoundName("renderer.json.jsonMaps"); - private static final CompoundName WRAP_WSETS = new CompoundName("renderer.json.jsonWsets"); - private static final CompoundName DEBUG_RENDERING_KEY = new CompoundName("renderer.json.debug"); - private static final CompoundName JSON_CALLBACK = new CompoundName("jsoncallback"); + private static final CompoundName WRAP_DEEP_MAPS = CompoundName.from("renderer.json.jsonMaps"); + private static final CompoundName WRAP_WSETS = CompoundName.from("renderer.json.jsonWsets"); + private static final CompoundName DEBUG_RENDERING_KEY = CompoundName.from("renderer.json.debug"); + private static final CompoundName JSON_CALLBACK = CompoundName.from("jsoncallback"); // if this must be optimized, simply use com.fasterxml.jackson.core.SerializableString private static final String BUCKET_LIMITS = "limits"; diff --git a/container-search/src/main/java/com/yahoo/search/searchers/CacheControlSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/CacheControlSearcher.java index a4f03b088f2..29e8afa41ac 100644 --- a/container-search/src/main/java/com/yahoo/search/searchers/CacheControlSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchers/CacheControlSearcher.java @@ -32,9 +32,9 @@ import com.yahoo.search.searchchain.Execution; */ public class CacheControlSearcher extends Searcher { - private static final CompoundName cachecontrolNocache=new CompoundName("cachecontrol.nocache"); - private static final CompoundName cachecontrolMaxage=new CompoundName("cachecontrol.maxage"); - private static final CompoundName cachecontrolStaleage=new CompoundName("cachecontrol.staleage"); + private static final CompoundName cachecontrolNocache=CompoundName.from("cachecontrol.nocache"); + private static final CompoundName cachecontrolMaxage=CompoundName.from("cachecontrol.maxage"); + private static final CompoundName cachecontrolStaleage=CompoundName.from("cachecontrol.staleage"); public static final String CACHE_CONTROL_HEADER = "Cache-Control"; diff --git a/container-search/src/main/java/com/yahoo/search/searchers/ConnectionControlSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/ConnectionControlSearcher.java index 90cb05be1f8..18588b2026d 100644 --- a/container-search/src/main/java/com/yahoo/search/searchers/ConnectionControlSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchers/ConnectionControlSearcher.java @@ -49,7 +49,7 @@ public class ConnectionControlSearcher extends Searcher { private final LongSupplier clock; - private static final CompoundName KEEPALIVE_MAXLIFETIMESECONDS = new CompoundName("connectioncontrol.maxlifetime"); + private static final CompoundName KEEPALIVE_MAXLIFETIMESECONDS = CompoundName.from("connectioncontrol.maxlifetime"); private static final String HTTP_CONNECTION_HEADER_NAME = "Connection"; private static final String HTTP_CONNECTION_CLOSE_ARGUMENT = "Close"; diff --git a/container-search/src/main/java/com/yahoo/search/searchers/RateLimitingSearcher.java b/container-search/src/main/java/com/yahoo/search/searchers/RateLimitingSearcher.java index cf81f62f64d..35a3c86f763 100755 --- a/container-search/src/main/java/com/yahoo/search/searchers/RateLimitingSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/searchers/RateLimitingSearcher.java @@ -54,11 +54,11 @@ public class RateLimitingSearcher extends Searcher { /** Constant containing the name this Provides - "rateLimiting", for ordering constraints */ public static final String RATE_LIMITING = "rateLimiting"; - public static final CompoundName idKey = new CompoundName("rate.id"); - public static final CompoundName costKey = new CompoundName("rate.cost"); - public static final CompoundName quotaKey = new CompoundName("rate.quota"); - public static final CompoundName idDimensionKey = new CompoundName("rate.idDimension"); - public static final CompoundName dryRunKey = new CompoundName("rate.dryRun"); + public static final CompoundName idKey = CompoundName.from("rate.id"); + public static final CompoundName costKey = CompoundName.from("rate.cost"); + public static final CompoundName quotaKey = CompoundName.from("rate.quota"); + public static final CompoundName idDimensionKey = CompoundName.from("rate.idDimension"); + public static final CompoundName dryRunKey = CompoundName.from("rate.dryRun"); private static final String requestsOverQuotaMetricName = "requestsOverQuota"; diff --git a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java index 833c1251a7b..6961a189d22 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java +++ b/container-search/src/main/java/com/yahoo/search/yql/FieldFiller.java @@ -27,7 +27,7 @@ public class FieldFiller extends Searcher { private final Set<String> intersectionOfAttributes; private final SchemaInfo schemaInfo; - public static final CompoundName FIELD_FILLER_DISABLE = new CompoundName("FieldFiller.disable"); + public static final CompoundName FIELD_FILLER_DISABLE = CompoundName.from("FieldFiller.disable"); public FieldFiller(SchemaInfo schemaInfo) { this.schemaInfo = schemaInfo; diff --git a/container-search/src/main/java/com/yahoo/search/yql/FieldFilter.java b/container-search/src/main/java/com/yahoo/search/yql/FieldFilter.java index 8cc6a1b42b9..df9722d3214 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/FieldFilter.java +++ b/container-search/src/main/java/com/yahoo/search/yql/FieldFilter.java @@ -2,13 +2,11 @@ package com.yahoo.search.yql; import java.util.Iterator; -import java.util.Map.Entry; import java.util.Set; import com.yahoo.api.annotations.Beta; import com.yahoo.component.chain.dependencies.After; import com.yahoo.component.chain.dependencies.Before; -import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.processing.request.CompoundName; import com.yahoo.search.Query; import com.yahoo.search.Result; @@ -27,7 +25,7 @@ import com.yahoo.search.searchchain.Execution; @Before("com.yahoo.search.yql.FieldFiller") public class FieldFilter extends Searcher { - public static final CompoundName FIELD_FILTER_DISABLE = new CompoundName("FieldFilter.disable"); + public static final CompoundName FIELD_FILTER_DISABLE = CompoundName.from("FieldFilter.disable"); /** Fields that should be kept even if not explicitly requested */ private static final Set<String> syntheticFields = Set.of("matchfeatures", "rankfeatures", "summaryfeatures"); diff --git a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java index e844bac21e8..ee4f931d532 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java +++ b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java @@ -39,10 +39,10 @@ public class MinimalQueryInserter extends Searcher { public static final String EXTERNAL_YQL = "ExternalYql"; - public static final CompoundName YQL = new CompoundName("yql"); + public static final CompoundName YQL = CompoundName.from("yql"); - private static final CompoundName MAX_HITS = new CompoundName("maxHits"); - private static final CompoundName MAX_OFFSET = new CompoundName("maxOffset"); + private static final CompoundName MAX_HITS = CompoundName.from("maxHits"); + private static final CompoundName MAX_OFFSET = CompoundName.from("maxOffset"); private static final Logger log = Logger.getLogger(MinimalQueryInserter.class.getName()); @Inject diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java index ab9da8ccee5..536355ab62d 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/MetricsSearcher.java @@ -22,8 +22,8 @@ import static com.yahoo.vespa.streamingvisitors.VdsStreamingSearcher.STREAMING_S */ public class MetricsSearcher extends Searcher { - private static final CompoundName metricsearcherId=new CompoundName("metricsearcher.id"); - private static final CompoundName streamingLoadtype=new CompoundName("streaming.loadtype"); + private static final CompoundName metricsearcherId = CompoundName.from("metricsearcher.id"); + private static final CompoundName streamingLoadtype = CompoundName.from("streaming.loadtype"); private static final Logger log = Logger.getLogger(MetricsSearcher.class.getName()); @@ -43,7 +43,8 @@ public class MetricsSearcher extends Searcher { public Result search(Query query, Execution execution) { long timeMs = System.currentTimeMillis(); - /** Backwards compatibility - convert metricsearcher.id to streaming.loadtype */ + // Backwards compatibility - convert metricsearcher.id to streaming.loadtype + // TODO Cleanup at some point String metricName = query.properties().getString(metricsearcherId); if (metricName != null) { query.properties().set(streamingLoadtype, metricName); diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java index ca199f37dd7..524001748c5 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java @@ -45,9 +45,9 @@ import java.util.logging.Logger; */ public class VdsStreamingSearcher extends VespaBackEndSearcher { - private static final CompoundName streamingUserid = new CompoundName("streaming.userid"); - private static final CompoundName streamingGroupname = new CompoundName("streaming.groupname"); - private static final CompoundName streamingSelection = new CompoundName("streaming.selection"); + private static final CompoundName streamingUserid = CompoundName.from("streaming.userid"); + private static final CompoundName streamingGroupname = CompoundName.from("streaming.groupname"); + private static final CompoundName streamingSelection = CompoundName.from("streaming.selection"); static final String STREAMING_STATISTICS = "streaming.statistics"; private final VisitorFactory visitorFactory; diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java index 85ef10b29f6..70dff6730ff 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsVisitor.java @@ -51,13 +51,13 @@ import java.util.logging.Level; */ class VdsVisitor extends VisitorDataHandler implements Visitor { - private static final CompoundName streamingUserid=new CompoundName("streaming.userid"); - private static final CompoundName streamingGroupname=new CompoundName("streaming.groupname"); - private static final CompoundName streamingSelection=new CompoundName("streaming.selection"); - private static final CompoundName streamingFromtimestamp=new CompoundName("streaming.fromtimestamp"); - private static final CompoundName streamingTotimestamp=new CompoundName("streaming.totimestamp"); - private static final CompoundName streamingPriority=new CompoundName("streaming.priority"); - private static final CompoundName streamingMaxbucketspervisitor=new CompoundName("streaming.maxbucketspervisitor"); + private static final CompoundName streamingUserid = CompoundName.from("streaming.userid"); + private static final CompoundName streamingGroupname = CompoundName.from("streaming.groupname"); + private static final CompoundName streamingSelection = CompoundName.from("streaming.selection"); + private static final CompoundName streamingFromtimestamp = CompoundName.from("streaming.fromtimestamp"); + private static final CompoundName streamingTotimestamp = CompoundName.from("streaming.totimestamp"); + private static final CompoundName streamingPriority = CompoundName.from("streaming.priority"); + private static final CompoundName streamingMaxbucketspervisitor = CompoundName.from("streaming.maxbucketspervisitor"); protected static final int MAX_BUCKETS_PER_VISITOR = 1024; diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java index 3a6be1521e2..1ff5574ec03 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/TokenizerTestCase.java @@ -13,6 +13,7 @@ import com.yahoo.prelude.query.parser.Tokenizer; import org.junit.jupiter.api.Test; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import static com.yahoo.prelude.query.parser.Token.Kind.COLON; @@ -28,9 +29,7 @@ import static com.yahoo.prelude.query.parser.Token.Kind.SPACE; import static com.yahoo.prelude.query.parser.Token.Kind.STAR; import static com.yahoo.prelude.query.parser.Token.Kind.UNDERSCORE; import static com.yahoo.prelude.query.parser.Token.Kind.WORD; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; /** * Tests the tokenizer @@ -284,7 +283,7 @@ public class TokenizerTestCase { sd.addIndex(index2); IndexFacts facts = new IndexFacts(new IndexModel(sd)); - IndexFacts.Session session = facts.newSession(); + IndexFacts.Session session = facts.newSession(Collections.emptySet(), Collections.emptySet()); Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); List<?> tokens = tokenizer.tokenize("normal a:b (normal testexact1:/,%#%&+-+ ) testexact2:ho_/&%&/()/aa*::*& b:c", "default", session); // tokenizer.print(); @@ -329,7 +328,7 @@ public class TokenizerTestCase { IndexFacts facts = new IndexFacts(new IndexModel(sd)); Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); - IndexFacts.Session session = facts.newSession(); + IndexFacts.Session session = facts.newSession(Collections.emptySet(), Collections.emptySet()); List<?> tokens = tokenizer.tokenize("normal a:b (normal testexact1:/,%#%&+-+ ) testexact2:ho_/&%&/()/aa*::*&", session); assertEquals(new Token(WORD, "normal"), tokens.get(0)); assertEquals(new Token(SPACE, " "), tokens.get(1)); @@ -366,7 +365,7 @@ public class TokenizerTestCase { IndexFacts facts = new IndexFacts(new IndexModel(sd)); Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); - IndexFacts.Session session = facts.newSession(); + IndexFacts.Session session = facts.newSession(Collections.emptySet(), Collections.emptySet()); List<?> tokens = tokenizer.tokenize("normal a:b (normal testexact1:/,%#%&+-+ ) testexact2:ho_/&%&/()/aa*::*", session); assertEquals(new Token(WORD, "normal"), tokens.get(0)); assertEquals(new Token(SPACE, " "), tokens.get(1)); @@ -403,7 +402,7 @@ public class TokenizerTestCase { IndexFacts facts = new IndexFacts(new IndexModel(sd)); Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); - IndexFacts.Session session = facts.newSession(); + IndexFacts.Session session = facts.newSession(Collections.emptySet(), Collections.emptySet()); List<?> tokens = tokenizer.tokenize("normal a:b (normal testexact1:!/%#%&+-+ ) testexact2:ho_/&%&/()/aa*::*&b:", session); assertEquals(new Token(WORD, "normal"), tokens.get(0)); assertEquals(new Token(SPACE, " "), tokens.get(1)); @@ -440,7 +439,7 @@ public class TokenizerTestCase { sd.addIndex(index2); IndexFacts indexFacts = new IndexFacts(new IndexModel(sd)); - IndexFacts.Session facts = indexFacts.newSession(); + IndexFacts.Session facts = indexFacts.newSession(Collections.emptySet(), Collections.emptySet()); Tokenizer tokenizer = new Tokenizer(new SimpleLinguistics()); List<?> tokens = tokenizer.tokenize("normal a:b (normal testexact1:foo) testexact2:bar", facts); diff --git a/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java index dbcb393c922..e6c5a18c9da 100644 --- a/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/test/IndexFactsTestCase.java @@ -15,12 +15,8 @@ import org.junit.jupiter.api.Test; import java.util.Collection; import java.util.List; import java.util.Map; -import java.util.Set; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; /** * Tests using synthetic index names for IndexFacts class. @@ -184,7 +180,7 @@ public class IndexFactsTestCase { query.getModel().getSources().add("one"); query.getModel().getRestrict().add("two"); - IndexFacts.Session indexFacts = createIndexFacts().newSession(List.of("clusterOne"), Set.of()); + IndexFacts.Session indexFacts = createIndexFacts().newSession(List.of("clusterOne"), List.of()); assertTrue(indexFacts.isIndex("a")); assertFalse(indexFacts.isIndex("b")); assertTrue(indexFacts.isIndex("d")); diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java index dcb41dc5e31..326c7985a5f 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/config/test/XmlReadingTestCase.java @@ -3,10 +3,8 @@ package com.yahoo.search.query.profile.config.test; import com.yahoo.jdisc.http.HttpRequest.Method; import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.language.Language; import com.yahoo.language.process.Embedder; import com.yahoo.processing.request.CompoundName; -import com.yahoo.search.query.profile.types.test.QueryProfileTypeTestCase; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import com.yahoo.yolean.Exceptions; @@ -25,7 +23,11 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; /** * @author bratseth @@ -77,8 +79,8 @@ public class XmlReadingTestCase { CompiledQueryProfile defaultProfile = cRegistry.getComponent("default"); assertNull(defaultProfile.getType()); assertEquals("20", defaultProfile.get("hits")); - assertFalse(defaultProfile.isOverridable(new CompoundName("hits"), null)); - assertFalse(defaultProfile.isOverridable(new CompoundName("user.trusted"), null)); + assertFalse(defaultProfile.isOverridable(CompoundName.from("hits"), null)); + assertFalse(defaultProfile.isOverridable(CompoundName.from("user.trusted"), null)); assertEquals("false", defaultProfile.get("user.trusted")); CompiledQueryProfile referencingProfile = cRegistry.getComponent("referencingModelSettings"); @@ -97,7 +99,7 @@ public class XmlReadingTestCase { assertEquals("rootType", rootProfile.getType().getId().getName()); assertEquals(30, rootProfile.get("hits")); //assertEquals(3, rootProfile.get("traceLevel")); - assertTrue(rootProfile.isOverridable(new CompoundName("hits"), null)); + assertTrue(rootProfile.isOverridable(CompoundName.from("hits"), null)); query = new Query(request, rootProfile); assertEquals(3, query.getTrace().getLevel()); @@ -231,8 +233,8 @@ public class XmlReadingTestCase { assertEquals("a.b.c-value", new Query("?d1=d1v", profile).properties().get("a.b.c")); assertEquals("a.b.c-variant-value", new Query("?d1=d1v&d2=d2v", profile).properties().get("a.b.c")); - assertTrue(profile.isOverridable(new CompoundName("a.b.c"), Map.of("d1", "d1v"))); - assertFalse(profile.isOverridable(new CompoundName("a.b.c"), Map.of("d1", "d1v", "d2", "d2v"))); + assertTrue(profile.isOverridable(CompoundName.from("a.b.c"), Map.of("d1", "d1v"))); + assertFalse(profile.isOverridable(CompoundName.from("a.b.c"), Map.of("d1", "d1v", "d2", "d2v"))); } @Test @@ -479,18 +481,18 @@ public class XmlReadingTestCase { QueryProfileType type1 = registry.getTypeRegistry().getComponent("type1"); assertEquals(TensorType.fromSpec("tensor<float>(x[1])"), - type1.getFieldType(new CompoundName("ranking.features.query(tensor_1)")).asTensorType()); - assertNull(type1.getFieldType(new CompoundName("ranking.features.query(tensor_2)"))); - assertNull(type1.getFieldType(new CompoundName("ranking.features.query(tensor_3)"))); + type1.getFieldType(CompoundName.from("ranking.features.query(tensor_1)")).asTensorType()); + assertNull(type1.getFieldType(CompoundName.from("ranking.features.query(tensor_2)"))); + assertNull(type1.getFieldType(CompoundName.from("ranking.features.query(tensor_3)"))); assertEquals(TensorType.fromSpec("tensor(key{})"), - type1.getFieldType(new CompoundName("ranking.features.query(tensor_4)")).asTensorType()); + type1.getFieldType(CompoundName.from("ranking.features.query(tensor_4)")).asTensorType()); QueryProfileType type2 = registry.getTypeRegistry().getComponent("type2"); - assertNull(type2.getFieldType(new CompoundName("ranking.features.query(tensor_1)"))); + assertNull(type2.getFieldType(CompoundName.from("ranking.features.query(tensor_1)"))); assertEquals(TensorType.fromSpec("tensor<float>(x[2])"), - type2.getFieldType(new CompoundName("ranking.features.query(tensor_2)")).asTensorType()); + type2.getFieldType(CompoundName.from("ranking.features.query(tensor_2)")).asTensorType()); assertEquals(TensorType.fromSpec("tensor<float>(x[3])"), - type2.getFieldType(new CompoundName("ranking.features.query(tensor_3)")).asTensorType()); + type2.getFieldType(CompoundName.from("ranking.features.query(tensor_3)")).asTensorType()); Query queryProfile1 = new Query.Builder().setQueryProfile(registry.getComponent("profile1")) .setRequest("?query=test&ranking.features.query(tensor_1)=[1.200]") diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileGetInComplexStructureMicroBenchmark.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileGetInComplexStructureMicroBenchmark.java index cd5a8e08aa9..080aef25c5f 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileGetInComplexStructureMicroBenchmark.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileGetInComplexStructureMicroBenchmark.java @@ -65,8 +65,8 @@ public class QueryProfileGetInComplexStructureMicroBenchmark { Map<String,String> dimensionValues=createDimensionValueMap(); String prefix=generatePrefix(); final int dotInterval=1000000; - final CompoundName found = new CompoundName(prefix + "a"); - final CompoundName notFound = new CompoundName(prefix + "nonexisting"); + final CompoundName found = CompoundName.from(prefix + "a"); + final CompoundName notFound = CompoundName.from(prefix + "nonexisting"); for (int i=0; i<count; i++) { if (count>dotInterval && i%(dotInterval)==0) System.out.print("."); diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileGetMicroBenchmark.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileGetMicroBenchmark.java index d38a1d64910..1123feb1b01 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileGetMicroBenchmark.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileGetMicroBenchmark.java @@ -57,8 +57,8 @@ public class QueryProfileGetMicroBenchmark { private void getValues(int count,Query query) { final int dotInterval=10000000; - CompoundName found = new CompoundName(propertyPrefix + "property1"); - CompoundName notFound = new CompoundName(propertyPrefix + "nonExisting"); + CompoundName found = CompoundName.from(propertyPrefix + "property1"); + CompoundName notFound = CompoundName.from(propertyPrefix + "nonExisting"); for (int i=0; i<count; i++) { if (count>dotInterval && i%(count/dotInterval)==0) System.out.print("."); diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java index cd4cb32df2e..7d5a2137770 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileTestCase.java @@ -534,7 +534,7 @@ public class QueryProfileTestCase { p.set("a.b", "a.b-value", null); { - Map<String, ValueWithSource> values = p.compile(null).listValuesWithSources(new CompoundName(""), new HashMap<>(), null); + Map<String, ValueWithSource> values = p.compile(null).listValuesWithSources(CompoundName.empty, new HashMap<>(), null); assertEquals(2, values.size()); assertEquals("a-value", values.get("a").value()); assertEquals("test", values.get("a").source()); @@ -543,7 +543,7 @@ public class QueryProfileTestCase { } { - Map<String, ValueWithSource> values = p.compile(null).listValuesWithSources(new CompoundName("a"), new HashMap<>(), null); + Map<String, ValueWithSource> values = p.compile(null).listValuesWithSources(CompoundName.from("a"), new HashMap<>(), null); assertEquals(1, values.size()); assertEquals("a.b-value", values.get("b").value()); assertEquals("test", values.get("b").source()); diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java index 1a6cfee75a5..36ff33ffd97 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/test/QueryProfileVariantsTestCase.java @@ -5,7 +5,6 @@ import ai.vespa.cloud.ApplicationId; import ai.vespa.cloud.Environment; import ai.vespa.cloud.Zone; import ai.vespa.cloud.ZoneInfo; -import com.yahoo.jdisc.application.Application; import com.yahoo.jdisc.http.HttpRequest.Method; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.processing.request.CompoundName; @@ -28,7 +27,9 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author bratseth @@ -133,7 +134,7 @@ public class QueryProfileVariantsTestCase { base.set("a.b", 1, new String[]{null, null, "d3-val"}, registry); QueryProfileVariant aVariants = base.getVariants().getVariants().get(0); assertEquals("[d1, d2, d3]", - ((QueryProfile) base.getVariants().getVariants().get(0).values().get("a")).getDimensions().toString(), + ((QueryProfile) aVariants.values().get("a")).getDimensions().toString(), "Variant dimensions are not overridden by the referenced dimensions"); } @@ -184,7 +185,7 @@ public class QueryProfileVariantsTestCase { { Map<String, ValueWithSource> values = cRegistry.findQueryProfile("test") - .listValuesWithSources(new CompoundName(""), + .listValuesWithSources(CompoundName.empty, new HashMap<>(), null); assertEquals(1, values.size()); @@ -195,7 +196,7 @@ public class QueryProfileVariantsTestCase { { Map<String, ValueWithSource> values = cRegistry.findQueryProfile("test") - .listValuesWithSources(new CompoundName(""), + .listValuesWithSources(CompoundName.empty, toMap("x=x1", "y=y1", "z=z1"), null); assertEquals(2, values.size()); @@ -1456,7 +1457,7 @@ public class QueryProfileVariantsTestCase { return context; } - public static final Map<String, String> toMap(String... bindings) { + public static Map<String, String> toMap(String... bindings) { Map<String, String> context = new HashMap<>(); for (String binding : bindings) { String[] entry = binding.split("="); diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java index 37991f7f14f..982ebd80fbd 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/NameTestCase.java @@ -55,7 +55,7 @@ public class NameTestCase { @Test void testComponentIdAsCompoundName() { String name = "a/b"; - assertEquals(new CompoundName(name), new QueryProfileType(name).getComponentIdAsCompoundName()); + assertEquals(CompoundName.from(name), new QueryProfileType(name).getComponentIdAsCompoundName()); } private void assertLegalName(String name) { @@ -89,10 +89,6 @@ public class NameTestCase { } } - private void assertIllegalFieldName(String name) { - assertIllegalFieldName(name,"Could not set '" + name + "' to 'anyValue'","Illegal name '" + name + "'"); - } - /** Checks that this is illegal both for profiles and types */ private void assertIllegalFieldName(String name, String expectedHighError, String expectedLowError) { try { diff --git a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java index 21a9b2fe399..4000c48bfed 100644 --- a/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/profile/types/test/QueryProfileTypeTestCase.java @@ -761,7 +761,7 @@ public class QueryProfileTypeTestCase { } private void assertNotPermitted(QueryProfile profile,String name,Object value) { - String localName = new CompoundName(name).last(); + String localName = CompoundName.from(name).last(); try { profile.set(name, value, registry); fail("Should fail setting " + name + " to " + value); diff --git a/container-search/src/test/java/com/yahoo/search/query/properties/test/SubPropertiesTestCase.java b/container-search/src/test/java/com/yahoo/search/query/properties/SubPropertiesTestCase.java index 0e9bf8d41ec..6b25dc526d9 100644 --- a/container-search/src/test/java/com/yahoo/search/query/properties/test/SubPropertiesTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/properties/SubPropertiesTestCase.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.query.properties.test; +package com.yahoo.search.query.properties; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; @@ -9,11 +9,12 @@ import java.util.HashSet; import com.yahoo.processing.request.properties.PropertyMap; import org.junit.jupiter.api.Test; -import com.yahoo.search.query.properties.SubProperties; /** * @author <a href="mailto:arnebef@yahoo-inc.com">Arne Bergene Fossaa</a> */ +@SuppressWarnings({"removal"}) + public class SubPropertiesTestCase { @Test diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java index 748a20801d1..63655da0784 100644 --- a/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/test/QueryTestCase.java @@ -2,7 +2,6 @@ package com.yahoo.search.test; import com.yahoo.component.chain.Chain; -import com.yahoo.data.JsonProducer; import com.yahoo.language.Language; import com.yahoo.language.Linguistics; import com.yahoo.language.detect.Detection; @@ -16,7 +15,6 @@ import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; -import com.yahoo.prelude.fastsearch.FastHit; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.AndSegmentItem; import com.yahoo.prelude.query.CompositeItem; @@ -145,7 +143,7 @@ public class QueryTestCase { @Test void testCloneWithConnectivity() { List<String> l = List.of("a", "b", "c", "a"); - printIt(l.stream().filter(i -> isA(i)).toList()); + printIt(l.stream().filter(this::isA).toList()); printIt(l.stream().filter(i -> !isA(i)).toList()); Query q = new Query(); @@ -314,7 +312,7 @@ public class QueryTestCase { profile.set("myField", "Profile: %{queryProfile}", null); Query query = new Query(QueryTestCase.httpEncode("/search?queryProfile=myProfile"), profile.compile(null)); - String source = query.properties().getInstance(com.yahoo.search.query.profile.QueryProfileProperties.class).getQueryProfile().listValuesWithSources(new CompoundName(""), query.getHttpRequest().propertyMap(), query.properties()).get("myField").source(); + String source = query.properties().getInstance(com.yahoo.search.query.profile.QueryProfileProperties.class).getQueryProfile().listValuesWithSources(CompoundName.empty, query.getHttpRequest().propertyMap(), query.properties()).get("myField").source(); assertEquals("myProfile", source); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java index ecce9b7ccfa..a389f771db6 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/PlanRegistryMock.java @@ -42,11 +42,15 @@ public class PlanRegistryMock implements PlanRegistry { private final boolean supported; public MockPlan(String planId, boolean billed, boolean supported, double cpuPrice, double memPrice, double dgbPrice, int quota, String description) { - this(PlanId.from(planId), billed, supported, new MockCostCalculator(cpuPrice, memPrice, dgbPrice), () -> Quota.unlimited().withBudget(quota), description); + this(PlanId.from(planId), billed, supported, new MockCostCalculator(cpuPrice, memPrice, dgbPrice), () -> createQuota(quota), description); + } + + private static Quota createQuota(int quota) { + return quota == 0 ? Quota.zero() : Quota.unlimited().withBudget(quota); } public MockPlan(String planId, boolean billed, boolean supported, String cpuPrice, String memPrice, String dgbPrice, int quota, String description) { - this(PlanId.from(planId), billed, supported, new MockCostCalculator(cpuPrice, memPrice, dgbPrice), () -> Quota.unlimited().withBudget(quota), description); + this(PlanId.from(planId), billed, supported, new MockCostCalculator(cpuPrice, memPrice, dgbPrice), () -> createQuota(quota), description); } public MockPlan(PlanId planId, boolean billed, boolean supported, MockCostCalculator calculator, QuotaCalculator quota, String description) { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java index ecea1ce6913..cff61f1a50a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateValidatorImpl.java @@ -69,7 +69,9 @@ public class EndpointCertificateValidatorImpl implements EndpointCertificateVali // Normally because the cert is in the process of being provisioned - this will cause a retry in InternalStepRunner throw new EndpointCertificateException(EndpointCertificateException.Type.CERT_NOT_AVAILABLE, "Certificate not found in secret store"); } catch (EndpointCertificateException e) { - log.log(Level.WARNING, "Certificate validation failure for " + serializedInstanceId, e); + if (!e.type().equals(EndpointCertificateException.Type.CERT_NOT_AVAILABLE)) { // such failures are normal and will be retried, it takes some time to show up in the secret store + log.log(Level.WARNING, "Certificate validation failure for " + serializedInstanceId, e); + } throw e; } catch (Exception e) { log.log(Level.WARNING, "Certificate validation failure for " + serializedInstanceId, e); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java index 945e0730fe6..34a7c1a6f7c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryNode.java @@ -113,6 +113,8 @@ public class NodeRepositoryNode { private String cloudAccount; @JsonProperty("wireguardPubKey") private String wireguardPubKey; + @JsonProperty("archiveUri") + private String archiveUri; public String getUrl() { return url; @@ -460,6 +462,10 @@ public class NodeRepositoryNode { public void setWireguardPubKey(String wireguardPubKey) { this.wireguardPubKey = wireguardPubKey; } + public String getArchiveUri() { return archiveUri; } + + public void setArchiveUri(String archiveUri) { this.archiveUri = archiveUri; } + // --- Helper methods for code that (wrongly) consume this directly public boolean hasType(NodeType type) { @@ -521,6 +527,7 @@ public class NodeRepositoryNode { ", switchHostname='" + switchHostname + '\'' + ", cloudAccount='" + cloudAccount + '\'' + ", wireguardPubKey='" + wireguardPubKey + '\'' + + ", archiveUri='" + archiveUri + '\'' + '}'; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index a1534ebc533..08a8440fbe2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -30,6 +30,7 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; +import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan; import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing; @@ -47,6 +48,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; +import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics.Warning; @@ -60,6 +62,7 @@ import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageValid import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade; import com.yahoo.vespa.hosted.controller.certificate.EndpointCertificates; import com.yahoo.vespa.hosted.controller.concurrent.Once; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.deployment.JobStatus; import com.yahoo.vespa.hosted.controller.deployment.Run; @@ -76,7 +79,6 @@ import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence; import com.yahoo.yolean.Exceptions; - import java.io.ByteArrayInputStream; import java.security.Principal; import java.security.cert.X509Certificate; @@ -586,7 +588,16 @@ public class ApplicationController { } // Validate new deployment spec thoroughly before storing it. - controller.jobController().deploymentStatus(application.get()); + DeploymentStatus status = controller.jobController().deploymentStatus(application.get()); + Change dummyChange = Change.of(RevisionId.forProduction(Long.MAX_VALUE)); // Should always run everywhere. + for (var jobs : status.jobsToRun(applicationPackage.deploymentSpec().instanceNames().stream() + .collect(toMap(name -> name, __ -> dummyChange))) + .entrySet()) { + for (var job : jobs.getValue()) { + decideCloudAccountOf(new DeploymentId(jobs.getKey().application(), job.type().zone()), + applicationPackage.deploymentSpec()); + } + } for (Notification notification : controller.notificationsDb().listNotifications(NotificationSource.from(application.get().id()), true)) { if ( notification.source().instance().isPresent() @@ -696,7 +707,7 @@ public class ApplicationController { throw new IllegalArgumentException("Requested cloud account '" + requestedAccount.get().value() + "' is not valid for tenant '" + tenant + "'"); } - if (!controller.zoneRegistry().hasZone(zoneId, requestedAccount.get())) { + if ( ! controller.zoneRegistry().hasZone(zoneId, requestedAccount.get())) { throw new IllegalArgumentException("Zone " + zoneId + " is not configured in requested cloud account '" + requestedAccount.get().value() + "'"); } @@ -1049,4 +1060,14 @@ public class ApplicationController { collectingAndThen(counting(), Long::intValue))); } + public void verifyPlan(TenantName tenantName) { + var planId = controller.serviceRegistry().billingController().getPlan(tenantName); + Optional<Plan> plan = controller.serviceRegistry().planRegistry().plan(planId); + if (plan.isEmpty()) + throw new IllegalArgumentException("Tenant '" + tenantName.value() + "' has no plan, not allowed to deploy. See https://cloud.vespa.ai/support"); + if (plan.get().quota().calculate().equals(Quota.zero())) + throw new IllegalArgumentException("Tenant '" + tenantName.value() + "' has a plan '" + + plan.get().displayName() + "' with zero quota, not allowed to deploy. See https://cloud.vespa.ai/support"); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index 1eb68c14353..80b52e0c7a4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -31,7 +31,8 @@ import static java.util.Comparator.comparing; */ public class Endpoint { - private static final String OATH_DNS_SUFFIX = ".vespa.oath.cloud"; + private static final String MAIN_OATH_DNS_SUFFIX = ".vespa.oath.cloud"; + private static final String CD_OATH_DNS_SUFFIX = ".cd.vespa.oath.cloud"; private static final String PUBLIC_DNS_SUFFIX = ".vespa-app.cloud"; private static final String PUBLIC_CD_DNS_SUFFIX = ".cd.vespa-app.cloud"; @@ -243,18 +244,13 @@ public class Endpoint { /** Returns the DNS suffix used for endpoints in given system */ private static String dnsSuffix(SystemName system) { - switch (system) { - case cd, main -> { - return OATH_DNS_SUFFIX; - } - case Public -> { - return PUBLIC_DNS_SUFFIX; - } - case PublicCd -> { - return PUBLIC_CD_DNS_SUFFIX; - } + return switch (system) { + case cd -> CD_OATH_DNS_SUFFIX; + case main -> MAIN_OATH_DNS_SUFFIX; + case Public -> PUBLIC_DNS_SUFFIX; + case PublicCd -> PUBLIC_CD_DNS_SUFFIX; default -> throw new IllegalArgumentException("No DNS suffix declared for system " + system); - } + }; } /** Returns the DNS suffix used for internal names (i.e. names not exposed to tenants) in given system */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java index 2441da19b90..c1bf083b26c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java @@ -21,7 +21,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.OptionalInt; import java.util.function.Function; import static java.util.Comparator.comparing; @@ -81,8 +80,7 @@ public class InstanceList extends AbstractFilteringList<ApplicationId, InstanceL /** Returns the subset of instances that are allowed to upgrade to the given version at the given time */ public InstanceList canUpgradeAt(Version version, Instant instant) { return matching(id -> instances.get(id).instanceSteps().get(id.instance()) - .readyAt(Change.of(version)) - .map(readyAt -> ! readyAt.isAfter(instant)).orElse(false)); + .readiness(Change.of(version)).okAt(instant)); } /** Returns the subset of instances which have at least one production deployment */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageStream.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageStream.java index f86073cfb25..3880b028eb0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageStream.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageStream.java @@ -5,6 +5,7 @@ import java.io.FilterInputStream; import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; +import java.nio.file.attribute.FileTime; import java.util.HashMap; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; @@ -32,6 +33,7 @@ public class ApplicationPackageStream { private final Supplier<Predicate<String>> filter; private final Supplier<InputStream> in; private final AtomicReference<ApplicationPackage> truncatedPackage = new AtomicReference<>(); + private final FileTime createdAt = FileTime.fromMillis(System.currentTimeMillis()); /** Stream that effectively copies the input stream to its {@link #truncatedPackage()} when exhausted. */ public ApplicationPackageStream(Supplier<InputStream> in) { @@ -60,7 +62,7 @@ public class ApplicationPackageStream { * and the first to be exhausted will populate the truncated application package. */ public InputStream zipStream() { - return new Stream(in.get(), replacer.get(), filter.get(), truncatedPackage); + return new Stream(in.get(), replacer.get(), filter.get(), createdAt, truncatedPackage); } /** @@ -85,6 +87,7 @@ public class ApplicationPackageStream { private final ZipInputStream inZip; private final Replacer replacer; private final Predicate<String> filter; + private final FileTime createdAt; private byte[] currentOut = new byte[0]; private InputStream currentIn = InputStream.nullInputStream(); private boolean includeCurrent = false; @@ -92,11 +95,12 @@ public class ApplicationPackageStream { private boolean closed = false; private boolean done = false; - private Stream(InputStream in, Replacer replacer, Predicate<String> filter, AtomicReference<ApplicationPackage> truncatedPackage) { + private Stream(InputStream in, Replacer replacer, Predicate<String> filter, FileTime createdAt, AtomicReference<ApplicationPackage> truncatedPackage) { this.in = in; this.inZip = new ZipInputStream(in); this.replacer = replacer; this.filter = filter; + this.createdAt = createdAt; this.truncatedPackage = truncatedPackage; } @@ -129,10 +133,12 @@ public class ApplicationPackageStream { ZipEntry next = inZip.getNextEntry(); String name; + FileTime modifiedAt; InputStream content = null; if (next == null) { // We may still have replacements to fill in, but if we don't, we're done filling, forever! name = replacer.next(); + modifiedAt = createdAt; if (name == null) { outZip.close(); // This typically makes new output available, so must check for that after this. teeZip.close(); @@ -144,6 +150,7 @@ public class ApplicationPackageStream { } else { name = next.getName(); + modifiedAt = next.getLastModifiedTime(); content = new FilterInputStream(inZip) { @Override public void close() { } }; // Protect inZip from replacements closing it. } @@ -153,8 +160,8 @@ public class ApplicationPackageStream { currentIn = InputStream.nullInputStream(); } else { - if (includeCurrent) teeZip.putNextEntry(new ZipEntry(name)); - outZip.putNextEntry(new ZipEntry(name)); + if (includeCurrent) teeZip.putNextEntry(new ZipEntry(name) {{ setLastModifiedTime(modifiedAt); }}); + outZip.putNextEntry(new ZipEntry(name) {{ setLastModifiedTime(modifiedAt); }}); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java index aa6e3b0c44d..cbd0f685d80 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java @@ -49,7 +49,7 @@ public record AuditLog(List<Entry> entries) { public record Entry(Instant at, String principal, Method method, String resource, Optional<String> data, Client client) implements Comparable<Entry> { - private final static int maxDataLength = 1024; + final static int maxDataLength = 1024; private final static Comparator<Entry> comparator = Comparator.comparing(Entry::at).reversed(); public Entry(Instant at, Client client, String principal, Method method, String resource, byte[] data) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java index 033cd0a52c9..13b3d9d170f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java @@ -4,11 +4,12 @@ package com.yahoo.vespa.hosted.controller.auditlog; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.jdisc.http.HttpHeaders; import com.yahoo.transaction.Mutex; +import com.yahoo.vespa.hosted.controller.auditlog.AuditLog.Entry; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.UncheckedIOException; +import java.io.InputStream; +import java.io.SequenceInputStream; import java.net.URI; import java.security.Principal; import java.time.Clock; @@ -17,6 +18,9 @@ import java.time.Instant; import java.util.Objects; import java.util.Optional; +import static com.yahoo.yolean.Exceptions.uncheck; +import static java.util.Objects.requireNonNullElse; + /** * This provides read and write operations for the audit log. * @@ -58,14 +62,8 @@ public class AuditLogger { "misconfiguration and should not happen"); } - byte[] data = new byte[0]; - try { - if (request.getData() != null) { - data = request.getData().readAllBytes(); - } - } catch (IOException e) { - throw new UncheckedIOException(e); - } + InputStream requestData = requireNonNullElse(request.getData(), InputStream.nullInputStream()); + byte[] data = uncheck(() -> requestData.readNBytes(Entry.maxDataLength)); AuditLog.Entry.Client client = parseClient(request); Instant now = clock.instant(); @@ -80,7 +78,9 @@ public class AuditLogger { } // Create a new input stream to allow callers to consume request body - return new HttpRequest(request.getJDiscRequest(), new ByteArrayInputStream(data), request.propertyMap()); + return new HttpRequest(request.getJDiscRequest(), + new SequenceInputStream(new ByteArrayInputStream(data), requestData), + request.propertyMap()); } private static AuditLog.Entry.Client parseClient(HttpRequest request) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index a76e76611c2..00da34fe2e4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -39,11 +39,11 @@ import java.util.Collections; import java.util.Deque; import java.util.HashMap; import java.util.HashSet; -import java.util.Iterator; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -303,7 +303,11 @@ public class DeploymentStatus { fallbackPlatform(change, job)); if (step.completedAt(change, firstProductionJobWithDeploymentInCloud).isEmpty()) { JobType typeWithZone = job.type().isSystemTest() ? JobType.systemTest(zones, cloud) : JobType.stagingTest(zones, cloud); - jobs.merge(job, List.of(new Job(typeWithZone, versions, step.readyAt(change), change)), DeploymentStatus::union); + Readiness readiness = step.readiness(change, firstProductionJobWithDeploymentInCloud); + jobs.merge(job, List.of(new Job(typeWithZone, + versions, + readiness.okAt(now) && jobs().get(job).get().isRunning() ? readiness.running() : readiness, + change)), DeploymentStatus::union); } }); }); @@ -390,7 +394,7 @@ public class DeploymentStatus { } /** The set of jobs that need to run for the given changes to be considered complete. */ - private Map<JobId, List<Job>> jobsToRun(Map<InstanceName, Change> changes) { + public Map<JobId, List<Job>> jobsToRun(Map<InstanceName, Change> changes) { return jobsToRun(changes, false); } @@ -498,21 +502,23 @@ public class DeploymentStatus { } /** Earliest instant when job was triggered with given versions, or both system and staging tests were successful. */ - public Optional<Instant> verifiedAt(JobId job, Versions versions) { - Optional<Instant> triggeredAt = allJobs.get(job) - .flatMap(status -> status.runs().values().stream() - .filter(run -> run.versions().equals(versions)) - .findFirst()) - .map(Run::start); - Optional<Instant> systemTestedAt = testedAt(job, systemTest(job.type()), versions); - Optional<Instant> stagingTestedAt = testedAt(job, stagingTest(job.type()), versions); - if (systemTestedAt.isEmpty() || stagingTestedAt.isEmpty()) return triggeredAt; - Optional<Instant> testedAt = systemTestedAt.get().isAfter(stagingTestedAt.get()) ? systemTestedAt : stagingTestedAt; - return triggeredAt.isPresent() && triggeredAt.get().isBefore(testedAt.get()) ? triggeredAt : testedAt; + public Readiness verifiedAt(JobId job, Versions versions) { + Readiness triggered = allJobs.get(job) + .flatMap(status -> status.runs().values().stream() + .filter(run -> run.versions().equals(versions)) + .findFirst()) + .map(Run::start) + .map(Readiness::new) + .orElse(Readiness.unverified); + Readiness systemTested = testedAt(job, systemTest(job.type()), versions); + Readiness stagingTested = testedAt(job, stagingTest(job.type()), versions); + if (! systemTested.ok() || ! stagingTested.ok()) return triggered; + Readiness tested = min(systemTested, stagingTested); + return triggered.ok() && triggered.at().isBefore(tested.at) ? triggered : tested; } /** Earliest instant when versions were tested for the given instance. */ - private Optional<Instant> testedAt(JobId job, JobType type, Versions versions) { + private Readiness testedAt(JobId job, JobType type, Versions versions) { return prerequisiteTests(job, type).stream() .map(test -> allJobs.get(test).stream() .flatMap(status -> RunList.from(status) @@ -522,19 +528,21 @@ public class DeploymentStatus { .asList().stream() .map(run -> run.end().get())) .min(naturalOrder())) - .reduce((o, n) -> o.isEmpty() || n.isEmpty() ? Optional.empty() : o.get().isBefore(n.get()) ? n : o) - .orElse(Optional.empty()); + .map(testedAt -> testedAt.map(Readiness::new).orElse(Readiness.unverified)) + .reduce(Readiness.empty, DeploymentStatus::max); } private Map<JobId, List<Job>> productionJobs(InstanceName instance, Change change, boolean assumeUpgradesSucceed) { Map<JobId, List<Job>> jobs = new LinkedHashMap<>(); - jobSteps.forEach((job, step) -> { + for (Entry<JobId, StepStatus> entry : reversed(List.copyOf(jobSteps.entrySet()))) { + JobId job = entry.getKey(); + StepStatus step = entry.getValue(); if ( ! job.application().instance().equals(instance) || ! job.type().isProduction()) - return; + continue; // Signal strict completion criterion by depending on job itself. if (step.completedAt(change, Optional.of(job)).isPresent()) - return; + continue; // When computing eager test jobs for outstanding changes, assume current change completes successfully. Optional<Deployment> deployment = deploymentFor(job); @@ -544,7 +552,7 @@ public class DeploymentStatus { || areIncompatible(change.platform(), existingRevision, job); if (assumeUpgradesSucceed) { if (deployingCompatibilityChange) // No eager tests for this. - return; + continue; Change currentChange = application.require(instance).change(); Versions target = Versions.from(currentChange, application, deployment, fallbackPlatform(currentChange, job)); @@ -554,21 +562,28 @@ public class DeploymentStatus { List<Job> toRun = new ArrayList<>(); List<Change> changes = deployingCompatibilityChange || allJobs.get(job).flatMap(status -> status.lastCompleted()).isEmpty() - ? List.of(change) - : changes(job, step, change); + ? List.of(change) + : changes(job, step, change); for (Change partial : changes) { - Job jobToRun = new Job(job.type(), - Versions.from(partial, application, existingPlatform, existingRevision, fallbackPlatform(partial, job)), - step.readyAt(partial, Optional.of(job)), - partial); - toRun.add(jobToRun); + Versions versions = Versions.from(partial, application, existingPlatform, existingRevision, fallbackPlatform(partial, job)); + Readiness readiness = step.readiness(partial, Optional.of(job)); + // This job is blocked if it is already running ... + readiness = jobs().get(job).get().isRunning() && readiness.okAt(now) ? readiness.running() : readiness; + // ... or if it is a deployment, and a test job for the current state is not yet complete, + // which is the case when the next versions to run that test with is not the same as we want to deploy here. + List<Job> tests = job.type().isTest() ? null : jobs.get(new JobId(job.application(), JobType.productionTestOf(job.type().zone()))); + readiness = tests != null && ! versions.targetsMatch(tests.get(0).versions) && readiness.okAt(now) ? readiness.blocked() : readiness; + toRun.add(new Job(job.type(), versions, readiness, partial)); // Assume first partial change is applied before the second. - existingPlatform = Optional.of(jobToRun.versions.targetPlatform()); - existingRevision = Optional.of(jobToRun.versions.targetRevision()); + existingPlatform = Optional.of(versions.targetPlatform()); + existingRevision = Optional.of(versions.targetRevision()); } jobs.put(job, toRun); - }); - return jobs; + } + Map<JobId, List<Job>> jobsInOrder = new LinkedHashMap<>(); + for (Entry<JobId, List<Job>> entry : reversed(List.copyOf(jobs.entrySet()))) + jobsInOrder.put(entry.getKey(), entry.getValue()); + return jobsInOrder; } private boolean areIncompatible(Optional<Version> platform, Optional<RevisionId> revision, JobId job) { @@ -606,8 +621,7 @@ public class DeploymentStatus { // the revision is now blocked by waiting for the production test to verify the upgrade. // In this case we must abandon the production test on the pure upgrade, so the revision can be deployed. if (platformDeployedAt.isPresent() && revisionDeployedAt.isEmpty()) { - if (jobSteps.get(deployment).readyAt(change, Optional.of(deployment)) - .map(ready -> ! now.isBefore(ready)).orElse(false)) { + if (jobSteps.get(deployment).readiness(change, Optional.of(deployment)).okAt(now)) { return switch (rollout) { // If separate rollout, this test should keep blocking the revision, unless there are failures. case separate -> hasFailures(jobSteps.get(deployment), jobSteps.get(job)) ? List.of(change) : List.of(change.withoutApplication(), change); @@ -676,12 +690,15 @@ public class DeploymentStatus { for (Job productionJob : versionsList) if (allJobs.successOn(testType, productionJob.versions()) .instance(testJob.application().instance()) - .asList().isEmpty()) + .asList().isEmpty()) { + Readiness readiness = jobSteps().get(testJob).readiness(productionJob.change, Optional.of(job)); testJobs.merge(testJob, List.of(new Job(testJob.type(), productionJob.versions(), - jobSteps().get(testJob).readyAt(productionJob.change), + readiness.okAt(now) && jobs().get(testJob).get().isRunning() ? readiness.running() : readiness, productionJob.change)), DeploymentStatus::union); + + } }); } } @@ -894,16 +911,17 @@ public class DeploymentStatus { abstract Optional<Instant> completedAt(Change change, Optional<JobId> dependent); /** The time at which this step is ready to run the specified change and / or versions. */ - public Optional<Instant> readyAt(Change change) { return readyAt(change, Optional.empty()); } + public Readiness readiness(Change change) { return readiness(change, Optional.empty()); } /** The time at which this step is ready to run the specified change and / or versions. */ - Optional<Instant> readyAt(Change change, Optional<JobId> dependent) { + Readiness readiness(Change change, Optional<JobId> dependent) { return dependenciesCompletedAt(change, dependent) + .map(Readiness::new) .map(ready -> Stream.of(blockedUntil(change), pausedUntil(), coolingDownUntil(change, dependent)) - .flatMap(Optional::stream) - .reduce(ready, maxBy(naturalOrder()))); + .reduce(ready, maxBy(naturalOrder()))) + .orElse(Readiness.notReady); } /** The time at which all dependencies completed on the given change and / or versions. */ @@ -918,13 +936,13 @@ public class DeploymentStatus { } /** The time until which this step is blocked by a change blocker. */ - public Optional<Instant> blockedUntil(Change change) { return Optional.empty(); } + public Readiness blockedUntil(Change change) { return Readiness.empty; } /** The time until which this step is paused by user intervention. */ - public Optional<Instant> pausedUntil() { return Optional.empty(); } + public Readiness pausedUntil() { return Readiness.empty; } /** The time until which this step is cooling down, due to consecutive failures. */ - public Optional<Instant> coolingDownUntil(Change change, Optional<JobId> dependent) { return Optional.empty(); } + public Readiness coolingDownUntil(Change change, Optional<JobId> dependent) { return Readiness.empty; } /** Whether this step is declared in the deployment spec, or is an implicit step. */ public boolean isDeclared() { return true; } @@ -940,7 +958,8 @@ public class DeploymentStatus { @Override Optional<Instant> completedAt(Change change, Optional<JobId> dependent) { - return readyAt(change, dependent).map(completion -> completion.plus(step().delay())); + return Optional.ofNullable(readiness(change, dependent).at()) + .map(completion -> completion.plus(step().delay())); } } @@ -964,12 +983,12 @@ public class DeploymentStatus { /** The time at which this step is ready to run the specified change and / or versions. */ @Override - public Optional<Instant> readyAt(Change change) { + public Readiness readiness(Change change) { return status.jobSteps.keySet().stream() .filter(job -> job.type().isProduction() && job.application().instance().equals(instance.name())) - .map(job -> super.readyAt(change, Optional.of(job))) - .reduce((o, n) -> o.isEmpty() || n.isEmpty() ? Optional.empty() : n.get().isBefore(o.get()) ? n : o) - .orElseGet(() -> super.readyAt(change, Optional.empty())); + .map(job -> super.readiness(change, Optional.of(job))) + .reduce((a, b) -> ! a.ok() ? a : ! b.ok() ? b : min(a, b)) + .orElseGet(() -> super.readiness(change, Optional.empty())); } /** @@ -986,7 +1005,7 @@ public class DeploymentStatus { } @Override - public Optional<Instant> blockedUntil(Change change) { + public Readiness blockedUntil(Change change) { for (Instant current = now; now.plus(Duration.ofDays(7)).isAfter(current); ) { boolean blocked = false; for (DeploymentSpec.ChangeBlocker blocker : spec.changeBlocker()) { @@ -999,9 +1018,9 @@ public class DeploymentStatus { } } if ( ! blocked) - return current == now ? Optional.empty() : Optional.of(current); + return current == now ? Readiness.empty : new Readiness(current, DelayCause.changeBlocked); } - return Optional.of(now.plusSeconds(1 << 30)); // Some time in the future that doesn't look like anything you'd expect. + return new Readiness(now.plusSeconds(1 << 30), DelayCause.changeBlocked); // Some time in the future that doesn't look like anything you'd expect. } } @@ -1023,31 +1042,34 @@ public class DeploymentStatus { public Optional<JobId> job() { return Optional.of(job.id()); } @Override - public Optional<Instant> pausedUntil() { - return status.application().require(job.id().application().instance()).jobPause(job.id().type()); + public Readiness pausedUntil() { + return status.application().require(job.id().application().instance()).jobPause(job.id().type()) + .map(pause -> new Readiness(pause, DelayCause.paused)) + .orElse(Readiness.empty); } @Override - public Optional<Instant> coolingDownUntil(Change change, Optional<JobId> dependent) { - if (job.lastTriggered().isEmpty()) return Optional.empty(); - if (job.lastCompleted().isEmpty()) return Optional.empty(); - if (job.firstFailing().isEmpty() || ! job.firstFailing().get().hasEnded()) return Optional.empty(); + public Readiness coolingDownUntil(Change change, Optional<JobId> dependent) { + if (job.lastTriggered().isEmpty()) return Readiness.empty; + if (job.lastCompleted().isEmpty()) return Readiness.empty; + if (job.firstFailing().isEmpty() || ! job.firstFailing().get().hasEnded()) return Readiness.empty; Versions lastVersions = job.lastCompleted().get().versions(); Versions toRun = Versions.from(change, status.application, dependent.flatMap(status::deploymentFor), status.fallbackPlatform(change, job.id())); - if ( ! toRun.targetsMatch(lastVersions)) return Optional.empty(); + if ( ! toRun.targetsMatch(lastVersions)) return Readiness.empty; if ( job.id().type().environment().isTest() && ! dependent.map(JobId::type).map(status::findCloud).map(List.of(CloudName.AWS, CloudName.GCP)::contains).orElse(true) - && job.isNodeAllocationFailure()) return Optional.empty(); + && job.isNodeAllocationFailure()) return Readiness.empty; - if (job.lastStatus().get() == invalidApplication) return Optional.of(status.now.plus(Duration.ofDays(36524))); // 100 years + if (job.lastStatus().get() == invalidApplication) return new Readiness(status.now.plus(Duration.ofSeconds(1 << 30)), DelayCause.invalidPackage); Instant firstFailing = job.firstFailing().get().end().get(); Instant lastCompleted = job.lastCompleted().get().end().get(); - return firstFailing.equals(lastCompleted) ? Optional.of(lastCompleted) - : Optional.of(lastCompleted.plus(Duration.ofMinutes(10)) - .plus(Duration.between(firstFailing, lastCompleted) - .dividedBy(2))) - .filter(status.now::isBefore); + Duration penalty = firstFailing.equals(lastCompleted) ? Duration.ZERO + : Duration.ofMinutes(10) + .plus(Duration.between(firstFailing, lastCompleted) + .dividedBy(2)); + return lastCompleted.plus(penalty).isAfter(status.now) ? new Readiness(lastCompleted.plus(penalty), DelayCause.coolingDown) + : Readiness.empty; } private static JobStepStatus ofProductionDeployment(DeclaredZone step, List<StepStatus> dependencies, @@ -1059,11 +1081,10 @@ public class DeploymentStatus { return new JobStepStatus(StepType.deployment, step, dependencies, job, status) { @Override - public Optional<Instant> readyAt(Change change, Optional<JobId> dependent) { - Optional<Instant> readyAt = super.readyAt(change, dependent); - Optional<Instant> testedAt = status.verifiedAt(job.id(), Versions.from(change, status.application, existingDeployment, status.fallbackPlatform(change, job.id()))); - if (readyAt.isEmpty() || testedAt.isEmpty()) return Optional.empty(); - return readyAt.get().isAfter(testedAt.get()) ? readyAt : testedAt; + public Readiness readiness(Change change, Optional<JobId> dependent) { + Readiness readyAt = super.readiness(change, dependent); + Readiness testedAt = status.verifiedAt(job.id(), Versions.from(change, status.application, existingDeployment, status.fallbackPlatform(change, job.id()))); + return max(readyAt, testedAt); } /** Complete if deployment is on pinned version, and last successful deployment, or if given versions is strictly a downgrade, and this isn't forced by a pin. */ @@ -1103,11 +1124,11 @@ public class DeploymentStatus { JobId prodId = new JobId(job.id().application(), JobType.deploymentTo(job.id().type().zone())); return new JobStepStatus(StepType.test, step, dependencies, job, status) { @Override - Optional<Instant> readyAt(Change change, Optional<JobId> dependent) { - Optional<Instant> readyAt = super.readyAt(change, dependent); - Optional<Instant> deployedAt = status.jobSteps().get(prodId).completedAt(change, Optional.of(prodId)); - if (readyAt.isEmpty() || deployedAt.isEmpty()) return Optional.empty(); - return readyAt.get().isAfter(deployedAt.get()) ? readyAt : deployedAt; + Readiness readiness(Change change, Optional<JobId> dependent) { + Readiness readyAt = super.readiness(change, dependent); + Readiness deployedAt = status.jobSteps().get(prodId).completedAt(change, Optional.of(prodId)) + .map(Readiness::new).orElse(Readiness.notReady); + return max(readyAt, deployedAt); } @Override @@ -1163,13 +1184,13 @@ public class DeploymentStatus { private final JobType type; private final Versions versions; - private final Optional<Instant> readyAt; + private final Readiness readiness; private final Change change; - public Job(JobType type, Versions versions, Optional<Instant> readyAt, Change change) { + public Job(JobType type, Versions versions, Readiness readiness, Change change) { this.type = type; this.versions = type.isSystemTest() ? versions.withoutSources() : versions; - this.readyAt = readyAt; + this.readiness = readiness; this.change = change; } @@ -1181,8 +1202,8 @@ public class DeploymentStatus { return versions; } - public Optional<Instant> readyAt() { - return readyAt; + public Readiness readiness() { + return readiness; } @Override @@ -1190,19 +1211,60 @@ public class DeploymentStatus { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; Job job = (Job) o; - return type.zone().equals(job.type.zone()) && versions.equals(job.versions) && readyAt.equals(job.readyAt) && change.equals(job.change); + return type.zone().equals(job.type.zone()) && versions.equals(job.versions) && readiness.equals(job.readiness) && change.equals(job.change); } @Override public int hashCode() { - return Objects.hash(type.zone(), versions, readyAt, change); + return Objects.hash(type.zone(), versions, readiness, change); } @Override public String toString() { - return change + " with versions " + versions + ", ready at " + readyAt; + return change + " with versions " + versions + ", " + readiness; } } + public enum DelayCause { none, unverified, notReady, blocked, running, coolingDown, invalidPackage, changeBlocked, paused } + public record Readiness(Instant at, DelayCause cause) implements Comparable<Readiness> { + public static final Readiness unverified = new Readiness(null, DelayCause.unverified); + public static final Readiness notReady = new Readiness(null, DelayCause.notReady); + public static final Readiness empty = new Readiness(Instant.EPOCH, DelayCause.none); + public Readiness(Instant at) { this(at, DelayCause.none); } + public Readiness blocked() { return new Readiness(at, DelayCause.blocked); } + public Readiness running() { return new Readiness(at, DelayCause.running); } + public boolean ok() { return at != null; } + public boolean okAt(Instant at) { return ok() && cause != DelayCause.running && cause != DelayCause.blocked && ! at.isBefore(this.at); } + @Override public int compareTo(Readiness o) { + return at == null ? o.at == null ? 0 : 1 + : o.at == null ? -1 : at.compareTo(o.at); + } + @Override public String toString() { + return ok() ? "ready at " + at + switch (cause) { + case none -> ""; + case coolingDown -> ": cooling down after repeated failures"; + case blocked -> ": waiting for verification test to complete"; + case running -> ": waiting for current run to complete"; + case invalidPackage -> ": invalid application package, must resubmit"; + case changeBlocked -> ": deployment configuration blocks changes"; + case paused -> ": manually paused"; + default -> throw new IllegalStateException(cause + " should not have an instant at which it is ready"); + } + : "not ready" + switch (cause) { + case unverified -> ": waiting for verification test to complete"; + case notReady -> ": waiting for dependencies to complete"; + default -> throw new IllegalStateException(cause + " should have an instant at which it is ready"); + }; + } + } + + static <T extends Comparable<T>> T min(T a, T b) { + return a.compareTo(b) > 0 ? b : a; + } + + static <T extends Comparable<T>> T max(T a, T b) { + return a.compareTo(b) < 0 ? b : a; + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index a5cb839e9c9..00a0e22f87d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -20,6 +20,8 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.DelayCause; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.Readiness; import java.math.BigDecimal; import java.time.Clock; @@ -39,6 +41,7 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import static java.util.Comparator.comparing; +import static java.util.Comparator.comparingDouble; import static java.util.stream.Collectors.groupingBy; import static java.util.stream.Collectors.toMap; @@ -80,8 +83,7 @@ public class DeploymentTrigger { Change outstanding = status.outstandingChange(instanceName); boolean deployOutstanding = outstanding.hasTargets() && status.instanceSteps().get(instanceName) - .readyAt(outstanding) - .map(readyAt -> ! readyAt.isAfter(clock.instant())).orElse(false) + .readiness(outstanding).okAt(clock.instant()) && acceptNewRevision(status, instanceName, outstanding.revision().get()); application = application.with(instanceName, instance -> withRemainingChange(instance, @@ -235,7 +237,7 @@ public class DeploymentTrigger { if ( ! upgradeRevision && change.revision().isPresent()) change = change.withoutApplication(); if ( ! upgradePlatform && change.platform().isPresent()) change = change.withoutPlatform(); Versions versions = Versions.from(change, application, status.deploymentFor(job), status.fallbackPlatform(change, job)); - DeploymentStatus.Job toTrigger = new DeploymentStatus.Job(job.type(), versions, Optional.of(controller.clock().instant()), instance.change()); + DeploymentStatus.Job toTrigger = new DeploymentStatus.Job(job.type(), versions, new Readiness(controller.clock().instant()), instance.change()); Map<JobId, List<DeploymentStatus.Job>> testJobs = status.testJobs(Map.of(job, List.of(toTrigger))); Map<JobId, List<DeploymentStatus.Job>> jobs = testJobs.isEmpty() || ! requireTests @@ -374,17 +376,17 @@ public class DeploymentTrigger { List<Job> jobs = new ArrayList<>(); Map<JobId, List<DeploymentStatus.Job>> jobsToRun = status.jobsToRun(); jobsToRun.forEach((jobId, jobsList) -> { + abortIfOutdated(status, jobsToRun, jobId); DeploymentStatus.Job job = jobsList.get(0); - if ( job.readyAt().isPresent() - && ! clock.instant().isBefore(job.readyAt().get()) + if ( job.readiness().okAt(clock.instant()) && ! controller.jobController().isDisabled(new JobId(jobId.application(), job.type())) - && ! (jobId.type().isProduction() && isUnhealthyInAnotherZone(status.application(), jobId)) - && abortIfRunning(status, jobsToRun, jobId)) // Abort and trigger this later if running with outdated parameters. + && ! (jobId.type().isProduction() && isUnhealthyInAnotherZone(status.application(), jobId))) { jobs.add(deploymentJob(status.application().require(jobId.application().instance()), job.versions(), job.type(), status.instanceJobs(jobId.application().instance()).get(jobId.type()).isNodeAllocationFailure(), - job.readyAt().get())); + job.readiness().at())); + } }); return Collections.unmodifiableList(jobs); } @@ -403,41 +405,29 @@ public class DeploymentTrigger { return false; } - private void abortIfOutdated(DeploymentStatus status, Map<JobId, List<DeploymentStatus.Job>> jobs, JobId job) { - status.jobs().get(job) - .flatMap(JobStatus::lastTriggered) - .filter(last -> ! last.hasEnded() && last.reason().isEmpty()) - .ifPresent(last -> { - if (jobs.get(job).stream().noneMatch(versions -> versions.versions().targetsMatch(last.versions()) - && versions.versions().sourcesMatchIfPresent(last.versions()))) { - String blocked = jobs.get(job).stream() - .map(scheduled -> scheduled.versions().toString()) - .collect(Collectors.joining(", ")); - log.log(Level.INFO, "Aborting outdated run " + last + ", which is blocking runs: " + blocked); - controller.jobController().abort(last.id(), "run no longer scheduled, and is blocking scheduled runs: " + blocked); - } - }); + private void abortIfOutdated(JobStatus job, List<DeploymentStatus.Job> jobs) { + job.lastTriggered() + .filter(last -> ! last.hasEnded() && last.reason().isEmpty()) + .ifPresent(last -> { + if (jobs.stream().noneMatch(versions -> versions.versions().targetsMatch(last.versions()) + && versions.versions().sourcesMatchIfPresent(last.versions()))) { + String blocked = jobs.stream() + .map(scheduled -> scheduled.versions().toString()) + .collect(Collectors.joining(", ")); + log.log(Level.INFO, "Aborting outdated run " + last + ", which is blocking runs: " + blocked); + controller.jobController().abort(last.id(), "run no longer scheduled, and is blocking scheduled runs: " + blocked); + } + }); } /** Returns whether the job is free to start, and also aborts it if it's running with outdated versions. */ - private boolean abortIfRunning(DeploymentStatus status, Map<JobId, List<DeploymentStatus.Job>> jobs, JobId job) { - abortIfOutdated(status, jobs, job); - boolean blocked = status.jobs().get(job).get().isRunning(); - - if ( ! job.type().isTest()) { - Optional<JobStatus> productionTest = status.jobs().get(new JobId(job.application(), JobType.productionTestOf(job.type().zone()))); - if (productionTest.isPresent()) { - abortIfOutdated(status, jobs, productionTest.get().id()); - // Production deployments are also blocked by their declared tests, if the next versions to run - // for those are not the same as the versions we're considering running in the deployment job now. - if (productionTest.map(JobStatus::id).map(jobs::get) - .map(versions -> ! versions.get(0).versions().targetsMatch(jobs.get(job).get(0).versions())) - .orElse(false)) - blocked = true; - } - } - - return ! blocked; + private void abortIfOutdated(DeploymentStatus status, Map<JobId, List<DeploymentStatus.Job>> jobs, JobId job) { + Readiness readiness = jobs.get(job).get(0).readiness(); + if (readiness.cause() == DelayCause.running) + abortIfOutdated(status.jobs().get(job).get(), jobs.get(job)); + if (readiness.cause() == DelayCause.blocked && ! job.type().isTest()) + status.jobs().get(new JobId(job.application(), JobType.productionTestOf(job.type().zone()))) + .ifPresent(jobStatus -> abortIfOutdated(jobStatus, jobs.get(jobStatus.id()))); } // ---------- Change management o_O ---------- diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 451f5555eb2..7b1a1e879d6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -279,7 +279,7 @@ public class InternalStepRunner implements StepRunner { switch (e.type()) { case CERT_NOT_AVAILABLE: // Same as CERTIFICATE_NOT_READY above, only from the controller - logger.log("Creating a CA signed certificate for the application. " + + logger.log("Retrieving CA signed certificate for the application. " + "This may take up to " + timeouts.endpointCertificate() + " on first deployment."); if (startTime.plus(timeouts.endpointCertificate()).isBefore(controller.clock().instant())) { logger.log(WARNING, "CA signed certificate for app not available within " + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 73c64be3e47..10e4052f067 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -713,6 +713,7 @@ public class JobController { // TODO(mpolden): Enable for public CD once all tests have been updated if (controller.system() != SystemName.PublicCd) { controller.applications().validatePackage(applicationPackage, application.get()); + controller.applications().decideCloudAccountOf(new DeploymentId(id, type.zone()), applicationPackage.deploymentSpec()); } controller.applications().store(application); }); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java index 02e1818932e..cbdfcf70123 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ApplicationOwnershipConfirmer.java @@ -79,7 +79,7 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer { log.log(Level.INFO, "Exception caught when attempting to file an issue for '" + application.id() + "': " + Exceptions.toMessageString(e)); } }); - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactorDeviation(attempts.get(), failures.get()); } private boolean isInCurrentShard(TenantAndApplicationId id) { @@ -122,7 +122,7 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer { log.log(Level.INFO, "Exception caught when attempting to escalate issue with id '" + issueId + "': " + Exceptions.toMessageString(e)); } }); - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactorDeviation(attempts.get(), failures.get()); } private double updateConfirmedApplicationOwners() { @@ -149,7 +149,7 @@ public class ApplicationOwnershipConfirmer extends ControllerMaintainer { log.log(Level.INFO, "Exception caught when attempting to find confirmed owner of issue with id '" + issueId + "': " + Exceptions.toMessageString(e)); } }); - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactorDeviation(attempts.get(), failures.get()); } private ApplicationList applications() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java index 518027f8099..c4f3c611cc5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java @@ -98,7 +98,7 @@ public class ArchiveUriUpdater extends ControllerMaintainer { } } - return asSuccessFactor(tenantsByZone.size(), failures); + return asSuccessFactorDeviation(tenantsByZone.size(), failures); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java index 25c4121c271..32d06286820 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.artifact.Artifact; import com.yahoo.vespa.hosted.controller.api.integration.artifact.ArtifactRegistry; @@ -13,12 +12,9 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import java.time.Duration; import java.time.Instant; import java.util.Comparator; -import java.util.EnumSet; import java.util.List; -import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.stream.Collectors; /** * Periodically expire unused artifacts, e.g. container images and RPMs. @@ -32,7 +28,7 @@ public class ArtifactExpirer extends ControllerMaintainer { private static final Duration MIN_AGE = Duration.ofDays(14); public ArtifactExpirer(Controller controller, Duration interval) { - super(controller, interval, null, expiringSystems()); + super(controller, interval); } @Override @@ -56,10 +52,10 @@ public class ArtifactExpirer extends ControllerMaintainer { log.log(Level.INFO, "Expiring " + artifactsToExpire.size() + " artifacts in " + cloudName + ": " + artifactsToExpire); artifactRegistry.deleteAll(artifactsToExpire); } - return 1; + return 0; } catch (RuntimeException e) { log.log(Level.WARNING, "Failed to expire artifacts in " + cloudName + ". Will retry in " + interval(), e); - return 0; + return 1; } } @@ -77,11 +73,4 @@ public class ArtifactExpirer extends ControllerMaintainer { return true; } - /** Returns systems where artifacts can be expired */ - private static Set<SystemName> expiringSystems() { - // Run only in public and main. Public systems have distinct container registries, while main and CD have - // shared registries. - return EnumSet.of(SystemName.Public, SystemName.PublicCd, SystemName.main); - } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java index 32e6ad0d557..fe1930a19ea 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdater.java @@ -42,11 +42,13 @@ public class BcpGroupUpdater extends ControllerMaintainer { private final ApplicationController applications; private final NodeRepository nodeRepository; + private final Double successFactorBaseline; - public BcpGroupUpdater(Controller controller, Duration duration) { - super(controller, duration); + public BcpGroupUpdater(Controller controller, Duration duration, Double successFactorBaseline) { + super(controller, duration, successFactorBaseline); this.applications = controller.applications(); this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository(); + this.successFactorBaseline = successFactorBaseline; } @Override @@ -58,7 +60,7 @@ public class BcpGroupUpdater extends ControllerMaintainer { for (var application : applications.asList()) { for (var instance : application.instances().values()) { for (var deployment : instance.productionDeployments().values()) { - if (shuttingDown()) return 1.0; + if (shuttingDown()) return 0.0; try { attempts++; var bcpGroups = BcpGroup.groupsFrom(instance, application.deploymentSpec()); @@ -75,12 +77,12 @@ public class BcpGroupUpdater extends ControllerMaintainer { } } } - double successFactor = asSuccessFactor(attempts, failures); - if ( successFactor == 0 ) + double successFactorDeviation = asSuccessFactorDeviation(attempts, failures); + if ( successFactorDeviation == -successFactorBaseline ) log.log(Level.WARNING, "Could not update traffic share on any applications", lastException); - else if ( successFactor < 0.9 ) + else if ( successFactorDeviation < -0.1 ) log.log(Level.FINE, "Could not update traffic share on all applications", lastException); - return successFactor; + return successFactorDeviation; } /** Adds deployment traffic share to the given patch. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java index a7ebaec7c09..b40078eef51 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingDatabaseMaintainer.java @@ -19,6 +19,6 @@ public class BillingDatabaseMaintainer extends ControllerMaintainer { @Override protected double maintain() { controller().serviceRegistry().billingDatabase().maintain(); - return 1; + return 0.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudDatabaseMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudDatabaseMaintainer.java index 914707aa318..68fd5c8bafe 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudDatabaseMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudDatabaseMaintainer.java @@ -19,8 +19,8 @@ public class CloudDatabaseMaintainer extends ControllerMaintainer { controller().serviceRegistry().billingController().updateCache(tenants); } catch (Exception e) { log.warning("Could not update cloud database cache: " + Exceptions.toMessageString(e)); - return 0.0; + return 1.0; } - return 1.0; + return 0.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java index 6ecad482cd2..f9c93a87c44 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java @@ -64,7 +64,7 @@ public class ContactInformationMaintainer extends ControllerMaintainer { interval()); } } - return asSuccessFactor(attempts, failures); + return asSuccessFactorDeviation(attempts, failures); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java index c861d522818..f21803283eb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainer.java @@ -12,7 +12,6 @@ import java.util.EnumSet; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.logging.Logger; /** * A maintainer is some job which runs at a fixed interval to perform some maintenance task in the controller. @@ -26,13 +25,22 @@ public abstract class ControllerMaintainer extends Maintainer { /** The systems in which this maintainer should run */ private final Set<SystemName> activeSystems; + public ControllerMaintainer(Controller controller, Duration interval) { - this(controller, interval, null, EnumSet.allOf(SystemName.class)); + this(controller, interval, null, EnumSet.allOf(SystemName.class), 1.0); + } + + public ControllerMaintainer(Controller controller, Duration interval, Double successFactorBaseline) { + this(controller, interval, null, EnumSet.allOf(SystemName.class), successFactorBaseline); } public ControllerMaintainer(Controller controller, Duration interval, String name, Set<SystemName> activeSystems) { + this(controller, interval, name, activeSystems, 1.0); + } + + public ControllerMaintainer(Controller controller, Duration interval, String name, Set<SystemName> activeSystems, Double successFactorBaseline) { super(name, interval, controller.clock(), controller.jobControl(), - new ControllerJobMetrics(controller.metric()), controller.curator().cluster(), true); + new ControllerJobMetrics(controller.metric()), controller.curator().cluster(), true, successFactorBaseline); this.controller = controller; this.activeSystems = Set.copyOf(Objects.requireNonNull(activeSystems)); } @@ -54,8 +62,8 @@ public abstract class ControllerMaintainer extends Maintainer { } @Override - public void completed(String job, double successFactor, long durationMs) { - metric.set("maintenance.successFactor", successFactor, metric.createContext(Map.of("job", job))); + public void completed(String job, double successFactorDeviation, long durationMs) { + metric.set("maintenance.successFactorDeviation", successFactorDeviation, metric.createContext(Map.of("job", job))); metric.set("maintenance.duration", durationMs, metric.createContext(Map.of("job", job))); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 05159b38ec6..7f48a1115c5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -40,6 +40,7 @@ public class ControllerMaintenance extends AbstractComponent { @SuppressWarnings("unused") // instantiated by Dependency Injection public ControllerMaintenance(Controller controller, Metric metric, UserManagement userManagement, AthenzClientFactory athenzClientFactory) { Intervals intervals = new Intervals(controller.system()); + SuccessFactorBaseline successFactorBaseline = new SuccessFactorBaseline(controller.system()); upgrader = new Upgrader(controller, intervals.defaultInterval); osUpgradeScheduler = new OsUpgradeScheduler(controller, intervals.osUpgradeScheduler); maintainers.add(upgrader); @@ -68,7 +69,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new HostInfoUpdater(controller, intervals.hostInfoUpdater)); maintainers.add(new ReindexingTriggerer(controller, intervals.reindexingTriggerer)); maintainers.add(new EndpointCertificateMaintainer(controller, intervals.endpointCertificateMaintainer)); - maintainers.add(new BcpGroupUpdater(controller, intervals.trafficFractionUpdater)); + maintainers.add(new BcpGroupUpdater(controller, intervals.trafficFractionUpdater, successFactorBaseline.trafficFractionUpdater)); maintainers.add(new ArchiveUriUpdater(controller, intervals.archiveUriUpdater)); maintainers.add(new ArchiveAccessMaintainer(controller, metric, intervals.archiveAccessMaintainer)); maintainers.add(new TenantRoleMaintainer(controller, intervals.tenantRoleMaintainer)); @@ -189,4 +190,15 @@ public class ControllerMaintenance extends AbstractComponent { } + private static class SuccessFactorBaseline { + + private final Double defaultSuccessFactorBaseline; + private final Double trafficFractionUpdater; + + public SuccessFactorBaseline(SystemName system) { + Objects.requireNonNull(system); + this.defaultSuccessFactorBaseline = 1.0; + this.trafficFractionUpdater = system.isCd() ? 0.5 : 0.7; + } + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java index 403b5aed1ce..668893d5a7e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java @@ -34,7 +34,7 @@ public class CostReportMaintainer extends ControllerMaintainer { protected double maintain() { var csv = CostCalculator.resourceShareByPropertyToCsv(nodeRepository, controller(), clock, consumer.fixedAllocations()); consumer.consume(csv); - return 1.0; + return 0.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java index 97f3f955a20..c22cb1efdb3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentExpirer.java @@ -5,8 +5,6 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.yolean.Exceptions; @@ -47,7 +45,7 @@ public class DeploymentExpirer extends ControllerMaintainer { } } } - return asSuccessFactor(attempts, failures); + return asSuccessFactorDeviation(attempts, failures); } /** Returns whether given deployment has expired according to its TTL */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java index f6029eade37..d8d89177a9e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java @@ -38,7 +38,7 @@ public class DeploymentInfoMaintainer extends ControllerMaintainer { } } } - return asSuccessFactor(attempts, failures); + return asSuccessFactorDeviation(attempts, failures); } private Collection<DeploymentId> instanceDeployments(Instance instance) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java index 6b058537c2d..c352fb053dc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporter.java @@ -77,7 +77,7 @@ public class DeploymentIssueReporter extends ControllerMaintainer { fileDeploymentIssueFor(application); else store(application.id(), null); - return 1.0; + return 0.0; } /** @@ -87,24 +87,24 @@ public class DeploymentIssueReporter extends ControllerMaintainer { */ private double maintainPlatformIssue(List<Application> applications) { if (controller().system() == SystemName.cd) - return 1.0; + return 0.0; VersionStatus versionStatus = controller().readVersionStatus(); Version systemVersion = controller().systemVersion(versionStatus); if (versionStatus.version(systemVersion).confidence() != broken) - return 1.0; + return 0.0; DeploymentStatusList statuses = controller().jobController().deploymentStatuses(ApplicationList.from(applications)); if (statuses.failingUpgradeToVersionSince(systemVersion, controller().clock().instant().minus(upgradeGracePeriod)).isEmpty()) - return 1.0; + return 0.0; List<ApplicationId> failingApplications = statuses.failingUpgradeToVersionSince(systemVersion, controller().clock().instant()) .mapToList(status -> status.application().id().defaultInstance()); // TODO jonmv: Send only tenant and application, here and elsewhere in this. deploymentIssues.fileUnlessOpen(failingApplications, systemVersion); - return 1.0; + return 0.0; } private Tenant ownerOf(TenantAndApplicationId applicationId) { @@ -145,7 +145,7 @@ public class DeploymentIssueReporter extends ControllerMaintainer { log.log(Level.INFO, "Exception caught when attempting to escalate issue with id '" + issueId + "': " + Exceptions.toMessageString(e)); } })); - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactorDeviation(attempts.get(), failures.get()); } private void store(TenantAndApplicationId id, IssueId issueId) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java index fa917d2eb4e..427ee7c4b06 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java @@ -96,7 +96,7 @@ public class DeploymentMetricsMaintainer extends ControllerMaintainer { } catch (InterruptedException e) { throw new RuntimeException(e); } - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactorDeviation(attempts.get(), failures.get()); } static DeploymentMetrics updateDeploymentMetrics(DeploymentMetrics current, List<ClusterMetrics> metrics) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java index 934a1b4fa2f..8a3a2a11e09 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java @@ -78,7 +78,7 @@ public class DeploymentUpgrader extends ControllerMaintainer { ": " + Exceptions.toMessageString(e) + ". Retrying in " + interval()); } - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactorDeviation(attempts.get(), failures.get()); } /** Returns whether query and feed metrics are ~zero, or currently platform has been deployed for a week. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java index 7af96d10f2f..5218da91c46 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EnclaveAccessMaintainer.java @@ -26,7 +26,7 @@ public class EnclaveAccessMaintainer extends ControllerMaintainer { return controller().serviceRegistry().enclaveAccessService().allowAccessFor(externalAccounts()); } catch (RuntimeException e) { logger.log(WARNING, "Failed sharing resources with enclave", e); - return 0; + return 1.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index 0b96d8adc1a..713782eb7b9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -75,10 +75,10 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { deleteOrReportUnmanagedCertificates(); } catch (Exception e) { log.log(Level.SEVERE, "Exception caught while maintaining endpoint certificates", e); - return 0.0; + return 1.0; } - return 1.0; + return 0.0; } private void updateRefreshedCertificates() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java index 1f21c688540..31236f4fcda 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/HostInfoUpdater.java @@ -73,7 +73,7 @@ public class HostInfoUpdater extends ControllerMaintainer { LOG.info("Updated information for " + hostsUpdated + " hosts(s)"); } } - return 1.0; + return 0.0; } private static Optional<String> modelNameOf(NodeEntity nodeEntity) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java index b051590ac5a..68c79fdca7e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java @@ -50,7 +50,7 @@ public abstract class InfrastructureUpgrader<TARGET extends VersionTarget> exten @Override protected double maintain() { return target().map(target -> upgradeAll(target, managedApplications)) - .orElse(1.0); + .orElse(0.0); } /** Deploy a list of system applications until they converge on the given version */ @@ -81,7 +81,7 @@ public abstract class InfrastructureUpgrader<TARGET extends VersionTarget> exten break; } } - return asSuccessFactor(attempts, failures); + return asSuccessFactorDeviation(attempts, failures); } /** Returns whether all applications have converged to the target version in zone */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 294d5bad42d..67188eb5e3a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -90,7 +90,7 @@ public class MetricsReporter extends ControllerMaintainer { reportBrokenSystemVersion(versionStatus); reportTenantMetrics(); reportZmsQuotaMetrics(); - return 1.0; + return 0.0; } private void reportBrokenSystemVersion(VersionStatus versionStatus) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java index f930e64fc5a..f0d218ae6cf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgradeScheduler.java @@ -40,7 +40,7 @@ public class OsUpgradeScheduler extends ControllerMaintainer { if (!change.get().scheduleAt(now)) continue; controller().upgradeOsIn(cloud, change.get().version(), false); } - return 1.0; + return 0.0; } /** Returns the wanted change for cloud at given instant, if any */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java index 119540eaa68..c643df6af68 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdater.java @@ -22,12 +22,12 @@ public class OsVersionStatusUpdater extends ControllerMaintainer { try { OsVersionStatus newStatus = OsVersionStatus.compute(controller()); controller().updateOsVersionStatus(newStatus); - return 1.0; + return 0.0; } catch (Exception e) { log.log(Level.WARNING, "Failed to compute OS version status: " + Exceptions.toMessageString(e) + ". Retrying in " + interval()); } - return 0.0; + return 1.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java index 400673bfd0c..945b6d32a30 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ReindexingTriggerer.java @@ -53,11 +53,11 @@ public class ReindexingTriggerer extends ControllerMaintainer { controller().applications().reindex(id, deployment.zone(), List.of(), List.of(), true, speed, "bakground reindexing, to account for changes in built-in linguistics components"); }); - return 1.0; + return 0.0; } catch (RuntimeException e) { log.log(Level.WARNING, "Failed to trigger reindexing: " + Exceptions.toMessageString(e)); - return 0.0; + return 1.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index 3f20c2eac8f..52206d41c00 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -98,13 +98,13 @@ public class ResourceMeterMaintainer extends ControllerMaintainer { } catch (Exception e) { log.log(Level.WARNING, "Failed to collect resource snapshots. Retrying in " + interval() + ". Error: " + Exceptions.toMessageString(e)); - return 0.0; + return 1.0; } if (systemName.isPublic()) reportResourceSnapshots(resourceSnapshots); if (systemName.isPublic()) reportAllScalingEvents(); updateDeploymentCost(resourceSnapshots); - return 1.0; + return 0.0; } void updateDeploymentCost(Collection<ResourceSnapshot> resourceSnapshots) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java index 18ed154fcf1..59871f716e0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceTagMaintainer.java @@ -43,7 +43,7 @@ public class ResourceTagMaintainer extends ControllerMaintainer { if (taggedResources > 0) log.log(Level.INFO, "Tagged " + taggedResources + " resources in " + zone.getId()); }); - return 1.0; + return 0.0; } private Map<HostName, ApplicationId> getTenantOfParentHosts(ZoneId zoneId) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java index 74bb89e4105..aaf730cc158 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RetriggerMaintainer.java @@ -47,9 +47,9 @@ public class RetriggerMaintainer extends ControllerMaintainer { controller().curator().writeRetriggerEntries(remaining); } catch (Exception e) { logger.log(Level.WARNING, "Exception while triggering jobs", e); - return 0.0; + return 1.0; } - return 1.0; + return 0.0; } /** Returns true if a job is ready to run, i.e. is currently not running */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleCleanupMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleCleanupMaintainer.java index f29df1bc0d5..e3a3415e170 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleCleanupMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleCleanupMaintainer.java @@ -26,6 +26,6 @@ public class TenantRoleCleanupMaintainer extends ControllerMaintainer { controller().serviceRegistry().tenantSecretService().cleanupSecretStores(deletedTenants); } - return 1.0; + return 0.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java index bd121871c7c..c7b236880fd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/TenantRoleMaintainer.java @@ -37,7 +37,7 @@ public class TenantRoleMaintainer extends ControllerMaintainer { controller().tenants().updateLastTenantRolesMaintained(t.name(), updated); }); - return 1.0; + return 0.0; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index 82b3141e503..edcfcc317a7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -64,7 +64,7 @@ public class Upgrader extends ControllerMaintainer { for (UpgradePolicy policy : UpgradePolicy.values()) updateTargets(versionStatus, deploymentStatuses, policy); - return 1.0; + return 0.0; } private DeploymentStatusList deploymentStatuses(VersionStatus versionStatus) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java index 03987efab8b..7c4645a6e48 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/UserManagementMaintainer.java @@ -11,6 +11,7 @@ import java.util.Optional; import java.util.logging.Logger; import java.util.stream.Collectors; + /** * Maintains user management resources. * For now, ensures there's no discrepnacy between expected tenant/application roles and auth0/athenz roles @@ -46,7 +47,7 @@ public class UserManagementMaintainer extends ControllerMaintainer { }); } - return 1.0; + return 0.0; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java index da0fa890960..9271f870390 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java @@ -90,7 +90,7 @@ public class VcmrMaintainer extends ControllerMaintainer { } }); updateMetrics(); - return 1.0; + return 0.0; } /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java index 154455c5198..1c4d13aa16d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java @@ -39,12 +39,12 @@ public class VersionStatusUpdater extends ControllerMaintainer { controller().serviceRegistry().systemMonitor().reportSystemVersion(version.versionNumber(), convert(version.confidence())); }); - return 1.0; + return 0.0; } catch (Exception e) { log.log(Level.WARNING, "Failed to compute version status: " + Exceptions.toMessageString(e) + ". Retrying in " + interval()); } - return 0.0; + return 1.0; } static SystemMonitor.Confidence convert(VespaVersion.Confidence confidence) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 8a0e2d01d8c..81988753621 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -131,7 +131,6 @@ import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; - import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -1897,7 +1896,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { JobControllerApiHandlerHelper.toSlime(response.setObject("applicationVersion"), application.revisions().get(deployment.revision())); if ( ! status.jobsToRun().containsKey(stepStatus.job().get())) response.setString("status", "complete"); - else if (stepStatus.readyAt(instance.change()).map(controller.clock().instant()::isBefore).orElse(true)) + else if ( ! stepStatus.readiness(instance.change()).okAt(controller.clock().instant())) response.setString("status", "pending"); else response.setString("status", "running"); @@ -3004,7 +3003,9 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { byte[] testPackage = dataParts.getOrDefault(EnvironmentResource.APPLICATION_TEST_ZIP, new byte[0]); Submission submission = new Submission(applicationPackage, testPackage, sourceUrl, sourceRevision, authorEmail, description, risk); - controller.applications().verifyApplicationIdentityConfiguration(TenantName.from(tenant), + TenantName tenantName = TenantName.from(tenant); + controller.applications().verifyPlan(tenantName); + controller.applications().verifyApplicationIdentityConfiguration(tenantName, Optional.empty(), Optional.empty(), applicationPackage, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index ce60e0054c4..804ae7b7805 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -15,7 +15,6 @@ import com.yahoo.text.Text; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.NotExistsException; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; @@ -26,6 +25,8 @@ import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.deployment.ConvergenceSummary; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.DelayCause; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.Readiness; import com.yahoo.vespa.hosted.controller.deployment.JobController; import com.yahoo.vespa.hosted.controller.deployment.JobStatus; import com.yahoo.vespa.hosted.controller.deployment.Run; @@ -269,17 +270,39 @@ class JobControllerApiHandlerHelper { stepObject.setString("instance", stepStatus.instance().value()); // TODO: recursively search dependents for what is the relevant partial change when this is a delay step ... - Optional<Instant> readyAt = stepStatus.job().map(jobsToRun::get).map(jobs -> jobs.get(0).readyAt()) - .orElse(stepStatus.readyAt(change)); - readyAt.ifPresent(ready -> stepObject.setLong("readyAt", ready.toEpochMilli())); - readyAt.filter(controller.clock().instant()::isBefore) - .ifPresent(until -> stepObject.setLong("delayedUntil", until.toEpochMilli())); - stepStatus.pausedUntil().ifPresent(until -> stepObject.setLong("pausedUntil", until.toEpochMilli())); - stepStatus.coolingDownUntil(change, Optional.empty()).ifPresent(until -> stepObject.setLong("coolingDownUntil", until.toEpochMilli())); - stepStatus.blockedUntil(Change.of(controller.systemVersion(versionStatus))) // Dummy version — just anything with a platform. - .ifPresent(until -> stepObject.setLong("platformBlockedUntil", until.toEpochMilli())); - stepStatus.blockedUntil(Change.of(RevisionId.forProduction(1))) // Dummy version — just anything with an application. - .ifPresent(until -> stepObject.setLong("applicationBlockedUntil", until.toEpochMilli())); + Readiness readiness = stepStatus.job().map(jobsToRun::get).map(job -> job.get(0).readiness()) + .orElse(stepStatus.readiness(change)); + if (readiness.ok()) { + stepObject.setLong("readyAt", readiness.at().toEpochMilli()); + if ( ! readiness.okAt(controller.clock().instant())) { + Instant until = readiness.at(); + stepObject.setLong("delayedUntil", readiness.at().toEpochMilli()); + switch (readiness.cause()) { + case paused -> stepObject.setLong("pausedUntil", until.toEpochMilli()); + case coolingDown -> stepObject.setLong("coolingDownUntil", until.toEpochMilli()); + case changeBlocked -> { + Readiness platformReadiness = stepStatus.readiness(Change.of(controller.systemVersion(versionStatus))); // Dummy version — just anything with a platform. + if (platformReadiness.cause() == DelayCause.changeBlocked) + stepObject.setLong("platformBlockedUntil", platformReadiness.at().toEpochMilli()); + Readiness applicationReadiness = stepStatus.readiness(Change.of(RevisionId.forProduction(1))); // Dummy version — just anything with an application. + if (applicationReadiness.cause() == DelayCause.changeBlocked) + stepObject.setLong("applicationBlockedUntil", applicationReadiness.at().toEpochMilli()); + } + } + } + } + stepObject.setString("delayCause", + switch (readiness.cause()) { + case none -> null; + case invalidPackage -> "invalidPackage"; + case paused -> "paused"; + case coolingDown -> "coolingDown"; + case changeBlocked -> "changeBlocked"; + case blocked -> "blocked"; + case running -> "running"; + case notReady -> "notReady"; + case unverified -> "unverified"; + }); if (stepStatus.type() == DeploymentStatus.StepType.delay) stepStatus.completedAt(change).ifPresent(completed -> stepObject.setLong("completedAt", completed.toEpochMilli())); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index 759b2366229..069ee58e9c5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -22,6 +22,8 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.DelayCause; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.Readiness; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.deployment.Versions; @@ -172,8 +174,9 @@ public class DeploymentApiHandler extends ThreadedHttpRequestHandler { instanceObject.setBool("pinned", status.application().require(instance.instance()).change().isPinned()); DeploymentStatus.StepStatus stepStatus = status.instanceSteps().get(instance.instance()); if (stepStatus != null) { // Instance may not have any steps, i.e. an empty deployment spec has been submitted - stepStatus.blockedUntil(Change.of(statistics.version())) - .ifPresent(until -> instanceObject.setLong("blockedUntil", until.toEpochMilli())); + Readiness platformReadiness = stepStatus.blockedUntil(Change.of(statistics.version())); + if (platformReadiness.cause() == DelayCause.changeBlocked) + instanceObject.setLong("blockedUntil", platformReadiness.at().toEpochMilli()); } instanceObject.setString("upgradePolicy", toString(status.application().deploymentSpec().instance(instance.instance()) .map(DeploymentInstanceSpec::upgradePolicy) @@ -185,10 +188,12 @@ public class DeploymentApiHandler extends ThreadedHttpRequestHandler { if ( ! job.application().equals(instance)) return; Cursor jobObject = jobsArray.addObject(); jobObject.setString("name", job.type().jobName()); - jobStatus.pausedUntil().ifPresent(until -> jobObject.setLong("pausedUntil", until.toEpochMilli())); - jobStatus.coolingDownUntil(status.application().require(instance.instance()).change(), Optional.empty()) - .ifPresent(until -> jobObject.setLong("coolingDownUntil", until.toEpochMilli())); if (jobsToRun.containsKey(job)) { + Readiness readiness = jobsToRun.get(job).get(0).readiness(); + switch (readiness.cause()) { + case paused -> jobObject.setLong("pausedUntil", readiness.at().toEpochMilli()); + case coolingDown -> jobObject.setLong("coolingDownUntil", readiness.at().toEpochMilli()); + } List<Versions> versionsOnThisPlatform = jobsToRun.get(job).stream() .map(DeploymentStatus.Job::versions) .filter(versions -> versions.targetPlatform().equals(statistics.version())) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java index e9947f3d565..45c00848407 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java @@ -41,8 +41,8 @@ public record VespaVersion(Version version, .not().upgradingTo(statistics.version()); InstanceList failingOnThis = all.matching(instance -> statistics.failingUpgrades().stream().anyMatch(run -> run.id().application().equals(instance))); - // 'broken' if any canary fails - if ( ! failingOnThis.with(UpgradePolicy.canary).isEmpty()) + // 'broken' if any canary fails, and no non-canary is upgraded + if ( ! failingOnThis.with(UpgradePolicy.canary).isEmpty() && productionOnThis.not().with(UpgradePolicy.canary).isEmpty()) return Confidence.broken; // 'broken' if 6 non-canary was broken by this, and that is at least 5% of all diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 8f0536480f5..a9a6fe602b6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -22,6 +22,7 @@ import com.yahoo.path.Path; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistryMock; import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; @@ -54,7 +55,6 @@ import com.yahoo.vespa.hosted.controller.routing.rotation.RotationLock; import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import org.junit.jupiter.api.Test; - import java.io.InputStream; import java.time.Duration; import java.time.Instant; @@ -1429,9 +1429,14 @@ public class ControllerTest { // Deployment fails because zone is not configured in requested cloud account tester.controllerTester().flagSource().withListFlag(PermanentFlags.CLOUD_ACCOUNTS.id(), List.of(cloudAccount), String.class); - context.submit(applicationPackage) - .runJobExpectingFailure(systemTest, "Zone test.us-east-1 is not configured in requested cloud account '012345678912'") - .abortJob(stagingTest); + assertEquals("Zone test.us-east-1 is not configured in requested cloud account '012345678912'", + assertThrows(IllegalArgumentException.class, + () -> context.submit(applicationPackage)) + .getMessage()); + assertEquals("Zone dev.us-east-1 is not configured in requested cloud account '012345678912'", + assertThrows(IllegalArgumentException.class, + () -> context.runJob(devUsEast1, applicationPackage)) + .getMessage()); // Deployment to prod succeeds once all zones are configured in requested account tester.controllerTester().zoneRegistry().configureCloudAccount(CloudAccount.from(cloudAccount), @@ -1508,4 +1513,19 @@ public class ControllerTest { assertFalse(tester.configServer().application(deployment.applicationId(), deployment.zoneId()).isPresent()); } + @Test + void testVerifyPlan() { + DeploymentId deployment = tester.newDeploymentContext().deploymentIdIn(ZoneId.from("prod", "us-west-1")); + TenantName tenant = deployment.applicationId().tenant(); + + tester.controller().serviceRegistry().billingController().setPlan(tenant, PlanRegistryMock.nonePlan.id(), false, false); + try { + tester.controller().applications().verifyPlan(tenant); + fail("should have thrown an exception"); + } catch (IllegalArgumentException e) { + assertEquals("Tenant 'tenant' has a plan 'None Plan - for testing purposes' with zero quota, not allowed to deploy. " + + "See https://cloud.vespa.ai/support", e.getMessage()); + } + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java index a76d2eca521..ca31ceebc17 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java @@ -126,7 +126,7 @@ public class EndpointTest { Endpoint.of(instance1).target(cluster, prodZone).on(Port.tls()).in(SystemName.main), // Prod endpoint in CD - "https://cd.a1.t1.us-north-1.vespa.oath.cloud/", + "https://cd.a1.t1.us-north-1.cd.vespa.oath.cloud/", Endpoint.of(instance1).target(cluster, prodZone).on(Port.tls()).in(SystemName.cd), // Test endpoint in main @@ -300,7 +300,7 @@ public class EndpointTest { .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) .in(SystemName.main), - "cd.a2.t2.us-east-3-r.vespa.oath.cloud", + "cd.a2.t2.us-east-3-r.cd.vespa.oath.cloud", Endpoint.of(app2) .targetApplication(EndpointId.defaultId(), ClusterSpec.Id.from("qrs"), Map.of(new DeploymentId(app2.instance("i1"), ZoneId.from("prod", "us-east-3")), 1)) @@ -335,7 +335,7 @@ public class EndpointTest { .routingMethod(RoutingMethod.exclusive) .on(Port.tls()) .in(SystemName.main), - "https://cd.a2.t2.a.vespa.oath.cloud/", + "https://cd.a2.t2.a.cd.vespa.oath.cloud/", Endpoint.of(app2) .targetApplication(EndpointId.defaultId(), ClusterSpec.Id.from("qrs"), Map.of(new DeploymentId(app2.instance("i1"), ZoneId.from("prod", "us-east-3")), 1)) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 226fb785bf6..afb92d84f3b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -756,8 +756,8 @@ public class DeploymentTriggerTest { // Last job has a different deployment target, so tests need to run again. app1.runJob(productionEuWest1) // Upgrade completes, and revision is the only change. - .runJob(productionUsCentral1) // With only revision change, central should run to cover a previous failure. - .runJob(productionEuWest1); // Finally, west changes revision. + .runJob(productionUsCentral1) // With only revision change, central should run to cover a previous failure. + .runJob(productionEuWest1); // Finally, west changes revision. assertEquals(Change.empty(), app1.instance().change()); assertEquals(Optional.of(RunStatus.success), app1.instanceJobs().get(productionUsCentral1).lastStatus()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdaterTest.java index 5deba19c5ea..b8b05bfcfc1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BcpGroupUpdaterTest.java @@ -45,7 +45,7 @@ public class BcpGroupUpdaterTest { tester.controllerTester().upgradeSystem(Version.fromString("7.1")); var context = tester.newDeploymentContext(); var deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(tester.controller(), Duration.ofDays(1)); - var updater = new BcpGroupUpdater(tester.controller(), Duration.ofDays(1)); + var updater = new BcpGroupUpdater(tester.controller(), Duration.ofDays(1), 1.0); ZoneId prod1 = ZoneId.from("prod", "ap-northeast-1"); ZoneId prod2 = ZoneId.from("prod", "us-east-3"); ZoneId prod3 = ZoneId.from("prod", "us-west-1"); @@ -57,7 +57,7 @@ public class BcpGroupUpdaterTest { setQpsMetric(50.0, context.application().id().defaultInstance(), prod1, tester); setBcpMetrics(1.5, 0.1, 0.45, context.instanceId(), prod1, "cluster1", tester); deploymentMetricsMaintainer.maintain(); - assertEquals(1.0, updater.maintain(), 0.0000001); + assertEquals(0.0, updater.maintain(), 0.0000001); assertTrafficFraction(1.0, 1.0, context.instanceId(), prod1, tester); assertNoBcpGroupInfo(context.instanceId(), prod1, "cluster1", tester, "No other regions in group"); @@ -67,7 +67,7 @@ public class BcpGroupUpdaterTest { setQpsMetric(20.0, context.application().id().defaultInstance(), prod2, tester); setBcpMetrics(100.0, 0.1, 0.45, context.instanceId(), prod1, "cluster1", tester); deploymentMetricsMaintainer.maintain(); - assertEquals(1.0, updater.maintain(), 0.0000001); + assertEquals(0.0, updater.maintain(), 0.0000001); assertTrafficFraction(0.75, 1.0, context.instanceId(), prod1, tester); assertTrafficFraction(0.25, 1.0, context.instanceId(), prod2, tester); assertNoBcpGroupInfo(context.instanceId(), prod1, "cluster1", tester, @@ -75,7 +75,7 @@ public class BcpGroupUpdaterTest { assertBcpGroupInfo(100.0, 0.1, 0.45, context.instanceId(), prod2, "cluster1", tester); setBcpMetrics(50.0, 0.2, 0.5, context.instanceId(), prod2, "cluster1", tester); - assertEquals(1.0, updater.maintain(), 0.0000001); + assertEquals(0.0, updater.maintain(), 0.0000001); assertBcpGroupInfo(50.0, 0.2, 0.5, context.instanceId(), prod1, "cluster1", tester); @@ -85,7 +85,7 @@ public class BcpGroupUpdaterTest { setQpsMetric(45.0, context.application().id().defaultInstance(), prod2, tester); setQpsMetric(02.0, context.application().id().defaultInstance(), prod3, tester); deploymentMetricsMaintainer.maintain(); - assertEquals(1.0, updater.maintain(), 0.0000001); + assertEquals(0.0, updater.maintain(), 0.0000001); assertTrafficFraction(0.53, 0.53 + (double)45/2 / 100, context.instanceId(), prod1, tester); assertTrafficFraction(0.45, 0.45 + (double)53/2 / 100, context.instanceId(), prod2, tester); assertTrafficFraction(0.02, 0.02 + (double)53/2 / 100, context.instanceId(), prod3, tester); @@ -129,7 +129,7 @@ public class BcpGroupUpdaterTest { locked -> tester.controller().applications().store(locked.with(deploymentSpec))); var deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(tester.controller(), Duration.ofDays(1)); - var updater = new BcpGroupUpdater(tester.controller(), Duration.ofDays(1)); + var updater = new BcpGroupUpdater(tester.controller(), Duration.ofDays(1), 1.0); ZoneId ap1 = ZoneId.from("prod", "ap-northeast-1"); ZoneId ap2 = ZoneId.from("prod", "ap-southeast-1"); @@ -150,7 +150,7 @@ public class BcpGroupUpdaterTest { setQpsMetric(40.0, context.application().id().defaultInstance(), eu1, tester); deploymentMetricsMaintainer.maintain(); - assertEquals(1.0, updater.maintain(), 0.0000001); + assertEquals(0.0, updater.maintain(), 0.0000001); assertTrafficFraction(0.5, 0.5, context.instanceId(), ap1, tester); assertTrafficFraction(0.0, 0.5, context.instanceId(), ap2, tester); assertTrafficFraction(0.1, 0.1, context.instanceId(), us1, tester); @@ -197,7 +197,7 @@ public class BcpGroupUpdaterTest { locked -> tester.controller().applications().store(locked.with(deploymentSpec))); var deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(tester.controller(), Duration.ofDays(1)); - var updater = new BcpGroupUpdater(tester.controller(), Duration.ofDays(1)); + var updater = new BcpGroupUpdater(tester.controller(), Duration.ofDays(1), 1.0); ZoneId ap1 = ZoneId.from("prod", "ap-northeast-1"); ZoneId ap2 = ZoneId.from("prod", "ap-southeast-1"); @@ -221,7 +221,7 @@ public class BcpGroupUpdaterTest { setQpsMetric(60.0, context.application().id().defaultInstance(), eu1, tester); deploymentMetricsMaintainer.maintain(); - assertEquals(1.0, updater.maintain(), 0.0000001); + assertEquals(0.0, updater.maintain(), 0.0000001); assertTrafficFraction(0.10, 0.10 + 50 / 200.0 / 1.5, context.instanceId(), ap1, tester); assertTrafficFraction(0.25, 0.25 + 30 / 200.0 / 1.5, context.instanceId(), ap2, tester); assertTrafficFraction(0.00, 0.00 + 40 / 200.0 / 2.5, context.instanceId(), us1, tester); @@ -242,7 +242,7 @@ public class BcpGroupUpdaterTest { setBcpMetrics(300, 0.3, 0.3, context.instanceId(), us3, "cluster2", tester); setBcpMetrics(100, 0.1, 0.1, context.instanceId(), eu1, "cluster2", tester); - assertEquals(1.0, updater.maintain(), 0.0000001); + assertEquals(0.0, updater.maintain(), 0.0000001); assertNoBcpGroupInfo(context.instanceId(), ap1, "cluster1", tester, "No info in ap"); assertNoBcpGroupInfo(context.instanceId(), ap2, "cluster1", tester, "No info in ap"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java index 63e2c99cb6e..6452edc9e61 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintainerTest.java @@ -38,14 +38,14 @@ public class ControllerMaintainerTest { void records_metric() { TestControllerMaintainer maintainer = new TestControllerMaintainer(tester.controller(), SystemName.main, new AtomicInteger()); maintainer.run(); - assertEquals(1.0, successFactorMetric(), 0.0000001); + assertEquals(0.0, successFactorDeviationMetric(), 0.0000001); maintainer.success = false; maintainer.run(); maintainer.run(); - assertEquals(0.0, successFactorMetric(), 0.0000001); + assertEquals(1.0, successFactorDeviationMetric(), 0.0000001); maintainer.success = true; maintainer.run(); - assertEquals(1.0, successFactorMetric(), 0.0000001); + assertEquals(0.0, successFactorDeviationMetric(), 0.0000001); } private long consecutiveFailuresMetric() { @@ -54,10 +54,10 @@ public class ControllerMaintainerTest { "maintenance.consecutiveFailures").get().longValue(); } - private long successFactorMetric() { + private long successFactorDeviationMetric() { MetricsMock metrics = (MetricsMock) tester.controller().metric(); return metrics.getMetric((context) -> "TestControllerMaintainer".equals(context.get("job")), - "maintenance.successFactor").get().longValue(); + "maintenance.successFactorDeviation").get().longValue(); } private static class TestControllerMaintainer extends ControllerMaintainer { @@ -73,7 +73,7 @@ public class ControllerMaintainerTest { @Override protected double maintain() { executions.incrementAndGet(); - return success ? 1.0 : 0.0; + return success ? 0.0 : 1.0; } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java index 934e15ad623..49cf8c634ba 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java @@ -49,7 +49,7 @@ public class EndpointCertificateMaintainerTest { @Test void old_and_unused_cert_is_deleted() { tester.curator().writeEndpointCertificateMetadata(ApplicationId.defaultId(), exampleMetadata); - assertEquals(1.0, maintainer.maintain(), 0.0000001); + assertEquals(0.0, maintainer.maintain(), 0.0000001); assertTrue(tester.curator().readEndpointCertificateMetadata(ApplicationId.defaultId()).isEmpty()); } @@ -57,7 +57,7 @@ public class EndpointCertificateMaintainerTest { void unused_but_recently_used_cert_is_not_deleted() { EndpointCertificateMetadata recentlyRequestedCert = exampleMetadata.withLastRequested(tester.clock().instant().minusSeconds(3600).getEpochSecond()); tester.curator().writeEndpointCertificateMetadata(ApplicationId.defaultId(), recentlyRequestedCert); - assertEquals(1.0, maintainer.maintain(), 0.0000001); + assertEquals(0.0, maintainer.maintain(), 0.0000001); assertEquals(Optional.of(recentlyRequestedCert), tester.curator().readEndpointCertificateMetadata(ApplicationId.defaultId())); } @@ -69,7 +69,7 @@ public class EndpointCertificateMaintainerTest { secretStore.setSecret(exampleMetadata.keyName(), "foo", 1); secretStore.setSecret(exampleMetadata.certName(), "bar", 1); - assertEquals(1.0, maintainer.maintain(), 0.0000001); + assertEquals(0.0, maintainer.maintain(), 0.0000001); var updatedCert = Optional.of(recentlyRequestedCert.withLastRefreshed(tester.clock().instant().getEpochSecond()).withVersion(1)); @@ -90,7 +90,7 @@ public class EndpointCertificateMaintainerTest { deploymentContext.submit(applicationPackage).runJob(systemTest).runJob(stagingTest).runJob(productionUsWest1); - assertEquals(1.0, maintainer.maintain(), 0.0000001); + assertEquals(0.0, maintainer.maintain(), 0.0000001); var metadata = tester.curator().readEndpointCertificateMetadata(appId).orElseThrow(); tester.controller().serviceRegistry().endpointCertificateProvider().certificateDetails(metadata.rootRequestId()); // cert should not be deleted, the app is deployed! } @@ -110,7 +110,7 @@ public class EndpointCertificateMaintainerTest { var originalMetadata = tester.curator().readEndpointCertificateMetadata(appId).orElseThrow(); // cert should not be deleted, the app is deployed! - assertEquals(1.0, maintainer.maintain(), 0.0000001); + assertEquals(0.0, maintainer.maintain(), 0.0000001); assertEquals(tester.curator().readEndpointCertificateMetadata(appId), Optional.of(originalMetadata)); tester.controller().serviceRegistry().endpointCertificateProvider().certificateDetails(originalMetadata.rootRequestId()); @@ -121,7 +121,7 @@ public class EndpointCertificateMaintainerTest { tester.controller().serviceRegistry().endpointCertificateProvider().requestCaSignedCertificate(appId, originalMetadata.requestedDnsSans(), Optional.of(originalMetadata)); // We should now pick up the new key and cert version + uuid, but not force trigger deployment yet - assertEquals(1.0, maintainer.maintain(), 0.0000001); + assertEquals(0.0, maintainer.maintain(), 0.0000001); deploymentContext.assertNotRunning(productionUsWest1); var updatedMetadata = tester.curator().readEndpointCertificateMetadata(appId).orElseThrow(); assertNotEquals(originalMetadata.leafRequestId().orElseThrow(), updatedMetadata.leafRequestId().orElseThrow()); @@ -130,7 +130,7 @@ public class EndpointCertificateMaintainerTest { // after another 4 days, we should force trigger deployment if it hasn't already happened tester.clock().advance(Duration.ofDays(4).plusSeconds(1)); deploymentContext.assertNotRunning(productionUsWest1); - assertEquals(1.0, maintainer.maintain(), 0.0000001); + assertEquals(0.0, maintainer.maintain(), 0.0000001); deploymentContext.assertRunning(productionUsWest1); } @@ -156,7 +156,7 @@ public class EndpointCertificateMaintainerTest { ApplicationId unknown = ApplicationId.fromSerializedForm("applicationid:is:unknown"); endpointCertificateProvider.requestCaSignedCertificate(unknown, List.of("a", "b", "c"), Optional.empty()); // Unknown to controller! - assertEquals(1.0, maintainer.maintain(), 0.0000001); + assertEquals(0.0, maintainer.maintain(), 0.0000001); assertTrue(endpointCertificateProvider.dnsNamesOf(unknown).isEmpty()); assertTrue(endpointCertificateProvider.listCertificates().isEmpty()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json index a02fb1fb375..6793553faca 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json @@ -9,6 +9,7 @@ "declared": true, "instance": "default", "readyAt": 0, + "delayCause": null, "deploying": { "application": { "build": 3, @@ -155,6 +156,7 @@ "declared": false, "instance": "default", "readyAt": 0, + "delayCause": null, "jobName": "system-test", "url": "https://some.url:43/instance/default/job/system-test", "environment": "test", @@ -344,6 +346,7 @@ "readyAt": 15153000, "delayedUntil": 15153000, "coolingDownUntil": 15153000, + "delayCause": "coolingDown", "jobName": "staging-test", "url": "https://some.url:43/instance/default/job/staging-test", "environment": "staging", @@ -777,6 +780,8 @@ "declared": true, "instance": "default", "readyAt": 14403000, + "delayedUntil": 14403000, + "delayCause": "running", "jobName": "production-us-central-1", "url": "https://some.url:43/instance/default/job/production-us-central-1", "environment": "prod", @@ -902,6 +907,7 @@ ], "declared": true, "instance": "default", + "delayCause": "notReady", "jobName": "test-us-central-1", "url": "https://some.url:43/instance/default/job/test-us-central-1", "environment": "prod", @@ -1042,6 +1048,7 @@ ], "declared": true, "instance": "default", + "delayCause": "notReady", "jobName": "production-us-west-1", "url": "https://some.url:43/instance/default/job/production-us-west-1", "environment": "prod", @@ -1150,6 +1157,7 @@ ], "declared": true, "instance": "default", + "delayCause": "notReady", "jobName": "production-us-east-3", "url": "https://some.url:43/instance/default/job/production-us-east-3", "environment": "prod", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json index 35dd6fc5398..ec6ccf3ecf2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json @@ -9,6 +9,7 @@ "declared": true, "instance": "instance1", "readyAt": 0, + "delayCause": null, "deploying": { "application": { "build": 4, @@ -59,6 +60,8 @@ "declared": false, "instance": "instance1", "readyAt": 0, + "delayedUntil": 0, + "delayCause": "running", "jobName": "system-test", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test", "environment": "test", @@ -187,6 +190,8 @@ "declared": false, "instance": "instance1", "readyAt": 0, + "delayedUntil": 0, + "delayCause": "running", "jobName": "staging-test", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test", "environment": "staging", @@ -348,6 +353,7 @@ ], "declared": true, "instance": "instance1", + "delayCause": "unverified", "jobName": "production-us-central-1", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-central-1", "environment": "prod", @@ -405,6 +411,7 @@ ], "declared": true, "instance": "instance1", + "delayCause": "notReady", "jobName": "production-us-west-1", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-west-1", "environment": "prod", @@ -462,6 +469,7 @@ ], "declared": true, "instance": "instance1", + "delayCause": "notReady", "jobName": "production-us-east-3", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3", "environment": "prod", @@ -547,6 +555,7 @@ ], "declared": true, "instance": "instance2", + "delayCause": "notReady", "deploying": { "application": { "build": 4, @@ -598,6 +607,7 @@ ], "declared": true, "instance": "instance2", + "delayCause": "unverified", "jobName": "production-us-central-1", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance2/job/production-us-central-1", "environment": "prod", @@ -624,6 +634,7 @@ ], "declared": true, "instance": "instance2", + "delayCause": "notReady", "jobName": "production-us-west-1", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance2/job/production-us-west-1", "environment": "prod", @@ -650,6 +661,7 @@ ], "declared": true, "instance": "instance2", + "delayCause": "notReady", "jobName": "production-us-east-3", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance2/job/production-us-east-3", "environment": "prod", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index 7ee5f6db9b9..c942a7ad63d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -18,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import org.junit.jupiter.api.Test; import java.io.File; +import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -76,7 +77,7 @@ public class DeploymentApiTest extends ControllerContainerTest { deploymentTester.upgrader().maintain(); deploymentTester.triggerJobs(); productionApp.runJob(DeploymentContext.systemTest).runJob(DeploymentContext.stagingTest).runJob(DeploymentContext.productionUsWest1); - failingApp.failDeployment(DeploymentContext.systemTest).failDeployment(DeploymentContext.stagingTest); + failingApp.failDeployment(DeploymentContext.systemTest).failDeployment(DeploymentContext.stagingTest).timeOutConvergence(DeploymentContext.stagingTest); deploymentTester.upgrader().maintain(); deploymentTester.triggerJobs(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json index 51398daa1d4..a1f386d51a7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json @@ -41,12 +41,11 @@ "compileVersion": "6.1.0", "jobs": [ { - "name": "system-test", - "coolingDownUntil": 1600000000000 + "name": "system-test" }, { "name": "staging-test", - "coolingDownUntil": 1600000000000 + "coolingDownUntil": 1600022201500 }, { "name": "production-us-west-1" @@ -141,7 +140,7 @@ "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1", "upgradePolicy": "default", "failing": "staging-test", - "status": "error" + "status": "installationFailed" } ], "productionApplications": [ @@ -165,14 +164,6 @@ "running": "system-test" }, { - "tenant": "tenant1", - "application": "application1", - "instance": "default", - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1", - "upgradePolicy": "default", - "running": "staging-test" - }, - { "tenant": "tenant2", "application": "application2", "instance": "i2", @@ -193,12 +184,11 @@ "jobs": [ { "name": "system-test", - "coolingDownUntil": 1600000000000, "pending": "application" }, { "name": "staging-test", - "coolingDownUntil": 1600000000000, + "coolingDownUntil": 1600022201500, "pending": "platform" }, { @@ -222,15 +212,10 @@ }, "staging-test": { "failing": { - "number": 2, - "start": 1600000000000, - "end": 1600000000000, - "status": "error" - }, - "running": { "number": 3, "start": 1600000000000, - "status": "running" + "end": 1600014401000, + "status": "installationFailed" } } }, @@ -250,15 +235,10 @@ }, "staging-test": { "failing": { - "number": 2, - "start": 1600000000000, - "end": 1600000000000, - "status": "error" - }, - "running": { "number": 3, "start": 1600000000000, - "status": "running" + "end": 1600014401000, + "status": "installationFailed" } } } @@ -289,15 +269,15 @@ "system-test": { "failing": { "number": 3, - "start": 1600000000000, - "end": 1600000000000, + "start": 1600014401000, + "end": 1600014401000, "status": "error" } }, "staging-test": { "running": { "number": 3, - "start": 1600000000000, + "start": 1600014401000, "status": "running" } }, @@ -341,7 +321,7 @@ "production-us-west-1": { "running": { "number": 2, - "start": 1600000000000, + "start": 1600014401000, "status": "running" } } @@ -350,7 +330,7 @@ "production-us-west-1": { "running": { "number": 2, - "start": 1600000000000, + "start": 1600014401000, "status": "running" } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java index 43ad01fc5c2..168a1345c39 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java @@ -18,7 +18,6 @@ import org.junit.jupiter.api.Test; import java.net.URI; import java.util.List; import java.util.Set; -import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -155,7 +154,7 @@ public class RotationRepositoryTest { var application2 = tester.newDeploymentContext("tenant2", "app2", "default"); application2.submit(applicationPackage).deploy(); assertEquals(List.of(new RotationId("foo-1")), rotationIds(application2.instance().rotations())); - assertEquals("https://cd.app2.tenant2.global.vespa.oath.cloud/", + assertEquals("https://cd.app2.tenant2.global.cd.vespa.oath.cloud/", tester.controller().routing().readDeclaredEndpointsOf(application2.instanceId()).primary().get().url().toString()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index f08e92a515d..7afa5c7f44a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -349,6 +349,13 @@ public class VersionStatusTest { assertEquals(Confidence.high, confidence(tester.controller(), version0), "Confidence remains unchanged for version0: High"); assertEquals(VespaVersion.Confidence.high, confidence(tester.controller(), version2), "90% of defaults deployed successfully: High"); + // Canary failing a new revision does not affect confidence + canary0.submit(canaryPolicy).failDeployment(systemTest); + tester.controllerTester().computeVersionStatus(); + assertEquals(Confidence.high, confidence(tester.controller(), version0), "Confidence remains unchanged for version0: High"); + assertEquals(VespaVersion.Confidence.high, confidence(tester.controller(), version2), "90% of defaults deployed successfully: High"); + canary0.deploy(); + // A new version is released, all canaries upgrade successfully, but enough "default" apps fail to mark version // as broken Version version3 = new Version("6.5"); diff --git a/default_build_settings.cmake b/default_build_settings.cmake index c82bb91accc..2785a98a396 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -4,11 +4,11 @@ include(VespaExtendedDefaultBuildSettings OPTIONAL) function(setup_vespa_default_build_settings_darwin) message("-- Setting up default build settings for darwin") - set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS_PREFIX}/lib" "/usr/local/opt/bison/lib" "/usr/local/opt/flex/lib" "/usr/local/opt/icu4c/lib" "/usr/local/opt/openssl@1.1/lib" "/usr/local/opt/openblas/lib") - list(APPEND DEFAULT_EXTRA_LINK_DIRECTORY "/usr/local/lib") + set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS_PREFIX}/lib" "${VESPA_HOMEBREW_PREFIX}/opt/bison/lib" "${VESPA_HOMEBREW_PREFIX}/opt/flex/lib" "${VESPA_HOMEBREW_PREFIX}/opt/icu4c/lib" "${VESPA_HOMEBREW_PREFIX}/opt/openssl@1.1/lib" "${VESPA_HOMEBREW_PREFIX}/opt/openblas/lib") + list(APPEND DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_HOMEBREW_PREFIX}/lib") set(DEFAULT_EXTRA_LINK_DIRECTORY "${DEFAULT_EXTRA_LINK_DIRECTORY}" PARENT_SCOPE) - set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS_PREFIX}/include" "/usr/local/opt/flex/include" "/usr/local/opt/icu4c/include" "/usr/local/opt/openssl@1.1/include" "/usr/local/opt/openblas/include") - list(APPEND DEFAULT_EXTRA_INCLUDE_DIRECTORY "/usr/local/include") + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS_PREFIX}/include" "${VESPA_HOMEBREW_PREFIX}/opt/flex/include" "${VESPA_HOMEBREW_PREFIX}/opt/icu4c/include" "${VESPA_HOMEBREW_PREFIX}/opt/openssl@1.1/include" "${VESPA_HOMEBREW_PREFIX}/opt/openblas/include") + list(APPEND DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_HOMEBREW_PREFIX}/include") set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${DEFAULT_EXTRA_INCLUDE_DIRECTORY}" PARENT_SCOPE) endfunction() @@ -84,7 +84,7 @@ endfunction() function(vespa_use_default_cmake_prefix_path) set(DEFAULT_CMAKE_PREFIX_PATH ${VESPA_DEPS_PREFIX}) if (APPLE) - list(APPEND DEFAULT_CMAKE_PREFIX_PATH "/usr/local/opt/bison" "/usr/local/opt/flex" "/usr/local/opt/openssl@1.1" "/usr/local/opt/openblas" "/usr/local/opt/icu4c") + list(APPEND DEFAULT_CMAKE_PREFIX_PATH "${VESPA_HOMEBREW_PREFIX}/opt/bison" "${VESPA_HOMEBREW_PREFIX}/opt/flex" "${VESPA_HOMEBREW_PREFIX}/opt/openssl@1.1" "${VESPA_HOMEBREW_PREFIX}/opt/openblas" "${VESPA_HOMEBREW_PREFIX}/opt/icu4c") endif() message("-- DEFAULT_CMAKE_PREFIX_PATH is ${DEFAULT_CMAKE_PREFIX_PATH}") if(NOT DEFINED CMAKE_PREFIX_PATH) @@ -203,8 +203,8 @@ function(vespa_use_default_cxx_compiler) unset(DEFAULT_CMAKE_CXX_COMPILER) if(NOT DEFINED VESPA_COMPILER_VARIANT OR VESPA_COMPILER_VARIANT STREQUAL "gcc") if(APPLE) - set(DEFAULT_CMAKE_C_COMPILER "/usr/local/bin/gcc-12") - set(DEFAULT_CMAKE_CXX_COMPILER "/usr/local/bin/g++-12") + set(DEFAULT_CMAKE_C_COMPILER "${VESPA_HOMEBREW_PREFIX}/bin/gcc-12") + set(DEFAULT_CMAKE_CXX_COMPILER "${VESPA_HOMEBREW_PREFIX}/bin/g++-12") elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "amzn 2") set(DEFAULT_CMAKE_C_COMPILER "/usr/bin/gcc10-gcc") set(DEFAULT_CMAKE_CXX_COMPILER "/usr/bin/gcc10-g++") @@ -215,8 +215,8 @@ function(vespa_use_default_cxx_compiler) endif() elseif(VESPA_COMPILER_VARIANT STREQUAL "clang") if(APPLE) - set(DEFAULT_CMAKE_C_COMPILER, "/usr/local/opt/llvm/bin/clang") - set(DEFAULT_CMAKE_CXX_COMPILER "/usr/local/opt/llvm/bin/clang++") + set(DEFAULT_CMAKE_C_COMPILER, "${VESPA_HOMEBREW_PREFIX}/opt/llvm/bin/clang") + set(DEFAULT_CMAKE_CXX_COMPILER "${VESPA_HOMEBREW_PREFIX}/opt/llvm/bin/clang++") elseif(EXISTS "/usr/bin/clang" AND EXISTS "/usr/bin/clang++") set(DEFAULT_CMAKE_C_COMPILER "/usr/bin/clang") set(DEFAULT_CMAKE_CXX_COMPILER "/usr/bin/clang++") diff --git a/document/src/vespa/document/select/parse_utils.cpp b/document/src/vespa/document/select/parse_utils.cpp index 95461442349..4c116d5bff4 100644 --- a/document/src/vespa/document/select/parse_utils.cpp +++ b/document/src/vespa/document/select/parse_utils.cpp @@ -24,7 +24,7 @@ parse_i64(const char* str, size_t len, int64_t& out) { } bool parse_double(const char* str, size_t len, double& out) { -#if defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 160000 +#if defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 170000 // Temporary workaround that also handles underflow (cf. issue 3081) // until libc++ supports std::from_chars for double char *str_end = const_cast<char*>(str) + len; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 76702d3791d..d55b58a1728 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -57,7 +57,7 @@ public class Flags { public static final UnboundBooleanFlag DROP_CACHES = defineFeatureFlag( "drop-caches", false, - List.of("hakonhall", "baldersheim"), "2023-03-06", "2023-04-05", + List.of("hakonhall", "baldersheim"), "2023-03-06", "2023-06-05", "Drop caches on tenant hosts", "Takes effect on next tick", ZONE_ID, @@ -79,6 +79,12 @@ public class Flags { " latency-amortized-over-requests, latency-amortized-over-time", "Takes effect at redeployment (requires restart)", ZONE_ID, APPLICATION_ID); + public static final UnboundStringFlag SUMMARY_DECODE_POLICY = defineStringFlag( + "summary-decode-policy", "eager", + List.of("baldersheim"), "2023-03-30", "2023-12-31", + "Select summary decoding policy, valid values are eager and on-demand/ondemand.", + "Takes effect at redeployment (requires restart)", + ZONE_ID, APPLICATION_ID); public static final UnboundStringFlag FEED_SEQUENCER_TYPE = defineStringFlag( "feed-sequencer-type", "THROUGHPUT", @@ -303,7 +309,7 @@ public class Flags { public static final UnboundBooleanFlag ENABLE_PROXY_PROTOCOL_MIXED_MODE = defineFeatureFlag( "enable-proxy-protocol-mixed-mode", true, - List.of("tokle"), "2022-05-09", "2023-03-31", + List.of("tokle"), "2022-05-09", "2023-04-30", "Enable or disable proxy protocol mixed mode", "Takes effect on redeployment", APPLICATION_ID); @@ -379,6 +385,19 @@ public class Flags { "Takes effect on next config server container start", ZONE_ID); + public static final UnboundBooleanFlag NODE_ADMIN_TENANT_SERVICE_REGISTRY = defineFeatureFlag( + "node-admin-tenant-service-registry", false, + List.of("olaa"), "2023-04-12", "2023-06-12", + "Whether AthenzCredentialsMaintainer in node-admin should create tenant service identity certificate", + "Takes effect on next tick", + ZONE_ID, HOSTNAME + ); + + public static final UnboundBooleanFlag ENABLE_CROWDSTRIKE = defineFeatureFlag( + "enable-crowdstrike", true, List.of("andreer"), "2023-04-13", "2023-06-13", + "Whether to enable CrowdStrike.", "Takes effect on next host admin tick", + HOSTNAME); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index 336edf74e8f..dea0dbd3623 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -347,6 +347,12 @@ public class PermanentFlags { "Takes effect immediately", TENANT_ID); + public static final UnboundIntFlag KEEP_FILE_REFERENCES_ON_TENANT_NODES = defineIntFlag( + "keep-file-references-on-tenant-nodes", 14, + "How many days to keep file references on tenant nodes (based on last modification time)", + "Takes effect on restart of Docker container", + ZONE_ID, APPLICATION_ID + ); private PermanentFlags() {} diff --git a/functions.cmake b/functions.cmake index 7fa0b0db954..7f217867314 100644 --- a/functions.cmake +++ b/functions.cmake @@ -746,6 +746,14 @@ function(vespa_detect_build_platform) elseif(APPLE) set(OS_DISTRO "darwin") set(OS_DISTRO_VERSION ${CMAKE_SYSTEM_VERSION}) + if(EXISTS "/opt/homebrew/bin/brew") + set(VESPA_HOMEBREW_PREFIX "/opt/homebrew") + elseif(EXISTS "/usr/local/bin/brew") + set(VESPA_HOMEBREW_PREFIX "/usr/local") + else() + message(FATAL_ERROR "-- Cannot determine homebrew prefix") + endif() + set(VESPA_HOMEBREW_PREFIX ${VESPA_HOMEBREW_PREFIX} PARENT_SCOPE) endif() if(OS_DISTRO) set(VESPA_OS_DISTRO ${OS_DISTRO} PARENT_SCOPE) diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionConverter.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionConverter.java index 45d8637aa3e..231f6fb7598 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionConverter.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ExpressionConverter.java @@ -1,122 +1,22 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.indexinglanguage; -import com.yahoo.collections.Pair; -import com.yahoo.vespa.indexinglanguage.expressions.*; - -import java.lang.reflect.InvocationTargetException; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import com.yahoo.vespa.indexinglanguage.expressions.Expression; /** * @author Simon Thoresen Hult */ -@SuppressWarnings({ "UnusedDeclaration" }) public abstract class ExpressionConverter implements Cloneable { - public final Expression convert(Expression exp) { - if (exp == null) { - return null; - } - if (shouldConvert(exp)) { - return doConvert(exp); - } - if (!(exp instanceof CompositeExpression)) { - return exp; - } - try { - // The class.getMethod here takes 8% of the cpu time in reading the SSBE application package - // TODO: Implement double dispatch through visitor instead? - return (Expression)ExpressionConverter.class.getMethod("innerConvert", exp.getClass()).invoke(this, exp); - } catch (IllegalAccessException | NoSuchMethodException e) { - throw new UnsupportedOperationException(exp.getClass().getName(), e); - } catch (InvocationTargetException e) { - Throwable t = e.getTargetException(); - throw t instanceof RuntimeException ? (RuntimeException)t : new RuntimeException(t); - } - } - - public Expression innerConvert(ArithmeticExpression exp) { - return new ArithmeticExpression(convert(exp.getLeftHandSide()), - exp.getOperator(), - convert(exp.getRightHandSide())); - } - - public Expression innerConvert(CatExpression exp) { - List<Expression> lst = new LinkedList<>(); - for (Expression innerExp : exp) { - Expression next = convert(innerExp); - if (next != null) { - lst.add(next); - } - } - return new CatExpression(lst); - } - - public Expression innerConvert(ForEachExpression exp) { - return new ForEachExpression(convert(exp.getInnerExpression())); - } - - public Expression innerConvert(GuardExpression exp) { - return new GuardExpression(convert(exp.getInnerExpression())); - } - - public Expression innerConvert(IfThenExpression exp) { - return new IfThenExpression(branch().convert(exp.getLeftHandSide()), - exp.getComparator(), - branch().convert(exp.getRightHandSide()), - branch().convert(exp.getIfTrueExpression()), - branch().convert(exp.getIfFalseExpression())); - } - - public Expression innerConvert(ParenthesisExpression exp) { - return new ParenthesisExpression(convert(exp.getInnerExpression())); - } - - public Expression innerConvert(ScriptExpression exp) { - List<StatementExpression> lst = new LinkedList<>(); - for (Expression innerExp : exp) { - StatementExpression next = (StatementExpression)branch().convert(innerExp); - if (next != null) { - lst.add(next); - } - } - return new ScriptExpression(lst); - } - - public Expression innerConvert(SelectInputExpression exp) { - List<Pair<String, Expression>> cases = new LinkedList<>(); - for (Pair<String, Expression> pair : exp.getCases()) { - cases.add(new Pair<>(pair.getFirst(), branch().convert(pair.getSecond()))); - } - return new SelectInputExpression(cases); - } - - public Expression innerConvert(StatementExpression exp) { - List<Expression> lst = new LinkedList<>(); - for (Expression innerExp : exp) { - Expression next = convert(innerExp); - if (next != null) { - lst.add(next); - } - } - return new StatementExpression(lst); - } - - public Expression innerConvert(SwitchExpression exp) { - Map<String, Expression> cases = new HashMap<>(); - for (Map.Entry<String, Expression> entry : exp.getCases().entrySet()) { - Expression next = branch().convert(entry.getValue()); - if (next != null) { - cases.put(entry.getKey(), next); - } - } - return new SwitchExpression(cases, branch().convert(exp.getDefaultExpression())); + public final Expression convert(Expression expression) { + if (expression == null) return null; + if (shouldConvert(expression)) + return doConvert(expression); + else + return expression.convertChildren(this); } - protected ExpressionConverter branch() { + public ExpressionConverter branch() { return this; } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ValueTransformProvider.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ValueTransformProvider.java index bbd8c5ebcb8..623f940b06b 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ValueTransformProvider.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/ValueTransformProvider.java @@ -20,7 +20,7 @@ public abstract class ValueTransformProvider extends ExpressionConverter { } @Override - protected final ExpressionConverter branch() { + public final ExpressionConverter branch() { return clone(); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java index e4bc2dae965..b7ee444975f 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ArithmeticExpression.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; import com.yahoo.document.NumericDataType; import com.yahoo.document.datatypes.*; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; @@ -55,6 +56,12 @@ public final class ArithmeticExpression extends CompositeExpression { this.rhs = rhs; } + @Override + public ArithmeticExpression convertChildren(ExpressionConverter converter) { + // TODO: branch()? + return new ArithmeticExpression(converter.convert(lhs), op, converter.convert(rhs)); + } + public Expression getLeftHandSide() { return lhs; } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java index 4c14c633fbf..564ab015e10 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CatExpression.java @@ -9,6 +9,7 @@ import com.yahoo.document.datatypes.Array; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import java.util.*; @@ -26,6 +27,11 @@ public final class CatExpression extends ExpressionList<Expression> { } @Override + public CatExpression convertChildren(ExpressionConverter converter) { + return new CatExpression(convertChildList(converter)); + } + + @Override protected void doExecute(ExecutionContext context) { FieldValue input = context.getValue(); DataType inputType = input != null ? input.getDataType() : null; diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java index 4f83cbfdd8c..86826770828 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceExpression.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import java.util.Arrays; import java.util.Collection; @@ -32,6 +33,11 @@ public class ChoiceExpression extends ExpressionList<Expression> { } @Override + public ChoiceExpression convertChildren(ExpressionConverter converter) { + return new ChoiceExpression(asList().stream().map(choice -> converter.branch().convert(choice)).toList()); + } + + @Override protected void doExecute(ExecutionContext context) { FieldValue input = context.getValue(); for (Expression expression : this) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpression.java index 27e5524f4ad..8c00aad6bb0 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/CompositeExpression.java @@ -2,12 +2,16 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; /** * @author Simon Thoresen Hult */ public abstract class CompositeExpression extends Expression { + @Override + public abstract CompositeExpression convertChildren(ExpressionConverter converter); + protected CompositeExpression(DataType inputType) { super(inputType); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java index bf8201ee7ee..f498b871096 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/Expression.java @@ -35,6 +35,12 @@ public abstract class Expression extends Selectable { this.inputType = inputType; } + /** + * Returns an expression where the children of this has been converted using the given converter. + * This default implementation returns this as it has no children. + */ + public Expression convertChildren(ExpressionConverter converter) { return this; } + /** Sets the document type and field the statement this expression is part of will write to */ public void setStatementOutput(DocumentType documentType, Field field) {} diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java index e2ff1de7126..57de66f80a0 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ExpressionList.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; import com.yahoo.document.Field; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; @@ -11,6 +12,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.LinkedList; import java.util.List; +import java.util.Objects; /** * @author Simon Thoresen Hult @@ -26,6 +28,10 @@ public abstract class ExpressionList<T extends Expression> extends CompositeExpr } } + protected List<Expression> convertChildList(ExpressionConverter converter) { + return asList().stream().map(converter::convert).filter(Objects::nonNull).toList(); + } + @Override public void setStatementOutput(DocumentType documentType, Field field) { for (Expression expression : expressions) diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java index 0f3a445bcb9..3053a391823 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ForEachExpression.java @@ -6,6 +6,7 @@ import com.yahoo.document.datatypes.Array; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.Struct; import com.yahoo.document.datatypes.WeightedSet; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.indexinglanguage.FieldValueConverter; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; @@ -27,6 +28,11 @@ public final class ForEachExpression extends CompositeExpression { } @Override + public ForEachExpression convertChildren(ExpressionConverter converter) { + return new ForEachExpression(converter.convert(exp)); + } + + @Override public void setStatementOutput(DocumentType documentType, Field field) { exp.setStatementOutput(documentType, field); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java index da7cfcdcaee..38a05c3056c 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/GuardExpression.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; import com.yahoo.document.Field; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; import com.yahoo.vespa.indexinglanguage.UpdateAdapter; import com.yahoo.vespa.objects.ObjectOperation; @@ -28,6 +29,11 @@ public final class GuardExpression extends CompositeExpression { } @Override + public GuardExpression convertChildren(ExpressionConverter converter) { + return new GuardExpression(converter.convert(exp)); + } + + @Override public void setStatementOutput(DocumentType documentType, Field field) { exp.setStatementOutput(documentType, field); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java index 8a29c8e8645..f05795aa234 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/IfThenExpression.java @@ -6,6 +6,7 @@ import com.yahoo.document.DocumentType; import com.yahoo.document.Field; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.NumericFieldValue; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; @@ -57,6 +58,15 @@ public final class IfThenExpression extends CompositeExpression { } @Override + public IfThenExpression convertChildren(ExpressionConverter converter) { + return new IfThenExpression(converter.branch().convert(lhs), + cmp, + converter.branch().convert(rhs), + converter.branch().convert(ifTrue), + converter.branch().convert(ifFalse)); + } + + @Override public void setStatementOutput(DocumentType documentType, Field field) { lhs.setStatementOutput(documentType, field); rhs.setStatementOutput(documentType, field); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java index 30c824d410d..bba1b09cda2 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/InputExpression.java @@ -21,7 +21,9 @@ public final class InputExpression extends Expression { public InputExpression(String fieldName) { super(null); - this.fieldName = Objects.requireNonNull(fieldName); + if (fieldName == null) + throw new IllegalArgumentException("'input' must be given a field name as argument"); + this.fieldName = fieldName; } public String getFieldName() { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java index 60b059f3ef1..6e476f5f7e4 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ParenthesisExpression.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; import com.yahoo.document.Field; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; @@ -24,6 +25,11 @@ public class ParenthesisExpression extends CompositeExpression { } @Override + public ParenthesisExpression convertChildren(ExpressionConverter converter) { + return new ParenthesisExpression(converter.convert(innerExp)); + } + + @Override public void setStatementOutput(DocumentType documentType, Field field) { innerExp.setStatementOutput(documentType, field); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java index f0c37960a99..1a640c9924e 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java @@ -6,6 +6,7 @@ import com.yahoo.document.datatypes.FieldValue; import com.yahoo.language.Linguistics; import com.yahoo.language.process.Embedder; import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.indexinglanguage.ScriptParser; import com.yahoo.vespa.indexinglanguage.ScriptParserContext; import com.yahoo.vespa.indexinglanguage.parser.IndexingInput; @@ -17,6 +18,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; /** * @author Simon Thoresen Hult @@ -36,6 +38,14 @@ public final class ScriptExpression extends ExpressionList<StatementExpression> } @Override + public ScriptExpression convertChildren(ExpressionConverter converter) { + return new ScriptExpression(asList().stream() + .map(child -> (StatementExpression)converter.branch().convert(child)) + .filter(Objects::nonNull) + .toList()); + } + + @Override protected void doExecute(ExecutionContext context) { FieldValue input = context.getValue(); for (StatementExpression statement : this) { diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java index 212b60525f9..bb8111f358e 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SelectInputExpression.java @@ -6,10 +6,12 @@ import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; import com.yahoo.document.Field; import com.yahoo.document.datatypes.FieldValue; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; @@ -32,6 +34,13 @@ public final class SelectInputExpression extends CompositeExpression { } @Override + public SelectInputExpression convertChildren(ExpressionConverter converter) { + return new SelectInputExpression(cases.stream() + .map(c -> new Pair<>(c.getFirst(), converter.branch().convert(c.getSecond()))) + .toList()); + } + + @Override public void setStatementOutput(DocumentType documentType, Field field) { for (var casePair : cases) casePair.getSecond().setStatementOutput(documentType, field); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java index 8516ddb883d..75f206ef47d 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/StatementExpression.java @@ -5,6 +5,7 @@ import com.yahoo.document.DataType; import com.yahoo.language.Linguistics; import com.yahoo.language.process.Embedder; import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.indexinglanguage.ScriptParser; import com.yahoo.vespa.indexinglanguage.ScriptParserContext; import com.yahoo.vespa.indexinglanguage.parser.IndexingInput; @@ -15,6 +16,7 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; /** @@ -45,6 +47,14 @@ public final class StatementExpression extends ExpressionList<Expression> { public List<String> getInputFields() { return inputFields; } @Override + public StatementExpression convertChildren(ExpressionConverter converter) { + return new StatementExpression(asList().stream() + .map(child -> converter.convert(child)) + .filter(Objects::nonNull) + .toList()); + } + + @Override protected void doExecute(ExecutionContext context) { for (Expression expression : this) { context.execute(expression); diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java index 86913d8c1ba..c7cf7066483 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/SwitchExpression.java @@ -7,6 +7,7 @@ import com.yahoo.document.Field; import com.yahoo.document.datatypes.FieldValue; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.text.StringUtilities; +import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.objects.ObjectOperation; import com.yahoo.vespa.objects.ObjectPredicate; @@ -32,6 +33,17 @@ public final class SwitchExpression extends CompositeExpression { this.cases.putAll(cases); } + @Override + public SwitchExpression convertChildren(ExpressionConverter converter) { + var convertedCases = new LinkedHashMap<String, Expression>(); + for (var entry : cases.entrySet()) { + var converted = converter.branch().convert(entry.getValue()); + if (converted != null) + convertedCases.put(entry.getKey(), converted); + } + return new SwitchExpression(convertedCases, converter.branch().convert(defaultExp)); + } + public boolean isEmpty() { return defaultExp == null && cases.isEmpty(); } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java index f1e1be0ae41..8aeaa084e1b 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/ExpressionConverterTestCase.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.indexinglanguage; import com.yahoo.collections.Pair; -import com.yahoo.document.DataType; import com.yahoo.document.datatypes.IntegerFieldValue; import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.vespa.indexinglanguage.expressions.ArithmeticExpression; @@ -11,9 +10,7 @@ import com.yahoo.vespa.indexinglanguage.expressions.Base64DecodeExpression; import com.yahoo.vespa.indexinglanguage.expressions.Base64EncodeExpression; import com.yahoo.vespa.indexinglanguage.expressions.CatExpression; import com.yahoo.vespa.indexinglanguage.expressions.ClearStateExpression; -import com.yahoo.vespa.indexinglanguage.expressions.CompositeExpression; import com.yahoo.vespa.indexinglanguage.expressions.EchoExpression; -import com.yahoo.vespa.indexinglanguage.expressions.ExecutionContext; import com.yahoo.vespa.indexinglanguage.expressions.Expression; import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression; import com.yahoo.vespa.indexinglanguage.expressions.GetFieldExpression; @@ -54,7 +51,6 @@ import com.yahoo.vespa.indexinglanguage.expressions.ToStringExpression; import com.yahoo.vespa.indexinglanguage.expressions.ToWsetExpression; import com.yahoo.vespa.indexinglanguage.expressions.TokenizeExpression; import com.yahoo.vespa.indexinglanguage.expressions.TrimExpression; -import com.yahoo.vespa.indexinglanguage.expressions.VerificationContext; import com.yahoo.vespa.indexinglanguage.expressions.ZCurveExpression; import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig; import org.junit.Test; @@ -73,7 +69,6 @@ import static org.junit.Assert.fail; */ public class ExpressionConverterTestCase { - @SuppressWarnings("unchecked") @Test public void requireThatAllExpressionTypesCanBeTraversed() { assertConvertable(new ArithmeticExpression(new InputExpression("foo"), ArithmeticExpression.Operator.ADD, @@ -167,16 +162,6 @@ public class ExpressionConverterTestCase { } @Test - public void requireThatUnknownCompositeThrows() { - try { - new MyTraverser().convert(new MyComposite()); - fail(); - } catch (UnsupportedOperationException e) { - assertEquals(NoSuchMethodException.class, e.getCause().getClass()); - } - } - - @Test public void requireThatConversionExceptionCanBeThrown() { final RuntimeException expectedCause = new RuntimeException(); try { @@ -254,24 +239,4 @@ public class ExpressionConverterTestCase { } } - private static class MyComposite extends CompositeExpression { - - MyComposite() { - super(null); - } - @Override - protected void doExecute(ExecutionContext context) { - - } - - @Override - protected void doVerify(VerificationContext context) { - - } - - @Override - public DataType createdOutputType() { - return null; - } - } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceTestCase.java index 351c925ed56..7ece841e9b7 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/ChoiceTestCase.java @@ -4,7 +4,13 @@ package com.yahoo.vespa.indexinglanguage.expressions; import com.yahoo.document.DataType; import com.yahoo.document.Field; import com.yahoo.document.datatypes.StringFieldValue; +import com.yahoo.language.Linguistics; +import com.yahoo.language.process.Embedder; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.vespa.indexinglanguage.ExpressionSearcher; import com.yahoo.vespa.indexinglanguage.SimpleTestAdapter; +import com.yahoo.vespa.indexinglanguage.parser.ParseException; +import com.yahoo.yolean.Exceptions; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -44,4 +50,24 @@ public class ChoiceTestCase { } } + @Test + public void testIllegalChoiceExpression() throws ParseException { + try { + parse("input (foo || 99999999) | attribute"); + } + catch (IllegalArgumentException e) { + assertEquals("'input' must be given a field name as argument", Exceptions.toMessageString(e)); + } + } + + @Test + public void testInnerConvert() throws ParseException { + var expression = parse("(input foo || 99999999) | attribute"); + new ExpressionSearcher<>(AttributeExpression.class).searchIn(expression); // trigger innerConvert + } + + private static Expression parse(String s) throws ParseException { + return Expression.fromString(s, new SimpleLinguistics(), Embedder.throwsOnUse.asMap()); + } + } diff --git a/model-integration/pom.xml b/model-integration/pom.xml index 9bb60827a68..c27ed9d2c31 100644 --- a/model-integration/pom.xml +++ b/model-integration/pom.xml @@ -106,6 +106,11 @@ </dependency> <dependency> + <groupId>org.lz4</groupId> + <artifactId>lz4-java</artifactId> + </dependency> + + <dependency> <groupId>junit</groupId> <artifactId>junit</artifactId> <scope>test</scope> diff --git a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java index 7cdc27b6d63..02fa7b68dc4 100644 --- a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java +++ b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java @@ -7,6 +7,7 @@ import ai.onnxruntime.OnnxTensor; import ai.onnxruntime.OnnxValue; import ai.onnxruntime.OrtException; import ai.onnxruntime.OrtSession; +import ai.vespa.modelintegration.evaluator.OnnxRuntime.ModelPathOrData; import ai.vespa.modelintegration.evaluator.OnnxRuntime.ReferencedOrtSession; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; @@ -28,7 +29,11 @@ public class OnnxEvaluator implements AutoCloseable { private final ReferencedOrtSession session; OnnxEvaluator(String modelPath, OnnxEvaluatorOptions options, OnnxRuntime runtime) { - session = createSession(modelPath, runtime, options, true); + session = createSession(ModelPathOrData.of(modelPath), runtime, options, true); + } + + OnnxEvaluator(byte[] data, OnnxEvaluatorOptions options, OnnxRuntime runtime) { + session = createSession(ModelPathOrData.of(data), runtime, options, true); } public Tensor evaluate(Map<String, Tensor> inputs, String output) { @@ -125,19 +130,20 @@ public class OnnxEvaluator implements AutoCloseable { } } - private static ReferencedOrtSession createSession(String modelPath, OnnxRuntime runtime, OnnxEvaluatorOptions options, boolean tryCuda) { + private static ReferencedOrtSession createSession( + ModelPathOrData model, OnnxRuntime runtime, OnnxEvaluatorOptions options, boolean tryCuda) { if (options == null) { options = new OnnxEvaluatorOptions(); } try { - return runtime.acquireSession(modelPath, options, tryCuda && options.requestingGpu()); + return runtime.acquireSession(model, options, tryCuda && options.requestingGpu()); } catch (OrtException e) { if (e.getCode() == OrtException.OrtErrorCode.ORT_NO_SUCHFILE) { - throw new IllegalArgumentException("No such file: " + modelPath); + throw new IllegalArgumentException("No such file: " + model.path().get()); } if (tryCuda && isCudaError(e) && !options.gpuDeviceRequired()) { // Failed in CUDA native code, but GPU device is optional, so we can proceed without it - return createSession(modelPath, runtime, options, false); + return createSession(model, runtime, options, false); } if (isCudaError(e)) { throw new IllegalArgumentException("GPU device is required, but CUDA initialization failed", e); @@ -146,6 +152,9 @@ public class OnnxEvaluator implements AutoCloseable { } } + // For unit testing + OrtSession ortSession() { return session.instance(); } + private String mapToInternalName(String outputName) throws OrtException { var info = session.instance().getOutputInfo(); var internalNames = info.keySet(); diff --git a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxRuntime.java b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxRuntime.java index 42830041c02..ece1db55c1e 100644 --- a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxRuntime.java +++ b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxRuntime.java @@ -10,9 +10,15 @@ import com.yahoo.component.annotation.Inject; import com.yahoo.jdisc.ResourceReference; import com.yahoo.jdisc.refcount.DebugReferencesWithStack; import com.yahoo.jdisc.refcount.References; +import net.jpountz.xxhash.XXHashFactory; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; @@ -26,14 +32,22 @@ import static com.yahoo.yolean.Exceptions.throwUnchecked; public class OnnxRuntime extends AbstractComponent { // For unit testing - @FunctionalInterface interface OrtSessionFactory { + interface OrtSessionFactory { OrtSession create(String path, OrtSession.SessionOptions opts) throws OrtException; + OrtSession create(byte[] data, OrtSession.SessionOptions opts) throws OrtException; } private static final Logger log = Logger.getLogger(OnnxRuntime.class.getName()); private static final OrtEnvironmentResult ortEnvironment = getOrtEnvironment(); - private static final OrtSessionFactory defaultFactory = (path, opts) -> ortEnvironment().createSession(path, opts); + private static final OrtSessionFactory defaultFactory = new OrtSessionFactory() { + @Override public OrtSession create(String path, OrtSession.SessionOptions opts) throws OrtException { + return ortEnvironment().createSession(path, opts); + } + @Override public OrtSession create(byte[] data, OrtSession.SessionOptions opts) throws OrtException { + return ortEnvironment().createSession(data, opts); + } + }; private final Object monitor = new Object(); private final Map<OrtSessionId, SharedOrtSession> sessions = new HashMap<>(); @@ -43,6 +57,14 @@ public class OnnxRuntime extends AbstractComponent { OnnxRuntime(OrtSessionFactory factory) { this.factory = factory; } + public OnnxEvaluator evaluatorOf(byte[] model) { + return new OnnxEvaluator(model, null, this); + } + + public OnnxEvaluator evaluatorOf(byte[] model, OnnxEvaluatorOptions options) { + return new OnnxEvaluator(model, options, this); + } + public OnnxEvaluator evaluatorOf(String modelPath) { return new OnnxEvaluator(modelPath, null, this); } @@ -105,8 +127,8 @@ public class OnnxRuntime extends AbstractComponent { }; } - ReferencedOrtSession acquireSession(String modelPath, OnnxEvaluatorOptions options, boolean loadCuda) throws OrtException { - var sessionId = new OrtSessionId(modelPath, options, loadCuda); + ReferencedOrtSession acquireSession(ModelPathOrData model, OnnxEvaluatorOptions options, boolean loadCuda) throws OrtException { + var sessionId = new OrtSessionId(calculateModelHash(model), options, loadCuda); synchronized (monitor) { var sharedSession = sessions.get(sessionId); if (sharedSession != null) { @@ -114,8 +136,9 @@ public class OnnxRuntime extends AbstractComponent { } } + var opts = options.getOptions(loadCuda); // Note: identical models loaded simultaneously will result in duplicate session instances - var session = factory.create(modelPath, options.getOptions(loadCuda)); + var session = model.path().isPresent() ? factory.create(model.path().get(), opts) : factory.create(model.data().get(), opts); log.fine(() -> "Created new session (%s)".formatted(System.identityHashCode(session))); var sharedSession = new SharedOrtSession(sessionId, session); @@ -125,25 +148,52 @@ public class OnnxRuntime extends AbstractComponent { return referencedSession; } + private static long calculateModelHash(ModelPathOrData model) { + if (model.path().isPresent()) { + try (var hasher = XXHashFactory.fastestInstance().newStreamingHash64(0); + var in = Files.newInputStream(Paths.get(model.path().get()))) { + byte[] buffer = new byte[8192]; + int bytesRead; + while ((bytesRead = in.read(buffer)) != -1) { + hasher.update(buffer, 0, bytesRead); + } + return hasher.getValue(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } else { + var data = model.data().get(); + return XXHashFactory.fastestInstance().hash64().hash(data, 0, data.length, 0); + } + } + int sessionsCached() { synchronized(monitor) { return sessions.size(); } } - public static class ReferencedOrtSession implements AutoCloseable { + static class ReferencedOrtSession implements AutoCloseable { private final OrtSession instance; private final ResourceReference ref; - public ReferencedOrtSession(OrtSession instance, ResourceReference ref) { + ReferencedOrtSession(OrtSession instance, ResourceReference ref) { this.instance = instance; this.ref = ref; } - public OrtSession instance() { return instance; } + OrtSession instance() { return instance; } @Override public void close() { ref.close(); } } + record ModelPathOrData(Optional<String> path, Optional<byte[]> data) { + static ModelPathOrData of(String path) { return new ModelPathOrData(Optional.of(path), Optional.empty()); } + static ModelPathOrData of(byte[] data) { return new ModelPathOrData(Optional.empty(), Optional.of(data)); } + ModelPathOrData { + if (path.isEmpty() == data.isEmpty()) throw new IllegalArgumentException("Either path or data must be non-empty"); + } + } + // Assumes options are never modified after being stored in `onnxSessions` - record OrtSessionId(String modelPath, OnnxEvaluatorOptions options, boolean loadCuda) {} + private record OrtSessionId(long modelHash, OnnxEvaluatorOptions options, boolean loadCuda) {} - record OrtEnvironmentResult(OrtEnvironment env, Throwable failure) {} + private record OrtEnvironmentResult(OrtEnvironment env, Throwable failure) {} private class SharedOrtSession { private final OrtSessionId id; diff --git a/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java b/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java index 5aba54de11b..5a367ef83e4 100644 --- a/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java +++ b/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java @@ -5,30 +5,26 @@ package ai.vespa.modelintegration.evaluator; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import org.junit.Test; -import org.junit.jupiter.api.BeforeAll; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.junit.Assume.assumeNotNull; +import static org.junit.Assume.assumeTrue; /** * @author lesters */ public class OnnxEvaluatorTest { - private static OnnxRuntime runtime; - - @BeforeAll - public static void beforeAll() { - if (OnnxRuntime.isRuntimeAvailable()) runtime = new OnnxRuntime(); - } - @Test public void testSimpleModel() { - assumeNotNull(runtime); + assumeTrue(OnnxRuntime.isRuntimeAvailable()); + var runtime = new OnnxRuntime(); OnnxEvaluator evaluator = runtime.evaluatorOf("src/test/models/onnx/simple/simple.onnx"); // Input types @@ -53,7 +49,8 @@ public class OnnxEvaluatorTest { @Test public void testBatchDimension() { - assumeNotNull(runtime); + assumeTrue(OnnxRuntime.isRuntimeAvailable()); + var runtime = new OnnxRuntime(); OnnxEvaluator evaluator = runtime.evaluatorOf("src/test/models/onnx/pytorch/one_layer.onnx"); // Input types @@ -72,21 +69,23 @@ public class OnnxEvaluatorTest { @Test public void testMatMul() { - assumeNotNull(runtime); + assumeTrue(OnnxRuntime.isRuntimeAvailable()); + var runtime = new OnnxRuntime(); String expected = "tensor<float>(d0[2],d1[4]):[38,44,50,56,83,98,113,128]"; String input1 = "tensor<float>(d0[2],d1[3]):[1,2,3,4,5,6]"; String input2 = "tensor<float>(d0[3],d1[4]):[1,2,3,4,5,6,7,8,9,10,11,12]"; - assertEvaluate("simple/matmul.onnx", expected, input1, input2); + assertEvaluate(runtime, "simple/matmul.onnx", expected, input1, input2); } @Test public void testTypes() { - assumeNotNull(runtime); - assertEvaluate("add_double.onnx", "tensor(d0[1]):[3]", "tensor(d0[1]):[1]", "tensor(d0[1]):[2]"); - assertEvaluate("add_float.onnx", "tensor<float>(d0[1]):[3]", "tensor<float>(d0[1]):[1]", "tensor<float>(d0[1]):[2]"); - assertEvaluate("add_int64.onnx", "tensor<double>(d0[1]):[3]", "tensor<double>(d0[1]):[1]", "tensor<double>(d0[1]):[2]"); - assertEvaluate("cast_int8_float.onnx", "tensor<float>(d0[1]):[-128]", "tensor<int8>(d0[1]):[128]"); - assertEvaluate("cast_float_int8.onnx", "tensor<int8>(d0[1]):[-1]", "tensor<float>(d0[1]):[255]"); + assumeTrue(OnnxRuntime.isRuntimeAvailable()); + var runtime = new OnnxRuntime(); + assertEvaluate(runtime, "add_double.onnx", "tensor(d0[1]):[3]", "tensor(d0[1]):[1]", "tensor(d0[1]):[2]"); + assertEvaluate(runtime, "add_float.onnx", "tensor<float>(d0[1]):[3]", "tensor<float>(d0[1]):[1]", "tensor<float>(d0[1]):[2]"); + assertEvaluate(runtime, "add_int64.onnx", "tensor<double>(d0[1]):[3]", "tensor<double>(d0[1]):[1]", "tensor<double>(d0[1]):[2]"); + assertEvaluate(runtime, "cast_int8_float.onnx", "tensor<float>(d0[1]):[-128]", "tensor<int8>(d0[1]):[128]"); + assertEvaluate(runtime, "cast_float_int8.onnx", "tensor<int8>(d0[1]):[-1]", "tensor<float>(d0[1]):[255]"); // ONNX Runtime 1.8.0 does not support much of bfloat16 yet // assertEvaluate("cast_bfloat16_float.onnx", "tensor<float>(d0[1]):[1]", "tensor<bfloat16>(d0[1]):[1]"); @@ -94,7 +93,8 @@ public class OnnxEvaluatorTest { @Test public void testNotIdentifiers() { - assumeNotNull(runtime); + assumeTrue(OnnxRuntime.isRuntimeAvailable()); + var runtime = new OnnxRuntime(); OnnxEvaluator evaluator = runtime.evaluatorOf("src/test/models/onnx/badnames.onnx"); var inputInfo = evaluator.getInputInfo(); var outputInfo = evaluator.getOutputInfo(); @@ -159,7 +159,18 @@ public class OnnxEvaluatorTest { assertEquals(3, allResults.size()); } - private void assertEvaluate(String model, String output, String... input) { + @Test + public void testLoadModelFromBytes() throws IOException { + assumeTrue(OnnxRuntime.isRuntimeAvailable()); + var runtime = new OnnxRuntime(); + var model = Files.readAllBytes(Paths.get("src/test/models/onnx/simple/simple.onnx")); + var evaluator = runtime.evaluatorOf(model); + assertEquals(3, evaluator.getInputs().size()); + assertEquals(1, evaluator.getOutputs().size()); + evaluator.close(); + } + + private void assertEvaluate(OnnxRuntime runtime, String model, String output, String... input) { OnnxEvaluator evaluator = runtime.evaluatorOf("src/test/models/onnx/" + model); Map<String, Tensor> inputs = new HashMap<>(); for (int i = 0; i < input.length; ++i) { diff --git a/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxRuntimeTest.java b/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxRuntimeTest.java index 81b1237e770..fdbd4fa4e5c 100644 --- a/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxRuntimeTest.java +++ b/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxRuntimeTest.java @@ -2,16 +2,18 @@ package ai.vespa.modelintegration.evaluator; -import ai.onnxruntime.OrtException; -import ai.onnxruntime.OrtSession; import org.junit.jupiter.api.Test; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; + import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotSame; import static org.junit.jupiter.api.Assertions.assertSame; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.verify; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; /** * @author bjorncs @@ -19,30 +21,81 @@ import static org.mockito.Mockito.verify; class OnnxRuntimeTest { @Test - void reuses_sessions_while_active() throws OrtException { - var runtime = new OnnxRuntime((__, ___) -> mock(OrtSession.class)); - var session1 = runtime.acquireSession("model1", new OnnxEvaluatorOptions(), false); - var session2 = runtime.acquireSession("model1", new OnnxEvaluatorOptions(), false); - var session3 = runtime.acquireSession("model2", new OnnxEvaluatorOptions(), false); - assertSame(session1.instance(), session2.instance()); - assertNotSame(session1.instance(), session3.instance()); + void reuses_sessions_while_active() { + assumeTrue(OnnxRuntime.isRuntimeAvailable()); + OnnxRuntime runtime = new OnnxRuntime(); + String model1 = "src/test/models/onnx/simple/simple.onnx"; + var evaluator1 = runtime.evaluatorOf(model1); + var evaluator2 = runtime.evaluatorOf(model1); + String model2 = "src/test/models/onnx/simple/matmul.onnx"; + var evaluator3 = runtime.evaluatorOf(model2); + assertSameSession(evaluator1, evaluator2); + assertNotSameSession(evaluator1, evaluator3); assertEquals(2, runtime.sessionsCached()); - session1.close(); - session2.close(); + evaluator1.close(); + evaluator2.close(); assertEquals(1, runtime.sessionsCached()); - verify(session1.instance()).close(); - verify(session3.instance(), never()).close(); + assertClosed(evaluator1); + assertNotClosed(evaluator3); - session3.close(); + evaluator3.close(); assertEquals(0, runtime.sessionsCached()); - verify(session3.instance()).close(); + assertClosed(evaluator3); - var session4 = runtime.acquireSession("model1", new OnnxEvaluatorOptions(), false); - assertNotSame(session1.instance(), session4.instance()); + var session4 = runtime.evaluatorOf(model1); + assertNotSameSession(evaluator1, session4); assertEquals(1, runtime.sessionsCached()); session4.close(); assertEquals(0, runtime.sessionsCached()); - verify(session4.instance()).close(); + assertClosed(session4); + } + + @Test + void loads_model_from_byte_array() throws IOException { + assumeTrue(OnnxRuntime.isRuntimeAvailable()); + var runtime = new OnnxRuntime(); + byte[] bytes = Files.readAllBytes(Paths.get("src/test/models/onnx/simple/simple.onnx")); + var evaluator1 = runtime.evaluatorOf(bytes); + var evaluator2 = runtime.evaluatorOf(bytes); + assertEquals(3, evaluator1.getInputs().size()); + assertEquals(1, runtime.sessionsCached()); + assertSameSession(evaluator1, evaluator2); + evaluator2.close(); + evaluator1.close(); + assertEquals(0, runtime.sessionsCached()); + assertClosed(evaluator1); + } + + @Test + void loading_same_model_from_bytes_and_file_resolve_to_same_instance() throws IOException { + assumeTrue(OnnxRuntime.isRuntimeAvailable()); + var runtime = new OnnxRuntime(); + String modelPath = "src/test/models/onnx/simple/simple.onnx"; + byte[] bytes = Files.readAllBytes(Paths.get(modelPath)); + try (var evaluator1 = runtime.evaluatorOf(bytes); + var evaluator2 = runtime.evaluatorOf(modelPath)) { + assertSameSession(evaluator1, evaluator2); + assertEquals(1, runtime.sessionsCached()); + } + } + + private static void assertClosed(OnnxEvaluator evaluator) { assertTrue(isClosed(evaluator), "Session is not closed"); } + private static void assertNotClosed(OnnxEvaluator evaluator) { assertFalse(isClosed(evaluator), "Session is closed"); } + private static void assertSameSession(OnnxEvaluator evaluator1, OnnxEvaluator evaluator2) { + assertSame(evaluator1.ortSession(), evaluator2.ortSession()); + } + private static void assertNotSameSession(OnnxEvaluator evaluator1, OnnxEvaluator evaluator2) { + assertNotSame(evaluator1.ortSession(), evaluator2.ortSession()); + } + + private static boolean isClosed(OnnxEvaluator evaluator) { + try { + evaluator.getInputs(); + return false; + } catch (IllegalStateException e) { + assertEquals("Asking for inputs from a closed OrtSession.", e.getMessage()); + return true; + } } }
\ No newline at end of file diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java index 7367a254b4a..92cccf86ecb 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoreDumpMetadata.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.node.admin.configserver.cores; import com.yahoo.config.provision.DockerImage; import java.nio.file.Path; +import java.time.Instant; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -12,7 +13,11 @@ import java.util.Optional; * @author hakonhall */ public class CoreDumpMetadata { + public enum Type { CORE_DUMP, JVM_HEAP, OOM } + + private Type type; private String binPath; + private Instant created; private List<String> backtrace; private List<String> backtraceAllThreads; private Path coreDumpPath; @@ -24,30 +29,36 @@ public class CoreDumpMetadata { public CoreDumpMetadata() {} - public Optional<String> binPath() { return Optional.ofNullable(binPath); }; - public Optional<List<String>> backtrace() { return Optional.ofNullable(backtrace); }; - public Optional<List<String>> backtraceAllThreads() { return Optional.ofNullable(backtraceAllThreads); }; - public Optional<Path> coredumpPath() { return Optional.ofNullable(coreDumpPath); }; + public Optional<Type> type() { return Optional.ofNullable(type); } + public Optional<String> binPath() { return Optional.ofNullable(binPath); } + public Optional<Instant> created() { return Optional.ofNullable(created); } + public Optional<List<String>> backtrace() { return Optional.ofNullable(backtrace); } + public Optional<List<String>> backtraceAllThreads() { return Optional.ofNullable(backtraceAllThreads); } + public Optional<Path> coredumpPath() { return Optional.ofNullable(coreDumpPath); } public Optional<String> decryptionToken() { return Optional.ofNullable(decryptionToken); } - public Optional<String> kernelVersion() { return Optional.ofNullable(kernelVersion); }; - public Optional<String> cpuMicrocodeVersion() { return Optional.ofNullable(cpuMicrocodeVersion); }; - public Optional<DockerImage> dockerImage() { return Optional.ofNullable(dockerImage); }; - public Optional<String> vespaVersion() { return Optional.ofNullable(vespaVersion); }; + public Optional<String> kernelVersion() { return Optional.ofNullable(kernelVersion); } + public Optional<String> cpuMicrocodeVersion() { return Optional.ofNullable(cpuMicrocodeVersion); } + public Optional<DockerImage> dockerImage() { return Optional.ofNullable(dockerImage); } + public Optional<String> vespaVersion() { return Optional.ofNullable(vespaVersion); } - public CoreDumpMetadata setBinPath(String binPath) { this.binPath = binPath; return this; }; - public CoreDumpMetadata setBacktrace(List<String> backtrace) { this.backtrace = backtrace; return this; }; - public CoreDumpMetadata setBacktraceAllThreads(List<String> backtraceAllThreads) { this.backtraceAllThreads = backtraceAllThreads; return this; }; - public CoreDumpMetadata setCoreDumpPath(Path coreDumpPath) { this.coreDumpPath = coreDumpPath; return this; }; + public CoreDumpMetadata setType(Type type) { this.type = type; return this; } + public CoreDumpMetadata setBinPath(String binPath) { this.binPath = binPath; return this; } + public CoreDumpMetadata setCreated(Instant created) { this.created = created; return this; } + public CoreDumpMetadata setBacktrace(List<String> backtrace) { this.backtrace = backtrace; return this; } + public CoreDumpMetadata setBacktraceAllThreads(List<String> backtraceAllThreads) { this.backtraceAllThreads = backtraceAllThreads; return this; } + public CoreDumpMetadata setCoreDumpPath(Path coreDumpPath) { this.coreDumpPath = coreDumpPath; return this; } public CoreDumpMetadata setDecryptionToken(String decryptionToken) { this.decryptionToken = decryptionToken; return this; } - public CoreDumpMetadata setKernelVersion(String kernelVersion) { this.kernelVersion = kernelVersion; return this; }; - public CoreDumpMetadata setCpuMicrocodeVersion(String cpuMicrocodeVersion) { this.cpuMicrocodeVersion = cpuMicrocodeVersion; return this; }; - public CoreDumpMetadata setDockerImage(DockerImage dockerImage) { this.dockerImage = dockerImage; return this; }; - public CoreDumpMetadata setVespaVersion(String vespaVersion) { this.vespaVersion = vespaVersion; return this; }; + public CoreDumpMetadata setKernelVersion(String kernelVersion) { this.kernelVersion = kernelVersion; return this; } + public CoreDumpMetadata setCpuMicrocodeVersion(String cpuMicrocodeVersion) { this.cpuMicrocodeVersion = cpuMicrocodeVersion; return this; } + public CoreDumpMetadata setDockerImage(DockerImage dockerImage) { this.dockerImage = dockerImage; return this; } + public CoreDumpMetadata setVespaVersion(String vespaVersion) { this.vespaVersion = vespaVersion; return this; } @Override public String toString() { return "CoreDumpMetadata{" + - "binPath=" + binPath + + "type=" + type + + ", binPath=" + binPath + + ", created=" + created + ", backtrace=" + backtrace + ", backtraceAllThreads=" + backtraceAllThreads + ", coreDumpPath=" + coreDumpPath + @@ -64,7 +75,9 @@ public class CoreDumpMetadata { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; CoreDumpMetadata metadata = (CoreDumpMetadata) o; - return Objects.equals(binPath, metadata.binPath) && + return type == metadata.type && + Objects.equals(binPath, metadata.binPath) && + Objects.equals(created, metadata.created) && Objects.equals(backtrace, metadata.backtrace) && Objects.equals(backtraceAllThreads, metadata.backtraceAllThreads) && Objects.equals(coreDumpPath, metadata.coreDumpPath) && @@ -77,7 +90,7 @@ public class CoreDumpMetadata { @Override public int hashCode() { - return Objects.hash(binPath, backtrace, backtraceAllThreads, coreDumpPath, decryptionToken, kernelVersion, + return Objects.hash(type, binPath, created, backtrace, backtraceAllThreads, coreDumpPath, decryptionToken, kernelVersion, cpuMicrocodeVersion, dockerImage, vespaVersion); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java index 27cf28b8e1e..a9620ebabc2 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/bindings/ReportCoreDumpRequest.java @@ -14,6 +14,7 @@ import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; +import java.time.Instant; import java.util.List; import java.util.Optional; @@ -31,6 +32,8 @@ public class ReportCoreDumpRequest { public List<String> backtrace; public List<String> backtrace_all_threads; + public Long created; + public String type; public String bin_path; public String coredump_path; public String cpu_microcode_version; @@ -44,7 +47,9 @@ public class ReportCoreDumpRequest { /** Fill this from metadata and return this. */ @JsonIgnore public ReportCoreDumpRequest fillFrom(CoreDumpMetadata metadata) { + metadata.type().ifPresent(type -> this.type = type.name()); metadata.binPath().ifPresent(binPath -> this.bin_path = binPath); + metadata.created().ifPresent(created -> this.created = created.toEpochMilli()); metadata.backtrace().ifPresent(backtrace -> this.backtrace = List.copyOf(backtrace)); metadata.backtraceAllThreads().ifPresent(backtraceAllThreads -> this.backtrace_all_threads = List.copyOf(backtraceAllThreads)); metadata.coredumpPath().ifPresent(coredumpPath -> this.coredump_path = coredumpPath.toString()); @@ -58,7 +63,9 @@ public class ReportCoreDumpRequest { @JsonIgnore public void populateMetadata(CoreDumpMetadata metadata, FileSystem fileSystem) { + if (type != null) metadata.setType(CoreDumpMetadata.Type.valueOf(type)); if (bin_path != null) metadata.setBinPath(bin_path); + if (created != null) metadata.setCreated(Instant.ofEpochMilli(created)); if (backtrace != null) metadata.setBacktrace(backtrace); if (backtrace_all_threads != null) metadata.setBacktraceAllThreads(backtrace_all_threads); if (coredump_path != null) metadata.setCoreDumpPath(fileSystem.getPath(coredump_path)); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java index 28773767d24..5d4628b41b6 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollector.java @@ -5,6 +5,7 @@ import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoreDumpMetadata; import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; @@ -96,17 +97,19 @@ public class CoreCollector { } CoreDumpMetadata collect(NodeAgentContext context, ContainerPath coredumpPath) { - var metadata = new CoreDumpMetadata(); + var metadata = new CoreDumpMetadata() + .setCreated(new UnixPath(coredumpPath).getLastModifiedTime()); if (JAVA_HEAP_DUMP_PATTERN.matcher(coredumpPath.getFileName().toString()).find()) { - metadata.setBinPath("java") + metadata.setType(CoreDumpMetadata.Type.JVM_HEAP) + .setBinPath("java") .setBacktrace(List.of("Heap dump, no backtrace available")); return metadata; } try { String binPath = readBinPath(context, coredumpPath); - metadata.setBinPath(binPath); + metadata.setType(CoreDumpMetadata.Type.CORE_DUMP).setBinPath(binPath); if (Path.of(binPath).getFileName().toString().equals("java")) { metadata.setBacktraceAllThreads(readJstack(context, coredumpPath, binPath)); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index 15be7accb7d..d22fd667202 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -19,6 +19,10 @@ import com.yahoo.vespa.athenz.identityprovider.client.CsrGenerator; import com.yahoo.vespa.athenz.identityprovider.client.DefaultIdentityDocumentClient; import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; import com.yahoo.vespa.athenz.utils.SiaUtils; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.node.admin.component.ConfigServerInfo; import com.yahoo.vespa.hosted.node.admin.container.ContainerName; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; @@ -47,6 +51,9 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.logging.Level; import java.util.logging.Logger; +import static com.yahoo.vespa.hosted.node.admin.maintenance.identity.AthenzCredentialsMaintainer.IdentityType.NODE; +import static com.yahoo.vespa.hosted.node.admin.maintenance.identity.AthenzCredentialsMaintainer.IdentityType.TENANT; + /** * A maintainer that is responsible for providing and refreshing Athenz credentials for a container. * @@ -68,7 +75,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { private final String certificateDnsSuffix; private final ServiceIdentityProvider hostIdentityProvider; private final IdentityDocumentClient identityDocumentClient; - private final boolean useInternalZts; + private final BooleanFlag tenantServiceIdentityFlag; // Used as an optimization to ensure ZTS is not DDoS'ed on continuously failing refresh attempts private final Map<ContainerName, Instant> lastRefreshAttempt = new ConcurrentHashMap<>(); @@ -78,7 +85,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { ConfigServerInfo configServerInfo, String certificateDnsSuffix, ServiceIdentityProvider hostIdentityProvider, - boolean useInternalZts, + FlagSource flagSource, Clock clock) { this.ztsEndpoint = ztsEndpoint; this.ztsTrustStorePath = ztsTrustStorePath; @@ -89,24 +96,33 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { hostIdentityProvider, new AthenzIdentityVerifier(Set.of(configServerInfo.getConfigServerIdentity()))); this.clock = clock; - this.useInternalZts = useInternalZts; + this.tenantServiceIdentityFlag = Flags.NODE_ADMIN_TENANT_SERVICE_REGISTRY.bindTo(flagSource); } public boolean converge(NodeAgentContext context) { + var modified = false; + modified |= maintain(context, NODE); + if (shouldWriteTenantServiceIdentity(context)) + modified |= maintain(context, TENANT); + return modified; + } + + private boolean maintain(NodeAgentContext context, IdentityType identityType) { if (context.isDisabled(NodeAgentTask.CredentialsMaintainer)) return false; try { context.log(logger, Level.FINE, "Checking certificate"); - ContainerPath containerSiaDirectory = context.paths().of(CONTAINER_SIA_DIRECTORY, context.users().vespa()); - ContainerPath privateKeyFile = (ContainerPath) SiaUtils.getPrivateKeyFile(containerSiaDirectory, context.identity()); - ContainerPath certificateFile = (ContainerPath) SiaUtils.getCertificateFile(containerSiaDirectory, context.identity()); - ContainerPath identityDocumentFile = containerSiaDirectory.resolve("vespa-node-identity-document.json"); + ContainerPath siaDirectory = context.paths().of(CONTAINER_SIA_DIRECTORY, context.users().vespa()); + ContainerPath identityDocumentFile = siaDirectory.resolve(identityType.getIdentityDocument()); + AthenzIdentity athenzIdentity = getAthenzIdentity(context, identityType, identityDocumentFile); + ContainerPath privateKeyFile = (ContainerPath) SiaUtils.getPrivateKeyFile(siaDirectory, athenzIdentity); + ContainerPath certificateFile = (ContainerPath) SiaUtils.getCertificateFile(siaDirectory, athenzIdentity); if (!Files.exists(privateKeyFile) || !Files.exists(certificateFile) || !Files.exists(identityDocumentFile)) { context.log(logger, "Certificate/private key/identity document file does not exist"); Files.createDirectories(privateKeyFile.getParent()); Files.createDirectories(certificateFile.getParent()); Files.createDirectories(identityDocumentFile.getParent()); - registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile); + registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, identityType); return true; } @@ -116,11 +132,11 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { var doc = EntityBindingsMapper.readSignedIdentityDocumentFromFile(identityDocumentFile); if (doc.outdated()) { context.log(logger, "Identity document is outdated (version=%d)", doc.documentVersion()); - registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile); + registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, identityType); return true; } else if (isCertificateExpired(expiry, now)) { context.log(logger, "Certificate has expired (expiry=%s)", expiry.toString()); - registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile); + registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, identityType); return true; } @@ -134,7 +150,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { return false; } else { lastRefreshAttempt.put(context.containerName(), now); - refreshIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, doc); + refreshIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, doc, identityType); return true; } } @@ -182,9 +198,9 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { now)) > 0; } - private void registerIdentity(NodeAgentContext context, ContainerPath privateKeyFile, ContainerPath certificateFile, ContainerPath identityDocumentFile) { + private void registerIdentity(NodeAgentContext context, ContainerPath privateKeyFile, ContainerPath certificateFile, ContainerPath identityDocumentFile, IdentityType identityType) { KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); - SignedIdentityDocument doc = identityDocumentClient.getNodeIdentityDocument(context.hostname().value()); + SignedIdentityDocument doc = signedIdentityDocument(context, identityType); CsrGenerator csrGenerator = new CsrGenerator(certificateDnsSuffix, doc.providerService().getFullName()); Pkcs10Csr csr = csrGenerator.generateInstanceCsr( context.identity(), doc.providerUniqueId(), doc.ipAddresses(), doc.clusterType(), keyPair); @@ -214,7 +230,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { .orElse(ztsEndpoint); } private void refreshIdentity(NodeAgentContext context, ContainerPath privateKeyFile, ContainerPath certificateFile, - ContainerPath identityDocumentFile, SignedIdentityDocument doc) { + ContainerPath identityDocumentFile, SignedIdentityDocument doc, IdentityType identityType) { KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); CsrGenerator csrGenerator = new CsrGenerator(certificateDnsSuffix, doc.providerService().getFullName()); Pkcs10Csr csr = csrGenerator.generateInstanceCsr( @@ -239,7 +255,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { } catch (ZtsClientException e) { if (e.getErrorCode() == 403 && e.getDescription().startsWith("Certificate revoked")) { context.log(logger, Level.SEVERE, "Certificate cannot be refreshed as it is revoked by ZTS - re-registering the instance now", e); - registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile); + registerIdentity(context, privateKeyFile, certificateFile, identityDocumentFile, identityType); } else { throw e; } @@ -272,4 +288,46 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { private static boolean isCertificateExpired(Instant expiry, Instant now) { return now.isAfter(expiry.minus(EXPIRY_MARGIN)); } + + private SignedIdentityDocument signedIdentityDocument(NodeAgentContext context, IdentityType identityType) { + return switch (identityType) { + case NODE -> identityDocumentClient.getNodeIdentityDocument(context.hostname().value()); + case TENANT -> identityDocumentClient.getTenantIdentityDocument(context.hostname().value()); + }; + } + + private AthenzIdentity getAthenzIdentity(NodeAgentContext context, IdentityType identityType, ContainerPath identityDocumentFile) { + return switch (identityType) { + case NODE -> context.identity(); + case TENANT -> getTenantIdentity(context, identityDocumentFile); + }; + } + + private AthenzIdentity getTenantIdentity(NodeAgentContext context, ContainerPath identityDocumentFile) { + if (Files.exists(identityDocumentFile)) { + return EntityBindingsMapper.readSignedIdentityDocumentFromFile(identityDocumentFile).serviceIdentity(); + } else { + return identityDocumentClient.getTenantIdentityDocument(context.hostname().value()).serviceIdentity(); + } + } + + private boolean shouldWriteTenantServiceIdentity(NodeAgentContext context) { + return tenantServiceIdentityFlag + .with(FetchVector.Dimension.HOSTNAME, context.hostname().value()) + .value(); + } + + enum IdentityType { + NODE("vespa-node-identity-document.json"), + TENANT("vespa-tenant-identity-document.json"); + + private String identityDocument; + IdentityType(String identityDocument) { + this.identityDocument = identityDocument; + } + + public String getIdentityDocument() { + return identityDocument; + } + } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java index 3a12191a0de..c743f1c8c85 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java @@ -71,11 +71,14 @@ public class SyncFileInfo { if (filename.startsWith("vespa.log")) { dir = "logs/vespa/"; compression = Compression.ZSTD; - minDurationBetweenSync = filename.length() == 9 ? rotatedOnly ? Duration.ofHours(1) : Duration.ZERO : null; + if (filename.length() == 9) { + if (!rotatedOnly) remoteFilename = "vespa.log-" + DATE_TIME_FORMATTER.format(new UnixPath(logFile).getLastModifiedTime()); + minDurationBetweenSync = rotatedOnly ? Duration.ofHours(1) : Duration.ZERO; + } } else if (filename.startsWith("zookeeper.") && filename.endsWith(".log")) { compression = Compression.ZSTD; dir = "logs/zookeeper/"; - remoteFilename = filename.endsWith(".0.log") ? "zookeeper.log" : + remoteFilename = rotatedOnly && filename.endsWith(".0.log") ? "zookeeper.log" : "zookeeper.log-" + DATE_TIME_FORMATTER.format(new UnixPath(logFile).getLastModifiedTime()); minDurationBetweenSync = filename.endsWith(".0.log") ? rotatedOnly ? Duration.ofHours(1) : Duration.ZERO : null; } else { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java index c7d34a12f43..9662d4184df 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/systemd/SystemCtl.java @@ -91,7 +91,7 @@ public class SystemCtl { public String getServiceProperty(TaskContext context, String unit, String property) { return newCommandLine(context) .add("systemctl", "show", "--property", property, "--value", unit + ".service") - .execute() + .executeSilently() .getOutput(); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java index f49dd2e705b..b35f4d6c790 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/cores/CoresTest.java @@ -16,6 +16,7 @@ import org.mockito.ArgumentCaptor; import java.nio.file.FileSystem; import java.nio.file.Path; +import java.time.Instant; import java.util.List; import java.util.Optional; @@ -40,6 +41,8 @@ class CoresTest { private final HostName hostname = HostName.of("foo.com"); private final String id = "5c987afb-347a-49ee-a0c5-bef56bbddeb0"; private final CoreDumpMetadata metadata = new CoreDumpMetadata() + .setType(CoreDumpMetadata.Type.OOM) + .setCreated(Instant.ofEpochMilli(12345678)) .setKernelVersion("4.18.0-372.26.1.el8_6.x86_64") .setCpuMicrocodeVersion("0x1000065") .setCoreDumpPath(fileSystem.getPath("/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813")) @@ -83,9 +86,11 @@ class CoresTest { "bin_path": "/usr/bin/java", "coredump_path": "/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813", "cpu_microcode_version": "0x1000065", + "created": 12345678, "decryption_token": "987def", "docker_image": "us-central1-docker.pkg.dev/vespa-external-cd/vespa-cloud/vespa/cloud-tenant-rhel8:8.68.8", "kernel_version": "4.18.0-372.26.1.el8_6.x86_64", + "type": "OOM", "vespa_version": "8.68.8" }""", JsonTestHelper.normalize(uncheck(() -> mapper.writeValueAsString(bodyJsonPojoCaptor.getValue())))); @@ -128,9 +133,11 @@ class CoresTest { "bin_path": "/usr/bin/java", "coredump_path": "/data/vespa/processed-coredumps/h7641a/5c987afb-347a-49ee-a0c5-bef56bbddeb0/dump_java.core.813", "cpu_microcode_version": "0x1000065", + "created": 12345678, "decryption_token": "987def", "docker_image": "us-central1-docker.pkg.dev/vespa-external-cd/vespa-cloud/vespa/cloud-tenant-rhel8:8.68.8", "kernel_version": "4.18.0-372.26.1.el8_6.x86_64", + "type": "OOM", "vespa_version": "8.68.8" }""", JsonTestHelper.normalize(new UnixPath(path).readUtf8File())); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java index 4fa18c71da0..b4a35d6012c 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoreCollectorTest.java @@ -5,11 +5,13 @@ import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoreDumpMetadata; import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.jupiter.api.Test; +import java.time.Instant; import java.util.List; import static com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoreCollector.GDB_PATH_RHEL8; @@ -22,12 +24,18 @@ import static org.mockito.Mockito.when; * @author freva */ public class CoreCollectorTest { + private static final Instant CORE_CREATED = Instant.ofEpochMilli(2233445566L); + private final ContainerOperations docker = mock(ContainerOperations.class); private final CoreCollector coreCollector = new CoreCollector(docker); private final NodeAgentContext context = NodeAgentContextImpl.builder("container-123.domain.tld") .fileSystem(TestFileSystem.create()).build(); - private final ContainerPath TEST_CORE_PATH = context.paths().of("/tmp/core.1234"); + private final ContainerPath TEST_CORE_PATH = (ContainerPath) new UnixPath(context.paths().of("/tmp/core.1234")) + .createParents() + .createNewFile() + .setLastModifiedTime(CORE_CREATED) + .toPath(); private final String TEST_BIN_PATH = "/usr/bin/program"; private final List<String> GDB_BACKTRACE = List.of("[New Thread 2703]", "Core was generated by `/usr/bin/program\'.", "Program terminated with signal 11, Segmentation fault.", @@ -143,6 +151,8 @@ public class CoreCollectorTest { String.join("\n", GDB_BACKTRACE)); var expected = new CoreDumpMetadata().setBinPath(TEST_BIN_PATH) + .setCreated(CORE_CREATED) + .setType(CoreDumpMetadata.Type.CORE_DUMP) .setBacktrace(GDB_BACKTRACE) .setBacktraceAllThreads(GDB_BACKTRACE); assertEquals(expected, coreCollector.collect(context, TEST_CORE_PATH)); @@ -156,7 +166,7 @@ public class CoreCollectorTest { mockExec(new String[]{GDB_PATH_RHEL8 + " -n -ex set print frame-arguments none -ex bt -batch /usr/bin/program /tmp/core.1234"}, "", "Failure"); - var expected = new CoreDumpMetadata().setBinPath(TEST_BIN_PATH); + var expected = new CoreDumpMetadata().setBinPath(TEST_BIN_PATH).setCreated(CORE_CREATED).setType(CoreDumpMetadata.Type.CORE_DUMP); assertEquals(expected, coreCollector.collect(context, TEST_CORE_PATH)); } @@ -174,6 +184,8 @@ public class CoreCollectorTest { jstack); var expected = new CoreDumpMetadata().setBinPath(jdkPath) + .setCreated(CORE_CREATED) + .setType(CoreDumpMetadata.Type.CORE_DUMP) .setBacktraceAllThreads(List.of(jstack)); assertEquals(expected, coreCollector.collect(context, TEST_CORE_PATH)); } @@ -181,9 +193,14 @@ public class CoreCollectorTest { @Test void metadata_for_java_heap_dump() { var expected = new CoreDumpMetadata().setBinPath("java") + .setType(CoreDumpMetadata.Type.JVM_HEAP) + .setCreated(CORE_CREATED) .setBacktrace(List.of("Heap dump, no backtrace available")); - assertEquals(expected, coreCollector.collect(context, context.paths().of("/dump_java_pid123.hprof"))); + assertEquals(expected, coreCollector.collect(context, (ContainerPath) new UnixPath(context.paths().of("/dump_java_pid123.hprof")) + .createNewFile() + .setLastModifiedTime(CORE_CREATED) + .toPath())); } private void mockExec(String[] cmd, String output) { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java index 3c91a9f32d1..b7aee6706b1 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java @@ -74,8 +74,9 @@ public class SyncFileInfoTest { @Test void vespa_logs() { + new UnixPath(vespaLogPath1).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z")); assertForLogFile(vespaLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log.zst", ZSTD, Duration.ofHours(1), true); - assertForLogFile(vespaLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log.zst", ZSTD, Duration.ZERO, false); + assertForLogFile(vespaLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2022-05-09.14-22-11.zst", ZSTD, Duration.ZERO, false); assertForLogFile(vespaLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2021-02-12.zst", ZSTD, true); assertForLogFile(vespaLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/vespa/vespa.log-2021-02-12.zst", ZSTD, false); @@ -83,8 +84,9 @@ public class SyncFileInfoTest { @Test void zookeeper_logs() { + new UnixPath(zkLogPath0).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-13T13:13:45Z")); assertForLogFile(zkLogPath0, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log.zst", ZSTD, Duration.ofHours(1), true); - assertForLogFile(zkLogPath0, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log.zst", ZSTD, Duration.ZERO, false); + assertForLogFile(zkLogPath0, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log-2022-05-13.13-13-45.zst", ZSTD, Duration.ZERO, false); new UnixPath(zkLogPath1).createParents().createNewFile().setLastModifiedTime(Instant.parse("2022-05-09T14:22:11Z")); assertForLogFile(zkLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/zookeeper/zookeeper.log-2022-05-09.14-22-11.zst", ZSTD, true); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index 4c9fab748d1..856d6e07156 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -59,7 +59,7 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { failures++; } } - return asSuccessFactor(attempts, failures); + return asSuccessFactorDeviation(attempts, failures); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DiskReplacer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DiskReplacer.java index acd5cb61d81..6f2eb726e91 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DiskReplacer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DiskReplacer.java @@ -51,6 +51,6 @@ public class DiskReplacer extends NodeRepositoryMaintainer { log.log(Level.WARNING, "Failed to rebuild " + host.hostname() + ", will retry in " + interval(), e); } } - return this.asSuccessFactor(nodes.size(), failures); + return this.asSuccessFactorDeviation(nodes.size(), failures); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java index 83dadddf76c..a4bc3a1aea5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java @@ -113,7 +113,7 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { } success++; } - return asSuccessFactor(attempts, attempts - success); + return asSuccessFactorDeviation(attempts, attempts - success); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisioner.java index 7ecfc8f7926..0fa16f22061 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostDeprovisioner.java @@ -53,7 +53,7 @@ public class HostDeprovisioner extends NodeRepositoryMaintainer { log.log(Level.WARNING, "Failed to deprovision " + host.hostname() + ", will retry in " + interval(), e); } } - return asSuccessFactor(hosts.size(), failures); + return asSuccessFactorDeviation(hosts.size(), failures); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java index 86c5a926900..3c77725298d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java @@ -63,7 +63,7 @@ public class HostResumeProvisioner extends NodeRepositoryMaintainer { } } } - return asSuccessFactor(hosts.size(), failures); + return asSuccessFactorDeviation(hosts.size(), failures); } private void setIpConfig(Node host, NodeList children, HostIpConfig hostIpConfig) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index f864ab18920..baa2e596b36 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -94,7 +94,7 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { .collect(Collectors.joining(", ")), interval())); } - return asSuccessFactor(attempts.get(), failed.size()); + return asSuccessFactorDeviation(attempts.get(), failed.size()); } /** Remove reals from inactive load balancers */ @@ -131,7 +131,7 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { interval()), lastException.get()); } - return asSuccessFactor(attempts.get(), failed.size()); + return asSuccessFactorDeviation(attempts.get(), failed.size()); } /** Patch load balancers matching given filter, while holding lock */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index 5af74214648..e6cfe8ca6b5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -176,6 +176,9 @@ public class MetricsReporter extends NodeRepositoryMaintainer { boolean converged = currentVersion.isPresent() && currentVersion.get().equals(wantedVersion); metric.set("wantToChangeVespaVersion", converged ? 0 : 1, context); + if (node.cloudAccount().isEnclave(nodeRepository().zone())) { + metric.set("hasWireguardKey", node.wireguardPubKey().isPresent() ? 1 : 0, context); + } } else { context = getContext(Map.of("state", node.state().name(), "host", node.hostname())); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java index afea08711fa..f9ff2f08375 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailer.java @@ -97,7 +97,7 @@ public class NodeFailer extends NodeRepositoryMaintainer { metric.set(throttlingActiveMetric, throttlingActive, null); metric.set(throttledHostFailuresMetric, throttledHostFailures, null); metric.set(throttledNodeFailuresMetric, throttledNodeFailures, null); - return asSuccessFactor(attempts, failures); + return asSuccessFactorDeviation(attempts, failures); } private Collection<FailingNode> findActiveFailingNodes() { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java index 781debe26a0..979a2771082 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java @@ -77,7 +77,7 @@ public class NodeHealthTracker extends NodeRepositoryMaintainer { failures.add(1); } }); - return asSuccessFactor(attempts.get(), failures.get()); + return asSuccessFactorDeviation(attempts.get(), failures.get()); } /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java index b299369db1a..e28dac1c915 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainer.java @@ -55,10 +55,10 @@ public class NodeMetricsDbMaintainer extends NodeRepositoryMaintainer { nodeRepository().metricsDb().gc(); - return asSuccessFactor(attempts, failures.get()); + return asSuccessFactorDeviation(attempts, failures.get()); } catch (InterruptedException e) { - return asSuccessFactor(attempts, failures.get()); + return asSuccessFactorDeviation(attempts, failures.get()); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java index 3c00e3b708d..b8d37c7eb5c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeRepositoryMaintainer.java @@ -59,7 +59,7 @@ public abstract class NodeRepositoryMaintainer extends Maintainer { @Override public void completed(String job, double successFactor, long duration) { var context = metric.createContext(Map.of("job", job)); - metric.set("maintenance.successFactor", successFactor, context); + metric.set("maintenance.successFactorDeviation", successFactor, context); metric.set("maintenance.duration", duration, context); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java index af368934188..4071559d841 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainer.java @@ -48,7 +48,7 @@ public class ScalingSuggestionsMaintainer extends NodeRepositoryMaintainer { failures++; } } - return asSuccessFactor(attempts, failures); + return asSuccessFactorDeviation(attempts, failures); } private Applications applications() { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java index 178ea6ed514..35b2fef2c78 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImpl.java @@ -1,14 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; -import com.yahoo.component.annotation.Inject; +import ai.vespa.http.DomainName; import com.yahoo.component.Version; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.ActivationContext; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.Deployment; import com.yahoo.config.provision.HostFilter; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InfraDeployer; import com.yahoo.config.provision.NodeType; @@ -116,7 +116,7 @@ public class InfraDeployerImpl implements InfraDeployer { duperModel.infraApplicationActivated( application.getApplicationId(), - hostSpecs.stream().map(HostSpec::hostname).map(HostName::of).toList()); + hostSpecs.stream().map(HostSpec::hostname).map(DomainName::of).toList()); logger.log(Level.FINE, () -> generateActivationLogMessage(hostSpecs, application.getApplicationId())); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index 04f64b070b3..890d190c24e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -109,7 +109,7 @@ public class LoadBalancerProvisioner { public void activate(Set<ClusterSpec> clusters, NodeList newActive, ApplicationTransaction transaction) { Map<ClusterSpec.Id, ZoneEndpoint> activatingClusters = clusters.stream() // .collect(Collectors.toMap(ClusterSpec::id, ClusterSpec::zoneEndpoint)); - // TODO: this dies with combined clusters Ü + // TODO: this dies with combined clusters .collect(groupingBy(LoadBalancerProvisioner::effectiveId, reducing(ZoneEndpoint.defaultEndpoint, ClusterSpec::zoneEndpoint, @@ -193,14 +193,13 @@ public class LoadBalancerProvisioner { Optional<LoadBalancer> loadBalancer = db.readLoadBalancer(id); LoadBalancer newLoadBalancer; LoadBalancer.State fromState = loadBalancer.map(LoadBalancer::state).orElse(null); - if ( loadBalancer.isPresent() - && ( ! inAccount(cloudAccount, loadBalancer.get()) - || ! hasCorrectVisibility(loadBalancer.get(), zoneEndpoint))) { - // We have a load balancer, but with the wrong account or visibility. - // Load balancer must be removed before we can provision a new one with the wanted visibility - newLoadBalancer = loadBalancer.get().with(LoadBalancer.State.removable, now); - } - else { + boolean recreateLoadBalancer = loadBalancer.isPresent() && (!inAccount(cloudAccount, loadBalancer.get()) + || !hasCorrectVisibility(loadBalancer.get(), zoneEndpoint)); + if (recreateLoadBalancer) { + // We have a load balancer, but with the wrong account or visibility. + // Load balancer must be removed before we can provision a new one with the wanted visibility + newLoadBalancer = loadBalancer.get().with(LoadBalancer.State.removable, now); + } else { Optional<LoadBalancerInstance> instance = provisionInstance(id, loadBalancer, zoneEndpoint, cloudAccount); newLoadBalancer = loadBalancer.isEmpty() ? new LoadBalancer(id, instance, LoadBalancer.State.reserved, now) : loadBalancer.get().with(instance); @@ -211,8 +210,8 @@ public class LoadBalancerProvisioner { } private static boolean hasCorrectVisibility(LoadBalancer newLoadBalancer, ZoneEndpoint zoneEndpoint) { - return newLoadBalancer.instance().isEmpty() - || newLoadBalancer.instance().get().settings().isPublicEndpoint() == zoneEndpoint.isPublicEndpoint(); + return newLoadBalancer.instance().isEmpty() || + newLoadBalancer.instance().get().settings().isPublicEndpoint() == zoneEndpoint.isPublicEndpoint(); } private void activate(ApplicationTransaction transaction, ClusterSpec.Id cluster, ZoneEndpoint settings, NodeList nodes) { @@ -320,14 +319,6 @@ public class LoadBalancerProvisioner { return loadBalancer.instance().isEmpty() || loadBalancer.instance().get().cloudAccount().equals(cloudAccount); } - /** Returns whether load balancer has given reals, and settings if specified */ - private static boolean isUpToDate(LoadBalancer loadBalancer, Set<Real> reals, ZoneEndpoint zoneEndpoint) { - if (loadBalancer.instance().isEmpty()) - throw new IllegalStateException("Expected a load balancer instance to be present, for " + loadBalancer.id()); - return loadBalancer.instance().get().reals().equals(reals) - && loadBalancer.instance().get().settings().equals(zoneEndpoint); - } - /** Find IP addresses reachable by the load balancer service */ private Set<String> reachableIpAddresses(Node node) { Set<String> reachable = new LinkedHashSet<>(node.ipConfig().primary()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java index 15126ed5845..06ce08df8c3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDuperModel.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; +import ai.vespa.http.DomainName; import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.HostName; import com.yahoo.vespa.service.monitor.DuperModelInfraApi; import com.yahoo.vespa.service.monitor.InfraApplicationApi; @@ -20,7 +20,7 @@ import java.util.concurrent.ConcurrentHashMap; public class MockDuperModel implements DuperModelInfraApi { private final Map<ApplicationId, InfraApplicationApi> supportedInfraApps = new HashMap<>(); - private final ConcurrentHashMap<ApplicationId, List<HostName>> activeApps = new ConcurrentHashMap<>(); + private final ConcurrentHashMap<ApplicationId, List<DomainName>> activeApps = new ConcurrentHashMap<>(); @Inject public MockDuperModel() { @@ -46,12 +46,12 @@ public class MockDuperModel implements DuperModelInfraApi { return activeApps.containsKey(applicationId); } - public List<HostName> hostnamesOf(ApplicationId applicationId) { + public List<DomainName> hostnamesOf(ApplicationId applicationId) { return activeApps.getOrDefault(applicationId, List.of()); } @Override - public void infraApplicationActivated(ApplicationId applicationId, List<HostName> hostnames) { + public void infraApplicationActivated(ApplicationId applicationId, List<DomainName> hostnames) { activeApps.put(applicationId, hostnames); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java index c7c6e770fe3..611594cc72e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMetricsDbMaintainerTest.java @@ -44,7 +44,7 @@ public class NodeMetricsDbMaintainerTest { fetcher, Duration.ofHours(1), new TestMetric()); - assertEquals(maintainer.maintain(), 1.0, 0.0000001); + assertEquals(maintainer.maintain(), 0.0, 0.0000001); List<NodeTimeseries> timeseriesList = tester.nodeRepository().metricsDb().getNodeTimeseries(Duration.ofDays(1), Set.of("host-1.yahoo.com", "host-2.yahoo.com")); assertEquals(2, timeseriesList.size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java index 58f4be18992..aaa4bffc000 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import ai.vespa.http.DomainName; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; @@ -268,7 +269,7 @@ public class RetiredExpirerTest { private Set<String> configServerHostnames(MockDuperModel duperModel) { return duperModel.hostnamesOf(new ConfigServerApplication().getApplicationId()).stream() - .map(com.yahoo.config.provision.HostName::value) + .map(DomainName::value) .collect(Collectors.toSet()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java index 9cd5adef5f4..7763459dd92 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; +import ai.vespa.http.DomainName; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Provisioner; @@ -133,7 +133,7 @@ public class InfraDeployerImplTest { @SuppressWarnings("unchecked") private void verifyActivated(String... hostnames) { verify(duperModelInfraApi).infraApplicationActivated( - eq(application.getApplicationId()), eq(Stream.of(hostnames).map(HostName::of).toList())); + eq(application.getApplicationId()), eq(Stream.of(hostnames).map(DomainName::of).toList())); ArgumentMatcher<ApplicationTransaction> transactionMatcher = t -> { assertEquals(application.getApplicationId(), t.application()); return true; diff --git a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java index 323ae678b0b..73774321ffb 100644 --- a/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java +++ b/orchestrator/src/test/java/com/yahoo/vespa/orchestrator/OrchestratorTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.orchestrator; +import ai.vespa.http.DomainName; import com.yahoo.config.model.api.SuperModel; import com.yahoo.config.model.api.SuperModelListener; import com.yahoo.config.model.api.SuperModelProvider; @@ -87,9 +88,9 @@ public class OrchestratorTest { // There is one config server application with 3 nodes ApplicationId applicationId = new ConfigServerApplication().getApplicationId(); - var cfg1 = com.yahoo.config.provision.HostName.of("cfg1"); - var cfg2 = com.yahoo.config.provision.HostName.of("cfg2"); - var cfg3 = com.yahoo.config.provision.HostName.of("cfg3"); + var cfg1 = DomainName.of("cfg1"); + var cfg2 = DomainName.of("cfg2"); + var cfg3 = DomainName.of("cfg3"); duperModelManager.infraApplicationActivated(applicationId, List.of(cfg1, cfg2, cfg3)); duperModelManager.infraApplicationsIsNowComplete(); @@ -135,7 +136,7 @@ public class OrchestratorTest { // etc (should be the same as for cfg1) } - private HostName toApplicationModelHostName(com.yahoo.config.provision.HostName hostname) { + private HostName toApplicationModelHostName(DomainName hostname) { return new HostName(hostname.value()); } diff --git a/parent/pom.xml b/parent/pom.xml index b0ba6e89f6e..ace6a47d6bc 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -1170,7 +1170,7 @@ <org.lz4.version>1.8.0</org.lz4.version> <prometheus.client.version>0.6.0</prometheus.client.version> <protobuf.version>3.21.7</protobuf.version> - <spifly.version>1.3.5</spifly.version> + <spifly.version>1.3.6</spifly.version> <surefire.version>3.0.0-M9</surefire.version> <wiremock.version>2.35.0</wiremock.version> <zero-allocation-hashing.version>0.16</zero-allocation-hashing.version> diff --git a/screwdriver/release-container-image.sh b/screwdriver/release-container-image.sh index 8b61dde4e1d..33231890626 100755 --- a/screwdriver/release-container-image.sh +++ b/screwdriver/release-container-image.sh @@ -57,7 +57,8 @@ for data in "Dockerfile vespa"; do --jobs 2 \ --layers=false \ --manifest "vespaengine/$IMAGE_NAME:$VESPA_VERSION" \ - --platform linux/amd64,linux/arm64 | cat + --platform linux/amd64,linux/arm64 \ + --squash | cat # Test buildah tag vespaengine/$IMAGE_NAME:$VESPA_VERSION vespaengine/$IMAGE_NAME:latest diff --git a/searchcore/src/tests/proton/common/attribute_updater/CMakeLists.txt b/searchcore/src/tests/proton/common/attribute_updater/CMakeLists.txt index 5955ee52756..6e285423931 100644 --- a/searchcore/src/tests/proton/common/attribute_updater/CMakeLists.txt +++ b/searchcore/src/tests/proton/common/attribute_updater/CMakeLists.txt @@ -4,5 +4,6 @@ vespa_add_executable(searchcore_attribute_updater_test_app TEST attribute_updater_test.cpp DEPENDS searchcore_pcommon + searchlib_test ) vespa_add_test(NAME searchcore_attribute_updater_test_app COMMAND searchcore_attribute_updater_test_app) diff --git a/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp b/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp index 037bf884e6b..892be2c874f 100644 --- a/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp +++ b/searchcore/src/tests/proton/common/attribute_updater/attribute_updater_test.cpp @@ -6,6 +6,7 @@ #include <vespa/searchlib/attribute/single_raw_attribute.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> #include <vespa/searchlib/tensor/serialized_fast_value_attribute.h> +#include <vespa/searchlib/test/attribute_builder.h> #include <vespa/searchlib/test/weighted_type_test_utils.h> #include <vespa/searchcommon/attribute/config.h> #include <vespa/document/base/testdocrepo.h> @@ -53,6 +54,7 @@ using search::attribute::Config; using search::attribute::Reference; using search::attribute::ReferenceAttribute; using search::attribute::SingleRawAttribute; +using search::attribute::test::AttributeBuilder; using search::tensor::ITensorAttribute; using search::tensor::DenseTensorAttribute; using search::tensor::SerializedFastValueAttribute; @@ -124,19 +126,19 @@ struct Fixture { void applyArrayUpdates(AttributeVector & vec, std::unique_ptr<FieldValue> assign, std::unique_ptr<FieldValue> first, std::unique_ptr<FieldValue> second) { - applyValueUpdate(vec, 0, std::make_unique<AssignValueUpdate>(std::move(assign))); - applyValueUpdate(vec, 1, std::make_unique<AddValueUpdate>(std::move(second))); - applyValueUpdate(vec, 2, std::make_unique<RemoveValueUpdate>(std::move(first))); - applyValueUpdate(vec, 3, std::make_unique<ClearValueUpdate>()); + applyValueUpdate(vec, 1, std::make_unique<AssignValueUpdate>(std::move(assign))); + applyValueUpdate(vec, 2, std::make_unique<AddValueUpdate>(std::move(second))); + applyValueUpdate(vec, 3, std::make_unique<RemoveValueUpdate>(std::move(first))); + applyValueUpdate(vec, 4, std::make_unique<ClearValueUpdate>()); } void applyWeightedSetUpdates(AttributeVector & vec, std::unique_ptr<FieldValue> assign, std::unique_ptr<FieldValue> first, std::unique_ptr<FieldValue> copyOfFirst, std::unique_ptr<FieldValue> second) { - applyValueUpdate(vec, 0, std::make_unique<AssignValueUpdate>(std::move(assign))); - applyValueUpdate(vec, 1, std::make_unique<AddValueUpdate>(std::move(second), 20)); - applyValueUpdate(vec, 2, std::make_unique<RemoveValueUpdate>(std::move(first))); - applyValueUpdate(vec, 3, std::make_unique<ClearValueUpdate>()); - applyValueUpdate(vec, 4, std::make_unique<MapValueUpdate>(std::move(copyOfFirst), std::make_unique<ArithmeticValueUpdate>(ArithmeticValueUpdate::Add, 10))); + applyValueUpdate(vec, 1, std::make_unique<AssignValueUpdate>(std::move(assign))); + applyValueUpdate(vec, 2, std::make_unique<AddValueUpdate>(std::move(second), 20)); + applyValueUpdate(vec, 3, std::make_unique<RemoveValueUpdate>(std::move(first))); + applyValueUpdate(vec, 4, std::make_unique<ClearValueUpdate>()); + applyValueUpdate(vec, 5, std::make_unique<MapValueUpdate>(std::move(copyOfFirst), std::make_unique<ArithmeticValueUpdate>(ArithmeticValueUpdate::Add, 10))); } void applyValue(AttributeVector& vec, uint32_t docid, std::unique_ptr<FieldValue> value) { @@ -144,35 +146,6 @@ struct Fixture { } }; -template <typename T, typename VectorType> -AttributePtr -create(uint32_t numDocs, T val, int32_t weight, - const std::string & baseName, - const Config &info) -{ - LOG(info, "create attribute vector: %s", baseName.c_str()); - AttributePtr vec = AttributeFactory::createAttribute(baseName, info); - VectorType * api = static_cast<VectorType *>(vec.get()); - for (uint32_t i = 0; i < numDocs; ++i) { - if (!api->addDoc(i)) { - LOG(info, "failed adding doc: %u", i); - return AttributePtr(); - } - if (api->hasMultiValue()) { - if (!api->append(i, val, weight)) { - LOG(info, "failed append to doc: %u", i); - } - } else { - if (!api->update(i, val)) { - LOG(info, "failed update doc: %u", i); - return AttributePtr(); - } - } - } - api->commit(); - return vec; -} - template <typename T> bool check(const AttributePtr &vec, uint32_t docId, const std::vector<T> &values) @@ -223,70 +196,67 @@ TEST_F("require that single attributes are updated", Fixture) using search::attribute::getUndefined; CollectionType ct(CollectionType::SINGLE); { - BasicType bt(BasicType::INT32); - AttributePtr vec = create<int32_t, IntegerAttribute>(3, 32, 0, "in1/int", Config(bt, ct)); - f.applyValueUpdate(*vec, 0, std::make_unique<AssignValueUpdate>(std::make_unique<IntFieldValue>(64))); - f.applyValueUpdate(*vec, 1, std::make_unique<ArithmeticValueUpdate>(ArithmeticValueUpdate::Add, 10)); - f.applyValueUpdate(*vec, 2, std::make_unique<ClearValueUpdate>()); - EXPECT_EQUAL(3u, vec->getNumDocs()); - EXPECT_TRUE(check(vec, 0, std::vector<WeightedInt>{WeightedInt(64)})); - EXPECT_TRUE(check(vec, 1, std::vector<WeightedInt>{WeightedInt(42)})); - EXPECT_TRUE(check(vec, 2, std::vector<WeightedInt>{WeightedInt(getUndefined<int32_t>())})); + auto vec = AttributeBuilder("in1/int", Config(BasicType::INT32)).fill({ 32, 32, 32}).get(); + f.applyValueUpdate(*vec, 1, std::make_unique<AssignValueUpdate>(std::make_unique<IntFieldValue>(64))); + f.applyValueUpdate(*vec, 2, std::make_unique<ArithmeticValueUpdate>(ArithmeticValueUpdate::Add, 10)); + f.applyValueUpdate(*vec, 3, std::make_unique<ClearValueUpdate>()); + EXPECT_EQUAL(4u, vec->getNumDocs()); + EXPECT_TRUE(check(vec, 1, std::vector<WeightedInt>{WeightedInt(64)})); + EXPECT_TRUE(check(vec, 2, std::vector<WeightedInt>{WeightedInt(42)})); + EXPECT_TRUE(check(vec, 3, std::vector<WeightedInt>{WeightedInt(getUndefined<int32_t>())})); } { - BasicType bt(BasicType::FLOAT); - AttributePtr vec = create<float, FloatingPointAttribute>(3, 55.5f, 0, "in1/float",Config(bt, ct)); - f.applyValueUpdate(*vec, 0, std::make_unique<AssignValueUpdate>(std::make_unique<FloatFieldValue>(77.7f))); - f.applyValueUpdate(*vec, 1, std::make_unique<ArithmeticValueUpdate>(ArithmeticValueUpdate::Add, 10)); - f.applyValueUpdate(*vec, 2, std::make_unique<ClearValueUpdate>()); - EXPECT_EQUAL(3u, vec->getNumDocs()); - EXPECT_TRUE(check(vec, 0, std::vector<WeightedFloat>{WeightedFloat(77.7f)})); - EXPECT_TRUE(check(vec, 1, std::vector<WeightedFloat>{WeightedFloat(65.5f)})); - EXPECT_TRUE(std::isnan(vec->getFloat(2))); + auto vec = AttributeBuilder("in1/float", Config(BasicType::FLOAT)).fill({55.5, 55.5, 55.5}).get(); + f.applyValueUpdate(*vec, 1, std::make_unique<AssignValueUpdate>(std::make_unique<FloatFieldValue>(77.7f))); + f.applyValueUpdate(*vec, 2, std::make_unique<ArithmeticValueUpdate>(ArithmeticValueUpdate::Add, 10)); + f.applyValueUpdate(*vec, 3, std::make_unique<ClearValueUpdate>()); + EXPECT_EQUAL(4u, vec->getNumDocs()); + EXPECT_TRUE(check(vec, 1, std::vector<WeightedFloat>{WeightedFloat(77.7f)})); + EXPECT_TRUE(check(vec, 2, std::vector<WeightedFloat>{WeightedFloat(65.5f)})); + EXPECT_TRUE(std::isnan(vec->getFloat(3))); } { - BasicType bt(BasicType::STRING); - AttributePtr vec = create<std::string, StringAttribute>(3, "first", 0, "in1/string",Config(bt, ct)); - f.applyValueUpdate(*vec, 0, std::make_unique<AssignValueUpdate>(StringFieldValue::make("second"))); - f.applyValueUpdate(*vec, 2, std::make_unique<ClearValueUpdate>()); - EXPECT_EQUAL(3u, vec->getNumDocs()); - EXPECT_TRUE(check(vec, 0, std::vector<WeightedString>{WeightedString("second")})); - EXPECT_TRUE(check(vec, 1, std::vector<WeightedString>{WeightedString("first")})); - EXPECT_TRUE(check(vec, 2, std::vector<WeightedString>{WeightedString("")})); + auto vec = AttributeBuilder("in1/string", Config(BasicType::STRING)).fill({"first", "first", "first"}).get(); + f.applyValueUpdate(*vec, 1, std::make_unique<AssignValueUpdate>(StringFieldValue::make("second"))); + f.applyValueUpdate(*vec, 3, std::make_unique<ClearValueUpdate>()); + EXPECT_EQUAL(4u, vec->getNumDocs()); + EXPECT_TRUE(check(vec, 1, std::vector<WeightedString>{WeightedString("second")})); + EXPECT_TRUE(check(vec, 2, std::vector<WeightedString>{WeightedString("first")})); + EXPECT_TRUE(check(vec, 3, std::vector<WeightedString>{WeightedString("")})); } { BasicType bt(BasicType::REFERENCE); Config cfg(bt, ct); AttributePtr vec = AttributeFactory::createAttribute("in1/ref", cfg); + vec->addReservedDoc(); uint32_t startDoc = 0; uint32_t endDoc = 0; EXPECT_TRUE(vec->addDocs(startDoc, endDoc, 3)); - EXPECT_EQUAL(0u, startDoc); - EXPECT_EQUAL(2u, endDoc); - for (uint32_t docId = 0; docId < 3; ++docId) { + EXPECT_EQUAL(1u, startDoc); + EXPECT_EQUAL(3u, endDoc); + for (uint32_t docId = 1; docId < 4; ++docId) { asReferenceAttribute(*vec).update(docId, toGid(doc1)); } vec->commit(); - f.applyValueUpdate(*vec, 0, std::make_unique<AssignValueUpdate>(std::make_unique<ReferenceFieldValue>(dynamic_cast<const ReferenceDataType &>(f.docType->getField("ref").getDataType()), DocumentId(doc2)))); - f.applyValueUpdate(*vec, 2, std::make_unique<ClearValueUpdate>()); - EXPECT_EQUAL(3u, vec->getNumDocs()); - TEST_DO(assertRef(*vec, doc2, 0)); - TEST_DO(assertRef(*vec, doc1, 1)); - TEST_DO(assertNoRef(*vec, 2)); + f.applyValueUpdate(*vec, 1, std::make_unique<AssignValueUpdate>(std::make_unique<ReferenceFieldValue>(dynamic_cast<const ReferenceDataType &>(f.docType->getField("ref").getDataType()), DocumentId(doc2)))); + f.applyValueUpdate(*vec, 3, std::make_unique<ClearValueUpdate>()); + EXPECT_EQUAL(4u, vec->getNumDocs()); + TEST_DO(assertRef(*vec, doc2, 1)); + TEST_DO(assertRef(*vec, doc1, 2)); + TEST_DO(assertNoRef(*vec, 3)); } { - BasicType bt(BasicType::RAW); vespalib::string first_backing("first"); vespalib::ConstArrayRef<char> first(first_backing.data(), first_backing.size()); - AttributePtr vec = create<vespalib::ConstArrayRef<char>, SingleRawAttribute>(4, first, 0, "in1/raw", Config(bt, ct)); - f.applyValueUpdate(*vec, 0, std::make_unique<AssignValueUpdate>(std::make_unique<RawFieldValue>("second"))); - f.applyValueUpdate(*vec, 2, std::make_unique<ClearValueUpdate>()); - f.applyValue(*vec, 3, std::make_unique<RawFieldValue>("third")); - EXPECT_EQUAL(4u, vec->getNumDocs()); - EXPECT_EQUAL(as_vector("second"), as_vector(vec->get_raw(0))); - EXPECT_EQUAL(as_vector("first"), as_vector(vec->get_raw(1))); - EXPECT_EQUAL(as_vector(""), as_vector(vec->get_raw(2))); - EXPECT_EQUAL(as_vector("third"), as_vector(vec->get_raw(3))); + auto vec = AttributeBuilder("in1/raw", Config(BasicType::RAW)).fill({first, first, first, first}).get(); + f.applyValueUpdate(*vec, 1, std::make_unique<AssignValueUpdate>(std::make_unique<RawFieldValue>("second"))); + f.applyValueUpdate(*vec, 3, std::make_unique<ClearValueUpdate>()); + f.applyValue(*vec, 4, std::make_unique<RawFieldValue>("third")); + EXPECT_EQUAL(5u, vec->getNumDocs()); + EXPECT_EQUAL(as_vector("second"), as_vector(vec->get_raw(1))); + EXPECT_EQUAL(as_vector("first"), as_vector(vec->get_raw(2))); + EXPECT_EQUAL(as_vector(""), as_vector(vec->get_raw(3))); + EXPECT_EQUAL(as_vector("third"), as_vector(vec->get_raw(4))); } } @@ -294,52 +264,51 @@ TEST_F("require that array attributes are updated", Fixture) { CollectionType ct(CollectionType::ARRAY); { - BasicType bt(BasicType::INT32); - AttributePtr vec = create<int32_t, IntegerAttribute>(5, 32, 1, "in1/aint", Config(bt, ct)); + using IL = AttributeBuilder::IntList; + auto vec = AttributeBuilder("in1/aint", Config(BasicType::INT32, ct)).fill_array({IL{32}, {32}, {32}, {32}, {32}}).get(); auto first = std::make_unique<IntFieldValue>(32); auto second = std::make_unique<IntFieldValue>(64); auto assign = std::make_unique<ArrayFieldValue>(f.docType->getField("aint").getDataType()); assign->add(*second); f.applyArrayUpdates(*vec, std::move(assign), std::move(first), std::move(second)); - EXPECT_EQUAL(5u, vec->getNumDocs()); - EXPECT_TRUE(check(vec, 0, std::vector<WeightedInt>{WeightedInt(64)})); - EXPECT_TRUE(check(vec, 1, std::vector<WeightedInt>{WeightedInt(32), WeightedInt(64)})); - EXPECT_TRUE(check(vec, 2, std::vector<WeightedInt>{})); + EXPECT_EQUAL(6u, vec->getNumDocs()); + EXPECT_TRUE(check(vec, 1, std::vector<WeightedInt>{WeightedInt(64)})); + EXPECT_TRUE(check(vec, 2, std::vector<WeightedInt>{WeightedInt(32), WeightedInt(64)})); EXPECT_TRUE(check(vec, 3, std::vector<WeightedInt>{})); - EXPECT_TRUE(check(vec, 4, std::vector<WeightedInt>{WeightedInt(32)})); + EXPECT_TRUE(check(vec, 4, std::vector<WeightedInt>{})); + EXPECT_TRUE(check(vec, 5, std::vector<WeightedInt>{WeightedInt(32)})); } { - BasicType bt(BasicType::FLOAT); - AttributePtr vec = create<float, FloatingPointAttribute>(5, 55.5f, 1, "in1/afloat", Config(bt, ct)); + using DL = AttributeBuilder::DoubleList; + auto vec = AttributeBuilder("in1/afloat", Config(BasicType::FLOAT, ct)).fill_array({DL{55.5}, {55.5}, {55.5}, {55.5}, {55.5}}).get(); auto first = std::make_unique<FloatFieldValue>(55.5f); auto second = std::make_unique<FloatFieldValue>(77.7f); auto assign = std::make_unique<ArrayFieldValue>(f.docType->getField("afloat").getDataType()); assign->add(*second); f.applyArrayUpdates(*vec, std::move(assign), std::move(first), std::move(second)); - EXPECT_EQUAL(5u, vec->getNumDocs()); - EXPECT_TRUE(check(vec, 0, std::vector<WeightedFloat>{WeightedFloat(77.7f)})); - EXPECT_TRUE(check(vec, 1, std::vector<WeightedFloat>{WeightedFloat(55.5f), WeightedFloat(77.7f)})); - EXPECT_TRUE(check(vec, 2, std::vector<WeightedFloat>{})); + EXPECT_EQUAL(6u, vec->getNumDocs()); + EXPECT_TRUE(check(vec, 1, std::vector<WeightedFloat>{WeightedFloat(77.7f)})); + EXPECT_TRUE(check(vec, 2, std::vector<WeightedFloat>{WeightedFloat(55.5f), WeightedFloat(77.7f)})); EXPECT_TRUE(check(vec, 3, std::vector<WeightedFloat>{})); - EXPECT_TRUE(check(vec, 4, std::vector<WeightedFloat>{WeightedFloat(55.5f)})); + EXPECT_TRUE(check(vec, 4, std::vector<WeightedFloat>{})); + EXPECT_TRUE(check(vec, 5, std::vector<WeightedFloat>{WeightedFloat(55.5f)})); } { - BasicType bt(BasicType::STRING); - AttributePtr vec = create<std::string, StringAttribute>(5, "first", 1, "in1/astring", Config(bt, ct)); + auto vec = AttributeBuilder("in1/astring", Config(BasicType::STRING, ct)).fill_array({{"first"}, {"first"}, {"first"}, {"first"}, {"first"}}).get(); auto first = StringFieldValue::make("first"); auto second = StringFieldValue::make("second"); auto assign = std::make_unique<ArrayFieldValue>(f.docType->getField("astring").getDataType()); assign->add(*second); f.applyArrayUpdates(*vec, std::move(assign), std::move(first), std::move(second)); - EXPECT_EQUAL(5u, vec->getNumDocs()); - EXPECT_TRUE(check(vec, 0, std::vector<WeightedString>{WeightedString("second")})); - EXPECT_TRUE(check(vec, 1, std::vector<WeightedString>{WeightedString("first"), WeightedString("second")})); - EXPECT_TRUE(check(vec, 2, std::vector<WeightedString>{})); + EXPECT_EQUAL(6u, vec->getNumDocs()); + EXPECT_TRUE(check(vec, 1, std::vector<WeightedString>{WeightedString("second")})); + EXPECT_TRUE(check(vec, 2, std::vector<WeightedString>{WeightedString("first"), WeightedString("second")})); EXPECT_TRUE(check(vec, 3, std::vector<WeightedString>{})); - EXPECT_TRUE(check(vec, 4, std::vector<WeightedString>{WeightedString("first")})); + EXPECT_TRUE(check(vec, 4, std::vector<WeightedString>{})); + EXPECT_TRUE(check(vec, 5, std::vector<WeightedString>{WeightedString("first")})); } } @@ -347,8 +316,8 @@ TEST_F("require that weighted set attributes are updated", Fixture) { CollectionType ct(CollectionType::WSET); { - BasicType bt(BasicType::INT32); - AttributePtr vec = create<int32_t, IntegerAttribute>(5, 32, 100, "in1/wsint", Config(bt, ct)); + using WIL = AttributeBuilder::WeightedIntList; + auto vec = AttributeBuilder("in1/wsint", Config(BasicType::INT32, ct)).fill_wset({WIL{{32, 100}}, {{32, 100}}, {{32, 100}}, {{32, 100}}, {{32, 100}}}).get(); auto first = std::make_unique<IntFieldValue>(32); auto copyOfFirst = std::make_unique<IntFieldValue>(32); auto second = std::make_unique<IntFieldValue>(64); @@ -356,16 +325,16 @@ TEST_F("require that weighted set attributes are updated", Fixture) assign->add(*second, 20); f.applyWeightedSetUpdates(*vec, std::move(assign), std::move(first), std::move(copyOfFirst), std::move(second)); - EXPECT_EQUAL(5u, vec->getNumDocs()); - EXPECT_TRUE(check(vec, 0, std::vector<WeightedInt>{WeightedInt(64, 20)})); - EXPECT_TRUE(check(vec, 1, std::vector<WeightedInt>{WeightedInt(32, 100), WeightedInt(64, 20)})); - EXPECT_TRUE(check(vec, 2, std::vector<WeightedInt>{})); + EXPECT_EQUAL(6u, vec->getNumDocs()); + EXPECT_TRUE(check(vec, 1, std::vector<WeightedInt>{WeightedInt(64, 20)})); + EXPECT_TRUE(check(vec, 2, std::vector<WeightedInt>{WeightedInt(32, 100), WeightedInt(64, 20)})); EXPECT_TRUE(check(vec, 3, std::vector<WeightedInt>{})); - EXPECT_TRUE(check(vec, 4, std::vector<WeightedInt>{WeightedInt(32, 110)})); + EXPECT_TRUE(check(vec, 4, std::vector<WeightedInt>{})); + EXPECT_TRUE(check(vec, 5, std::vector<WeightedInt>{WeightedInt(32, 110)})); } { - BasicType bt(BasicType::FLOAT); - AttributePtr vec = create<float, FloatingPointAttribute>(5, 55.5f, 100, "in1/wsfloat", Config(bt, ct)); + using WDL = AttributeBuilder::WeightedDoubleList; + auto vec = AttributeBuilder("in1/wsfloat", Config(BasicType::FLOAT, ct)).fill_wset({WDL{{55.5, 100}}, {{55.5, 100}}, {{55.5, 100}}, {{55.5, 100}}, {{55.5, 100}}}).get(); auto first = std::make_unique<FloatFieldValue>(55.5f); auto copyOfFirst = std::make_unique<FloatFieldValue>(55.5f); auto second = std::make_unique<FloatFieldValue>(77.7f); @@ -373,16 +342,15 @@ TEST_F("require that weighted set attributes are updated", Fixture) assign->add(*second, 20); f.applyWeightedSetUpdates(*vec, std::move(assign), std::move(first), std::move(copyOfFirst), std::move(second)); - EXPECT_EQUAL(5u, vec->getNumDocs()); - EXPECT_TRUE(check(vec, 0, std::vector<WeightedFloat>{WeightedFloat(77.7f, 20)})); - EXPECT_TRUE(check(vec, 1, std::vector<WeightedFloat>{WeightedFloat(55.5f, 100), WeightedFloat(77.7f, 20)})); - EXPECT_TRUE(check(vec, 2, std::vector<WeightedFloat>{})); + EXPECT_EQUAL(6u, vec->getNumDocs()); + EXPECT_TRUE(check(vec, 1, std::vector<WeightedFloat>{WeightedFloat(77.7f, 20)})); + EXPECT_TRUE(check(vec, 2, std::vector<WeightedFloat>{WeightedFloat(55.5f, 100), WeightedFloat(77.7f, 20)})); EXPECT_TRUE(check(vec, 3, std::vector<WeightedFloat>{})); - EXPECT_TRUE(check(vec, 4, std::vector<WeightedFloat>{WeightedFloat(55.5f, 110)})); + EXPECT_TRUE(check(vec, 4, std::vector<WeightedFloat>{})); + EXPECT_TRUE(check(vec, 5, std::vector<WeightedFloat>{WeightedFloat(55.5f, 110)})); } { - BasicType bt(BasicType::STRING); - AttributePtr vec = create<std::string, StringAttribute>(5, "first", 100, "in1/wsstring", Config(bt, ct)); + auto vec = AttributeBuilder("in1/wsstring", Config(BasicType::STRING, ct)).fill_wset({{{"first", 100}}, {{"first", 100}}, {{"first", 100}}, {{"first", 100}}, {{"first", 100}}}).get(); auto first = StringFieldValue::make("first"); auto copyOfFirst = StringFieldValue::make("first"); auto second = StringFieldValue::make("second"); @@ -390,12 +358,12 @@ TEST_F("require that weighted set attributes are updated", Fixture) assign->add(*second, 20); f.applyWeightedSetUpdates(*vec, std::move(assign), std::move(first), std::move(copyOfFirst), std::move(second)); - EXPECT_EQUAL(5u, vec->getNumDocs()); - EXPECT_TRUE(check(vec, 0, std::vector<WeightedString>{WeightedString("second", 20)})); - EXPECT_TRUE(check(vec, 1, std::vector<WeightedString>{WeightedString("first", 100), WeightedString("second", 20)})); - EXPECT_TRUE(check(vec, 2, std::vector<WeightedString>{})); + EXPECT_EQUAL(6u, vec->getNumDocs()); + EXPECT_TRUE(check(vec, 1, std::vector<WeightedString>{WeightedString("second", 20)})); + EXPECT_TRUE(check(vec, 2, std::vector<WeightedString>{WeightedString("first", 100), WeightedString("second", 20)})); EXPECT_TRUE(check(vec, 3, std::vector<WeightedString>{})); - EXPECT_TRUE(check(vec, 4, std::vector<WeightedString>{WeightedString("first", 110)})); + EXPECT_TRUE(check(vec, 4, std::vector<WeightedString>{})); + EXPECT_TRUE(check(vec, 5, std::vector<WeightedString>{WeightedString("first", 110)})); } } diff --git a/searchcore/src/tests/proton/docsummary/docsummary_test.cpp b/searchcore/src/tests/proton/docsummary/docsummary_test.cpp index 020f4ff42c1..70f81d629d5 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary_test.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary_test.cpp @@ -107,7 +107,10 @@ namespace proton { class MockDocsumFieldWriterFactory : public search::docsummary::IDocsumFieldWriterFactory { public: - std::unique_ptr<DocsumFieldWriter> create_docsum_field_writer(const vespalib::string&, const vespalib::string&, const vespalib::string&) override { + std::unique_ptr<DocsumFieldWriter> create_docsum_field_writer(const vespalib::string&, + const vespalib::string&, + const vespalib::string&, + std::shared_ptr<search::MatchingElementsFields>) override { return {}; } diff --git a/searchlib/src/tests/attribute/attribute_test.cpp b/searchlib/src/tests/attribute/attribute_test.cpp index 7eb43faac18..d2d3ccaad23 100644 --- a/searchlib/src/tests/attribute/attribute_test.cpp +++ b/searchlib/src/tests/attribute/attribute_test.cpp @@ -912,8 +912,8 @@ AttributeTest::testSingle() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("sv-post-int32", cfg); ptr->updateStat(true); - EXPECT_EQ(339020u, ptr->getStatus().getAllocated()); - EXPECT_EQ(101852u, ptr->getStatus().getUsed()); + EXPECT_EQ(338972u, ptr->getStatus().getAllocated()); + EXPECT_EQ(101492u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testSingle<IntegerAttribute, AttributeVector::largeint_t, int32_t>(ptr, values); } @@ -934,8 +934,8 @@ AttributeTest::testSingle() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("sv-post-float", cfg); ptr->updateStat(true); - EXPECT_EQ(339020, ptr->getStatus().getAllocated()); - EXPECT_EQ(101852u, ptr->getStatus().getUsed()); + EXPECT_EQ(338972u, ptr->getStatus().getAllocated()); + EXPECT_EQ(101492u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testSingle<FloatingPointAttribute, double, float>(ptr, values); } @@ -947,8 +947,8 @@ AttributeTest::testSingle() { AttributePtr ptr = createAttribute("sv-string", Config(BasicType::STRING, CollectionType::SINGLE)); ptr->updateStat(true); - EXPECT_EQ(117256u + sizeof_large_string_entry, ptr->getStatus().getAllocated()); - EXPECT_EQ(53240u + sizeof_large_string_entry, ptr->getStatus().getUsed()); + EXPECT_EQ(116528u + sizeof_large_string_entry, ptr->getStatus().getAllocated()); + EXPECT_EQ(52760u + sizeof_large_string_entry, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testSingle<StringAttribute, string, string>(ptr, values); } @@ -957,8 +957,8 @@ AttributeTest::testSingle() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("sv-fs-string", cfg); ptr->updateStat(true); - EXPECT_EQ(345624u + sizeof_large_string_entry, ptr->getStatus().getAllocated()); - EXPECT_EQ(105176u + sizeof_large_string_entry, ptr->getStatus().getUsed()); + EXPECT_EQ(344848u + sizeof_large_string_entry, ptr->getStatus().getAllocated()); + EXPECT_EQ(104408u + sizeof_large_string_entry, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testSingle<StringAttribute, string, string>(ptr, values); } @@ -1089,8 +1089,8 @@ AttributeTest::testArray() { AttributePtr ptr = createAttribute("a-int32", Config(BasicType::INT32, CollectionType::ARRAY)); ptr->updateStat(true); - EXPECT_EQ(512056u, ptr->getStatus().getAllocated()); - EXPECT_EQ(504392u, ptr->getStatus().getUsed()); + EXPECT_EQ(487480u, ptr->getStatus().getAllocated()); + EXPECT_EQ(479720u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<IntegerAttribute, AttributeVector::largeint_t>(ptr, values); } @@ -1099,8 +1099,8 @@ AttributeTest::testArray() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("flags", cfg); ptr->updateStat(true); - EXPECT_EQ(512056u, ptr->getStatus().getAllocated()); - EXPECT_EQ(504392u, ptr->getStatus().getUsed()); + EXPECT_EQ(487480u, ptr->getStatus().getAllocated()); + EXPECT_EQ(479720u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<IntegerAttribute, AttributeVector::largeint_t>(ptr, values); } @@ -1109,8 +1109,8 @@ AttributeTest::testArray() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("a-fs-int32", cfg); ptr->updateStat(true); - EXPECT_EQ(868788u, ptr->getStatus().getAllocated()); - EXPECT_EQ(606264u, ptr->getStatus().getUsed()); + EXPECT_EQ(844116u, ptr->getStatus().getAllocated()); + EXPECT_EQ(581232u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<IntegerAttribute, AttributeVector::largeint_t>(ptr, values); } @@ -1128,8 +1128,8 @@ AttributeTest::testArray() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("a-fs-float", cfg); ptr->updateStat(true); - EXPECT_EQ(868788u, ptr->getStatus().getAllocated()); - EXPECT_EQ(606264u, ptr->getStatus().getUsed()); + EXPECT_EQ(844116u, ptr->getStatus().getAllocated()); + EXPECT_EQ(581232u, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<FloatingPointAttribute, double>(ptr, values); } @@ -1140,8 +1140,8 @@ AttributeTest::testArray() { AttributePtr ptr = createAttribute("a-string", Config(BasicType::STRING, CollectionType::ARRAY)); ptr->updateStat(true); - EXPECT_EQ(625088u + sizeof_large_string_entry, ptr->getStatus().getAllocated()); - EXPECT_EQ(557632u + sizeof_large_string_entry, ptr->getStatus().getUsed()); + EXPECT_EQ(599784u + sizeof_large_string_entry, ptr->getStatus().getAllocated()); + EXPECT_EQ(532480u + sizeof_large_string_entry, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<StringAttribute, string>(ptr, values); } @@ -1150,8 +1150,8 @@ AttributeTest::testArray() cfg.setFastSearch(true); AttributePtr ptr = createAttribute("afs-string", cfg); ptr->updateStat(true); - EXPECT_EQ(875392u + sizeof_large_string_entry, ptr->getStatus().getAllocated()); - EXPECT_EQ(609588u + sizeof_large_string_entry, ptr->getStatus().getUsed()); + EXPECT_EQ(849992u + sizeof_large_string_entry, ptr->getStatus().getAllocated()); + EXPECT_EQ(584148u + sizeof_large_string_entry, ptr->getStatus().getUsed()); addDocs(ptr, numDocs); testArray<StringAttribute, string>(ptr, values); } @@ -2334,6 +2334,10 @@ AttributeTest::test_paged_attribute(const vespalib::string& name, const vespalib size_t rounded_size = vespalib::round_up_to_page_size(1); size_t lid_mapping_size = 1200; size_t sv_maxlid = 1200; + if (rounded_size == 16_Ki) { + lid_mapping_size = 4200; + sv_maxlid = 1300; + } if (rounded_size == 64_Ki) { lid_mapping_size = 17000; sv_maxlid = 1500; diff --git a/searchlib/src/tests/attribute/extendattributes/extendattribute.cpp b/searchlib/src/tests/attribute/extendattributes/extendattribute.cpp index 8f056323733..3f775e99891 100644 --- a/searchlib/src/tests/attribute/extendattributes/extendattribute.cpp +++ b/searchlib/src/tests/attribute/extendattributes/extendattribute.cpp @@ -1,9 +1,25 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/gtest/gtest.h> +#include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/value_codec.h> +#include <vespa/searchcommon/attribute/config.h> #include <vespa/searchlib/attribute/extendableattributes.h> #include <vespa/searchlib/attribute/single_raw_ext_attribute.h> +#include <vespa/searchlib/tensor/tensor_ext_attribute.h> +#include <vespa/searchlib/tensor/vector_bundle.h> +#include <vespa/vespalib/stllike/asciistream.h> +using search::attribute::Config; +using search::attribute::BasicType; +using search::attribute::CollectionType; using search::attribute::SingleRawExtAttribute; +using search::tensor::TensorExtAttribute; +using vespalib::eval::FastValueBuilderFactory; +using vespalib::eval::TensorSpec; +using vespalib::eval::Value; +using vespalib::eval::ValueType; namespace search { @@ -15,8 +31,46 @@ std::vector<char> as_vector(vespalib::ConstArrayRef<char> value) { return {value.data(), value.data() + value.size()}; } +std::vector<double> as_vector(vespalib::ConstArrayRef<double> value) { + return {value.data(), value.data() + value.size()}; +} + +vespalib::string vec_2d_spec("tensor(x[2])"); +vespalib::string vec_mixed_2d_spec("tensor(a{},x[2])"); + +TensorSpec +vec_2d(double x0, double x1) +{ + return TensorSpec(vec_2d_spec).add({{"x", 0}}, x0).add({{"x", 1}}, x1); +} + +TensorSpec +vec_mixed_2d(std::vector<std::vector<double>> val) +{ + TensorSpec spec(vec_mixed_2d_spec); + for (uint32_t a = 0; a < val.size(); ++a) { + vespalib::asciistream a_stream; + a_stream << a; + vespalib::string a_as_string = a_stream.str(); + for (uint32_t x = 0; x < val[a].size(); ++x) { + spec.add({{"a", a_as_string.c_str()},{"x", x}}, val[a][x]); + } + } + return spec; +} + +void add_doc(AttributeVector& attr, uint32_t exp_docid) +{ + uint32_t docid(0); + EXPECT_EQ(exp_docid, attr.getNumDocs()); + attr.addDoc(docid); + EXPECT_EQ(exp_docid, docid); + EXPECT_EQ(exp_docid + 1, attr.getNumDocs()); +} + class ExtendAttributeTest : public ::testing::Test { + std::vector<std::unique_ptr<Value>> _tensors; protected: ExtendAttributeTest() = default; ~ExtendAttributeTest() override = default; @@ -27,16 +81,22 @@ protected: template <typename Attribute> void testExtendString(Attribute & attr); void testExtendRaw(AttributeVector& attr); + void testExtendTensor(AttributeVector& attr); + const Value& create_tensor(const TensorSpec &spec); }; +const Value& +ExtendAttributeTest::create_tensor(const TensorSpec &spec) +{ + auto value = value_from_spec(spec, FastValueBuilderFactory::get()); + _tensors.emplace_back(std::move(value)); + return *_tensors.back(); +} + template <typename Attribute> void ExtendAttributeTest::testExtendInteger(Attribute & attr) { - uint32_t docId(0); - EXPECT_EQ(attr.getNumDocs(), 0u); - attr.addDoc(docId); - EXPECT_EQ(docId, 0u); - EXPECT_EQ(attr.getNumDocs(), 1u); + add_doc(attr, 0); attr.add(1, 10); EXPECT_EQ(attr.getInt(0), 1); attr.add(2, 20); @@ -51,9 +111,7 @@ void ExtendAttributeTest::testExtendInteger(Attribute & attr) EXPECT_EQ(v[1].getWeight(), 20); } } - attr.addDoc(docId); - EXPECT_EQ(docId, 1u); - EXPECT_EQ(attr.getNumDocs(), 2u); + add_doc(attr, 1); attr.add(3, 30); EXPECT_EQ(attr.getInt(1), 3); if (attr.hasMultiValue()) { @@ -69,11 +127,7 @@ void ExtendAttributeTest::testExtendInteger(Attribute & attr) template <typename Attribute> void ExtendAttributeTest::testExtendFloat(Attribute & attr) { - uint32_t docId(0); - EXPECT_EQ(attr.getNumDocs(), 0u); - attr.addDoc(docId); - EXPECT_EQ(docId, 0u); - EXPECT_EQ(attr.getNumDocs(), 1u); + add_doc(attr, 0); attr.add(1.7, 10); EXPECT_EQ(attr.getInt(0), 1); EXPECT_EQ(attr.getFloat(0), 1.7); @@ -89,9 +143,7 @@ void ExtendAttributeTest::testExtendFloat(Attribute & attr) EXPECT_EQ(v[1].getWeight(), 20); } } - attr.addDoc(docId); - EXPECT_EQ(docId, 1u); - EXPECT_EQ(attr.getNumDocs(), 2u); + add_doc(attr, 1); attr.add(3.6, 30); EXPECT_EQ(attr.getFloat(1), 3.6); if (attr.hasMultiValue()) { @@ -107,11 +159,7 @@ void ExtendAttributeTest::testExtendFloat(Attribute & attr) template <typename Attribute> void ExtendAttributeTest::testExtendString(Attribute & attr) { - uint32_t docId(0); - EXPECT_EQ(attr.getNumDocs(), 0u); - attr.addDoc(docId); - EXPECT_EQ(docId, 0u); - EXPECT_EQ(attr.getNumDocs(), 1u); + add_doc(attr, 0); attr.add("1.7", 10); auto buf = attr.get_raw(0); EXPECT_EQ(std::string(buf.data(), buf.size()), "1.7"); @@ -128,9 +176,7 @@ void ExtendAttributeTest::testExtendString(Attribute & attr) EXPECT_EQ(v[1].getWeight(), 20); } } - attr.addDoc(docId); - EXPECT_EQ(docId, 1u); - EXPECT_EQ(attr.getNumDocs(), 2u); + add_doc(attr, 1); attr.add("3.6", 30); buf = attr.get_raw(1); EXPECT_EQ(std::string(buf.data(), buf.size()), "3.6"); @@ -150,41 +196,77 @@ void ExtendAttributeTest::testExtendRaw(AttributeVector& attr) std::vector<char> zeros{10, 0, 0, 11}; auto* ext_attr = attr.getExtendInterface(); EXPECT_NE(nullptr, ext_attr); - uint32_t docId(0); - EXPECT_EQ(0u, attr.getNumDocs()); - attr.addDoc(docId); - EXPECT_EQ(0u, docId); - EXPECT_EQ(1u, attr.getNumDocs()); + add_doc(attr, 0); ext_attr->add(as_vector("1.7")); auto buf = attr.get_raw(0); EXPECT_EQ(as_vector("1.7"), as_vector(buf)); ext_attr->add(vespalib::ConstArrayRef<char>(as_vector("2.3"))); buf = attr.get_raw(0); EXPECT_EQ(as_vector("2.3"), as_vector(buf)); - attr.addDoc(docId); - EXPECT_EQ(1u, docId); - EXPECT_EQ(attr.getNumDocs(), 2u); + add_doc(attr, 1); ext_attr->add(as_vector("3.6")); buf = attr.get_raw(1); EXPECT_EQ(as_vector("3.6"), as_vector(buf)); buf = attr.get_raw(0); EXPECT_EQ(as_vector("2.3"), as_vector(buf)); - attr.addDoc(docId); - EXPECT_EQ(2u, docId); + add_doc(attr, 2); ext_attr->add(zeros); buf = attr.get_raw(2); EXPECT_EQ(zeros, as_vector(buf)); - attr.addDoc(docId); - EXPECT_EQ(3u, docId); + add_doc(attr, 3); buf = attr.get_raw(3); EXPECT_EQ(empty, as_vector(buf)); - attr.addDoc(docId); - EXPECT_EQ(4u, docId); + add_doc(attr, 4); ext_attr->add(empty); buf = attr.get_raw(4); EXPECT_EQ(empty, as_vector(buf)); } +void ExtendAttributeTest::testExtendTensor(AttributeVector& attr) +{ + std::vector<double> empty_cells{0.0, 0.0}; + std::vector<double> spec0_dense_cells{1.0, 2.0}; + std::vector<double> spec0_mixed_cells0{3.0, 4.0}; + std::vector<double> spec0_mixed_cells1{5.0, 6.0}; + bool dense = attr.getConfig().tensorType().is_dense(); + auto* ext_attr = attr.getExtendInterface(); + EXPECT_NE(nullptr, ext_attr); + auto* tensor_attr = attr.asTensorAttribute(); + EXPECT_NE(nullptr, tensor_attr); + add_doc(attr, 0); + TensorSpec spec0 = dense ? vec_2d(1.0, 2.0) : vec_mixed_2d({{3.0, 4.0}, {5.0, 6.0}}); + EXPECT_TRUE(ext_attr->add(create_tensor(spec0))); + auto tensor = tensor_attr->getTensor(0); + EXPECT_NE(nullptr, tensor.get()); + EXPECT_EQ(spec0, TensorSpec::from_value(*tensor)); + EXPECT_EQ(dense, tensor_attr->supports_extract_cells_ref()); + if (dense) { + EXPECT_EQ(spec0_dense_cells, as_vector(tensor_attr->extract_cells_ref(0).typify<double>())); + } + EXPECT_TRUE(tensor_attr->supports_get_tensor_ref()); + EXPECT_EQ(spec0, TensorSpec::from_value(tensor_attr->get_tensor_ref(0))); + EXPECT_FALSE(tensor_attr->supports_get_serialized_tensor_ref()); + auto vectors = tensor_attr->get_vectors(0); + if (dense) { + EXPECT_EQ(1, vectors.subspaces()); + EXPECT_EQ(spec0_dense_cells, as_vector(vectors.cells(0).typify<double>())); + EXPECT_EQ(spec0_dense_cells, as_vector(tensor_attr->get_vector(0, 0).typify<double>())); + EXPECT_EQ(empty_cells, as_vector(tensor_attr->get_vector(0, 1).typify<double>())); + } else { + EXPECT_EQ(2, vectors.subspaces()); + EXPECT_EQ(spec0_mixed_cells0, as_vector(vectors.cells(0).typify<double>())); + EXPECT_EQ(spec0_mixed_cells1, as_vector(vectors.cells(1).typify<double>())); + EXPECT_EQ(spec0_mixed_cells0, as_vector(tensor_attr->get_vector(0, 0).typify<double>())); + EXPECT_EQ(spec0_mixed_cells1, as_vector(tensor_attr->get_vector(0, 1).typify<double>())); + EXPECT_EQ(empty_cells, as_vector(tensor_attr->get_vector(0, 2).typify<double>())); + } + add_doc(attr, 1); + vectors = tensor_attr->get_vectors(1); + EXPECT_EQ(0, vectors.subspaces()); + EXPECT_EQ(empty_cells, as_vector(tensor_attr->get_vector(1, 0).typify<double>())); + EXPECT_EQ(nullptr, tensor_attr->getTensor(1).get()); +} + TEST_F(ExtendAttributeTest, single_integer_ext_attribute) { SingleIntegerExtAttribute siattr("si1"); @@ -255,6 +337,24 @@ TEST_F(ExtendAttributeTest, single_raw_ext_attribute) testExtendRaw(srattr); } +TEST_F(ExtendAttributeTest, tensor_ext_attribute_dense) +{ + Config cfg(BasicType::TENSOR, CollectionType::SINGLE); + cfg.setTensorType(ValueType::from_spec(vec_2d_spec)); + TensorExtAttribute tattr("td1", cfg); + EXPECT_TRUE(! tattr.hasMultiValue()); + testExtendTensor(tattr); +} + +TEST_F(ExtendAttributeTest, tensor_ext_attribute_mixed) +{ + Config cfg(BasicType::TENSOR, CollectionType::SINGLE); + cfg.setTensorType(ValueType::from_spec(vec_mixed_2d_spec)); + TensorExtAttribute tattr("tm1", cfg); + EXPECT_TRUE(! tattr.hasMultiValue()); + testExtendTensor(tattr); +} + } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp b/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp index df2a5ea2ac9..17e852c8b92 100644 --- a/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp +++ b/searchlib/src/tests/attribute/multi_value_mapping/multi_value_mapping_test.cpp @@ -22,11 +22,11 @@ using vespalib::datastore::CompactionStrategy; using vespalib::alloc::test::MemoryAllocatorObserver; using AllocStats = MemoryAllocatorObserver::Stats; -template <typename EntryT> +template <typename ElemT> void -assertArray(const std::vector<EntryT> &exp, vespalib::ConstArrayRef<EntryT> values) +assertArray(const std::vector<ElemT> &exp, vespalib::ConstArrayRef<ElemT> values) { - EXPECT_EQ(exp, std::vector<EntryT>(values.cbegin(), values.cend())); + EXPECT_EQ(exp, std::vector<ElemT>(values.cbegin(), values.cend())); } template <class MvMapping> @@ -69,10 +69,10 @@ public: constexpr float ALLOC_GROW_FACTOR = 0.2; -template <typename EntryT> +template <typename ElemT> class MappingTestBase : public ::testing::Test { protected: - using MvMapping = search::attribute::MultiValueMapping<EntryT>; + using MvMapping = search::attribute::MultiValueMapping<ElemT>; using AttributeType = MyAttribute<MvMapping>; AllocStats _stats; std::unique_ptr<MvMapping> _mvMapping; @@ -82,8 +82,8 @@ protected: using generation_t = vespalib::GenerationHandler::generation_t; public: - using ArrayRef = vespalib::ArrayRef<EntryT>; - using ConstArrayRef = vespalib::ConstArrayRef<EntryT>; + using ArrayRef = vespalib::ArrayRef<ElemT>; + using ConstArrayRef = vespalib::ConstArrayRef<ElemT>; MappingTestBase() : _stats(), _mvMapping(), @@ -99,9 +99,9 @@ public: _attr = std::make_unique<AttributeType>(*_mvMapping); _maxSmallArraySize = maxSmallArraySize; } - void setup(uint32_t maxSmallArraySize, size_t minArrays, size_t maxArrays, size_t numArraysForNewBuffer, bool enable_free_lists = true) { + void setup(uint32_t maxSmallArraySize, size_t min_entries, size_t max_entries, size_t num_entries_for_new_buffer, bool enable_free_lists = true) { ArrayStoreConfig config(maxSmallArraySize, - ArrayStoreConfig::AllocSpec(minArrays, maxArrays, numArraysForNewBuffer, ALLOC_GROW_FACTOR)); + ArrayStoreConfig::AllocSpec(min_entries, max_entries, num_entries_for_new_buffer, ALLOC_GROW_FACTOR)); config.enable_free_lists(enable_free_lists); _mvMapping = std::make_unique<MvMapping>(config, vespalib::GrowStrategy(), std::make_unique<MemoryAllocatorObserver>(_stats)); _attr = std::make_unique<AttributeType>(*_mvMapping); @@ -109,12 +109,12 @@ public: } ~MappingTestBase() { } - void set(uint32_t docId, const std::vector<EntryT> &values) { _mvMapping->set(docId, values); } + void set(uint32_t docId, const std::vector<ElemT> &values) { _mvMapping->set(docId, values); } ConstArrayRef get(uint32_t docId) { return _mvMapping->get(docId); } ArrayRef get_writable(uint32_t docId) { return _mvMapping->get_writable(docId); } - void assertGet(uint32_t docId, const std::vector<EntryT> &exp) { + void assertGet(uint32_t docId, const std::vector<ElemT> &exp) { ConstArrayRef act = get(docId); - EXPECT_EQ(exp, std::vector<EntryT>(act.cbegin(), act.cend())); + EXPECT_EQ(exp, std::vector<ElemT>(act.cbegin(), act.cend())); } void assign_generation(generation_t current_gen) { _mvMapping->assign_generation(current_gen); } void reclaim_memory(generation_t oldest_used_gen) { _mvMapping->reclaim_memory(oldest_used_gen); } diff --git a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp index b57a2f42ea7..69478c09a25 100644 --- a/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp +++ b/searchlib/src/tests/memoryindex/field_index/field_index_test.cpp @@ -1032,14 +1032,14 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working) auto beforeStats = getFeatureStoreMemStats(_fic); LOG(info, - "Before feature compaction: allocElems=%zu, usedElems=%zu" - ", deadElems=%zu, holdElems=%zu" + "Before feature compaction: alloc_entries=%zu, used_entries=%zu" + ", dead_entries=%zu, hold_entries=%zu" ", freeBuffers=%u, activeBuffers=%u" ", holdBuffers=%u", - beforeStats._allocElems, - beforeStats._usedElems, - beforeStats._deadElems, - beforeStats._holdElems, + beforeStats._alloc_entries, + beforeStats._used_entries, + beforeStats._dead_entries, + beforeStats._hold_entries, beforeStats._freeBuffers, beforeStats._activeBuffers, beforeStats._holdBuffers); @@ -1052,14 +1052,14 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working) myCommit(_fic, *_pushThreads); auto duringStats = getFeatureStoreMemStats(_fic); LOG(info, - "During feature compaction: allocElems=%zu, usedElems=%zu" - ", deadElems=%zu, holdElems=%zu" + "During feature compaction: alloc_entries=%zu, used_entries=%zu" + ", dead_entries=%zu, hold_entries=%zu" ", freeBuffers=%u, activeBuffers=%u" ", holdBuffers=%u", - duringStats._allocElems, - duringStats._usedElems, - duringStats._deadElems, - duringStats._holdElems, + duringStats._alloc_entries, + duringStats._used_entries, + duringStats._dead_entries, + duringStats._hold_entries, duringStats._freeBuffers, duringStats._activeBuffers, duringStats._holdBuffers); @@ -1067,14 +1067,14 @@ TEST_F(BasicInverterTest, require_that_inversion_is_working) myCommit(_fic, *_pushThreads); auto afterStats = getFeatureStoreMemStats(_fic); LOG(info, - "After feature compaction: allocElems=%zu, usedElems=%zu" - ", deadElems=%zu, holdElems=%zu" + "After feature compaction: alloc_entries=%zu, used_entries=%zu" + ", dead_entries=%zu, hold_entries=%zu" ", freeBuffers=%u, activeBuffers=%u" ", holdBuffers=%u", - afterStats._allocElems, - afterStats._usedElems, - afterStats._deadElems, - afterStats._holdElems, + afterStats._alloc_entries, + afterStats._used_entries, + afterStats._dead_entries, + afterStats._hold_entries, afterStats._freeBuffers, afterStats._activeBuffers, afterStats._holdBuffers); diff --git a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp index 8073fb8d232..e3b9cf9702d 100644 --- a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp +++ b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp @@ -462,8 +462,8 @@ TEST(MemoryIndexTest, require_that_num_docs_and_doc_id_limit_is_returned) TEST(MemoryIndexTest, require_that_we_understand_the_memory_footprint) { - constexpr size_t BASE_ALLOCATED = 361032u; - constexpr size_t BASE_USED = 151188u; + constexpr size_t BASE_ALLOCATED = 360936u; + constexpr size_t BASE_USED = 150804u; { MySetup setup; Index index(setup); diff --git a/searchlib/src/tests/predicate/document_features_store_test.cpp b/searchlib/src/tests/predicate/document_features_store_test.cpp index 4ac4bdc32f0..d30df9dba6e 100644 --- a/searchlib/src/tests/predicate/document_features_store_test.cpp +++ b/searchlib/src/tests/predicate/document_features_store_test.cpp @@ -165,17 +165,17 @@ TEST("require that both features and ranges are removed by 'remove'") { TEST("require that both features and ranges counts towards memory usage") { DocumentFeaturesStore features_store(10); - EXPECT_EQUAL(50136u, features_store.getMemoryUsage().usedBytes()); + EXPECT_EQUAL(50064u, features_store.getMemoryUsage().usedBytes()); PredicateTreeAnnotations annotations; annotations.features.push_back(PredicateHash::hash64("foo=100-199")); features_store.insert(annotations, doc_id); - EXPECT_EQUAL(50144u, features_store.getMemoryUsage().usedBytes()); + EXPECT_EQUAL(50072u, features_store.getMemoryUsage().usedBytes()); annotations.features.clear(); annotations.range_features.push_back({"foo", 100, 199}); features_store.insert(annotations, doc_id + 1); - EXPECT_EQUAL(50240u, features_store.getMemoryUsage().usedBytes()); + EXPECT_EQUAL(50168u, features_store.getMemoryUsage().usedBytes()); } TEST("require that DocumentFeaturesStore can be serialized") { @@ -205,17 +205,17 @@ TEST("require that serialization cleans up wordstore") { PredicateTreeAnnotations annotations; annotations.range_features.push_back({"foo", 100, 199}); features_store.insert(annotations, doc_id); - EXPECT_EQUAL(50232u, features_store.getMemoryUsage().usedBytes()); + EXPECT_EQUAL(50160u, features_store.getMemoryUsage().usedBytes()); annotations.range_features.push_back({"bar", 100, 199}); features_store.insert(annotations, doc_id + 1); - EXPECT_EQUAL(50620u, features_store.getMemoryUsage().usedBytes()); + EXPECT_EQUAL(50548u, features_store.getMemoryUsage().usedBytes()); features_store.remove(doc_id + 1); - EXPECT_EQUAL(50572u, features_store.getMemoryUsage().usedBytes()); + EXPECT_EQUAL(50500u, features_store.getMemoryUsage().usedBytes()); vespalib::DataBuffer buffer; features_store.serialize(buffer); DocumentFeaturesStore features_store2(buffer); - EXPECT_EQUAL(50232u, features_store2.getMemoryUsage().usedBytes()); + EXPECT_EQUAL(50160u, features_store2.getMemoryUsage().usedBytes()); } diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index 3d14622ca02..e40785911ea 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -38,6 +38,8 @@ namespace vespalib::alloc { class Alloc; } +namespace vespalib::eval { struct Value; } + namespace search { template <typename T> class ComponentGuard; @@ -86,6 +88,7 @@ public: virtual bool add(double, int32_t = 1) { return false; } virtual bool add(const char *, int32_t = 1) { return false; } virtual bool add(vespalib::ConstArrayRef<char>, int32_t = 1) { return false; } + virtual bool add(const vespalib::eval::Value&, int32_t = 1) { return false; } virtual ~IExtendAttribute() = default; }; diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h index bd624e9f388..4fce64aa762 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.h @@ -12,18 +12,18 @@ namespace search::attribute { /** * Class for mapping from document id to an array of values. */ -template <typename EntryT, typename RefT = vespalib::datastore::EntryRefT<19> > +template <typename ElemT, typename RefT = vespalib::datastore::EntryRefT<19> > class MultiValueMapping : public MultiValueMappingBase { public: - using MultiValueType = EntryT; + using MultiValueType = ElemT; using RefType = RefT; - using ReadView = MultiValueMappingReadView<EntryT, RefT>; + using ReadView = MultiValueMappingReadView<ElemT, RefT>; private: - using ArrayRef = vespalib::ArrayRef<EntryT>; - using ArrayStore = vespalib::datastore::ArrayStore<EntryT, RefT>; + using ArrayRef = vespalib::ArrayRef<ElemT>; + using ArrayStore = vespalib::datastore::ArrayStore<ElemT, RefT>; using generation_t = vespalib::GenerationHandler::generation_t; - using ConstArrayRef = vespalib::ConstArrayRef<EntryT>; + using ConstArrayRef = vespalib::ConstArrayRef<ElemT>; ArrayStore _store; public: @@ -73,7 +73,7 @@ public: static vespalib::datastore::ArrayStoreConfig optimizedConfigForHugePage(size_t maxSmallArraySize, size_t hugePageSize, size_t smallPageSize, - size_t minNumArraysForNewBuffer, + size_t min_num_entries_for_new_buffer, float allocGrowFactor, bool enable_free_lists); }; diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp index b486fa60265..ab68bea58cc 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping.hpp @@ -7,8 +7,8 @@ namespace search::attribute { -template <typename EntryT, typename RefT> -MultiValueMapping<EntryT,RefT>::MultiValueMapping(const vespalib::datastore::ArrayStoreConfig &storeCfg, +template <typename ElemT, typename RefT> +MultiValueMapping<ElemT,RefT>::MultiValueMapping(const vespalib::datastore::ArrayStoreConfig &storeCfg, const vespalib::GrowStrategy &gs, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator) : MultiValueMappingBase(gs, ArrayStore::getGenerationHolderLocation(_store), memory_allocator), @@ -16,12 +16,12 @@ MultiValueMapping<EntryT,RefT>::MultiValueMapping(const vespalib::datastore::Arr { } -template <typename EntryT, typename RefT> -MultiValueMapping<EntryT,RefT>::~MultiValueMapping() = default; +template <typename ElemT, typename RefT> +MultiValueMapping<ElemT,RefT>::~MultiValueMapping() = default; -template <typename EntryT, typename RefT> +template <typename ElemT, typename RefT> void -MultiValueMapping<EntryT,RefT>::set(uint32_t docId, ConstArrayRef values) +MultiValueMapping<ElemT,RefT>::set(uint32_t docId, ConstArrayRef values) { _indices.ensure_size(docId + 1); EntryRef oldRef(_indices[docId].load_relaxed()); @@ -31,18 +31,18 @@ MultiValueMapping<EntryT,RefT>::set(uint32_t docId, ConstArrayRef values) _store.remove(oldRef); } -template <typename EntryT, typename RefT> +template <typename ElemT, typename RefT> vespalib::MemoryUsage -MultiValueMapping<EntryT,RefT>::update_stat(const CompactionStrategy& compaction_strategy) +MultiValueMapping<ElemT,RefT>::update_stat(const CompactionStrategy& compaction_strategy) { auto retval = _store.update_stat(compaction_strategy); retval.merge(_indices.getMemoryUsage()); return retval; } -template <typename EntryT, typename RefT> +template <typename ElemT, typename RefT> void -MultiValueMapping<EntryT,RefT>::compact_worst(const CompactionStrategy& compaction_strategy) +MultiValueMapping<ElemT,RefT>::compact_worst(const CompactionStrategy& compaction_strategy) { vespalib::datastore::ICompactionContext::UP compactionContext(_store.compact_worst(compaction_strategy)); if (compactionContext) { @@ -50,29 +50,29 @@ MultiValueMapping<EntryT,RefT>::compact_worst(const CompactionStrategy& compacti } } -template <typename EntryT, typename RefT> +template <typename ElemT, typename RefT> vespalib::MemoryUsage -MultiValueMapping<EntryT,RefT>::getArrayStoreMemoryUsage() const +MultiValueMapping<ElemT,RefT>::getArrayStoreMemoryUsage() const { return _store.getMemoryUsage(); } -template <typename EntryT, typename RefT> +template <typename ElemT, typename RefT> vespalib::AddressSpace -MultiValueMapping<EntryT, RefT>::getAddressSpaceUsage() const { +MultiValueMapping<ElemT, RefT>::getAddressSpaceUsage() const { return _store.addressSpaceUsage(); } -template <typename EntryT, typename RefT> +template <typename ElemT, typename RefT> vespalib::datastore::ArrayStoreConfig -MultiValueMapping<EntryT, RefT>::optimizedConfigForHugePage(size_t maxSmallArraySize, +MultiValueMapping<ElemT, RefT>::optimizedConfigForHugePage(size_t maxSmallArraySize, size_t hugePageSize, size_t smallPageSize, - size_t minNumArraysForNewBuffer, + size_t min_num_entries_for_new_buffer, float allocGrowFactor, bool enable_free_lists) { - auto result = ArrayStore::optimizedConfigForHugePage(maxSmallArraySize, hugePageSize, smallPageSize, minNumArraysForNewBuffer, allocGrowFactor); + auto result = ArrayStore::optimizedConfigForHugePage(maxSmallArraySize, hugePageSize, smallPageSize, min_num_entries_for_new_buffer, allocGrowFactor); result.enable_free_lists(enable_free_lists); return result; } diff --git a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_read_view.h b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_read_view.h index 116e069e8b4..41138ff0890 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_read_view.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_value_mapping_read_view.h @@ -11,12 +11,12 @@ namespace search::attribute { /** * Class for mapping from document id to an array of values as reader. */ -template <typename EntryT, typename RefT = vespalib::datastore::EntryRefT<19> > +template <typename ElemT, typename RefT = vespalib::datastore::EntryRefT<19> > class MultiValueMappingReadView { using AtomicEntryRef = vespalib::datastore::AtomicEntryRef; using Indices = vespalib::ConstArrayRef<AtomicEntryRef>; - using ArrayStore = vespalib::datastore::ArrayStore<EntryT, RefT>; + using ArrayStore = vespalib::datastore::ArrayStore<ElemT, RefT>; Indices _indices; const ArrayStore* _store; @@ -31,7 +31,7 @@ public: _store(store) { } - vespalib::ConstArrayRef<EntryT> get(uint32_t doc_id) const { return _store->get(_indices[doc_id].load_acquire()); } + vespalib::ConstArrayRef<ElemT> get(uint32_t doc_id) const { return _store->get(_indices[doc_id].load_acquire()); } bool valid() const noexcept { return _store != nullptr; } }; diff --git a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp index ea449300aef..1009fa2fb5f 100644 --- a/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multinumericpostattribute.hpp @@ -51,7 +51,7 @@ template <typename B, typename M> MultiValueNumericPostingAttribute<B, M>::~MultiValueNumericPostingAttribute() { this->disableFreeLists(); - this->disableElemHoldList(); + this->disable_entry_hold_list(); clearAllPostings(); } diff --git a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp index cd46bbb5a8a..19840b5a474 100644 --- a/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/multistringpostattribute.hpp @@ -27,7 +27,7 @@ template <typename B, typename T> MultiValueStringPostingAttributeT<B, T>::~MultiValueStringPostingAttributeT() { this->disableFreeLists(); - this->disableElemHoldList(); + this->disable_entry_hold_list(); clearAllPostings(); } diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h index 29440b6ce43..ecf7a46f21e 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/postinglistattribute.h @@ -58,7 +58,7 @@ protected: void updatePostings(PostingMap &changePost, const vespalib::datastore::EntryComparator &cmp); void clearAllPostings(); void disableFreeLists() { _postingList.disableFreeLists(); } - void disableElemHoldList() { _postingList.disableElemHoldList(); } + void disable_entry_hold_list() { _postingList.disable_entry_hold_list(); } void handle_load_posting_lists_and_update_enum_store(enumstore::EnumeratedPostingsLoader& loader); bool forwardedOnAddDoc(DocId doc, size_t wantSize, size_t wantCapacity); diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp index 94720212faf..2703201b292 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp @@ -231,7 +231,7 @@ PostingStore<DataT>::dropBitVector(EntryRef &ref) (void) tree; (void) docFreq; _bvs.erase(ref.ref()); - _store.holdElem(iRef, 1); + _store.hold_entry(iRef); _status.decBitVectors(); _bvExtraBytes -= bv->writer().extraByteSize(); ref = ref2; @@ -267,7 +267,7 @@ PostingStore<DataT>::makeBitVector(EntryRef &ref) if (_enableOnlyBitVector) { BTreeType *tree = getWTreeEntry(iRef); tree->clear(_allocator); - _store.holdElem(ref, 1); + _store.hold_entry(ref); } else { bve->_tree = ref; } @@ -590,19 +590,19 @@ PostingStore<DataT>::clear(const EntryRef ref) assert(isBTree(iRef2)); BTreeType *tree = getWTreeEntry(iRef2); tree->clear(_allocator); - _store.holdElem(iRef2, 1); + _store.hold_entry(iRef2); } _bvs.erase(ref.ref()); _status.decBitVectors(); _bvExtraBytes -= bve->_bv->writer().extraByteSize(); - _store.holdElem(ref, 1); + _store.hold_entry(ref); } else { BTreeType *tree = getWTreeEntry(iRef); tree->clear(_allocator); - _store.holdElem(ref, 1); + _store.hold_entry(ref); } } else { - _store.holdElem(ref, clusterSize); + _store.hold_entry(ref); } } diff --git a/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp index 1775774171d..de4a7157dae 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlenumericpostattribute.hpp @@ -13,7 +13,7 @@ template <typename B> SingleValueNumericPostingAttribute<B>::~SingleValueNumericPostingAttribute() { this->disableFreeLists(); - this->disableElemHoldList(); + this->disable_entry_hold_list(); clearAllPostings(); } diff --git a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp index eef72984e79..1ec9b54a73b 100644 --- a/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp +++ b/searchlib/src/vespa/searchlib/attribute/singlestringpostattribute.hpp @@ -27,7 +27,7 @@ template <typename B> SingleValueStringPostingAttributeT<B>::~SingleValueStringPostingAttributeT() { this->disableFreeLists(); - this->disableElemHoldList(); + this->disable_entry_hold_list(); clearAllPostings(); } diff --git a/searchlib/src/vespa/searchlib/memoryindex/feature_store.cpp b/searchlib/src/vespa/searchlib/memoryindex/feature_store.cpp index 72f4a7ae579..4bc7f5b1144 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/feature_store.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/feature_store.cpp @@ -36,7 +36,7 @@ EntryRef FeatureStore::addFeatures(const uint8_t *src, uint64_t byteLen) { uint32_t pad = Aligner::pad(byteLen); - auto result = _store.rawAllocator<uint8_t>(_typeId).alloc(byteLen + pad, DECODE_SAFETY); + auto result = _store.rawAllocator<uint8_t>(_typeId).alloc((byteLen + pad) / buffer_array_size, DECODE_SAFETY_ENTRIES); uint8_t *dst = result.data; memcpy(dst, src, byteLen); dst += byteLen; @@ -113,7 +113,7 @@ FeatureStore::add_features_guard_bytes() { uint32_t len = DECODE_SAFETY; uint32_t pad = Aligner::pad(len); - auto result = _store.rawAllocator<uint8_t>(_typeId).alloc(len + pad); + auto result = _store.rawAllocator<uint8_t>(_typeId).alloc((len + pad) / buffer_array_size); memset(result.data, 0, len + pad); } diff --git a/searchlib/src/vespa/searchlib/memoryindex/feature_store.h b/searchlib/src/vespa/searchlib/memoryindex/feature_store.h index 53588fa2894..1e48189987e 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/feature_store.h +++ b/searchlib/src/vespa/searchlib/memoryindex/feature_store.h @@ -29,6 +29,7 @@ private: using PosOccFieldsParams = bitcompression::PosOccFieldsParams; static constexpr uint32_t DECODE_SAFETY = 16; + static constexpr uint32_t DECODE_SAFETY_ENTRIES = 16 / buffer_array_size; DataStoreType _store; @@ -117,7 +118,7 @@ public: * overrun beyond the compressed data either goes into other features * already written or into the guard area. * - * If buffer type is changed to have a nonzero numArraysForNewBuffer then + * If buffer type is changed to have a nonzero num_entries_for_new_buffer then * extra logic to add guard bytes is needed when switching primary buffer * to avoid issues if the buffer is resumed as primary buffer later on. */ diff --git a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp index 4be3031303e..8dd76a90b14 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/field_index.cpp @@ -55,9 +55,9 @@ template <bool interleaved_features> FieldIndex<interleaved_features>::~FieldIndex() { _postingListStore.disableFreeLists(); - _postingListStore.disableElemHoldList(); + _postingListStore.disable_entry_hold_list(); _dict.disableFreeLists(); - _dict.disableElemHoldList(); + _dict.disable_entry_hold_list(); // XXX: Kludge for (DictionaryTree::Iterator it = _dict.begin(); it.valid(); ++it) { diff --git a/searchlib/src/vespa/searchlib/memoryindex/word_store.cpp b/searchlib/src/vespa/searchlib/memoryindex/word_store.cpp index 3e4c38ceb0e..e330dc83055 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/word_store.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/word_store.cpp @@ -27,7 +27,7 @@ WordStore::addWord(const vespalib::stringref word) { size_t wordSize = word.size() + 1; size_t bufferSize = wordSize + Aligner::pad(wordSize); - auto result = _store.rawAllocator<char>(_typeId).alloc(bufferSize); + auto result = _store.rawAllocator<char>(_typeId).alloc(bufferSize / buffer_array_size); char *be = result.data; for (size_t i = 0; i < word.size(); ++i) { *be++ = word[i]; diff --git a/searchlib/src/vespa/searchlib/predicate/document_features_store.cpp b/searchlib/src/vespa/searchlib/predicate/document_features_store.cpp index 604a467a6e6..a6a82ec09f8 100644 --- a/searchlib/src/vespa/searchlib/predicate/document_features_store.cpp +++ b/searchlib/src/vespa/searchlib/predicate/document_features_store.cpp @@ -102,7 +102,7 @@ DocumentFeaturesStore::DocumentFeaturesStore(DataBuffer &buffer) DocumentFeaturesStore::~DocumentFeaturesStore() { _word_index.disableFreeLists(); - _word_index.disableElemHoldList(); + _word_index.disable_entry_hold_list(); _word_index.getAllocator().freeze(); _word_index.clear(); } diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_interval_store.cpp b/searchlib/src/vespa/searchlib/predicate/predicate_interval_store.cpp index af809b2fa69..af5aae6e519 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_interval_store.cpp +++ b/searchlib/src/vespa/searchlib/predicate/predicate_interval_store.cpp @@ -95,7 +95,7 @@ PredicateIntervalStore::remove(EntryRef ref) { // BufferState &state = _store.getBufferState(buffer_id); // uint32_t type_id = state.getTypeId(); // uint32_t size = type_id <= MAX_ARRAY_SIZE ? type_id : 1; - // _store.holdElem(ref, size); + // _store.hold_entries(ref, size); } } diff --git a/searchlib/src/vespa/searchlib/predicate/simple_index.hpp b/searchlib/src/vespa/searchlib/predicate/simple_index.hpp index c6f640d72ed..9320488f88e 100644 --- a/searchlib/src/vespa/searchlib/predicate/simple_index.hpp +++ b/searchlib/src/vespa/searchlib/predicate/simple_index.hpp @@ -41,7 +41,7 @@ SimpleIndex<Posting, Key, DocId>::insertIntoVectorPosting(vespalib::datastore::E template <typename Posting, typename Key, typename DocId> SimpleIndex<Posting, Key, DocId>::~SimpleIndex() { _btree_posting_lists.disableFreeLists(); - _btree_posting_lists.disableElemHoldList(); + _btree_posting_lists.disable_entry_hold_list(); for (auto it = _dictionary.begin(); it.valid(); ++it) { vespalib::datastore::EntryRef ref(it.getData()); @@ -51,13 +51,13 @@ SimpleIndex<Posting, Key, DocId>::~SimpleIndex() { } _vector_posting_lists.disableFreeLists(); - _vector_posting_lists.disableElemHoldList(); + _vector_posting_lists.disable_entry_hold_list(); _vector_posting_lists.clear(); _vector_posting_lists.getAllocator().freeze(); _vector_posting_lists.getAllocator().reclaim_all_memory(); _dictionary.disableFreeLists(); - _dictionary.disableElemHoldList(); + _dictionary.disable_entry_hold_list(); _dictionary.clear(); _dictionary.getAllocator().freeze(); _dictionary.getAllocator().reclaim_all_memory(); diff --git a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt index c8c5d4d4257..313863d8dcb 100644 --- a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt @@ -40,6 +40,7 @@ vespa_add_library(searchlib_tensor OBJECT tensor_buffer_store.cpp tensor_buffer_type_mapper.cpp tensor_deserialize.cpp + tensor_ext_attribute.cpp tensor_store.cpp DEPENDS ) diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp index c373f6bdcd0..c51d0ec7fd3 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp @@ -62,10 +62,10 @@ DenseTensorStore::BufferType::BufferType(const TensorSizeCalc &tensorSizeCalc, s DenseTensorStore::BufferType::~BufferType() = default; void -DenseTensorStore::BufferType::cleanHold(void *buffer, size_t offset, - ElemCount numElems, CleanContext) +DenseTensorStore::BufferType::clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext) { - memset(static_cast<char *>(buffer) + offset, 0, numElems); + auto num_elems = num_entries * getArraySize(); + memset(static_cast<char *>(buffer) + offset * getArraySize(), 0, num_elems); } const vespalib::alloc::MemoryAllocator* @@ -107,7 +107,7 @@ DenseTensorStore::allocRawBuffer() { size_t bufSize = getBufSize(); size_t alignedBufSize = _tensorSizeCalc.alignedSize(); - auto result = _concreteStore.freeListRawAllocator<char>(0u).alloc(alignedBufSize); + auto result = _concreteStore.freeListRawAllocator<char>(0u).alloc(1); clearPadAreaAfterBuffer(result.data, bufSize, alignedBufSize); return result; } @@ -118,7 +118,7 @@ DenseTensorStore::holdTensor(EntryRef ref) if (!ref.valid()) { return; } - _concreteStore.holdElem(ref, _tensorSizeCalc.alignedSize()); + _concreteStore.hold_entry(ref); } TensorStore::EntryRef diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h index 9e326e0ab1e..0dd483e7f08 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h @@ -44,7 +44,7 @@ public: public: BufferType(const TensorSizeCalc &tensorSizeCalc, std::shared_ptr<vespalib::alloc::MemoryAllocator> allocator); ~BufferType() override; - void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override; + void clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx) override; const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override; }; private: diff --git a/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.cpp index fa13ab6303c..8526138fd31 100644 --- a/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.cpp @@ -30,11 +30,11 @@ DirectTensorStore::TensorBufferType::TensorBufferType() } void -DirectTensorStore::TensorBufferType::cleanHold(void* buffer, size_t offset, ElemCount num_elems, CleanContext clean_ctx) +DirectTensorStore::TensorBufferType::clean_hold(void* buffer, size_t offset, EntryCount num_entries, CleanContext clean_ctx) { TensorSP* elem = static_cast<TensorSP*>(buffer) + offset; const auto& empty = empty_entry(); - for (size_t i = 0; i < num_elems; ++i) { + for (size_t i = 0; i < num_entries; ++i) { clean_ctx.extraBytesCleaned((*elem)->get_memory_usage().allocatedBytes()); *elem = empty; ++elem; @@ -69,7 +69,7 @@ DirectTensorStore::holdTensor(EntryRef ref) } const auto& tensor = _tensor_store.getEntry(ref); assert(tensor); - _tensor_store.holdElem(ref, 1, tensor->get_memory_usage().allocatedBytes()); + _tensor_store.hold_entry(ref, tensor->get_memory_usage().allocatedBytes()); } EntryRef diff --git a/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h b/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h index 01084e89776..1230494fe41 100644 --- a/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h +++ b/searchlib/src/vespa/searchlib/tensor/direct_tensor_store.h @@ -20,7 +20,7 @@ namespace search::tensor { */ class DirectTensorStore : public TensorStore { private: - // Note: Must use SP (instead of UP) because of fallbackCopy() and initializeReservedElements() in BufferType, + // Note: Must use SP (instead of UP) because of fallback_copy() and initialize_reserved_entries() in BufferType, // and implementation of move(). using TensorSP = std::shared_ptr<vespalib::eval::Value>; using TensorStoreType = vespalib::datastore::DataStore<TensorSP>; @@ -32,7 +32,7 @@ private: using CleanContext = typename ParentType::CleanContext; public: TensorBufferType(); - void cleanHold(void* buffer, size_t offset, ElemCount num_elems, CleanContext clean_ctx) override; + void clean_hold(void* buffer, size_t offset, EntryCount num_entries, CleanContext clean_ctx) override; }; TensorStoreType _tensor_store; diff --git a/searchlib/src/vespa/searchlib/tensor/large_subspaces_buffer_type.cpp b/searchlib/src/vespa/searchlib/tensor/large_subspaces_buffer_type.cpp index cb074348f08..5d3b2206703 100644 --- a/searchlib/src/vespa/searchlib/tensor/large_subspaces_buffer_type.cpp +++ b/searchlib/src/vespa/searchlib/tensor/large_subspaces_buffer_type.cpp @@ -12,7 +12,7 @@ using vespalib::alloc::MemoryAllocator; namespace search::tensor { LargeSubspacesBufferType::LargeSubspacesBufferType(const AllocSpec& spec, std::shared_ptr<MemoryAllocator> memory_allocator, TensorBufferTypeMapper& type_mapper) noexcept - : ParentType(1u, spec.minArraysInBuffer, spec.maxArraysInBuffer, spec.numArraysForNewBuffer, spec.allocGrowFactor), + : ParentType(1u, spec.min_entries_in_buffer, spec.max_entries_in_buffer, spec.num_entries_for_new_buffer, spec.allocGrowFactor), _memory_allocator(std::move(memory_allocator)), _ops(type_mapper.get_tensor_buffer_operations()) { @@ -21,10 +21,10 @@ LargeSubspacesBufferType::LargeSubspacesBufferType(const AllocSpec& spec, std::s LargeSubspacesBufferType::~LargeSubspacesBufferType() = default; void -LargeSubspacesBufferType::cleanHold(void* buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) +LargeSubspacesBufferType::clean_hold(void* buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx) { auto elem = static_cast<ArrayType*>(buffer) + offset; - for (size_t i = 0; i < numElems; ++i) { + for (size_t i = 0; i < num_entries; ++i) { if (!elem->empty()) { cleanCtx.extraBytesCleaned(elem->size()); _ops.reclaim_labels({elem->data(), elem->size()}); @@ -35,10 +35,10 @@ LargeSubspacesBufferType::cleanHold(void* buffer, size_t offset, ElemCount numEl } void -LargeSubspacesBufferType::destroyElements(void *buffer, ElemCount numElems) +LargeSubspacesBufferType::destroy_entries(void *buffer, EntryCount num_entries) { auto elem = static_cast<ArrayType*>(buffer); - for (size_t i = 0; i < numElems; ++i) { + for (size_t i = 0; i < num_entries; ++i) { if (!elem->empty()) { _ops.reclaim_labels({elem->data(), elem->size()}); ArrayType().swap(*elem); @@ -48,11 +48,11 @@ LargeSubspacesBufferType::destroyElements(void *buffer, ElemCount numElems) } void -LargeSubspacesBufferType::fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems) +LargeSubspacesBufferType::fallback_copy(void *newBuffer, const void *oldBuffer, EntryCount num_entries) { auto old_elems = static_cast<const ArrayType*>(oldBuffer); auto new_elems = static_cast<ArrayType*>(newBuffer); - for (size_t i = 0; i < numElems; ++i) { + for (size_t i = 0; i < num_entries; ++i) { auto& old_elem = old_elems[i]; new (new_elems + i) ArrayType(old_elem); if (!old_elem.empty()) { @@ -62,12 +62,12 @@ LargeSubspacesBufferType::fallbackCopy(void *newBuffer, const void *oldBuffer, E } void -LargeSubspacesBufferType::initializeReservedElements(void *buffer, ElemCount reservedElements) +LargeSubspacesBufferType::initialize_reserved_entries(void *buffer, EntryCount reserved_entries) { - auto new_elems = static_cast<ArrayType*>(buffer); + auto new_entries = static_cast<ArrayType*>(buffer); const auto& empty = empty_entry(); - for (size_t i = 0; i < reservedElements; ++i) { - new (new_elems + i) ArrayType(empty); + for (size_t i = 0; i < reserved_entries; ++i) { + new (new_entries + i) ArrayType(empty); } } diff --git a/searchlib/src/vespa/searchlib/tensor/large_subspaces_buffer_type.h b/searchlib/src/vespa/searchlib/tensor/large_subspaces_buffer_type.h index cfab8ef20af..8cce08e9d81 100644 --- a/searchlib/src/vespa/searchlib/tensor/large_subspaces_buffer_type.h +++ b/searchlib/src/vespa/searchlib/tensor/large_subspaces_buffer_type.h @@ -30,10 +30,10 @@ class LargeSubspacesBufferType : public vespalib::datastore::BufferType<vespalib public: LargeSubspacesBufferType(const AllocSpec& spec, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator, TensorBufferTypeMapper& type_mapper) noexcept; ~LargeSubspacesBufferType() override; - void cleanHold(void* buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override; - void destroyElements(void *buffer, ElemCount numElems) override; - void fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems) override; - void initializeReservedElements(void *buffer, ElemCount reservedElements) override; + void clean_hold(void* buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx) override; + void destroy_entries(void *buffer, EntryCount num_entries) override; + void fallback_copy(void *newBuffer, const void *oldBuffer, EntryCount num_entries) override; + void initialize_reserved_entries(void *buffer, EntryCount reserved_entries) override; const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override; }; diff --git a/searchlib/src/vespa/searchlib/tensor/small_subspaces_buffer_type.cpp b/searchlib/src/vespa/searchlib/tensor/small_subspaces_buffer_type.cpp index 6a71388a3b9..7b54182f062 100644 --- a/searchlib/src/vespa/searchlib/tensor/small_subspaces_buffer_type.cpp +++ b/searchlib/src/vespa/searchlib/tensor/small_subspaces_buffer_type.cpp @@ -10,7 +10,7 @@ using vespalib::alloc::MemoryAllocator; namespace search::tensor { SmallSubspacesBufferType::SmallSubspacesBufferType(uint32_t array_size, const AllocSpec& spec, std::shared_ptr<MemoryAllocator> memory_allocator, TensorBufferTypeMapper& type_mapper) noexcept - : ParentType(array_size, spec.minArraysInBuffer, spec.maxArraysInBuffer, spec.numArraysForNewBuffer, spec.allocGrowFactor), + : ParentType(array_size, spec.min_entries_in_buffer, spec.max_entries_in_buffer, spec.num_entries_for_new_buffer, spec.allocGrowFactor), _memory_allocator(std::move(memory_allocator)), _ops(type_mapper.get_tensor_buffer_operations()) { @@ -19,45 +19,45 @@ SmallSubspacesBufferType::SmallSubspacesBufferType(uint32_t array_size, const Al SmallSubspacesBufferType::~SmallSubspacesBufferType() = default; void -SmallSubspacesBufferType::cleanHold(void* buffer, size_t offset, ElemCount numElems, CleanContext) +SmallSubspacesBufferType::clean_hold(void* buffer, size_t offset, EntryCount num_entries, CleanContext) { - char* elem = static_cast<char *>(buffer) + offset; - while (numElems >= getArraySize()) { + char* elem = static_cast<char *>(buffer) + offset * getArraySize(); + while (num_entries >= 1) { _ops.reclaim_labels(vespalib::ArrayRef<char>(elem, getArraySize())); elem += getArraySize(); - numElems -= getArraySize(); + --num_entries; } } void -SmallSubspacesBufferType::destroyElements(void *buffer, ElemCount numElems) +SmallSubspacesBufferType::destroy_entries(void *buffer, EntryCount num_entries) { char* elem = static_cast<char *>(buffer); - while (numElems >= getArraySize()) { + while (num_entries >= 1) { _ops.reclaim_labels(vespalib::ArrayRef<char>(elem, getArraySize())); elem += getArraySize(); - numElems -= getArraySize(); + --num_entries; } } void -SmallSubspacesBufferType::fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems) +SmallSubspacesBufferType::fallback_copy(void *newBuffer, const void *oldBuffer, EntryCount num_entries) { - if (numElems > 0) { - memcpy(newBuffer, oldBuffer, numElems); + if (num_entries > 0) { + memcpy(newBuffer, oldBuffer, num_entries * getArraySize()); } const char *elem = static_cast<const char *>(oldBuffer); - while (numElems >= getArraySize()) { + while (num_entries >= 1) { _ops.copied_labels(unconstify(vespalib::ConstArrayRef<char>(elem, getArraySize()))); elem += getArraySize(); - numElems -= getArraySize(); + --num_entries; } } void -SmallSubspacesBufferType::initializeReservedElements(void *buffer, ElemCount reservedElements) +SmallSubspacesBufferType::initialize_reserved_entries(void *buffer, EntryCount reserved_entries) { - memset(buffer, 0, reservedElements); + memset(buffer, 0, reserved_entries * getArraySize()); } const vespalib::alloc::MemoryAllocator* diff --git a/searchlib/src/vespa/searchlib/tensor/small_subspaces_buffer_type.h b/searchlib/src/vespa/searchlib/tensor/small_subspaces_buffer_type.h index 5622e9970b8..2f287ef1f3d 100644 --- a/searchlib/src/vespa/searchlib/tensor/small_subspaces_buffer_type.h +++ b/searchlib/src/vespa/searchlib/tensor/small_subspaces_buffer_type.h @@ -30,10 +30,10 @@ public: SmallSubspacesBufferType& operator=(SmallSubspacesBufferType&&) noexcept = delete; SmallSubspacesBufferType(uint32_t array_size, const AllocSpec& spec, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator, TensorBufferTypeMapper& type_mapper) noexcept; ~SmallSubspacesBufferType() override; - void cleanHold(void* buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override; - void destroyElements(void *buffer, ElemCount numElems) override; - void fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems) override; - void initializeReservedElements(void *buffer, ElemCount reservedElements) override; + void clean_hold(void* buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx) override; + void destroy_entries(void *buffer, EntryCount num_entries) override; + void fallback_copy(void *newBuffer, const void *oldBuffer, EntryCount num_entries) override; + void initialize_reserved_entries(void *buffer, EntryCount reserved_entries) override; const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override; }; diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_ext_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_ext_attribute.cpp new file mode 100644 index 00000000000..19c8cf6053b --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/tensor_ext_attribute.cpp @@ -0,0 +1,181 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "tensor_ext_attribute.h" +#include "serialized_tensor_ref.h" +#include "vector_bundle.h" +#include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/value_codec.h> +#include <vespa/searchcommon/attribute/config.h> + +#include <vespa/log/log.h> +LOG_SETUP(".searchlib.tensor.tensor_ext_attribute"); + +using vespalib::eval::FastValueBuilderFactory; +using vespalib::eval::TensorSpec; +using vespalib::eval::TypedCells; +using vespalib::eval::Value; +using vespalib::eval::ValueType; + +namespace search::tensor { + +namespace { + +std::unique_ptr<Value> +create_empty_tensor(const ValueType& type) +{ + const auto &factory = FastValueBuilderFactory::get(); + TensorSpec empty_spec(type.to_spec()); + return vespalib::eval::value_from_spec(empty_spec, factory); +} + +} + +TensorExtAttribute::TensorExtAttribute(const vespalib::string& name, const Config& cfg) + : NotImplementedAttribute(name, cfg), + ITensorAttribute(), + IExtendAttribute(), + _subspace_type(cfg.tensorType()), + _empty(_subspace_type), + _empty_tensor(create_empty_tensor(cfg.tensorType())) +{ +} + +TensorExtAttribute::~TensorExtAttribute() = default; + +const ITensorAttribute* +TensorExtAttribute::asTensorAttribute() const +{ + return this; +} + +void +TensorExtAttribute::onCommit() +{ + LOG_ABORT("should not be reached"); +} + +void +TensorExtAttribute::onUpdateStat() +{ +} + +bool +TensorExtAttribute::addDoc(DocId& docId) +{ + docId = _data.size(); + _data.emplace_back(nullptr); + incNumDocs(); + setCommittedDocIdLimit(getNumDocs()); + return true; +} + +bool +TensorExtAttribute::add(const vespalib::eval::Value& v, int32_t) +{ + _data.back() = &v; + return true; +} + +IExtendAttribute* +TensorExtAttribute::getExtendInterface() +{ + return this; +} + +TypedCells +TensorExtAttribute::get_vector(uint32_t docid, uint32_t subspace) const +{ + auto vectors = get_vectors(docid); + return (subspace < vectors.subspaces()) ? vectors.cells(subspace) : _empty.cells(); +} + +VectorBundle +TensorExtAttribute::get_vectors(uint32_t docid) const +{ + auto tensor = _data[docid]; + if (tensor == nullptr) { + return VectorBundle(); + } + return VectorBundle(tensor->cells().data, tensor->index().size(), _subspace_type); +} + +std::unique_ptr<Value> +TensorExtAttribute::getTensor(uint32_t docid) const +{ + auto tensor = _data[docid]; + if (tensor == nullptr) { + return {}; + } + return FastValueBuilderFactory::get().copy(*tensor); +} + +std::unique_ptr<Value> +TensorExtAttribute::getEmptyTensor() const +{ + return FastValueBuilderFactory::get().copy(*_empty_tensor); +} + +TypedCells +TensorExtAttribute::extract_cells_ref(uint32_t docid) const +{ + return get_vector(docid, 0); +} + +const vespalib::eval::Value& +TensorExtAttribute::get_tensor_ref(uint32_t docid) const +{ + auto tensor = _data[docid]; + return (tensor == nullptr) ? *_empty_tensor : *tensor; +} + +SerializedTensorRef +TensorExtAttribute::get_serialized_tensor_ref(uint32_t) const +{ + notImplemented(); +} + +bool +TensorExtAttribute::supports_extract_cells_ref() const +{ + return getConfig().tensorType().is_dense(); +} + +bool +TensorExtAttribute::supports_get_tensor_ref() const +{ + return true; +} + +bool +TensorExtAttribute::supports_get_serialized_tensor_ref() const +{ + return false; +} + +const ValueType& +TensorExtAttribute::getTensorType() const +{ + return getConfig().tensorType(); +} + +TensorExtAttribute::DistanceMetric +TensorExtAttribute::distance_metric() const +{ + return getConfig().distance_metric(); +} + +uint32_t +TensorExtAttribute::get_num_docs() const +{ + return _data.size(); +} + +void +TensorExtAttribute::get_state(const vespalib::slime::Inserter& inserter) const +{ + (void) inserter; +} + +} diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_ext_attribute.h b/searchlib/src/vespa/searchlib/tensor/tensor_ext_attribute.h new file mode 100644 index 00000000000..a58426cd146 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/tensor_ext_attribute.h @@ -0,0 +1,54 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_tensor_attribute.h" +#include "empty_subspace.h" +#include "subspace_type.h" +#include <vespa/searchlib/attribute/not_implemented_attribute.h> +#include <vespa/vespalib/stllike/allocator.h> + +namespace search::tensor { + +/** + * Attribute vector storing a pointer to single tensor value per + * document in streaming search. The tensor is not owned by this + * attribute vector. + */ +class TensorExtAttribute : public NotImplementedAttribute, + public ITensorAttribute, + public IExtendAttribute +{ + std::vector<const vespalib::eval::Value*> _data; + SubspaceType _subspace_type; + EmptySubspace _empty; + std::unique_ptr<vespalib::eval::Value> _empty_tensor; +public: + TensorExtAttribute(const vespalib::string& name, const Config& cfg); + ~TensorExtAttribute() override; + const ITensorAttribute* asTensorAttribute() const override; + void onCommit() override; + void onUpdateStat() override; + bool addDoc(DocId& docId) override; + bool add(const vespalib::eval::Value& v, int32_t) override; + IExtendAttribute* getExtendInterface() override; + // DocVectorAccess API + vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const override; + VectorBundle get_vectors(uint32_t docid) const override; + + // ITensorAttribute API + std::unique_ptr<vespalib::eval::Value> getTensor(uint32_t docid) const override; + std::unique_ptr<vespalib::eval::Value> getEmptyTensor() const override; + vespalib::eval::TypedCells extract_cells_ref(uint32_t docid) const override; + const vespalib::eval::Value& get_tensor_ref(uint32_t docid) const override; + SerializedTensorRef get_serialized_tensor_ref(uint32_t docid) const override; + bool supports_extract_cells_ref() const override; + bool supports_get_tensor_ref() const override; + bool supports_get_serialized_tensor_ref() const override; + const vespalib::eval::ValueType & getTensorType() const override; + search::attribute::DistanceMetric distance_metric() const override; + uint32_t get_num_docs() const override; + void get_state(const vespalib::slime::Inserter& inserter) const override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/test/attribute_builder.cpp b/searchlib/src/vespa/searchlib/test/attribute_builder.cpp index cc84355385d..ce8c86367b1 100644 --- a/searchlib/src/vespa/searchlib/test/attribute_builder.cpp +++ b/searchlib/src/vespa/searchlib/test/attribute_builder.cpp @@ -7,6 +7,7 @@ #include <vespa/searchlib/attribute/attributevector.h> #include <vespa/searchlib/attribute/floatbase.h> #include <vespa/searchlib/attribute/integerbase.h> +#include <vespa/searchlib/attribute/single_raw_attribute.h> #include <vespa/searchlib/attribute/stringbase.h> #include <vespa/searchlib/tensor/tensor_attribute.h> #include <cassert> @@ -158,6 +159,13 @@ AttributeBuilder::fill_wset(std::initializer_list<WeightedStringList> values) } AttributeBuilder& +AttributeBuilder::fill(std::initializer_list<vespalib::ConstArrayRef<char>> values) +{ + fill_helper<SingleRawAttribute, vespalib::ConstArrayRef<char>>(_attr, values); + return *this; +} + +AttributeBuilder& AttributeBuilder::fill_tensor(const std::vector<vespalib::string>& values) { add_docs(_attr, values.size()); diff --git a/searchlib/src/vespa/searchlib/test/attribute_builder.h b/searchlib/src/vespa/searchlib/test/attribute_builder.h index 339af4e22f5..e6730b74249 100644 --- a/searchlib/src/vespa/searchlib/test/attribute_builder.h +++ b/searchlib/src/vespa/searchlib/test/attribute_builder.h @@ -4,6 +4,7 @@ #include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/stllike/string.h> +#include <vespa/vespalib/util/arrayref.h> #include <memory> #include <utility> #include <vector> @@ -52,6 +53,9 @@ public: AttributeBuilder& fill_array(std::initializer_list<StringList> values); AttributeBuilder& fill_wset(std::initializer_list<WeightedStringList> values); + // Fill function for raw attributes + AttributeBuilder& fill(std::initializer_list<vespalib::ConstArrayRef<char>> values); + /** * Fill this tensor attribute with the given tensor values. * diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp index d6f06c9161e..6ac4fea2921 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp @@ -23,8 +23,7 @@ namespace search::docsummary { DocsumFieldWriterFactory::DocsumFieldWriterFactory(bool use_v8_geo_positions, const IDocsumEnvironment& env, const IQueryTermFilterFactory& query_term_filter_factory) : _use_v8_geo_positions(use_v8_geo_positions), _env(env), - _query_term_filter_factory(query_term_filter_factory), - _matching_elems_fields(std::make_shared<MatchingElementsFields>()) + _query_term_filter_factory(query_term_filter_factory) { } @@ -58,7 +57,8 @@ throw_missing_source(const vespalib::string& command) std::unique_ptr<DocsumFieldWriter> DocsumFieldWriterFactory::create_docsum_field_writer(const vespalib::string& field_name, const vespalib::string& command, - const vespalib::string& source) + const vespalib::string& source, + std::shared_ptr<MatchingElementsFields> matching_elems_fields) { std::unique_ptr<DocsumFieldWriter> fieldWriter; if (command == command::dynamic_teaser) { @@ -116,9 +116,9 @@ DocsumFieldWriterFactory::create_docsum_field_writer(const vespalib::string& fie if (has_attribute_manager()) { auto attr_ctx = getEnvironment().getAttributeManager()->createContext(); if (attr_ctx->getAttribute(source_field) != nullptr) { - fieldWriter = AttributeDFWFactory::create(*getEnvironment().getAttributeManager(), source_field, true, _matching_elems_fields); + fieldWriter = AttributeDFWFactory::create(*getEnvironment().getAttributeManager(), source_field, true, matching_elems_fields); } else { - fieldWriter = AttributeCombinerDFW::create(source_field, *attr_ctx, true, _matching_elems_fields); + fieldWriter = AttributeCombinerDFW::create(source_field, *attr_ctx, true, matching_elems_fields); } throw_if_nullptr(fieldWriter, command); } @@ -126,7 +126,7 @@ DocsumFieldWriterFactory::create_docsum_field_writer(const vespalib::string& fie const vespalib::string& source_field = source.empty() ? field_name : source; if (has_attribute_manager()) { auto attr_ctx = getEnvironment().getAttributeManager()->createContext(); - fieldWriter = MatchedElementsFilterDFW::create(source_field,*attr_ctx, _matching_elems_fields); + fieldWriter = MatchedElementsFilterDFW::create(source_field,*attr_ctx, matching_elems_fields); throw_if_nullptr(fieldWriter, command); } } else if (command == command::documentid) { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.h b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.h index e50fb85cca6..7175f043701 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.h @@ -20,7 +20,6 @@ class DocsumFieldWriterFactory : public IDocsumFieldWriterFactory const IDocsumEnvironment& _env; const IQueryTermFilterFactory& _query_term_filter_factory; protected: - std::shared_ptr<MatchingElementsFields> _matching_elems_fields; const IDocsumEnvironment& getEnvironment() const noexcept { return _env; } bool has_attribute_manager() const noexcept; public: @@ -28,7 +27,8 @@ public: ~DocsumFieldWriterFactory() override; std::unique_ptr<DocsumFieldWriter> create_docsum_field_writer(const vespalib::string& field_name, const vespalib::string& command, - const vespalib::string& source) override; + const vespalib::string& source, + std::shared_ptr<MatchingElementsFields> matching_elems_fields) override; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_field_writer_factory.h b/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_field_writer_factory.h index 6a5cd691857..bc2ebe3c40c 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_field_writer_factory.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_field_writer_factory.h @@ -5,6 +5,8 @@ #include <memory> #include <vespa/vespalib/stllike/string.h> +namespace search { class MatchingElementsFields; } + namespace search::docsummary { class DocsumFieldWriter; @@ -21,7 +23,8 @@ public: */ virtual std::unique_ptr<DocsumFieldWriter> create_docsum_field_writer(const vespalib::string& field_name, const vespalib::string& command, - const vespalib::string& source) = 0; + const vespalib::string& source, + std::shared_ptr<MatchingElementsFields> matching_elems_fields) = 0; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp index f620dcb1df5..eddb67f5822 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/resultconfig.cpp @@ -5,6 +5,7 @@ #include "docsum_field_writer_factory.h" #include "resultclass.h" #include <vespa/config-summary.h> +#include <vespa/searchlib/common/matching_elements_fields.h> #include <vespa/vespalib/stllike/hash_map.hpp> #include <vespa/vespalib/util/exceptions.h> #include <atomic> @@ -124,6 +125,7 @@ ResultConfig::readConfig(const SummaryConfig &cfg, const char *configId, IDocsum break; } resClass->set_omit_summary_features(cfg_class.omitsummaryfeatures); + auto matching_elems_fields = std::make_shared<MatchingElementsFields>(); for (const auto & field : cfg_class.fields) { const char *fieldname = field.name.c_str(); vespalib::string command = field.command; @@ -134,7 +136,8 @@ ResultConfig::readConfig(const SummaryConfig &cfg, const char *configId, IDocsum try { docsum_field_writer = docsum_field_writer_factory.create_docsum_field_writer(fieldname, command, - source_name); + source_name, + matching_elems_fields); } catch (const vespalib::IllegalArgumentException& ex) { LOG(error, "Exception during setup of summary result class '%s': field='%s', command='%s', source='%s': %s", cfg_class.name.c_str(), fieldname, command.c_str(), source_name.c_str(), ex.getMessage().c_str()); diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java index 3a9c2b1f1e7..d76b4ecc0b5 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModel.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.duper; +import ai.vespa.http.DomainName; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.HostName; import java.util.logging.Level; import com.yahoo.vespa.service.monitor.DuperModelListener; @@ -26,8 +26,8 @@ public class DuperModel { private static Logger logger = Logger.getLogger(DuperModel.class.getName()); private final Map<ApplicationId, ApplicationInfo> applicationsById = new HashMap<>(); - private final Map<HostName, ApplicationId> idsByHostname = new HashMap<>(); - private final Map<ApplicationId, HashSet<HostName>> hostnamesById = new HashMap<>(); + private final Map<DomainName, ApplicationId> idsByHostname = new HashMap<>(); + private final Map<ApplicationId, HashSet<DomainName>> hostnamesById = new HashMap<>(); private final List<DuperModelListener> listeners = new ArrayList<>(); private boolean isComplete = false; @@ -67,7 +67,7 @@ public class DuperModel { return Optional.ofNullable(applicationsById.get(applicationId)); } - public Optional<ApplicationInfo> getApplicationInfo(HostName hostName) { + public Optional<ApplicationInfo> getApplicationInfo(DomainName hostName) { return Optional.ofNullable(idsByHostname.get(hostName)).map(applicationsById::get); } @@ -76,12 +76,12 @@ public class DuperModel { } /** Note: Returns an empty set for unknown application. */ - public Set<HostName> getHostnames(ApplicationId applicationId) { - HashSet<HostName> set = hostnamesById.get(applicationId); + public Set<DomainName> getHostnames(ApplicationId applicationId) { + HashSet<DomainName> set = hostnamesById.get(applicationId); return set == null ? Set.of() : Set.copyOf(set); } - public Optional<ApplicationId> getApplicationId(HostName hostname) { + public Optional<ApplicationId> getApplicationId(DomainName hostname) { return Optional.ofNullable(idsByHostname.get(hostname)); } @@ -103,7 +103,7 @@ public class DuperModel { } public void remove(ApplicationId applicationId) { - Set<HostName> hostnames = hostnamesById.remove(applicationId); + Set<DomainName> hostnames = hostnamesById.remove(applicationId); if (hostnames != null) { hostnames.forEach(idsByHostname::remove); } @@ -117,11 +117,11 @@ public class DuperModel { /** Update hostnamesById and idsByHostname based on a new applicationInfo. */ private void updateHostnameVsIdMaps(ApplicationInfo applicationInfo, ApplicationId id) { - Set<HostName> removedHosts = new HashSet<>(hostnamesById.computeIfAbsent(id, k -> new HashSet<>())); + Set<DomainName> removedHosts = new HashSet<>(hostnamesById.computeIfAbsent(id, k -> new HashSet<>())); applicationInfo.getModel().getHosts().stream() .map(HostInfo::getHostname) - .map(HostName::of) + .map(DomainName::of) .forEach(hostname -> { if (!removedHosts.remove(hostname)) { // hostname has been added @@ -135,7 +135,7 @@ public class DuperModel { logger.log(Level.WARNING, hostname + " has been reassigned from " + previousId.toFullString() + " to " + id.toFullString()); - Set<HostName> previousHostnames = hostnamesById.get(previousId); + Set<DomainName> previousHostnames = hostnamesById.get(previousId); if (previousHostnames != null) { previousHostnames.remove(hostname); } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java index b8c980a8760..400c4f4b907 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/DuperModelManager.java @@ -1,14 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.duper; -import com.yahoo.component.annotation.Inject; +import ai.vespa.http.DomainName; import com.yahoo.cloud.config.ConfigserverConfig; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.SuperModel; import com.yahoo.config.model.api.SuperModelListener; import com.yahoo.config.model.api.SuperModelProvider; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.HostName; import com.yahoo.vespa.service.monitor.CriticalRegion; import com.yahoo.vespa.service.monitor.DuperModelInfraApi; import com.yahoo.vespa.service.monitor.DuperModelListener; @@ -133,7 +133,7 @@ public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi } @Override - public void infraApplicationActivated(ApplicationId applicationId, List<HostName> hostnames) { + public void infraApplicationActivated(ApplicationId applicationId, List<DomainName> hostnames) { InfraApplication application = supportedInfraApplications.get(applicationId); if (application == null) { throw new IllegalArgumentException("There is no infrastructure application with ID '" + applicationId + "'"); @@ -172,7 +172,7 @@ public class DuperModelManager implements DuperModelProvider, DuperModelInfraApi return lockedSupplier(() -> duperModel.getApplicationInfo(applicationId)); } - public Optional<ApplicationInfo> getApplicationInfo(HostName hostname) { + public Optional<ApplicationInfo> getApplicationInfo(DomainName hostname) { return lockedSupplier(() -> duperModel.getApplicationInfo(hostname)); } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java index e82e2cc0265..07f9d7bddc5 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/duper/InfraApplication.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.duper; +import ai.vespa.http.DomainName; import com.yahoo.component.Version; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.HostInfo; @@ -9,7 +10,6 @@ import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.applicationmodel.ApplicationInstanceId; import com.yahoo.vespa.applicationmodel.ClusterId; @@ -93,12 +93,12 @@ public abstract class InfraApplication implements InfraApplicationApi { return new TenantId(application.id().tenant().value()); } - public ApplicationInfo makeApplicationInfo(List<HostName> hostnames) { + public ApplicationInfo makeApplicationInfo(List<DomainName> hostnames) { List<HostInfo> hostInfos = hostnames.stream().map(this::makeHostInfo).toList(); return new ApplicationInfo(application.id(), 0, new HostsModel(hostInfos)); } - private HostInfo makeHostInfo(HostName hostname) { + private HostInfo makeHostInfo(DomainName hostname) { PortInfo portInfo = new PortInfo(healthPort, StateV1HealthModel.HTTP_HEALTH_PORT_TAGS); Map<String, String> properties = new HashMap<>(); @@ -116,7 +116,7 @@ public abstract class InfraApplication implements InfraApplicationApi { return new HostInfo(hostname.value(), Collections.singletonList(serviceInfo)); } - public ConfigId configIdFor(HostName hostname) { + public ConfigId configIdFor(DomainName hostname) { // Not necessarily unique, but service monitor doesn't require it to be unique. return new ConfigId(String.format("%s/%s", clusterSpecId.value(), prefixTo(hostname.value(), '.'))); } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java index a654a4557ef..1d832cc0eec 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthEndpoint.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.health; -import com.yahoo.config.provision.HostName; +import ai.vespa.http.DomainName; import com.yahoo.vespa.service.executor.RunletExecutor; import com.yahoo.vespa.service.monitor.ServiceId; @@ -22,7 +22,7 @@ class StateV1HealthEndpoint implements HealthEndpoint { private final RunletExecutor executor; StateV1HealthEndpoint(ServiceId serviceId, - HostName hostname, + DomainName hostname, int port, Duration delay, Duration requestTimeout, diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java index 3391bbcf82a..3969ef141bd 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/health/StateV1HealthModel.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.health; +import ai.vespa.http.DomainName; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.PortInfo; import com.yahoo.config.model.api.ServiceInfo; -import com.yahoo.config.provision.HostName; import com.yahoo.vespa.service.executor.RunletExecutor; import com.yahoo.vespa.service.model.ApplicationInstanceGenerator; import com.yahoo.vespa.service.monitor.ServiceId; @@ -45,7 +45,7 @@ public class StateV1HealthModel implements AutoCloseable { Map<ServiceId, HealthEndpoint> endpoints = new HashMap<>(); for (HostInfo hostInfo : application.getModel().getHosts()) { - HostName hostname = HostName.of(hostInfo.getHostname()); + DomainName hostname = DomainName.of(hostInfo.getHostname()); for (ServiceInfo serviceInfo : hostInfo.getServices()) { ServiceId serviceId = ApplicationInstanceGenerator.getServiceId(application, serviceInfo); for (PortInfo portInfo : serviceInfo.getPorts()) { diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java index 815f8de26b4..53db7cc135c 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/model/ServiceMonitorImpl.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.model; +import ai.vespa.http.DomainName; import com.yahoo.component.annotation.Inject; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.provision.ApplicationId; @@ -68,7 +69,7 @@ public class ServiceMonitorImpl implements ServiceMonitor, AntiServiceMonitor { @Override public Optional<ApplicationInstanceReference> getApplicationInstanceReference(HostName hostname) { - return duperModelManager.getApplicationInfo(toConfigProvisionHostName(hostname)) + return duperModelManager.getApplicationInfo(DomainName.of(hostname.s())) .map(ApplicationInfo::getApplicationId) .map(modelGenerator::toApplicationInstanceReference); } @@ -113,11 +114,7 @@ public class ServiceMonitorImpl implements ServiceMonitor, AntiServiceMonitor { } private Optional<ApplicationInfo> getApplicationInfo(HostName hostname) { - return duperModelManager.getApplicationInfo(toConfigProvisionHostName(hostname)); + return duperModelManager.getApplicationInfo(DomainName.of(hostname.s())); } - /** The duper model uses HostName from config.provision. */ - private static com.yahoo.config.provision.HostName toConfigProvisionHostName(HostName hostname) { - return com.yahoo.config.provision.HostName.of(hostname.s()); - } } diff --git a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java index 91782d5a582..783f0a27e51 100644 --- a/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java +++ b/service-monitor/src/main/java/com/yahoo/vespa/service/monitor/DuperModelInfraApi.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.monitor; +import ai.vespa.http.DomainName; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.HostName; import java.util.List; import java.util.Optional; @@ -24,7 +24,7 @@ public interface DuperModelInfraApi { boolean infraApplicationIsActive(ApplicationId applicationId); /** Update the DuperModel: A supported infrastructure application has been (re)activated or is active. */ - void infraApplicationActivated(ApplicationId applicationId, List<HostName> hostnames); + void infraApplicationActivated(ApplicationId applicationId, List<DomainName> hostnames); /** Update the DuperModel: A supported infrastructure application has been removed or is not active. */ void infraApplicationRemoved(ApplicationId applicationId); diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java index 6cf7d4d9f26..ad7f0f188c0 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelManagerTest.java @@ -1,17 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.duper; +import ai.vespa.http.DomainName; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.SuperModel; import com.yahoo.config.model.api.SuperModelListener; import com.yahoo.config.model.api.SuperModelProvider; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.HostName; import org.junit.Test; import org.mockito.ArgumentCaptor; import java.util.List; -import java.util.stream.Collectors; import java.util.stream.Stream; import static com.yahoo.vespa.service.duper.DuperModelManager.configServerApplication; @@ -64,7 +63,7 @@ public class DuperModelManagerTest { makeManager(false); ApplicationId id = proxyHostApplication.getApplicationId(); - List<HostName> proxyHostHosts = Stream.of("proxyhost1", "proxyhost2").map(HostName::of).toList(); + List<DomainName> proxyHostHosts = Stream.of("proxyhost1", "proxyhost2").map(DomainName::of).toList(); verify(duperModel, times(0)).add(any()); manager.infraApplicationActivated(id, proxyHostHosts); verify(duperModel, times(1)).add(any()); @@ -91,12 +90,12 @@ public class DuperModelManagerTest { } private void testEnabledConfigServerLikeInfraApplication(ApplicationId firstId, ApplicationId secondId) { - List<HostName> hostnames1 = Stream.of("node11", "node12").map(HostName::of).toList(); + List<DomainName> hostnames1 = Stream.of("node11", "node12").map(DomainName::of).toList(); manager.infraApplicationActivated(firstId, hostnames1); verify(duperModel, times(1)).add(any()); // Adding the second config server like application will be ignored - List<HostName> hostnames2 = Stream.of("node21", "node22").map(HostName::of).toList(); + List<DomainName> hostnames2 = Stream.of("node21", "node22").map(DomainName::of).toList(); assertThrows(IllegalArgumentException.class, () -> manager.infraApplicationActivated(secondId, hostnames2)); verify(duperModel, times(1)).add(any()); diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java index 73a49ca8717..568dc3640e2 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/duper/DuperModelTest.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.duper; +import ai.vespa.http.DomainName; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.Model; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.HostName; import com.yahoo.vespa.service.monitor.DuperModelListener; import org.junit.Before; import org.junit.Test; @@ -14,7 +14,6 @@ import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; import java.util.stream.Stream; import static org.junit.Assert.assertEquals; @@ -34,12 +33,12 @@ public class DuperModelTest { private final ApplicationId id1 = ApplicationId.fromSerializedForm("tenant:app1:default"); private final ApplicationInfo application1 = mock(ApplicationInfo.class); - private final HostName hostname1_1 = HostName.of("hostname1-1"); - private final HostName hostname1_2 = HostName.of("hostname1-2"); + private final DomainName hostname1_1 = DomainName.of("hostname1-1"); + private final DomainName hostname1_2 = DomainName.of("hostname1-2"); private final ApplicationId id2 = ApplicationId.fromSerializedForm("tenant:app2:default"); private final ApplicationInfo application2 = mock(ApplicationInfo.class); - private final HostName hostname2_1 = HostName.of("hostname2-1"); + private final DomainName hostname2_1 = DomainName.of("hostname2-1"); private final DuperModelListener listener1 = mock(DuperModelListener.class); @@ -49,7 +48,7 @@ public class DuperModelTest { setUpApplication(id2, application2, hostname2_1); } - private void setUpApplication(ApplicationId id, ApplicationInfo info, HostName... hostnames) { + private void setUpApplication(ApplicationId id, ApplicationInfo info, DomainName... hostnames) { when(info.getApplicationId()).thenReturn(id); Model model = mock(Model.class); @@ -129,7 +128,7 @@ public class DuperModelTest { addAndVerifyApplication1("host1"); addAndVerifyApplication1("host1", "host2"); addAndVerifyApplication1("host2", "host3"); - assertEquals(Optional.empty(), duperModel.getApplicationId(HostName.of("host1"))); + assertEquals(Optional.empty(), duperModel.getApplicationId(DomainName.of("host1"))); duperModel.remove(id1); assertEquals(0, duperModel.numberOfApplications()); @@ -138,7 +137,7 @@ public class DuperModelTest { } private void addAndVerifyApplication1(String... hostnameStrings) { - HostName[] hostnameArray = Stream.of(hostnameStrings).map(HostName::of).toArray(HostName[]::new); + DomainName[] hostnameArray = Stream.of(hostnameStrings).map(DomainName::of).toArray(DomainName[]::new); setUpApplication(id1, application1, hostnameArray); duperModel.add(application1); diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java index d99e882985f..9f46b5dfe8f 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/ApplicationHealthMonitorTest.java @@ -1,13 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.health; +import ai.vespa.http.DomainName; import com.yahoo.config.model.api.ApplicationInfo; -import com.yahoo.config.provision.HostName; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.service.duper.ConfigServerApplication; -import com.yahoo.vespa.service.monitor.ServiceId; import com.yahoo.vespa.service.monitor.ConfigserverUtil; +import com.yahoo.vespa.service.monitor.ServiceId; import org.junit.Test; import java.util.HashMap; @@ -87,7 +87,7 @@ public class ApplicationHealthMonitorTest { return new ServiceId(configServerApplication.getApplicationId(), configServerApplication.getClusterId(), configServerApplication.getServiceType(), - configServerApplication.configIdFor(HostName.of(hostname))); + configServerApplication.configIdFor(DomainName.of(hostname))); } private ServiceStatus getStatus(ApplicationHealthMonitor monitor, String hostname) { @@ -95,7 +95,7 @@ public class ApplicationHealthMonitorTest { configServerApplication.getApplicationId(), configServerApplication.getClusterId(), configServerApplication.getServiceType(), - configServerApplication.configIdFor(HostName.of(hostname))) + configServerApplication.configIdFor(DomainName.of(hostname))) .serviceStatus(); } }
\ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java index f7ee64b52a6..f9129b5e4f4 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/HealthMonitorManagerTest.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.health; +import ai.vespa.http.DomainName; import com.yahoo.config.model.api.ApplicationInfo; -import com.yahoo.config.provision.HostName; import com.yahoo.vespa.applicationmodel.ServiceStatus; import com.yahoo.vespa.applicationmodel.ServiceStatusInfo; import com.yahoo.vespa.service.duper.ControllerHostApplication; @@ -14,7 +14,6 @@ import org.junit.Before; import org.junit.Test; import java.util.List; -import java.util.stream.Collectors; import java.util.stream.Stream; import static org.junit.Assert.assertEquals; @@ -48,7 +47,7 @@ public class HealthMonitorManagerTest { public void infrastructureApplication() { ProxyHostApplication proxyHostApplication = new ProxyHostApplication(); when(duperModel.isSupportedInfraApplication(proxyHostApplication.getApplicationId())).thenReturn(true); - List<HostName> hostnames = Stream.of("proxyhost1", "proxyhost2").map(HostName::of).toList(); + List<DomainName> hostnames = Stream.of("proxyhost1", "proxyhost2").map(DomainName::of).toList(); ApplicationInfo proxyHostApplicationInfo = proxyHostApplication.makeApplicationInfo(hostnames); manager.applicationActivated(proxyHostApplicationInfo); @@ -77,7 +76,7 @@ public class HealthMonitorManagerTest { infraApplication.getApplicationId(), infraApplication.getClusterId(), infraApplication.getServiceType(), - infraApplication.configIdFor(HostName.of(hostname))).serviceStatus(); + infraApplication.configIdFor(DomainName.of(hostname))).serviceStatus(); assertEquals(expected, actual); @@ -85,7 +84,7 @@ public class HealthMonitorManagerTest { infraApplication.getApplicationId(), infraApplication.getClusterId(), infraApplication.getServiceType(), - infraApplication.configIdFor(HostName.of(hostname))); + infraApplication.configIdFor(DomainName.of(hostname))); } }
\ No newline at end of file diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java index 18b56f76e88..01b7930a74f 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/health/StateV1HealthModelTest.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.health; +import ai.vespa.http.DomainName; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.PortInfo; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.HostName; import com.yahoo.vespa.applicationmodel.ClusterId; import com.yahoo.vespa.applicationmodel.ConfigId; import com.yahoo.vespa.applicationmodel.ServiceStatus; @@ -18,7 +18,6 @@ import org.junit.Test; import java.time.Duration; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import java.util.stream.Stream; import static org.junit.Assert.assertEquals; @@ -38,7 +37,7 @@ public class StateV1HealthModelTest { private final Duration requestTimeout = Duration.ofSeconds(2); private final Duration keepAlive = Duration.ofSeconds(3); private final ProxyHostApplication proxyHostApplication = new ProxyHostApplication(); - private final List<HostName> hostnames = Stream.of("host1", "host2").map(HostName::of).toList(); + private final List<DomainName> hostnames = Stream.of("host1", "host2").map(DomainName::of).toList(); private final ApplicationInfo proxyHostApplicationInfo = proxyHostApplication.makeApplicationInfo(hostnames); private final StateV1HealthModel model = new StateV1HealthModel(healthStaleness, requestTimeout, keepAlive, executor); diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java index a2532590f52..e85a00d732c 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ApplicationInstanceGeneratorTest.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.model; +import ai.vespa.http.DomainName; import com.yahoo.config.model.api.ApplicationInfo; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.applicationmodel.ApplicationInstance; import com.yahoo.vespa.applicationmodel.ServiceStatus; @@ -12,7 +12,6 @@ import com.yahoo.vespa.service.monitor.ServiceStatusProvider; import org.junit.Test; import java.util.List; -import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -34,7 +33,7 @@ public class ApplicationInstanceGeneratorTest { when(statusProvider.getStatus(any(), any(), any(), any())).thenReturn(new ServiceStatusInfo(ServiceStatus.NOT_CHECKED)); Zone zone = mock(Zone.class); ApplicationInfo configServer = configServerApplication.makeApplicationInfo( - configServerList.stream().map(HostName::of).toList()); + configServerList.stream().map(DomainName::of).toList()); ApplicationInstance applicationInstance = new ApplicationInstanceGenerator(configServer, zone) .makeApplicationInstance(statusProvider); diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceHostListenerAdapterTest.java b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceHostListenerAdapterTest.java index c93e4dfb7fe..23422640af9 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceHostListenerAdapterTest.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/model/ServiceHostListenerAdapterTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.model; +import ai.vespa.http.DomainName; import com.yahoo.config.model.api.ApplicationInfo; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.model.api.Model; @@ -100,7 +101,7 @@ public class ServiceHostListenerAdapterTest { } private Optional<ApplicationInfo> getDuperModelApplicationInfo(String hostname) { - return duperModel.getApplicationInfo(com.yahoo.config.provision.HostName.of(hostname)); + return duperModel.getApplicationInfo(DomainName.of(hostname)); } private void removeAndVerify(ApplicationId id, boolean listenerInvoked) { diff --git a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ConfigserverUtil.java b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ConfigserverUtil.java index 498a9dfc15e..d2480cd6520 100644 --- a/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ConfigserverUtil.java +++ b/service-monitor/src/test/java/com/yahoo/vespa/service/monitor/ConfigserverUtil.java @@ -1,18 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.service.monitor; +import ai.vespa.http.DomainName; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.model.api.ApplicationInfo; -import com.yahoo.config.provision.HostName; import com.yahoo.vespa.service.duper.ConfigServerApplication; -import java.util.stream.Collectors; -import java.util.stream.Stream; +import java.util.List; /** * @author hakon */ public class ConfigserverUtil { + /** Create a ConfigserverConfig with the given settings. */ public static ConfigserverConfig create( boolean multitenant, @@ -31,16 +31,10 @@ public class ConfigserverUtil { return create(true, "cfg1", "cfg2", "cfg3"); } - public static ApplicationInfo makeConfigServerApplicationInfo( - String configServerHostname1, - String configServerHostname2, - String configServerHostname3) { - return new ConfigServerApplication().makeApplicationInfo( - Stream.of(configServerHostname1, configServerHostname2, configServerHostname3) - .map(HostName::of).toList()); - } - public static ApplicationInfo makeExampleConfigServer() { - return makeConfigServerApplicationInfo("cfg1", "cfg2", "cfg3"); + return new ConfigServerApplication().makeApplicationInfo(List.of(DomainName.of("cfg1"), + DomainName.of("cfg2"), + DomainName.of("cfg3"))); } + } diff --git a/storage/src/vespa/storage/bucketdb/btree_lockable_map.hpp b/storage/src/vespa/storage/bucketdb/btree_lockable_map.hpp index 1c0f9ec7a59..e24e97ff2a4 100644 --- a/storage/src/vespa/storage/bucketdb/btree_lockable_map.hpp +++ b/storage/src/vespa/storage/bucketdb/btree_lockable_map.hpp @@ -49,7 +49,7 @@ struct BTreeLockableMap<T>::ValueTraits { return store.addEntry(value).ref(); } static void remove_by_wrapped_value(DataStoreType& store, uint64_t value) noexcept { - store.holdElem(entry_ref_from_value(value), 1); + store.hold_entry(entry_ref_from_value(value)); } static ValueType unwrap_from_key_value(const DataStoreType& store, [[maybe_unused]] uint64_t key, uint64_t value) { return store.getEntry(entry_ref_from_value(value)); diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index 92b9e832a23..1d29a8795d5 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -1054,7 +1054,7 @@ FileStorHandlerImpl::Stripe::waitUntilNoLocks() const { std::unique_lock guard(*_lock); while (!_lockedBuckets.empty()) { - _cond->wait(guard); + _cond->wait_for(guard, 100ms); } } @@ -1062,7 +1062,7 @@ void FileStorHandlerImpl::Stripe::waitInactive(const AbortBucketOperationsCommand& cmd) const { std::unique_lock guard(*_lock); while (hasActive(guard, cmd)) { - _cond->wait(guard); + _cond->wait_for(guard, 100ms); } } diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index d0e3d1f038b..2caf2de1d0b 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -7,6 +7,7 @@ #include <vespa/persistence/spi/docentry.h> #include <vespa/document/datatype/positiondatatype.h> #include <vespa/document/datatype/documenttype.h> +#include <vespa/document/datatype/tensor_data_type.h> #include <vespa/document/datatype/weightedsetdatatype.h> #include <vespa/document/datatype/mapdatatype.h> #include <vespa/searchlib/aggregation/modifiers.h> @@ -14,6 +15,7 @@ #include <vespa/searchlib/common/packets.h> #include <vespa/searchlib/uca/ucaconverter.h> #include <vespa/searchlib/features/setup.h> +#include <vespa/searchlib/tensor/tensor_ext_attribute.h> #include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/geo/zcurve.h> #include <vespa/vespalib/objects/nbostream.h> @@ -99,6 +101,16 @@ createMultiValueAttribute(const vespalib::string & name, const document::FieldVa return {}; } +const document::TensorDataType* +get_tensor_type(const document::FieldValue& fv) +{ + auto tfv = dynamic_cast<const document::TensorFieldValue*>(&fv); + if (tfv == nullptr) { + return nullptr; + } + return dynamic_cast<const document::TensorDataType*>(tfv->getDataType()); +} + AttributeVector::SP createAttribute(const vespalib::string & name, const document::FieldValue & fv) { @@ -111,6 +123,12 @@ createAttribute(const vespalib::string & name, const document::FieldValue & fv) return std::make_shared<search::SingleStringExtAttribute>(name); } else if (fv.isA(document::FieldValue::Type::RAW)) { return std::make_shared<search::attribute::SingleRawExtAttribute>(name); + } else if (fv.isA(document::FieldValue::Type::TENSOR) && get_tensor_type(fv) != nullptr) { + search::attribute::Config cfg(search::attribute::BasicType::TENSOR, search::attribute::CollectionType::SINGLE); + auto tdt = get_tensor_type(fv); + assert(tdt != nullptr); + cfg.setTensorType(tdt->getTensorType()); + return std::make_shared<search::tensor::TensorExtAttribute>(name, cfg); } else { LOG(debug, "Can not make an attribute out of %s of type '%s'.", name.c_str(), fv.className()); } @@ -444,6 +462,14 @@ SearchVisitor::AttributeInserter::onPrimitive(uint32_t, const Content & c) } else if (_attribute.is_raw_type()) { auto raw_value = value.getAsRaw(); attr.add(vespalib::ConstArrayRef<char>(raw_value.first, raw_value.second), c.getWeight()); + } else if (_attribute.isTensorType()) { + auto tfvalue = dynamic_cast<const document::TensorFieldValue*>(&value); + if (tfvalue != nullptr) { + auto tensor = tfvalue->getAsTensorPtr(); + if (tensor != nullptr) { + attr.add(*tensor, c.getWeight()); + } + } } else { assert(false && "We got an attribute vector that is of an unknown type"); } diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.cpp b/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.cpp index 36873b713aa..b103d7d85b2 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.cpp @@ -48,7 +48,8 @@ DocsumFieldWriterFactory::~DocsumFieldWriterFactory() = default; std::unique_ptr<DocsumFieldWriter> DocsumFieldWriterFactory::create_docsum_field_writer(const vespalib::string& field_name, const vespalib::string& command, - const vespalib::string& source) + const vespalib::string& source, + std::shared_ptr<MatchingElementsFields> matching_elems_fields) { std::unique_ptr<DocsumFieldWriter> fieldWriter; using namespace search::docsummary; @@ -65,10 +66,10 @@ DocsumFieldWriterFactory::create_docsum_field_writer(const vespalib::string& fie } else if ((command == command::matched_attribute_elements_filter) || (command == command::matched_elements_filter)) { vespalib::string source_field = source.empty() ? field_name : source; - populate_fields(*_matching_elems_fields, _vsm_fields_config, source_field); - fieldWriter = MatchedElementsFilterDFW::create(source_field, _matching_elems_fields); + populate_fields(*matching_elems_fields, _vsm_fields_config, source_field); + fieldWriter = MatchedElementsFilterDFW::create(source_field, matching_elems_fields); } else { - return search::docsummary::DocsumFieldWriterFactory::create_docsum_field_writer(field_name, command, source); + return search::docsummary::DocsumFieldWriterFactory::create_docsum_field_writer(field_name, command, source, matching_elems_fields); } return fieldWriter; } diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.h b/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.h index 078c466d3d2..ac5cae8c49d 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.h +++ b/streamingvisitors/src/vespa/vsm/vsm/docsum_field_writer_factory.h @@ -21,7 +21,8 @@ public: std::unique_ptr<search::docsummary::DocsumFieldWriter> create_docsum_field_writer(const vespalib::string& field_name, const vespalib::string& command, - const vespalib::string& source) override; + const vespalib::string& source, + std::shared_ptr<search::MatchingElementsFields> matching_elems_fields) override; }; } diff --git a/vdslib/src/main/java/com/yahoo/vdslib/VisitorStatistics.java b/vdslib/src/main/java/com/yahoo/vdslib/VisitorStatistics.java index fda456ace05..14b44ede12a 100644 --- a/vdslib/src/main/java/com/yahoo/vdslib/VisitorStatistics.java +++ b/vdslib/src/main/java/com/yahoo/vdslib/VisitorStatistics.java @@ -2,6 +2,7 @@ package com.yahoo.vdslib; public class VisitorStatistics { + int bucketsVisited = 0; long documentsVisited = 0; long bytesVisited = 0; @@ -20,8 +21,8 @@ public class VisitorStatistics { public void setBucketsVisited(int bucketsVisited) { this.bucketsVisited = bucketsVisited; } /** - * @return the number of documents matching the document selection in the backend and that - * has been passed to the client-specified visitor instance (dumpvisitor, searchvisitor etc). + * Returns the number of documents matching the document selection in the backend that + * has been passed to the client-specified visitor instance (dumpvisitor, searchvisitor etc). */ public long getDocumentsVisited() { return documentsVisited; } public void setDocumentsVisited(long documentsVisited) { this.documentsVisited = documentsVisited; } @@ -30,9 +31,9 @@ public class VisitorStatistics { public void setBytesVisited(long bytesVisited) { this.bytesVisited = bytesVisited; } /** - * @return Number of documents returned to the visitor client by the backend. This number may - * be lower than that returned by getDocumentsVisited() since the client-specified visitor - * instance may further have filtered the set of documents returned by the backend. + * Returns the number of documents returned to the visitor client by the backend. This number may + * be lower than that returned by getDocumentsVisited() since the client-specified visitor + * instance may further have filtered the set of documents returned by the backend. */ public long getDocumentsReturned() { return documentsReturned; } public void setDocumentsReturned(long documentsReturned) { this.documentsReturned = documentsReturned; } @@ -40,15 +41,14 @@ public class VisitorStatistics { public long getBytesReturned() { return bytesReturned; } public void setBytesReturned(long bytesReturned) { this.bytesReturned = bytesReturned; } + @Override public String toString() { - String out = + return "Buckets visited: " + bucketsVisited + "\n" + "Documents visited: " + documentsVisited + "\n" + "Bytes visited: " + bytesVisited + "\n" + "Documents returned: " + documentsReturned + "\n" + "Bytes returned: " + bytesReturned + "\n"; - - return out; } } diff --git a/vdslib/src/tests/distribution/distributiontest.cpp b/vdslib/src/tests/distribution/distributiontest.cpp index a10aca45b33..b5c756aece9 100644 --- a/vdslib/src/tests/distribution/distributiontest.cpp +++ b/vdslib/src/tests/distribution/distributiontest.cpp @@ -17,8 +17,9 @@ #include <vespa/vespalib/util/size_literals.h> #include <gmock/gmock.h> #include <chrono> -#include <thread> #include <fstream> +#include <iterator> +#include <thread> using namespace ::testing; @@ -434,7 +435,8 @@ TEST(DistributionTest, test_distribution) _distribution[i].second = distr.getIdealStorageNodes( systemState, document::BucketId(26, i)); sort(_distribution[i].second.begin(), _distribution[i].second.end()); - unique(_distribution[i].second.begin(), _distribution[i].second.end()); + auto unique_nodes = std::distance(_distribution[i].second.begin(), unique(_distribution[i].second.begin(), _distribution[i].second.end())); + _distribution[i].second.resize(unique_nodes); for (unsigned j = 0; j < _distribution[i].second.size(); j++) { _nodeCount[_distribution[i].second[j]]++; diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index 18475a8e4d9..95f1179b1a3 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -80,7 +80,7 @@ net.java.dev.jna:jna:5.11.0 net.openhft:zero-allocation-hashing:0.16 org.antlr:antlr-runtime:3.5.3 org.antlr:antlr4-runtime:4.11.1 -org.apache.aries.spifly:org.apache.aries.spifly.dynamic.bundle:1.3.5 +org.apache.aries.spifly:org.apache.aries.spifly.dynamic.bundle:1.3.6 org.apache.commons:commons-compress:1.22 org.apache.commons:commons-csv:1.8 org.apache.commons:commons-exec:1.3 diff --git a/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/ApacheClusterTest.java b/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/ApacheClusterTest.java index 9195b5ab858..cf9a36f2aa8 100644 --- a/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/ApacheClusterTest.java +++ b/vespa-feed-client/src/test/java/ai/vespa/feed/client/impl/ApacheClusterTest.java @@ -35,7 +35,7 @@ class ApacheClusterTest { final WireMockExtension server = new WireMockExtension(); @Test - void testClient() throws IOException, ExecutionException, InterruptedException, TimeoutException { + void testClient() throws Exception { for (Compression compression : Compression.values()) { try (ApacheCluster cluster = new ApacheCluster(new FeedClientBuilderImpl(List.of(URI.create("http://localhost:" + server.port()))) .setCompression(compression))) { @@ -48,25 +48,28 @@ class ApacheClusterTest { Map.of("name1", () -> "value1", "name2", () -> "value2"), "content".getBytes(UTF_8), - Duration.ofSeconds(20)), + Duration.ofSeconds(10)), vessel); - HttpResponse response = vessel.get(15, TimeUnit.SECONDS); - assertEquals("{}", new String(response.body(), UTF_8)); - assertEquals(200, response.code()); - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - try (OutputStream zip = new GZIPOutputStream(buffer)) { zip.write("content".getBytes(UTF_8)); } - server.verify(1, anyRequestedFor(anyUrl())); - RequestPatternBuilder expected = postRequestedFor(urlEqualTo("/path")).withHeader("name1", equalTo("value1")) - .withHeader("name2", equalTo("value2")) - .withHeader("Content-Type", equalTo("application/json; charset=UTF-8")) - .withRequestBody(equalTo("content")); - expected = switch (compression) { - case auto, none -> expected.withoutHeader("Content-Encoding"); - case gzip -> expected.withHeader("Content-Encoding", equalTo("gzip")); + AutoCloseable verifyResponse = () -> { + HttpResponse response = vessel.get(15, TimeUnit.SECONDS); + assertEquals("{}", new String(response.body(), UTF_8)); + assertEquals(200, response.code()); }; - server.verify(1, expected); - server.resetRequests(); + AutoCloseable verifyServer = () -> { + server.verify(1, anyRequestedFor(anyUrl())); + RequestPatternBuilder expected = postRequestedFor(urlEqualTo("/path")).withHeader("name1", equalTo("value1")) + .withHeader("name2", equalTo("value2")) + .withHeader("Content-Type", equalTo("application/json; charset=UTF-8")) + .withRequestBody(equalTo("content")); + expected = switch (compression) { + case auto, none -> expected.withoutHeader("Content-Encoding"); + case gzip -> expected.withHeader("Content-Encoding", equalTo("gzip")); + }; + server.verify(1, expected); + server.resetRequests(); + }; + try (verifyServer; verifyResponse) { } } } } diff --git a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java index fe1e2b46830..2a688ad078b 100644 --- a/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java +++ b/vespaclient-container-plugin/src/main/java/com/yahoo/document/restapi/resource/DocumentV1ApiHandler.java @@ -1226,7 +1226,7 @@ public class DocumentV1ApiHandler extends AbstractRequestHandler { getProperty(request, TRACELEVEL, integerParser).ifPresent(parameters::setTraceLevel); - getProperty(request, CONTINUATION).map(ProgressToken::fromSerializedString).ifPresent(parameters::setResumeToken); + getProperty(request, CONTINUATION, ProgressToken::fromSerializedString).ifPresent(parameters::setResumeToken); parameters.setPriority(DocumentProtocol.Priority.NORMAL_4); getProperty(request, FROM_TIMESTAMP, unsignedLongParser).ifPresent(parameters::setFromTimestamp); diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java index 33e46ebc75f..31f4038c16e 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java @@ -37,15 +37,17 @@ public abstract class Maintainer implements Runnable { private final AtomicBoolean shutDown = new AtomicBoolean(); private final boolean ignoreCollision; private final Clock clock; + private final Double successFactorBaseline; public Maintainer(String name, Duration interval, Clock clock, JobControl jobControl, - JobMetrics jobMetrics, List<String> clusterHostnames, boolean ignoreCollision) { + JobMetrics jobMetrics, List<String> clusterHostnames, boolean ignoreCollision, Double successFactorBaseline) { this.name = name; this.interval = requireInterval(interval); this.jobControl = Objects.requireNonNull(jobControl); this.jobMetrics = Objects.requireNonNull(jobMetrics); this.ignoreCollision = ignoreCollision; this.clock = clock; + this.successFactorBaseline = successFactorBaseline; var startedAt = clock.instant(); Objects.requireNonNull(clusterHostnames); Duration initialDelay = staggeredDelay(interval, startedAt, HostName.getLocalhost(), clusterHostnames) @@ -55,6 +57,10 @@ public abstract class Maintainer implements Runnable { jobControl.started(name(), this); } + public Maintainer(String name, Duration interval, Clock clock, JobControl jobControl, + JobMetrics jobMetrics, List<String> clusterHostnames, boolean ignoreCollision) { + this(name, interval, clock, jobControl, jobMetrics, clusterHostnames, ignoreCollision, 1.0); + } @Override public void run() { lockAndMaintain(false); @@ -91,16 +97,16 @@ public abstract class Maintainer implements Runnable { /** * Called once each time this maintenance job should run. * - * @return the degree to which the run was successful - a number between 0 (no success), to 1 (complete success). + * @return the degree to which the run was deviated from the successFactorBaseline - a number between -1 (no success), to 0 (complete success). * Note that this indicates whether something is wrong, so e.g if the call did nothing because it should do - * nothing, 1.0 should be returned. + * nothing, 0.0 should be returned. */ protected abstract double maintain(); - /** Convenience methods to convert attempts and failures into a success factor */ - protected final double asSuccessFactor(int attempts, int failures) { + /** Convenience methods to convert attempts and failures into a success factor, and return */ + protected final double asSuccessFactorDeviation(int attempts, int failures) { double factor = attempts == 0 ? 1.0 : 1 - (double)failures / attempts; - return new BigDecimal(factor).setScale(2, RoundingMode.HALF_UP).doubleValue(); + return new BigDecimal(factor - successFactorBaseline).setScale(2, RoundingMode.HALF_UP).doubleValue(); } /** Returns the interval at which this job is set to run */ @@ -111,14 +117,14 @@ public abstract class Maintainer implements Runnable { if (!force && !jobControl.isActive(name())) return; log.log(Level.FINE, () -> "Running " + this.getClass().getSimpleName()); - double successFactor = 0; + double successFactorDeviation = 0; long startTime = clock.millis(); try (var lock = jobControl.lockJob(name())) { - successFactor = maintain(); + successFactorDeviation = maintain(); } catch (UncheckedTimeoutException e) { if (ignoreCollision) - successFactor = 1; + successFactorDeviation = 0; else log.log(Level.WARNING, this + " collided with another run. Will retry in " + interval); } @@ -127,7 +133,7 @@ public abstract class Maintainer implements Runnable { } finally { long endTime = clock.millis(); - jobMetrics.completed(name(), successFactor, endTime - startTime); + jobMetrics.completed(name(), successFactorDeviation, endTime - startTime); } log.log(Level.FINE, () -> "Finished " + this.getClass().getSimpleName()); } diff --git a/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java b/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java index f9230ab6df6..6acd0679da2 100644 --- a/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java +++ b/vespajlib/src/main/java/com/yahoo/slime/SlimeUtils.java @@ -166,81 +166,32 @@ public class SlimeUtils { false); } - private static class Equal { - protected final Inspector rhsInspector; - - protected boolean equal = true; - - public Equal(Inspector rhsInspector) { this.rhsInspector = rhsInspector; } - - public boolean isEqual() { return equal; } - } - - private static class EqualArray extends Equal implements ArrayTraverser { - public EqualArray(Inspector rhsInspector) { super(rhsInspector); } - - @Override - public void entry(int idx, Inspector inspector) { - if (equal) { - equal = inspector.equalTo(rhsInspector.entry(idx)); - } - } - } - - private static class EqualObject extends Equal implements ObjectTraverser { - public EqualObject(Inspector rhsInspector) { super(rhsInspector); } - - @Override - public void field(String name, Inspector inspector) { - if (equal) { - equal = inspector.equalTo(rhsInspector.field(name)); - } - } - } - public static boolean equalTo(Inspector a, Inspector b) { - boolean equal = a.type() == b.type(); - - if (equal) { - switch (a.type()) { - case NIX: - equal = a.valid() == b.valid(); - break; - case BOOL: - equal = a.asBool() == b.asBool(); - break; - case LONG: - equal = a.asLong() == b.asLong(); - break; - case DOUBLE: - equal = Double.compare(a.asDouble(), b.asDouble()) == 0; - break; - case STRING: - equal = a.asString().equals(b.asString()); - break; - case DATA: - equal = Arrays.equals(a.asData(), b.asData()); - break; - case ARRAY: - { - var traverser = new EqualArray(b); - a.traverse(traverser); - equal = traverser.isEqual() && (a.entries() == b.entries()); - } - break; - case OBJECT: - { - var traverser = new EqualObject(b); - a.traverse(traverser); - equal = traverser.isEqual() && (a.fields() == b.fields()); + if (a.type() != b.type()) return false; + + switch (a.type()) { + case NIX: return a.valid() == b.valid(); + case BOOL: return a.asBool() == b.asBool(); + case LONG: return a.asLong() == b.asLong(); + case DOUBLE: return Double.compare(a.asDouble(), b.asDouble()) == 0; + case STRING: return a.asString().equals(b.asString()); + case DATA: return Arrays.equals(a.asData(), b.asData()); + case ARRAY: { + if (a.entries() != b.entries()) return false; + for (int i = 0; i < a.entries(); i++) { + if (!equalTo(a.entry(i), b.entry(i))) return false; } - break; - default: - assert(false); - break; + return true; } + case OBJECT: { + if (a.fields() != b.fields()) return false; + boolean[] equal = new boolean[]{ true }; + a.traverse((String key, Inspector value) -> { + if (equal[0] && !equalTo(value, b.field(key))) equal[0] = false; + }); + return equal[0]; + } + default: throw new IllegalStateException("Unexpected type: " + a.type()); } - - return equal; } } diff --git a/vespajlib/src/test/java/com/yahoo/slime/ArrayValueTestCase.java b/vespajlib/src/test/java/com/yahoo/slime/ArrayValueTestCase.java index c9ff86e7c2e..730c3909b4c 100644 --- a/vespajlib/src/test/java/com/yahoo/slime/ArrayValueTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/slime/ArrayValueTestCase.java @@ -2,11 +2,11 @@ package com.yahoo.slime; import org.junit.Test; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.not; -import static org.hamcrest.CoreMatchers.sameInstance; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertNotSame; import java.util.List; import java.util.ArrayList; @@ -20,17 +20,17 @@ public class ArrayValueTestCase { @Test public void testSymbolTableForwarding() { SymbolTable names = new SymbolTable(); - assertThat(names.symbols(), is(0)); + assertEquals(0, names.symbols()); new ArrayValue(names).addArray().addObject().setLong("foo", 3); - assertThat(names.symbols(), is(1)); + assertEquals(1, names.symbols()); } @Test public void testOutOfBoundsAccess() { var array = makeArray(); array.addBool(true); - assertThat(array.entry(-1).valid(), is(false)); - assertThat(array.entry(1).valid(), is(false)); + assertFalse(array.entry(-1).valid()); + assertFalse(array.entry(1).valid()); } @Test @@ -44,8 +44,8 @@ public class ArrayValueTestCase { var e1 = array.entry(i); var e2 = array.entry(i); var e3 = added.get(i); - assertThat(e1, sameInstance(e2)); - assertThat(e1, sameInstance(e3)); + assertSame(e2, e1); + assertSame(e3, e1); } } @@ -61,12 +61,12 @@ public class ArrayValueTestCase { var e1 = array.entry(i); var e2 = array.entry(i); var e3 = added.get(i); - assertThat(e1, not(sameInstance(e2))); - assertThat(e1, not(sameInstance(e3))); - assertThat(e1.equalTo(e2), is(true)); - assertThat(e1.equalTo(e3), is(true)); - assertThat(e1.type(), is(Type.LONG)); - assertThat(e1.asLong(), is(expect)); + assertNotSame(e2, e1); + assertNotSame(e3, e1); + assertTrue(e1.equalTo(e2)); + assertTrue(e1.equalTo(e3)); + assertEquals(Type.LONG, e1.type()); + assertEquals(expect, e1.asLong()); } } @@ -82,12 +82,12 @@ public class ArrayValueTestCase { var e1 = array.entry(i); var e2 = array.entry(i); var e3 = added.get(i); - assertThat(e1, not(sameInstance(e2))); - assertThat(e1, not(sameInstance(e3))); - assertThat(e1.equalTo(e2), is(true)); - assertThat(e1.equalTo(e3), is(true)); - assertThat(e1.type(), is(Type.DOUBLE)); - assertThat(e1.asDouble(), is(expect)); + assertNotSame(e2, e1); + assertNotSame(e3, e1); + assertTrue(e1.equalTo(e2)); + assertTrue(e1.equalTo(e3)); + assertEquals(Type.DOUBLE, e1.type()); + assertEquals(expect, e1.asDouble(), 0.0); } } @@ -109,20 +109,20 @@ public class ArrayValueTestCase { case ARRAY: added.add(array.addArray()); break; case OBJECT: added.add(array.addObject()); break; } - assertThat(array.entries(), is(65)); - assertThat(array.entry(64).type(), is(type)); - assertThat(added.get(64), sameInstance(array.entry(64))); + assertEquals(65, array.entries()); + assertEquals(type, array.entry(64).type()); + assertSame(array.entry(64), added.get(64)); for (int i = 0; i < 64; ++i) { var e1 = array.entry(i); var e2 = array.entry(i); var e3 = added.get(i); long expect = i; - assertThat(e1, sameInstance(e2)); - assertThat(e1, not(sameInstance(e3))); - assertThat(e1.equalTo(e2), is(true)); - assertThat(e1.equalTo(e3), is(true)); - assertThat(e1.type(), is(Type.LONG)); - assertThat(e1.asLong(), is(expect)); + assertSame(e2, e1); + assertNotSame(e3, e1); + assertTrue(e1.equalTo(e2)); + assertTrue(e1.equalTo(e3)); + assertEquals(Type.LONG, e1.type()); + assertEquals(expect, e1.asLong()); } } } @@ -146,20 +146,20 @@ public class ArrayValueTestCase { case ARRAY: added.add(array.addArray()); break; case OBJECT: added.add(array.addObject()); break; } - assertThat(array.entries(), is(65)); - assertThat(array.entry(64).type(), is(type)); - assertThat(added.get(64), sameInstance(array.entry(64))); + assertEquals(65, array.entries()); + assertEquals(type, array.entry(64).type()); + assertSame(array.entry(64), added.get(64)); for (int i = 0; i < 64; ++i) { var e1 = array.entry(i); var e2 = array.entry(i); var e3 = added.get(i); double expect = i; - assertThat(e1, sameInstance(e2)); - assertThat(e1, not(sameInstance(e3))); - assertThat(e1.equalTo(e2), is(true)); - assertThat(e1.equalTo(e3), is(true)); - assertThat(e1.type(), is(Type.DOUBLE)); - assertThat(e1.asDouble(), is(expect)); + assertSame(e2, e1); + assertNotSame(e3, e1); + assertTrue(e1.equalTo(e2)); + assertTrue(e1.equalTo(e3)); + assertEquals(Type.DOUBLE, e1.type()); + assertEquals(expect, e1.asDouble(), 0.0); } } } @@ -179,9 +179,9 @@ public class ArrayValueTestCase { case ARRAY: added = array.addArray(); break; case OBJECT: added = array.addObject(); break; } - assertThat(array.entries(), is(1)); - assertThat(array.entry(0).type(), is(type)); - assertThat(added, sameInstance(array.entry(0))); + assertEquals(1, array.entries()); + assertEquals(type, array.entry(0).type()); + assertSame(array.entry(0), added); } } } diff --git a/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java b/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java index db001a9276b..5f96247b77e 100644 --- a/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/slime/BinaryFormatTestCase.java @@ -12,9 +12,10 @@ import static com.yahoo.slime.BinaryFormat.decode_zigzag; import static com.yahoo.slime.BinaryFormat.encode_double; import static com.yahoo.slime.BinaryFormat.encode_type_and_meta; import static com.yahoo.slime.BinaryFormat.encode_zigzag; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.MatcherAssert.assertThat; - +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; public class BinaryFormatTestCase { @@ -36,50 +37,50 @@ public class BinaryFormatTestCase { BinaryEncoder bof = new BinaryEncoder(output); bof.encode_cmpr_int(value); byte[] actual = output.toArray(); - assertThat(actual, is(expect)); + assertArrayEquals(expect, actual); BinaryDecoder bif = new BinaryDecoder(); bif.in = new BufferedInput(expect); int got = bif.in.read_cmpr_int(); - assertThat(got, is(value)); - assertThat(bif.in.failed(), is(false)); + assertEquals(value, got); + assertFalse(bif.in.failed()); bif = new BinaryDecoder(); bif.in = new BufferedInput(expect); got = bif.in.skip_cmpr_int(); - assertThat(got, is(expect.length - 1)); - assertThat(bif.in.getPosition(), is(expect.length)); - assertThat(bif.in.failed(), is(false)); + assertEquals(expect.length - 1, got); + assertEquals(expect.length, bif.in.getPosition()); + assertFalse(bif.in.failed()); - assertThat(BinaryView.peek_cmpr_int_for_testing(expect, 0), is(value)); - assertThat(BinaryView.skip_cmpr_int_for_testing(expect, 0), is(expect.length)); + assertEquals(value, BinaryView.peek_cmpr_int_for_testing(expect, 0)); + assertEquals(expect.length, BinaryView.skip_cmpr_int_for_testing(expect, 0)); } void verify_read_cmpr_int_fails(byte[] data) { BinaryDecoder bif = new BinaryDecoder(); bif.in = new BufferedInput(data); int got = bif.in.read_cmpr_int(); - assertThat(got, is(0)); - assertThat(bif.in.failed(), is(true)); + assertEquals(0, got); + assertTrue(bif.in.failed()); bif = new BinaryDecoder(); bif.in = new BufferedInput(data); got = bif.in.skip_cmpr_int(); - assertThat(got, is(data.length - 1)); - assertThat(bif.in.getPosition(), is(data.length)); - assertThat(bif.in.failed(), is(false)); + assertEquals(data.length - 1, got); + assertEquals(data.length, bif.in.getPosition()); + assertFalse(bif.in.failed()); - assertThat(BinaryView.skip_cmpr_int_for_testing(data, 0), is(data.length)); + assertEquals(data.length, BinaryView.skip_cmpr_int_for_testing(data, 0)); } // was verifyBasic void verifyEncoding(Slime slime, byte[] expect) { - assertThat(BinaryFormat.encode(slime), is(expect)); - assertThat(slime.get().equalTo(BinaryView.inspect(expect)), is(true)); + assertArrayEquals(expect, BinaryFormat.encode(slime)); + assertTrue(slime.get().equalTo(BinaryView.inspect(expect))); Compressor compressor = new Compressor(CompressionType.LZ4, 3, 2, 0); Compressor.Compression result = BinaryFormat.encode_and_compress(slime, compressor); byte [] decompressed = compressor.decompress(result); - assertThat(decompressed, is(expect)); + assertArrayEquals(expect, decompressed); verifyMultiEncode(expect); } @@ -90,65 +91,65 @@ public class BinaryFormatTestCase { for (int i = 0; i < 5; ++i) { Slime slime = BinaryFormat.decode(buffers[i]); buffers[i+1] = BinaryFormat.encode(slime); - assertThat(buffers[i+1], is(expect)); + assertArrayEquals(expect, buffers[i+1]); } } @Test public void testZigZagConversion() { - assertThat(encode_zigzag(0), is(0L)); - assertThat(decode_zigzag(encode_zigzag(0)), is(0L)); + assertEquals(0L, encode_zigzag(0)); + assertEquals(0L, decode_zigzag(encode_zigzag(0))); - assertThat(encode_zigzag(-1), is(1L)); - assertThat(decode_zigzag(encode_zigzag(-1)), is(-1L)); + assertEquals(1L, encode_zigzag(-1)); + assertEquals(-1L, decode_zigzag(encode_zigzag(-1))); - assertThat(encode_zigzag(1), is(2L)); - assertThat(decode_zigzag(encode_zigzag(1)), is(1L)); + assertEquals(2L, encode_zigzag(1)); + assertEquals(1L, decode_zigzag(encode_zigzag(1))); - assertThat(encode_zigzag(-2), is(3L)); - assertThat(decode_zigzag(encode_zigzag(-2)), is(-2L)); + assertEquals(3L, encode_zigzag(-2)); + assertEquals(-2L, decode_zigzag(encode_zigzag(-2))); - assertThat(encode_zigzag(2), is(4L)); - assertThat(decode_zigzag(encode_zigzag(2)), is(2L)); + assertEquals(4L, encode_zigzag(2)); + assertEquals(2L, decode_zigzag(encode_zigzag(2))); - assertThat(encode_zigzag(-1000), is(1999L)); - assertThat(decode_zigzag(encode_zigzag(-1000)), is(-1000L)); + assertEquals(1999L, encode_zigzag(-1000)); + assertEquals(-1000L, decode_zigzag(encode_zigzag(-1000))); - assertThat(encode_zigzag(1000), is(2000L)); - assertThat(decode_zigzag(encode_zigzag(1000)), is(1000L)); + assertEquals(2000L, encode_zigzag(1000)); + assertEquals(1000L, decode_zigzag(encode_zigzag(1000))); - assertThat(encode_zigzag(-0x8000000000000000L), is(-1L)); - assertThat(decode_zigzag(encode_zigzag(-0x8000000000000000L)), is(-0x8000000000000000L)); + assertEquals(-1L, encode_zigzag(-0x8000000000000000L)); + assertEquals(-0x8000000000000000L, decode_zigzag(encode_zigzag(-0x8000000000000000L))); - assertThat(encode_zigzag(0x7fffffffffffffffL), is(-2L)); - assertThat(decode_zigzag(encode_zigzag(0x7fffffffffffffffL)), is(0x7fffffffffffffffL)); + assertEquals(-2L, encode_zigzag(0x7fffffffffffffffL)); + assertEquals(0x7fffffffffffffffL, decode_zigzag(encode_zigzag(0x7fffffffffffffffL))); } @Test public void testDoubleConversion() { - assertThat(encode_double(0.0), is(0L)); - assertThat(decode_double(encode_double(0.0)), is(0.0)); + assertEquals(0L, encode_double(0.0)); + assertEquals(0.0, decode_double(encode_double(0.0)), 0.0); - assertThat(encode_double(1.0), is(0x3ff0000000000000L)); - assertThat(decode_double(encode_double(1.0)), is(1.0)); + assertEquals(0x3ff0000000000000L, encode_double(1.0)); + assertEquals(1.0, decode_double(encode_double(1.0)), 0.0); - assertThat(encode_double(-1.0), is(0xbff0000000000000L)); - assertThat(decode_double(encode_double(-1.0)), is(-1.0)); + assertEquals(0xbff0000000000000L, encode_double(-1.0)); + assertEquals(-1.0, decode_double(encode_double(-1.0)), 0.0); - assertThat(encode_double(2.0), is(0x4000000000000000L)); - assertThat(decode_double(encode_double(2.0)), is(2.0)); + assertEquals(0x4000000000000000L, encode_double(2.0)); + assertEquals(2.0, decode_double(encode_double(2.0)), 0.0); - assertThat(encode_double(-2.0), is(0xc000000000000000L)); - assertThat(decode_double(encode_double(-2.0)), is(-2.0)); + assertEquals(0xc000000000000000L, encode_double(-2.0)); + assertEquals(-2.0, decode_double(encode_double(-2.0)), 0.0); - assertThat(encode_double(-0.0), is(0x8000000000000000L)); - assertThat(decode_double(encode_double(-0.0)), is(-0.0)); + assertEquals(0x8000000000000000L, encode_double(-0.0)); + assertEquals(-0.0, decode_double(encode_double(-0.0)), 0.0); - assertThat(encode_double(3.5), is(0x400c000000000000L)); - assertThat(decode_double(encode_double(3.5)), is(3.5)); + assertEquals(0x400c000000000000L, encode_double(3.5)); + assertEquals(3.5, decode_double(encode_double(3.5)), 0.0); - assertThat(encode_double(65535.875), is(0x40EFFFFC00000000L)); - assertThat(decode_double(encode_double(65535.875)), is(65535.875)); + assertEquals(0x40EFFFFC00000000L, encode_double(65535.875)); + assertEquals(65535.875, decode_double(encode_double(65535.875)), 0.0); } @Test @@ -156,8 +157,8 @@ public class BinaryFormatTestCase { for (byte type = 0; type < TYPE_LIMIT; ++type) { for (int meta = 0; meta < META_LIMIT; ++meta) { byte mangled = encode_type_and_meta(type, meta); - assertThat(decode_type(mangled).ID, is(type)); - assertThat(decode_meta(mangled), is(meta)); + assertEquals(type, decode_type(mangled).ID); + assertEquals(meta, decode_meta(mangled)); } } } @@ -241,7 +242,7 @@ public class BinaryFormatTestCase { BinaryEncoder encoder = new BinaryEncoder(actual); encoder.write_type_and_size(type, size); } - assertThat(actual.toArray(), is(expect.toArray())); + assertArrayEquals(expect.toArray(), actual.toArray()); byte[] got = expect.toArray(); BinaryDecoder bif = new BinaryDecoder(); @@ -249,12 +250,12 @@ public class BinaryFormatTestCase { byte b = bif.in.getByte(); Type decodedType = decode_type(b); int decodedSize = bif.in.read_size(decode_meta(b)); - assertThat(decodedType.ID, is(type)); - assertThat(decodedSize, is(size)); - assertThat(bif.in.getConsumedSize(), is(got.length)); - assertThat(bif.in.failed(), is(false)); + assertEquals(type, decodedType.ID); + assertEquals(size, decodedSize); + assertEquals(got.length, bif.in.getConsumedSize()); + assertFalse(bif.in.failed()); - assertThat(BinaryView.extract_children_for_testing(got, 0), is(size)); + assertEquals(size, BinaryView.extract_children_for_testing(got, 0)); } } @@ -292,21 +293,21 @@ public class BinaryFormatTestCase { bof.write_type_and_bytes_le(type, bits); } byte[] actual = output.toArray(); - assertThat(actual, is(expect)); + assertArrayEquals(expect, actual); // test input: BinaryDecoder bif = new BinaryDecoder(); bif.in = new BufferedInput(expect); int size = decode_meta(bif.in.getByte()); long decodedBits = (hi != 0) ? bif.read_bytes_be(size) : bif.read_bytes_le(size); - assertThat(decodedBits, is(bits)); - assertThat(bif.in.getConsumedSize(), is(expect.length)); - assertThat(bif.in.failed(), is(false)); + assertEquals(bits, decodedBits); + assertEquals(expect.length, bif.in.getConsumedSize()); + assertFalse(bif.in.failed()); if (hi != 0) { - assertThat(encode_double(BinaryView.extract_double_for_testing(expect, 0)), is(bits)); + assertEquals(bits, encode_double(BinaryView.extract_double_for_testing(expect, 0))); } else { - assertThat(encode_zigzag(BinaryView.extract_long_for_testing(expect, 0)), is(bits)); + assertEquals(bits, encode_zigzag(BinaryView.extract_long_for_testing(expect, 0))); } } } @@ -322,7 +323,7 @@ public class BinaryFormatTestCase { expect.put((byte)0); // nix byte[] actual = BinaryFormat.encode(slime); - assertThat(actual, is(expect.toArray())); + assertArrayEquals(expect.toArray(), actual); verifyMultiEncode(expect.toArray()); } @@ -428,7 +429,7 @@ public class BinaryFormatTestCase { System.arraycopy(expect, 0, overlappingBuffer, 1, expect.length); overlappingBuffer[overlappingBuffer.length - 1] = 0; Slime copy = BinaryFormat.decode(overlappingBuffer, 1, expect.length); - assertThat(BinaryFormat.encode(slime), is(BinaryFormat.encode(copy))); + assertArrayEquals(BinaryFormat.encode(copy), BinaryFormat.encode(slime)); } @Test @@ -550,18 +551,17 @@ public class BinaryFormatTestCase { BinaryDecoder decoder = new BinaryDecoder(); Slime slime = decoder.decode(data); int consumed = decoder.in.getConsumedSize(); - assertThat(consumed, is(data.length)); + assertEquals(data.length, consumed); Cursor c = slime.get(); - assertThat(c.valid(), is(true)); - assertThat(c.type(), is(Type.OBJECT)); - assertThat(c.children(), is(5)); - assertThat(c.field("b").asBool(), is(true)); - assertThat(c.field("c").asLong(), is(5L)); - assertThat(c.field("d").asDouble(), is(3.5)); - assertThat(c.field("e").asString(), is("string")); + assertTrue(c.valid()); + assertEquals(Type.OBJECT, c.type()); + assertEquals(5, c.children()); + assertTrue(c.field("b").asBool()); + assertEquals(5L, c.field("c").asLong()); + assertEquals(3.5, c.field("d").asDouble(), 0.0); + assertEquals("string", c.field("e").asString()); byte[] expd = { 'd', 'a', 't', 'a' }; - assertThat(c.field("f").asData(), is(expd)); - assertThat(c.entry(5).valid(), is(false)); // not ARRAY + assertArrayEquals(expd, c.field("f").asData()); + assertFalse(c.entry(5).valid()); // not ARRAY } - } diff --git a/vespajlib/src/test/java/com/yahoo/slime/BinaryViewTest.java b/vespajlib/src/test/java/com/yahoo/slime/BinaryViewTest.java index 568124369d4..920a25b96c9 100644 --- a/vespajlib/src/test/java/com/yahoo/slime/BinaryViewTest.java +++ b/vespajlib/src/test/java/com/yahoo/slime/BinaryViewTest.java @@ -140,99 +140,43 @@ public class BinaryViewTest { } } - class MyVisitor implements Visitor { - enum Called { NONE, INVALID, NIX, BOOL, LONG, DOUBLE, UTF8, DATA, ARRAY, OBJECT } - Called called = Called.NONE; - boolean boolValue; - long longValue; - double doubleValue; - byte[] bytes; - Inspector stuff; - @Override public void visitInvalid() { - assertEquals(ctx, Called.NONE, called); - called = Called.INVALID; - } - @Override public void visitNix() { - assertEquals(ctx, Called.NONE, called); - called = Called.NIX; - } - @Override public void visitBool(boolean bit) { - assertEquals(ctx, Called.NONE, called); - called = Called.BOOL; - boolValue = bit; - } - @Override public void visitLong(long l) { - assertEquals(ctx, Called.NONE, called); - called = Called.LONG; - longValue = l; - } - @Override public void visitDouble(double d) { - assertEquals(ctx, Called.NONE, called); - called = Called.DOUBLE; - doubleValue = d; - } - @Override public void visitString(String str) { - fail(ctx + ", strings are never utf-16 in binary view"); - } - @Override public void visitString(byte[] utf8) { - assertEquals(ctx, Called.NONE, called); - called = Called.UTF8; - bytes = utf8; - } - @Override public void visitData(byte[] data) { - assertEquals(ctx, Called.NONE, called); - called = Called.DATA; - bytes = data; - } - @Override public void visitArray(Inspector arr) { - assertEquals(ctx, Called.NONE, called); - called = Called.ARRAY; - stuff = arr; - } - @Override public void visitObject(Inspector obj) { - assertEquals(ctx, Called.NONE, called); - called = Called.OBJECT; - stuff = obj; - } - }; - void checkVisitor(Inspector view) { - var visitor = new MyVisitor(); + var visitor = new MockVisitor(ctx); view.accept(visitor); if (!view.valid()) { - assertEquals(ctx, MyVisitor.Called.INVALID, visitor.called); + assertEquals(ctx, MockVisitor.Called.INVALID, visitor.called); return; } switch (view.type()) { case NIX: - assertEquals(ctx, MyVisitor.Called.NIX, visitor.called); + assertEquals(ctx, MockVisitor.Called.NIX, visitor.called); break; case BOOL: - assertEquals(ctx, MyVisitor.Called.BOOL, visitor.called); + assertEquals(ctx, MockVisitor.Called.BOOL, visitor.called); assertEquals(ctx, view.asBool(), visitor.boolValue); break; case LONG: - assertEquals(ctx, MyVisitor.Called.LONG, visitor.called); + assertEquals(ctx, MockVisitor.Called.LONG, visitor.called); assertEquals(ctx, view.asLong(), visitor.longValue); break; case DOUBLE: - assertEquals(ctx, MyVisitor.Called.DOUBLE, visitor.called); + assertEquals(ctx, MockVisitor.Called.DOUBLE, visitor.called); assertEquals(ctx, view.asDouble(), visitor.doubleValue, 0.0); break; case STRING: - assertEquals(ctx, MyVisitor.Called.UTF8, visitor.called); + assertEquals(ctx, MockVisitor.Called.UTF8, visitor.called); assertArrayEquals(ctx, view.asUtf8(), visitor.bytes); break; case DATA: - assertEquals(ctx, MyVisitor.Called.DATA, visitor.called); + assertEquals(ctx, MockVisitor.Called.DATA, visitor.called); assertArrayEquals(ctx, view.asData(), visitor.bytes); break; case ARRAY: - assertEquals(ctx, MyVisitor.Called.ARRAY, visitor.called); + assertEquals(ctx, MockVisitor.Called.ARRAY, visitor.called); assertSame(ctx, view, visitor.stuff); break; case OBJECT: - assertEquals(ctx, MyVisitor.Called.OBJECT, visitor.called); + assertEquals(ctx, MockVisitor.Called.OBJECT, visitor.called); assertSame(ctx, view, visitor.stuff); break; default: diff --git a/vespajlib/src/test/java/com/yahoo/slime/MockVisitor.java b/vespajlib/src/test/java/com/yahoo/slime/MockVisitor.java new file mode 100644 index 00000000000..a4354adbcd0 --- /dev/null +++ b/vespajlib/src/test/java/com/yahoo/slime/MockVisitor.java @@ -0,0 +1,69 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.slime; + +import static org.junit.Assert.assertEquals; + +class MockVisitor implements Visitor { + enum Called { NONE, INVALID, NIX, BOOL, LONG, DOUBLE, STRING, UTF8, DATA, ARRAY, OBJECT } + Called called = Called.NONE; + boolean boolValue; + long longValue; + double doubleValue; + String string; + byte[] bytes; + Inspector stuff; + String ctx; + + MockVisitor(String context) { + ctx = context; + } + MockVisitor() { this(""); } + @Override public void visitInvalid() { + assertEquals(ctx, Called.NONE, called); + called = Called.INVALID; + } + @Override public void visitNix() { + assertEquals(ctx, Called.NONE, called); + called = Called.NIX; + } + @Override public void visitBool(boolean bit) { + assertEquals(ctx, Called.NONE, called); + called = Called.BOOL; + boolValue = bit; + } + @Override public void visitLong(long l) { + assertEquals(ctx, Called.NONE, called); + called = Called.LONG; + longValue = l; + } + @Override public void visitDouble(double d) { + assertEquals(ctx, Called.NONE, called); + called = Called.DOUBLE; + doubleValue = d; + } + @Override public void visitString(String str) { + assertEquals(ctx, Called.NONE, called); + called = Called.STRING; + string = str; + } + @Override public void visitString(byte[] utf8) { + assertEquals(ctx, Called.NONE, called); + called = Called.UTF8; + bytes = utf8; + } + @Override public void visitData(byte[] data) { + assertEquals(ctx, Called.NONE, called); + called = Called.DATA; + bytes = data; + } + @Override public void visitArray(Inspector arr) { + assertEquals(ctx, Called.NONE, called); + called = Called.ARRAY; + stuff = arr; + } + @Override public void visitObject(Inspector obj) { + assertEquals(ctx, Called.NONE, called); + called = Called.OBJECT; + stuff = obj; + } +} diff --git a/vespajlib/src/test/java/com/yahoo/slime/SlimeTestCase.java b/vespajlib/src/test/java/com/yahoo/slime/SlimeTestCase.java index 6ee8fb6c7e2..0a5c69b2d73 100644 --- a/vespajlib/src/test/java/com/yahoo/slime/SlimeTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/slime/SlimeTestCase.java @@ -2,10 +2,11 @@ package com.yahoo.slime; import org.junit.Test; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.CoreMatchers.is; -import static org.hamcrest.CoreMatchers.sameInstance; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; public class SlimeTestCase { @@ -21,25 +22,25 @@ public class SlimeTestCase { public void testTypeIds() { System.out.println("testing type identifiers..."); - assertThat(Type.NIX.ID, is((byte)0)); - assertThat(Type.BOOL.ID, is((byte)1)); - assertThat(Type.LONG.ID, is((byte)2)); - assertThat(Type.DOUBLE.ID, is((byte)3)); - assertThat(Type.STRING.ID, is((byte)4)); - assertThat(Type.DATA.ID, is((byte)5)); - assertThat(Type.ARRAY.ID, is((byte)6)); - assertThat(Type.OBJECT.ID, is((byte)7)); + assertEquals((byte)0, Type.NIX.ID); + assertEquals((byte)1, Type.BOOL.ID); + assertEquals((byte)2, Type.LONG.ID); + assertEquals((byte)3, Type.DOUBLE.ID); + assertEquals((byte)4, Type.STRING.ID); + assertEquals((byte)5, Type.DATA.ID); + assertEquals((byte)6, Type.ARRAY.ID); + assertEquals((byte)7, Type.OBJECT.ID); - assertThat(Type.values().length, is(8)); + assertEquals(8, Type.values().length); - assertThat(Type.values()[0], sameInstance(Type.NIX)); - assertThat(Type.values()[1], sameInstance(Type.BOOL)); - assertThat(Type.values()[2], sameInstance(Type.LONG)); - assertThat(Type.values()[3], sameInstance(Type.DOUBLE)); - assertThat(Type.values()[4], sameInstance(Type.STRING)); - assertThat(Type.values()[5], sameInstance(Type.DATA)); - assertThat(Type.values()[6], sameInstance(Type.ARRAY)); - assertThat(Type.values()[7], sameInstance(Type.OBJECT)); + assertSame(Type.NIX, Type.values()[0]); + assertSame(Type.BOOL, Type.values()[1]); + assertSame(Type.LONG, Type.values()[2]); + assertSame(Type.DOUBLE, Type.values()[3]); + assertSame(Type.STRING, Type.values()[4]); + assertSame(Type.DATA, Type.values()[5]); + assertSame(Type.ARRAY, Type.values()[6]); + assertSame(Type.OBJECT, Type.values()[7]); } @Test @@ -50,41 +51,41 @@ public class SlimeTestCase { for (int i = 0; i < 2; i++) { if (i == 0) { cur = slime.get(); - assertThat(cur.valid(), is(true)); + assertTrue(cur.valid()); } else { cur = NixValue.invalid(); - assertThat(cur.valid(), is(false)); + assertFalse(cur.valid()); } - assertThat(cur.type(), is(Type.NIX)); - assertThat(cur.children(), is(0)); - assertThat(cur.asBool(), is(false)); - assertThat(cur.asLong(), is((long)0)); - assertThat(cur.asDouble(), is(0.0)); - assertThat(cur.asString(), is("")); - assertThat(cur.asData(), is(new byte[0])); - assertThat(cur.entry(0).valid(), is(false)); - assertThat(cur.field(0).valid(), is(false)); - assertThat(cur.field("foo").valid(), is(false)); + assertEquals(Type.NIX, cur.type()); + assertEquals(0, cur.children()); + assertFalse(cur.asBool()); + assertEquals(0L, cur.asLong()); + assertEquals(0.0, cur.asDouble(), 0.0); + assertEquals("", cur.asString()); + assertArrayEquals(new byte[0], cur.asData()); + assertFalse(cur.entry(0).valid()); + assertFalse(cur.field(0).valid()); + assertFalse(cur.field("foo").valid()); } Inspector insp; for (int i = 0; i < 2; i++) { if (i == 0) { insp = slime.get(); - assertThat(insp.valid(), is(true)); + assertTrue(insp.valid()); } else { insp = NixValue.invalid(); - assertThat(insp.valid(), is(false)); + assertFalse(insp.valid()); } - assertThat(insp.type(), is(Type.NIX)); - assertThat(insp.children(), is(0)); - assertThat(insp.asBool(), is(false)); - assertThat(insp.asLong(), is((long)0)); - assertThat(insp.asDouble(), is(0.0)); - assertThat(insp.asString(), is("")); - assertThat(insp.asData(), is(new byte[0])); - assertThat(insp.entry(0).valid(), is(false)); - assertThat(insp.field(0).valid(), is(false)); - assertThat(insp.field("foo").valid(), is(false)); + assertEquals(Type.NIX, insp.type()); + assertEquals(0, insp.children()); + assertFalse(insp.asBool()); + assertEquals(0L, insp.asLong()); + assertEquals(0.0, insp.asDouble(), 0.0); + assertEquals("", insp.asString()); + assertArrayEquals(new byte[0], insp.asData()); + assertFalse(insp.entry(0).valid()); + assertFalse(insp.field(0).valid()); + assertFalse(insp.field("foo").valid()); } } @@ -96,63 +97,63 @@ public class SlimeTestCase { System.out.println("testing boolean value"); slime.setBool(true); Inspector insp = slime.get(); - assertThat(insp.valid(), is(true)); - assertThat(insp.type(), sameInstance(Type.BOOL)); - assertThat(insp.asBool(), is(true)); + assertTrue(insp.valid()); + assertSame(Type.BOOL, insp.type()); + assertTrue(insp.asBool()); Cursor cur = slime.get(); - assertThat(cur.valid(), is(true)); - assertThat(cur.type(), sameInstance(Type.BOOL)); - assertThat(cur.asBool(), is(true)); + assertTrue(cur.valid()); + assertSame(Type.BOOL, cur.type()); + assertTrue(cur.asBool()); System.out.println("testing long value"); slime.setLong(42); cur = slime.get(); insp = slime.get(); - assertThat(cur.valid(), is(true)); - assertThat(insp.valid(), is(true)); - assertThat(cur.type(), sameInstance(Type.LONG)); - assertThat(insp.type(), sameInstance(Type.LONG)); - assertThat(cur.asLong(), is((long)42)); - assertThat(insp.asLong(), is((long)42)); + assertTrue(cur.valid()); + assertTrue(insp.valid()); + assertSame(Type.LONG, cur.type()); + assertSame(Type.LONG, insp.type()); + assertEquals(42L, cur.asLong()); + assertEquals(42L, insp.asLong()); System.out.println("testing double value"); slime.setDouble(4.2); cur = slime.get(); insp = slime.get(); - assertThat(cur.valid(), is(true)); - assertThat(insp.valid(), is(true)); - assertThat(cur.type(), sameInstance(Type.DOUBLE)); - assertThat(insp.type(), sameInstance(Type.DOUBLE)); - assertThat(cur.asDouble(), is(4.2)); - assertThat(insp.asDouble(), is(4.2)); + assertTrue(cur.valid()); + assertTrue(insp.valid()); + assertSame(Type.DOUBLE, cur.type()); + assertSame(Type.DOUBLE, insp.type()); + assertEquals(4.2, cur.asDouble(), 0.0); + assertEquals(4.2, insp.asDouble(), 0.0); System.out.println("testing string value"); slime.setString("fortytwo"); cur = slime.get(); insp = slime.get(); - assertThat(cur.valid(), is(true)); - assertThat(insp.valid(), is(true)); - assertThat(cur.type(), sameInstance(Type.STRING)); - assertThat(insp.type(), sameInstance(Type.STRING)); - assertThat(cur.asString(), is("fortytwo")); - assertThat(insp.asString(), is("fortytwo")); + assertTrue(cur.valid()); + assertTrue(insp.valid()); + assertSame(Type.STRING, cur.type()); + assertSame(Type.STRING, insp.type()); + assertEquals("fortytwo", cur.asString()); + assertEquals("fortytwo", insp.asString()); System.out.println("testing data value"); byte[] data = { (byte)4, (byte)2 }; slime.setData(data); cur = slime.get(); insp = slime.get(); - assertThat(cur.valid(), is(true)); - assertThat(insp.valid(), is(true)); - assertThat(cur.type(), sameInstance(Type.DATA)); - assertThat(insp.type(), sameInstance(Type.DATA)); - assertThat(cur.asData(), is(data)); - assertThat(insp.asData(), is(data)); + assertTrue(cur.valid()); + assertTrue(insp.valid()); + assertSame(Type.DATA, cur.type()); + assertSame(Type.DATA, insp.type()); + assertArrayEquals(data, cur.asData()); + assertArrayEquals(data, insp.asData()); data[0] = 10; data[1] = 20; byte[] data2 = { 10, 20 }; - assertThat(cur.asData(), is(data2)); - assertThat(insp.asData(), is(data2)); + assertArrayEquals(data2, cur.asData()); + assertArrayEquals(data2, insp.asData()); } @Test @@ -160,13 +161,13 @@ public class SlimeTestCase { System.out.println("testing array values..."); Slime slime = new Slime(); Cursor c = slime.setArray(); - assertThat(c.valid(), is(true)); - assertThat(c.type(), is(Type.ARRAY)); - assertThat(c.children(), is(0)); + assertTrue(c.valid()); + assertEquals(Type.ARRAY, c.type()); + assertEquals(0, c.children()); Inspector i = slime.get(); - assertThat(i.valid(), is(true)); - assertThat(i.type(), is(Type.ARRAY)); - assertThat(i.children(), is(0)); + assertTrue(i.valid()); + assertEquals(Type.ARRAY, i.type()); + assertEquals(0, i.children()); c.addNix(); c.addBool(true); c.addLong(5); @@ -174,23 +175,23 @@ public class SlimeTestCase { c.addString("string"); byte[] data = { (byte)'d', (byte)'a', (byte)'t', (byte)'a' }; c.addData(data); - assertThat(c.children(), is(6)); - assertThat(c.entry(0).valid(), is(true)); - assertThat(c.entry(1).asBool(), is(true)); - assertThat(c.entry(2).asLong(), is((long)5)); - assertThat(c.entry(3).asDouble(), is(3.5)); - assertThat(c.entry(4).asString(), is("string")); - assertThat(c.entry(5).asData(), is(data)); - assertThat(c.field(5).valid(), is(false)); // not OBJECT + assertEquals(6, c.children()); + assertTrue(c.entry(0).valid()); + assertTrue(c.entry(1).asBool()); + assertEquals(5L, c.entry(2).asLong()); + assertEquals(3.5, c.entry(3).asDouble(), 0.0); + assertEquals("string", c.entry(4).asString()); + assertArrayEquals(data, c.entry(5).asData()); + assertFalse(c.field(5).valid()); // not OBJECT - assertThat(i.children(), is(6)); - assertThat(i.entry(0).valid(), is(true)); - assertThat(i.entry(1).asBool(), is(true)); - assertThat(i.entry(2).asLong(), is((long)5)); - assertThat(i.entry(3).asDouble(), is(3.5)); - assertThat(i.entry(4).asString(), is("string")); - assertThat(i.entry(5).asData(), is(data)); - assertThat(i.field(5).valid(), is(false)); // not OBJECT + assertEquals(6, i.children()); + assertTrue(i.entry(0).valid()); + assertTrue(i.entry(1).asBool()); + assertEquals(5L, i.entry(2).asLong()); + assertEquals(3.5, i.entry(3).asDouble(), 0.0); + assertEquals("string", i.entry(4).asString()); + assertArrayEquals(data, i.entry(5).asData()); + assertFalse(i.field(5).valid()); // not OBJECT } @Test @@ -199,13 +200,13 @@ public class SlimeTestCase { Slime slime = new Slime(); Cursor c = slime.setObject(); - assertThat(c.valid(), is(true)); - assertThat(c.type(), is(Type.OBJECT)); - assertThat(c.children(), is(0)); + assertTrue(c.valid()); + assertEquals(Type.OBJECT, c.type()); + assertEquals(0, c.children()); Inspector i = slime.get(); - assertThat(i.valid(), is(true)); - assertThat(i.type(), is(Type.OBJECT)); - assertThat(i.children(), is(0)); + assertTrue(i.valid()); + assertEquals(Type.OBJECT, i.type()); + assertEquals(0, i.children()); c.setNix("a"); c.setBool("b", true); @@ -215,23 +216,23 @@ public class SlimeTestCase { byte[] data = { (byte)'d', (byte)'a', (byte)'t', (byte)'a' }; c.setData("f", data); - assertThat(c.children(), is(6)); - assertThat(c.field("a").valid(), is(true)); - assertThat(c.field("b").asBool(), is(true)); - assertThat(c.field("c").asLong(), is((long)5)); - assertThat(c.field("d").asDouble(), is(3.5)); - assertThat(c.field("e").asString(), is("string")); - assertThat(c.field("f").asData(), is(data)); - assertThat(c.entry(4).valid(), is(false)); // not ARRAY + assertEquals(6, c.children()); + assertTrue(c.field("a").valid()); + assertTrue(c.field("b").asBool()); + assertEquals(5L, c.field("c").asLong()); + assertEquals(3.5, c.field("d").asDouble(), 0.0); + assertEquals("string", c.field("e").asString()); + assertArrayEquals(data, c.field("f").asData()); + assertFalse(c.entry(4).valid()); // not ARRAY - assertThat(i.children(), is(6)); - assertThat(i.field("a").valid(), is(true)); - assertThat(i.field("b").asBool(), is(true)); - assertThat(i.field("c").asLong(), is((long)5)); - assertThat(i.field("d").asDouble(), is(3.5)); - assertThat(i.field("e").asString(), is("string")); - assertThat(i.field("f").asData(), is(data)); - assertThat(i.entry(4).valid(), is(false)); // not ARRAY + assertEquals(6, i.children()); + assertTrue(i.field("a").valid()); + assertTrue(i.field("b").asBool()); + assertEquals(5L, i.field("c").asLong()); + assertEquals(3.5, i.field("d").asDouble(), 0.0); + assertEquals("string", i.field("e").asString()); + assertArrayEquals(data, i.field("f").asData()); + assertFalse(i.entry(4).valid()); // not ARRAY } @Test @@ -240,12 +241,12 @@ public class SlimeTestCase { { Slime slime = new Slime(); Cursor c = slime.setArray(); - assertThat(c.addLong(5).asLong(), is((long)5)); + assertEquals(5L, c.addLong(5).asLong()); } { Slime slime = new Slime(); Cursor c = slime.setObject(); - assertThat(c.setLong("a", 5).asLong(), is((long)5)); + assertEquals(5L, c.setLong("a", 5).asLong()); } } @@ -256,10 +257,10 @@ public class SlimeTestCase { Slime slime = new Slime(); Cursor c = slime.setLong(10); Inspector i1 = c; - assertThat(i1.asLong(), is((long)10)); + assertEquals(10L, i1.asLong()); Inspector i2 = slime.get(); - assertThat(i2.asLong(), is((long)10)); + assertEquals(10L, i2.asLong()); } @Test @@ -275,14 +276,14 @@ public class SlimeTestCase { c3.setLong("answer", 42); } Inspector i = slime.get(); - assertThat(i.field("bar").asLong(), is((long)10)); - assertThat(i.field("foo").entry(0).asLong(), is((long)20)); - assertThat(i.field("foo").entry(1).field("answer").asLong(), is((long)42)); + assertEquals(10L, i.field("bar").asLong()); + assertEquals(20L, i.field("foo").entry(0).asLong()); + assertEquals(42L, i.field("foo").entry(1).field("answer").asLong()); Cursor c = slime.get(); - assertThat(c.field("bar").asLong(), is((long)10)); - assertThat(c.field("foo").entry(0).asLong(), is((long)20)); - assertThat(c.field("foo").entry(1).field("answer").asLong(), is((long)42)); + assertEquals(10L, c.field("bar").asLong()); + assertEquals(20L, c.field("foo").entry(0).asLong()); + assertEquals(42L, c.field("foo").entry(1).field("answer").asLong()); } @Test @@ -293,19 +294,19 @@ public class SlimeTestCase { Cursor c = slime.setObject(); for (int i = 0; i < n; i++) { String str = ("" + i + "_str_" + i); - assertThat(slime.lookup(str), is(SymbolTable.INVALID)); - assertThat(c.field(str).type(), sameInstance(Type.NIX)); + assertEquals(SymbolTable.INVALID, slime.lookup(str)); + assertSame(Type.NIX, c.field(str).type()); switch (i % 2) { - case 0: assertThat((int)c.setLong(str, i).asLong(), is(i)); break; - case 1: assertThat(slime.insert(str), is(i)); break; + case 0: assertEquals(i, (int)c.setLong(str, i).asLong()); break; + case 1: assertEquals(i, slime.insert(str)); break; } } for (int i = 0; i < n; i++) { String str = ("" + i + "_str_" + i); - assertThat(slime.lookup(str), is(i)); + assertEquals(i, slime.lookup(str)); switch (i % 2) { - case 0: assertThat((int)c.field(str).asLong(), is(i)); break; - case 1: assertThat((int)c.field(str).asLong(), is(0)); break; + case 0: assertEquals(i, (int)c.field(str).asLong()); break; + case 1: assertEquals(0, (int)c.field(str).asLong()); break; } } } @@ -317,12 +318,12 @@ public class SlimeTestCase { Slime slime = new Slime(); Cursor c = slime.setArray(); for (int i = 0; i < n; i++) { - assertThat((int)c.addLong(i).asLong(), is(i)); + assertEquals(i, (int)c.addLong(i).asLong()); } for (int i = 0; i < n; i++) { - assertThat((int)c.entry(i).asLong(), is(i)); + assertEquals(i, (int)c.entry(i).asLong()); } - assertThat((int)c.entry(n).asLong(), is(0)); + assertEquals(0, (int)c.entry(n).asLong()); } @Test @@ -332,8 +333,8 @@ public class SlimeTestCase { c1.addLong(20); Cursor c2 = c1.addObject(); c2.setLong("answer", 42); - assertThat(slime.get().toString(), is("[20,{\"answer\":42}]")); + assertEquals("[20,{\"answer\":42}]", slime.get().toString()); c1.addString("\u2008"); - assertThat(slime.get().toString(), is("[20,{\"answer\":42},\"\u2008\"]")); + assertEquals("[20,{\"answer\":42},\"\u2008\"]", slime.get().toString()); } } diff --git a/vespajlib/src/test/java/com/yahoo/slime/VisitorTestCase.java b/vespajlib/src/test/java/com/yahoo/slime/VisitorTestCase.java index 916cddac56b..cc4bf5d16e2 100644 --- a/vespajlib/src/test/java/com/yahoo/slime/VisitorTestCase.java +++ b/vespajlib/src/test/java/com/yahoo/slime/VisitorTestCase.java @@ -2,100 +2,97 @@ package com.yahoo.slime; import org.junit.Test; -import org.mockito.Mockito; - -import static org.hamcrest.CoreMatchers.sameInstance; -import static org.mockito.hamcrest.MockitoHamcrest.argThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertSame; public class VisitorTestCase { @Test public void testVisitInvalid() { - Visitor visitor = Mockito.mock(Visitor.class); + var visitor = new MockVisitor(); Inspector inspector = new Slime().get().field("invalid"); inspector.accept(visitor); - Mockito.verify(visitor).visitInvalid(); - Mockito.verifyNoMoreInteractions(visitor); + assertEquals(MockVisitor.Called.INVALID, visitor.called); } @Test public void testVisitNix() { - Visitor visitor = Mockito.mock(Visitor.class); + var visitor = new MockVisitor(); Inspector inspector = new Slime().get(); inspector.accept(visitor); - Mockito.verify(visitor).visitNix(); - Mockito.verifyNoMoreInteractions(visitor); + assertEquals(MockVisitor.Called.NIX, visitor.called); } @Test public void testVisitBool() { - Visitor visitor = Mockito.mock(Visitor.class); + var visitor = new MockVisitor(); Inspector inspector = new Slime().setBool(true); inspector.accept(visitor); - Mockito.verify(visitor).visitBool(true); - Mockito.verifyNoMoreInteractions(visitor); + assertEquals(MockVisitor.Called.BOOL, visitor.called); + assertEquals(true, visitor.boolValue); } @Test public void testVisitLong() { - Visitor visitor = Mockito.mock(Visitor.class); + var visitor = new MockVisitor(); Inspector inspector = new Slime().setLong(123); inspector.accept(visitor); - Mockito.verify(visitor).visitLong(123); - Mockito.verifyNoMoreInteractions(visitor); + assertEquals(MockVisitor.Called.LONG, visitor.called); + assertEquals(123, visitor.longValue); } @Test public void testVisitDouble() { - Visitor visitor = Mockito.mock(Visitor.class); + var visitor = new MockVisitor(); Inspector inspector = new Slime().setDouble(123.0); inspector.accept(visitor); - Mockito.verify(visitor).visitDouble(123.0); - Mockito.verifyNoMoreInteractions(visitor); + assertEquals(MockVisitor.Called.DOUBLE, visitor.called); + assertEquals(123.0, visitor.doubleValue, 0.0); } @Test public void testVisitStringUtf16() { - Visitor visitor = Mockito.mock(Visitor.class); + var visitor = new MockVisitor(); Inspector inspector = new Slime().setString("abc"); inspector.accept(visitor); - Mockito.verify(visitor).visitString("abc"); - Mockito.verifyNoMoreInteractions(visitor); + assertEquals(MockVisitor.Called.STRING, visitor.called); + assertEquals("abc", visitor.string); } @Test public void testVisitStringUtf8() { - Visitor visitor = Mockito.mock(Visitor.class); + var visitor = new MockVisitor(); Inspector inspector = new Slime().setString(new byte[] {65,66,67}); inspector.accept(visitor); - Mockito.verify(visitor).visitString(new byte[] {65,66,67}); - Mockito.verifyNoMoreInteractions(visitor); + assertEquals(MockVisitor.Called.UTF8, visitor.called); + assertArrayEquals(new byte[] {65,66,67}, visitor.bytes); } @Test public void testVisitData() { - Visitor visitor = Mockito.mock(Visitor.class); + var visitor = new MockVisitor(); Inspector inspector = new Slime().setData(new byte[] {1,2,3}); inspector.accept(visitor); - Mockito.verify(visitor).visitData(new byte[] {1,2,3}); - Mockito.verifyNoMoreInteractions(visitor); + assertEquals(MockVisitor.Called.DATA, visitor.called); + assertArrayEquals(new byte[] {1,2,3}, visitor.bytes); } @Test public void testVisitArray() { - Visitor visitor = Mockito.mock(Visitor.class); + var visitor = new MockVisitor(); Inspector inspector = new Slime().setArray(); inspector.accept(visitor); - Mockito.verify(visitor).visitArray(argThat(sameInstance(inspector))); - Mockito.verifyNoMoreInteractions(visitor); + assertEquals(MockVisitor.Called.ARRAY, visitor.called); + assertSame(inspector, visitor.stuff); } @Test public void testVisitObject() { - Visitor visitor = Mockito.mock(Visitor.class); + var visitor = new MockVisitor(); Inspector inspector = new Slime().setObject(); inspector.accept(visitor); - Mockito.verify(visitor).visitObject(argThat(sameInstance(inspector))); - Mockito.verifyNoMoreInteractions(visitor); + assertEquals(MockVisitor.Called.OBJECT, visitor.called); + assertSame(inspector, visitor.stuff); } } diff --git a/vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp b/vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp index 44decb9bf91..31113f2b4f2 100644 --- a/vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp +++ b/vespalib/src/tests/btree/btree-stress/btree_stress_test.cpp @@ -53,7 +53,7 @@ public: ~RealIntStore(); EntryRef add(uint32_t value) { return _store.addEntry(value); } AtomicEntryRef add_relaxed(uint32_t value) { return AtomicEntryRef(add(value)); } - void hold(const AtomicEntryRef& ref) { _store.holdElem(ref.load_relaxed(), 1); } + void hold(const AtomicEntryRef& ref) { _store.hold_entry(ref.load_relaxed()); } EntryRef move(EntryRef ref); void assign_generation(generation_t current_gen) { _store.assign_generation(current_gen); } void reclaim_memory(generation_t gen) { _store.reclaim_memory(gen); } diff --git a/vespalib/src/tests/btree/btree_test.cpp b/vespalib/src/tests/btree/btree_test.cpp index b8da9ea6042..afad3523fa3 100644 --- a/vespalib/src/tests/btree/btree_test.cpp +++ b/vespalib/src/tests/btree/btree_test.cpp @@ -1065,7 +1065,7 @@ adjustAllocatedBytes(size_t nodeCount, size_t nodeSize) TEST_F(BTreeTest, require_that_memory_usage_is_calculated) { constexpr size_t BASE_ALLOCATED = 28744u; - constexpr size_t BASE_USED = 24984; + constexpr size_t BASE_USED = 24936; typedef BTreeNodeAllocator<int32_t, int8_t, btree::NoAggregated, MyTraits::INTERNAL_SLOTS, MyTraits::LEAF_SLOTS> NodeAllocator; diff --git a/vespalib/src/tests/datastore/array_store/array_store_test.cpp b/vespalib/src/tests/datastore/array_store/array_store_test.cpp index ccc3ab88c31..1df03f6eb0a 100644 --- a/vespalib/src/tests/datastore/array_store/array_store_test.cpp +++ b/vespalib/src/tests/datastore/array_store/array_store_test.cpp @@ -29,16 +29,16 @@ constexpr float ALLOC_GROW_FACTOR = 0.2; } -template <typename TestT, typename EntryT, typename RefT = EntryRefT<19> > +template <typename TestT, typename ElemT, typename RefT = EntryRefT<19> > struct ArrayStoreTest : public TestT { using EntryRefType = RefT; - using ArrayStoreType = ArrayStore<EntryT, RefT>; + using ArrayStoreType = ArrayStore<ElemT, RefT>; using LargeArray = typename ArrayStoreType::LargeArray; using ConstArrayRef = typename ArrayStoreType::ConstArrayRef; - using EntryVector = std::vector<EntryT>; - using value_type = EntryT; - using ReferenceStore = vespalib::hash_map<EntryRef, EntryVector>; + using ElemVector = std::vector<ElemT>; + using value_type = ElemT; + using ReferenceStore = vespalib::hash_map<EntryRef, ElemVector>; AllocStats stats; ArrayStoreType store; @@ -61,11 +61,11 @@ struct ArrayStoreTest : public TestT add_using_allocate(false) {} ~ArrayStoreTest() override; - void assertAdd(const EntryVector &input) { + void assertAdd(const ElemVector &input) { EntryRef ref = add(input); assertGet(ref, input); } - EntryRef add(const EntryVector &input) { + EntryRef add(const ElemVector &input) { EntryRef result; if (add_using_allocate) { result = store.allocate(input.size()); @@ -82,16 +82,16 @@ struct ArrayStoreTest : public TestT refStore.insert(std::make_pair(result, input)); return result; } - void assertGet(EntryRef ref, const EntryVector &exp) const { + void assertGet(EntryRef ref, const ElemVector &exp) const { ConstArrayRef act = store.get(ref); - EXPECT_EQ(exp, EntryVector(act.begin(), act.end())); + EXPECT_EQ(exp, ElemVector(act.begin(), act.end())); } void remove(EntryRef ref) { ASSERT_EQ(1u, refStore.count(ref)); store.remove(ref); refStore.erase(ref); } - void remove(const EntryVector &input) { + void remove(const ElemVector &input) { remove(getEntryRef(input)); } uint32_t getBufferId(EntryRef ref) const { @@ -99,14 +99,14 @@ struct ArrayStoreTest : public TestT } void assertBufferState(EntryRef ref, const MemStats& expStats) { EXPECT_EQ(expStats._used, store.bufferState(ref).size()); - EXPECT_EQ(expStats._hold, store.bufferState(ref).stats().hold_elems()); - EXPECT_EQ(expStats._dead, store.bufferState(ref).stats().dead_elems()); + EXPECT_EQ(expStats._hold, store.bufferState(ref).stats().hold_entries()); + EXPECT_EQ(expStats._dead, store.bufferState(ref).stats().dead_entries()); } void assert_buffer_stats(EntryRef ref, const TestBufferStats& exp_stats) { const auto& state = store.bufferState(ref); EXPECT_EQ(exp_stats._used, state.size()); - EXPECT_EQ(exp_stats._hold, state.stats().hold_elems()); - EXPECT_EQ(exp_stats._dead, state.stats().dead_elems()); + EXPECT_EQ(exp_stats._hold, state.stats().hold_entries()); + EXPECT_EQ(exp_stats._dead, state.stats().dead_entries()); EXPECT_EQ(exp_stats._extra_used, state.stats().extra_used_bytes()); EXPECT_EQ(exp_stats._extra_hold, state.stats().extra_hold_bytes()); } @@ -121,7 +121,7 @@ struct ArrayStoreTest : public TestT assertGet(elem.first, elem.second); } } - void assert_ref_reused(const EntryVector& first, const EntryVector& second, bool should_reuse) { + void assert_ref_reused(const ElemVector& first, const ElemVector& second, bool should_reuse) { EntryRef ref1 = add(first); remove(ref1); reclaim_memory(); @@ -129,7 +129,7 @@ struct ArrayStoreTest : public TestT EXPECT_EQ(should_reuse, (ref2 == ref1)); assertGet(ref2, second); } - EntryRef getEntryRef(const EntryVector &input) { + EntryRef getEntryRef(const ElemVector &input) { for (auto itr = refStore.begin(); itr != refStore.end(); ++itr) { if (itr->second == input) { return itr->first; @@ -160,12 +160,12 @@ struct ArrayStoreTest : public TestT } refStore = compactedRefStore; } - size_t entrySize() const { return sizeof(EntryT); } + size_t elem_size() const { return sizeof(ElemT); } size_t largeArraySize() const { return sizeof(LargeArray); } }; -template <typename TestT, typename EntryT, typename RefT> -ArrayStoreTest<TestT, EntryT, RefT>::~ArrayStoreTest() = default; +template <typename TestT, typename ElemT, typename RefT> +ArrayStoreTest<TestT, ElemT, RefT>::~ArrayStoreTest() = default; struct TestParam { bool add_using_allocate; @@ -214,8 +214,8 @@ TEST_P(NumberStoreTest, control_static_sizes) { EXPECT_EQ(240u + sizeof_deque, sizeof(NumberStoreTest::ArrayStoreType::DataStoreType)); EXPECT_EQ(104u, sizeof(NumberStoreTest::ArrayStoreType::SmallBufferType)); MemoryUsage usage = store.getMemoryUsage(); - EXPECT_EQ(202120u, usage.allocatedBytes()); - EXPECT_EQ(197752u, usage.usedBytes()); + EXPECT_EQ(202116u, usage.allocatedBytes()); + EXPECT_EQ(197656u, usage.usedBytes()); } TEST_P(NumberStoreTest, add_and_get_small_arrays_of_trivial_type) @@ -246,15 +246,15 @@ TEST_F(StringStoreTest, add_and_get_large_arrays_of_non_trivial_type) assertAdd({"ddd", "eee", "ffff", "gggg", "hhhh"}); } -TEST_P(NumberStoreTest, elements_are_put_on_hold_when_a_small_array_is_removed) +TEST_P(NumberStoreTest, entries_are_put_on_hold_when_a_small_array_is_removed) { EntryRef ref = add({1,2,3}); - assertBufferState(ref, MemStats().used(3).hold(0)); + assertBufferState(ref, MemStats().used(1).hold(0)); store.remove(ref); - assertBufferState(ref, MemStats().used(3).hold(3)); + assertBufferState(ref, MemStats().used(1).hold(1)); } -TEST_P(NumberStoreTest, elements_are_put_on_hold_when_a_large_array_is_removed) +TEST_P(NumberStoreTest, entries_are_put_on_hold_when_a_large_array_is_removed) { EntryRef ref = add({1,2,3,4}); // Note: The first buffer has the first element reserved -> we expect 2 elements used here. @@ -319,7 +319,7 @@ test_compaction(NumberStoreBasicTest &f) f.remove(f.add({5,5})); f.reclaim_memory(); f.assertBufferState(size1Ref, MemStats().used(1).dead(0)); - f.assertBufferState(size2Ref, MemStats().used(4).dead(2)); + f.assertBufferState(size2Ref, MemStats().used(2).dead(1)); f.assertBufferState(size3Ref, MemStats().used(2).dead(1)); // Note: First element is reserved uint32_t size1BufferId = f.getBufferId(size1Ref); uint32_t size2BufferId = f.getBufferId(size2Ref); @@ -363,8 +363,8 @@ void testCompaction(NumberStoreTest &f, bool compactMemory, bool compactAddressS f.remove(f.add({7})); f.reclaim_memory(); f.assertBufferState(size1Ref, MemStats().used(3).dead(2)); - f.assertBufferState(size2Ref, MemStats().used(2).dead(0)); - f.assertBufferState(size3Ref, MemStats().used(6).dead(3)); + f.assertBufferState(size2Ref, MemStats().used(1).dead(0)); + f.assertBufferState(size3Ref, MemStats().used(2).dead(1)); uint32_t size1BufferId = f.getBufferId(size1Ref); uint32_t size2BufferId = f.getBufferId(size2Ref); uint32_t size3BufferId = f.getBufferId(size3Ref); @@ -434,22 +434,22 @@ TEST_P(NumberStoreTest, used_onHold_and_dead_memory_usage_is_tracked_for_small_a { MemStats exp(store.getMemoryUsage()); add({1,2,3}); - assertMemoryUsage(exp.used(entrySize() * 3)); + assertMemoryUsage(exp.used(elem_size() * 3)); remove({1,2,3}); - assertMemoryUsage(exp.hold(entrySize() * 3)); + assertMemoryUsage(exp.hold(elem_size() * 3)); reclaim_memory(); - assertMemoryUsage(exp.holdToDead(entrySize() * 3)); + assertMemoryUsage(exp.holdToDead(elem_size() * 3)); } TEST_P(NumberStoreTest, used_onHold_and_dead_memory_usage_is_tracked_for_large_arrays) { MemStats exp(store.getMemoryUsage()); add({1,2,3,4}); - assertMemoryUsage(exp.used(largeArraySize() + entrySize() * 4)); + assertMemoryUsage(exp.used(largeArraySize() + elem_size() * 4)); remove({1,2,3,4}); - assertMemoryUsage(exp.hold(largeArraySize() + entrySize() * 4)); + assertMemoryUsage(exp.hold(largeArraySize() + elem_size() * 4)); reclaim_memory(); - assertMemoryUsage(exp.decUsed(entrySize() * 4).decHold(largeArraySize() + entrySize() * 4). + assertMemoryUsage(exp.decUsed(elem_size() * 4).decHold(largeArraySize() + elem_size() * 4). dead(largeArraySize())); } diff --git a/vespalib/src/tests/datastore/array_store_config/array_store_config_test.cpp b/vespalib/src/tests/datastore/array_store_config/array_store_config_test.cpp index fcf1acd5cd3..171e7216638 100644 --- a/vespalib/src/tests/datastore/array_store_config/array_store_config_test.cpp +++ b/vespalib/src/tests/datastore/array_store_config/array_store_config_test.cpp @@ -22,32 +22,32 @@ struct Fixture Fixture(uint32_t maxSmallArrayTypeId, size_t hugePageSize, size_t smallPageSize, - size_t minNumArraysForNewBuffer) + size_t min_num_entries_for_new_buffer) : cfg(ArrayStoreConfig::optimizeForHugePage(maxSmallArrayTypeId, [](size_t type_id) noexcept { return type_id; }, hugePageSize, smallPageSize, sizeof(int), EntryRefType::offsetSize(), - minNumArraysForNewBuffer, + min_num_entries_for_new_buffer, ALLOC_GROW_FACTOR)) { } - void assertSpec(uint32_t type_id, uint32_t numArraysForNewBuffer) { + void assertSpec(uint32_t type_id, uint32_t num_entries_for_new_buffer) { assertSpec(type_id, AllocSpec(0, EntryRefType::offsetSize(), - numArraysForNewBuffer, ALLOC_GROW_FACTOR)); + num_entries_for_new_buffer, ALLOC_GROW_FACTOR)); } void assertSpec(uint32_t type_id, const AllocSpec &expSpec) { const auto& actSpec = cfg.spec_for_type_id(type_id); - EXPECT_EQUAL(expSpec.minArraysInBuffer, actSpec.minArraysInBuffer); - EXPECT_EQUAL(expSpec.maxArraysInBuffer, actSpec.maxArraysInBuffer); - EXPECT_EQUAL(expSpec.numArraysForNewBuffer, actSpec.numArraysForNewBuffer); + EXPECT_EQUAL(expSpec.min_entries_in_buffer, actSpec.min_entries_in_buffer); + EXPECT_EQUAL(expSpec.max_entries_in_buffer, actSpec.max_entries_in_buffer); + EXPECT_EQUAL(expSpec.num_entries_for_new_buffer, actSpec.num_entries_for_new_buffer); EXPECT_EQUAL(expSpec.allocGrowFactor, actSpec.allocGrowFactor); } }; AllocSpec -makeSpec(size_t minArraysInBuffer, - size_t maxArraysInBuffer, - size_t numArraysForNewBuffer) +makeSpec(size_t min_entries_in_buffer, + size_t max_entries_in_buffer, + size_t num_entries_for_new_buffer) { - return AllocSpec(minArraysInBuffer, maxArraysInBuffer, numArraysForNewBuffer, ALLOC_GROW_FACTOR); + return AllocSpec(min_entries_in_buffer, max_entries_in_buffer, num_entries_for_new_buffer, ALLOC_GROW_FACTOR); } constexpr size_t KB = 1024; diff --git a/vespalib/src/tests/datastore/buffer_stats/buffer_stats_test.cpp b/vespalib/src/tests/datastore/buffer_stats/buffer_stats_test.cpp index 09b2590a5f3..fec8d5949f8 100644 --- a/vespalib/src/tests/datastore/buffer_stats/buffer_stats_test.cpp +++ b/vespalib/src/tests/datastore/buffer_stats/buffer_stats_test.cpp @@ -9,10 +9,10 @@ using namespace vespalib::datastore; TEST(BufferStatsTest, buffer_stats_to_memory_stats) { InternalBufferStats buf; - buf.set_alloc_elems(17); + buf.set_alloc_entries(17); buf.pushed_back(7); - buf.set_dead_elems(5); - buf.set_hold_elems(3); + buf.set_dead_entries(5); + buf.set_hold_entries(3); buf.inc_extra_used_bytes(13); buf.inc_extra_hold_bytes(11); @@ -20,10 +20,10 @@ TEST(BufferStatsTest, buffer_stats_to_memory_stats) constexpr size_t es = 8; buf.add_to_mem_stats(es, mem); - EXPECT_EQ(17, mem._allocElems); - EXPECT_EQ(7, mem._usedElems); - EXPECT_EQ(5, mem._deadElems); - EXPECT_EQ(3, mem._holdElems); + EXPECT_EQ(17, mem._alloc_entries); + EXPECT_EQ(7, mem._used_entries); + EXPECT_EQ(5, mem._dead_entries); + EXPECT_EQ(3, mem._hold_entries); EXPECT_EQ(17 * es + 13, mem._allocBytes); EXPECT_EQ(7 * es + 13, mem._usedBytes); EXPECT_EQ(5 * es, mem._deadBytes); diff --git a/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp b/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp index de7d899e68a..9f7535a3676 100644 --- a/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp +++ b/vespalib/src/tests/datastore/buffer_type/buffer_type_test.cpp @@ -7,40 +7,40 @@ using namespace vespalib::datastore; using IntBufferType = BufferType<int>; constexpr uint32_t ARRAYS_SIZE(4); -constexpr uint32_t MAX_ARRAYS(128); -constexpr uint32_t NUM_ARRAYS_FOR_NEW_BUFFER(0); +constexpr uint32_t MAX_ENTRIES(128); +constexpr uint32_t NUM_ENTRIES_FOR_NEW_BUFFER(0); struct Setup { - uint32_t _minArrays; - std::atomic<ElemCount> _usedElems; - ElemCount _neededElems; - std::atomic<ElemCount> _deadElems; + uint32_t _min_entries; + std::atomic<EntryCount> _used_entries; + EntryCount _needed_entries; + std::atomic<EntryCount> _dead_entries; uint32_t _bufferId; float _allocGrowFactor; bool _resizing; Setup() - : _minArrays(0), - _usedElems(0), - _neededElems(0), - _deadElems(0), + : _min_entries(0), + _used_entries(0), + _needed_entries(0), + _dead_entries(0), _bufferId(1), _allocGrowFactor(0.5), _resizing(false) {} Setup(const Setup& rhs) noexcept; - Setup &minArrays(uint32_t value) { _minArrays = value; return *this; } - Setup &used(size_t value) { _usedElems = value; return *this; } - Setup &needed(size_t value) { _neededElems = value; return *this; } - Setup &dead(size_t value) { _deadElems = value; return *this; } + Setup &min_entries(uint32_t value) { _min_entries = value; return *this; } + Setup &used(size_t value) { _used_entries = value; return *this; } + Setup &needed(size_t value) { _needed_entries = value; return *this; } + Setup &dead(size_t value) { _dead_entries = value; return *this; } Setup &bufferId(uint32_t value) { _bufferId = value; return *this; } Setup &resizing(bool value) { _resizing = value; return *this; } }; Setup::Setup(const Setup& rhs) noexcept - : _minArrays(rhs._minArrays), - _usedElems(rhs._usedElems.load(std::memory_order_relaxed)), - _neededElems(rhs._neededElems), - _deadElems(rhs._deadElems.load(std::memory_order_relaxed)), + : _min_entries(rhs._min_entries), + _used_entries(rhs._used_entries.load(std::memory_order_relaxed)), + _needed_entries(rhs._needed_entries), + _dead_entries(rhs._dead_entries.load(std::memory_order_relaxed)), _bufferId(rhs._bufferId), _allocGrowFactor(rhs._allocGrowFactor), _resizing(rhs._resizing) @@ -53,7 +53,7 @@ struct Fixture { int buffer[ARRAYS_SIZE]; Fixture(const Setup &setup_) : setups(), - bufferType(ARRAYS_SIZE, setup_._minArrays, MAX_ARRAYS, NUM_ARRAYS_FOR_NEW_BUFFER, setup_._allocGrowFactor), + bufferType(ARRAYS_SIZE, setup_._min_entries, MAX_ENTRIES, NUM_ENTRIES_FOR_NEW_BUFFER, setup_._allocGrowFactor), buffer() { setups.reserve(4); @@ -61,121 +61,121 @@ struct Fixture { } ~Fixture() { for (auto& setup : setups) { - bufferType.onHold(setup._bufferId, &setup._usedElems, &setup._deadElems); - bufferType.onFree(setup._usedElems); + bufferType.on_hold(setup._bufferId, &setup._used_entries, &setup._dead_entries); + bufferType.on_free(setup._used_entries); } } Setup& curr_setup() { return setups.back(); } void add_setup(const Setup& setup_in) { - // The buffer type stores pointers to ElemCount (from Setup) and we must ensure these do not move in memory. + // The buffer type stores pointers to EntryCount (from Setup) and we must ensure these do not move in memory. assert(setups.size() < setups.capacity()); setups.push_back(setup_in); } void onActive() { - bufferType.onActive(curr_setup()._bufferId, &curr_setup()._usedElems, &curr_setup()._deadElems, &buffer[0]); + bufferType.on_active(curr_setup()._bufferId, &curr_setup()._used_entries, &curr_setup()._dead_entries, &buffer[0]); } - size_t arraysToAlloc() { - return bufferType.calcArraysToAlloc(curr_setup()._bufferId, curr_setup()._neededElems, curr_setup()._resizing); + size_t entries_to_alloc() { + return bufferType.calc_entries_to_alloc(curr_setup()._bufferId, curr_setup()._needed_entries, curr_setup()._resizing); } - void assertArraysToAlloc(size_t exp) { + void assert_entries_to_alloc(size_t exp) { onActive(); - EXPECT_EQUAL(exp, arraysToAlloc()); + EXPECT_EQUAL(exp, entries_to_alloc()); } }; void -assertArraysToAlloc(size_t exp, const Setup &setup) +assert_entries_to_alloc(size_t exp, const Setup &setup) { Fixture f(setup); - f.assertArraysToAlloc(exp); + f.assert_entries_to_alloc(exp); } -TEST("require that complete arrays are allocated") +TEST("require that entries are allocated") { - TEST_DO(assertArraysToAlloc(1, Setup().needed(1))); - TEST_DO(assertArraysToAlloc(1, Setup().needed(2))); - TEST_DO(assertArraysToAlloc(1, Setup().needed(3))); - TEST_DO(assertArraysToAlloc(1, Setup().needed(4))); - TEST_DO(assertArraysToAlloc(2, Setup().needed(5))); + TEST_DO(assert_entries_to_alloc(1, Setup().needed(1))); + TEST_DO(assert_entries_to_alloc(2, Setup().needed(2))); + TEST_DO(assert_entries_to_alloc(3, Setup().needed(3))); + TEST_DO(assert_entries_to_alloc(4, Setup().needed(4))); + TEST_DO(assert_entries_to_alloc(5, Setup().needed(5))); } -TEST("require that reserved elements are taken into account when not resizing") +TEST("require that reserved entries are taken into account when not resizing") { - TEST_DO(assertArraysToAlloc(2, Setup().needed(1).bufferId(0))); - TEST_DO(assertArraysToAlloc(2, Setup().needed(4).bufferId(0))); - TEST_DO(assertArraysToAlloc(3, Setup().needed(5).bufferId(0))); + TEST_DO(assert_entries_to_alloc(2, Setup().needed(1).bufferId(0))); + TEST_DO(assert_entries_to_alloc(5, Setup().needed(4).bufferId(0))); + TEST_DO(assert_entries_to_alloc(6, Setup().needed(5).bufferId(0))); } -TEST("require that arrays to alloc is based on currently used elements (no resizing)") +TEST("require that entries to alloc is based on currently used entries (no resizing)") { - TEST_DO(assertArraysToAlloc(2, Setup().used(4 * 4).needed(4))); - TEST_DO(assertArraysToAlloc(4, Setup().used(8 * 4).needed(4))); + TEST_DO(assert_entries_to_alloc(2, Setup().used(4).needed(1))); + TEST_DO(assert_entries_to_alloc(4, Setup().used(8).needed(1))); } -TEST("require that arrays to alloc is based on currently used elements (with resizing)") +TEST("require that entries to alloc is based on currently used entries (with resizing)") { - TEST_DO(assertArraysToAlloc(4 + 2, Setup().used(4 * 4).needed(4).resizing(true))); - TEST_DO(assertArraysToAlloc(8 + 4, Setup().used(8 * 4).needed(4).resizing(true))); - TEST_DO(assertArraysToAlloc(4 + 3, Setup().used(4 * 4).needed(3 * 4).resizing(true))); + TEST_DO(assert_entries_to_alloc(4 + 2, Setup().used(4).needed(1).resizing(true))); + TEST_DO(assert_entries_to_alloc(8 + 4, Setup().used(8).needed(1).resizing(true))); + TEST_DO(assert_entries_to_alloc(4 + 3, Setup().used(4).needed(3).resizing(true))); } -TEST("require that arrays to alloc always contain elements needed") +TEST("require that entries to alloc always contain entries needed") { - TEST_DO(assertArraysToAlloc(2, Setup().used(4 * 4).needed(2 * 4))); - TEST_DO(assertArraysToAlloc(3, Setup().used(4 * 4).needed(3 * 4))); - TEST_DO(assertArraysToAlloc(4, Setup().used(4 * 4).needed(4 * 4))); + TEST_DO(assert_entries_to_alloc(2, Setup().used(4).needed(2))); + TEST_DO(assert_entries_to_alloc(3, Setup().used(4).needed(3))); + TEST_DO(assert_entries_to_alloc(4, Setup().used(4).needed(4))); } -TEST("require that arrays to alloc is capped to max arrays") +TEST("require that entries to alloc is capped to max entries") { - TEST_DO(assertArraysToAlloc(127, Setup().used(254 * 4).needed(4))); - TEST_DO(assertArraysToAlloc(128, Setup().used(256 * 4).needed(4))); - TEST_DO(assertArraysToAlloc(128, Setup().used(258 * 4).needed(8))); + TEST_DO(assert_entries_to_alloc(127, Setup().used(254).needed(1))); + TEST_DO(assert_entries_to_alloc(128, Setup().used(256).needed(1))); + TEST_DO(assert_entries_to_alloc(128, Setup().used(258).needed(2))); } TEST("require that arrays to alloc is capped to min arrays") { - TEST_DO(assertArraysToAlloc(16, Setup().used(30 * 4).needed(4).minArrays(16))); - TEST_DO(assertArraysToAlloc(16, Setup().used(32 * 4).needed(4).minArrays(16))); - TEST_DO(assertArraysToAlloc(17, Setup().used(34 * 4).needed(4).minArrays(16))); + TEST_DO(assert_entries_to_alloc(16, Setup().used(30).needed(1).min_entries(16))); + TEST_DO(assert_entries_to_alloc(16, Setup().used(32).needed(1).min_entries(16))); + TEST_DO(assert_entries_to_alloc(17, Setup().used(34).needed(1).min_entries(16))); } -TEST("arrays to alloc considers used elements across all active buffers of same type (no resizing)") +TEST("entries to alloc considers used entries across all active buffers of same type (no resizing)") { - Fixture f(Setup().used(6 * 4)); - f.assertArraysToAlloc(6 * 0.5); - f.add_setup(Setup().used(8 * 4).bufferId(2)); - f.assertArraysToAlloc((6 + 8) * 0.5); - f.add_setup(Setup().used(10 * 4).bufferId(3)); - f.assertArraysToAlloc((6 + 8 + 10) * 0.5); + Fixture f(Setup().used(6)); + f.assert_entries_to_alloc(6 * 0.5); + f.add_setup(Setup().used(8).bufferId(2)); + f.assert_entries_to_alloc((6 + 8) * 0.5); + f.add_setup(Setup().used(10).bufferId(3)); + f.assert_entries_to_alloc((6 + 8 + 10) * 0.5); } -TEST("arrays to alloc considers used elements across all active buffers of same type when resizing") +TEST("entries to alloc considers used entries across all active buffers of same type when resizing") { - Fixture f(Setup().used(6 * 4)); - f.assertArraysToAlloc(6 * 0.5); - f.add_setup(Setup().used(8 * 4).resizing(true).bufferId(2)); - f.assertArraysToAlloc(8 + (6 + 8) * 0.5); + Fixture f(Setup().used(6)); + f.assert_entries_to_alloc(6 * 0.5); + f.add_setup(Setup().used(8).resizing(true).bufferId(2)); + f.assert_entries_to_alloc(8 + (6 + 8) * 0.5); } -TEST("arrays to alloc considers (and subtracts) dead elements across all active buffers of same type (no resizing)") +TEST("entries to alloc considers (and subtracts) dead entries across all active buffers of same type (no resizing)") { - Fixture f(Setup().used(6 * 4).dead(2 * 4)); - f.assertArraysToAlloc((6 - 2) * 0.5); - f.add_setup(Setup().used(12 * 4).dead(4 * 4).bufferId(2)); - f.assertArraysToAlloc((6 - 2 + 12 - 4) * 0.5); - f.add_setup(Setup().used(20 * 4).dead(6 * 4).bufferId(3)); - f.assertArraysToAlloc((6 - 2 + 12 - 4 + 20 - 6) * 0.5); + Fixture f(Setup().used(6).dead(2)); + f.assert_entries_to_alloc((6 - 2) * 0.5); + f.add_setup(Setup().used(12).dead(4).bufferId(2)); + f.assert_entries_to_alloc((6 - 2 + 12 - 4) * 0.5); + f.add_setup(Setup().used(20).dead(6).bufferId(3)); + f.assert_entries_to_alloc((6 - 2 + 12 - 4 + 20 - 6) * 0.5); } TEST("arrays to alloc considers (and subtracts) dead elements across all active buffers of same type when resizing") { - Fixture f(Setup().used(6 * 4).dead(2 * 4)); - f.assertArraysToAlloc((6 - 2) * 0.5); - f.add_setup(Setup().used(12 * 4).dead(4 * 4).resizing(true).bufferId(2)); - f.assertArraysToAlloc(12 + (6 - 2 + 12 - 4) * 0.5); + Fixture f(Setup().used(6).dead(2)); + f.assert_entries_to_alloc((6 - 2) * 0.5); + f.add_setup(Setup().used(12).dead(4).resizing(true).bufferId(2)); + f.assert_entries_to_alloc(12 + (6 - 2 + 12 - 4) * 0.5); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/vespalib/src/tests/datastore/datastore/datastore_test.cpp b/vespalib/src/tests/datastore/datastore/datastore_test.cpp index df347267c7e..9e27ed37dd3 100644 --- a/vespalib/src/tests/datastore/datastore/datastore_test.cpp +++ b/vespalib/src/tests/datastore/datastore/datastore_test.cpp @@ -26,8 +26,8 @@ public: void holdBuffer(uint32_t bufferId) { ParentType::holdBuffer(bufferId); } - void holdElem(EntryRef ref, uint64_t len) { - ParentType::holdElem(ref, len); + void hold_entry(EntryRef ref) { + ParentType::hold_entry(ref); } void assign_generation(generation_t current_gen) { ParentType::assign_generation(current_gen); @@ -35,8 +35,8 @@ public: void reclaim_entry_refs(generation_t oldest_used_gen) override { ParentType::reclaim_entry_refs(oldest_used_gen); } - void ensureBufferCapacity(size_t sizeNeeded) { - ParentType::ensureBufferCapacity(0, sizeNeeded); + void ensure_buffer_capacity(size_t entries_needed) { + ParentType::ensure_buffer_capacity(0, entries_needed); } void enableFreeLists() { ParentType::enableFreeLists(); @@ -66,10 +66,10 @@ class GrowStore BufferType<DataType> _type; uint32_t _typeId; public: - GrowStore(size_t arraySize, size_t minArrays, size_t maxArrays, size_t numArraysForNewBuffer) + GrowStore(size_t arraySize, size_t min_entries, size_t max_entries, size_t num_entries_for_new_buffer) : _store(), - _firstType(1, 1, maxArrays, 0, ALLOC_GROW_FACTOR), - _type(arraySize, minArrays, maxArrays, numArraysForNewBuffer, ALLOC_GROW_FACTOR), + _firstType(1, 1, max_entries, 0, ALLOC_GROW_FACTOR), + _type(arraySize, min_entries, max_entries, num_entries_for_new_buffer, ALLOC_GROW_FACTOR), _typeId(0) { (void) _store.addType(&_firstType); @@ -87,7 +87,7 @@ public: while (sizes.size() < bufs) { RefType iRef = (_type.getArraySize() == 1) ? (_store.template allocator<DataType>(_typeId).alloc().ref) : - (_store.template allocator<DataType>(_typeId).allocArray(_type.getArraySize()).ref); + (_store.template allocator<DataType>(_typeId).allocArray().ref); int bufferId = iRef.bufferId(); if (bufferId != prevBufferId) { if (prevBufferId >= 0) { @@ -126,7 +126,7 @@ public: while (buffers.size() < bufs) { RefType iRef = (_type.getArraySize() == 1) ? (_store.template allocator<DataType>(_typeId).alloc().ref) : - (_store.template allocator<DataType>(_typeId).allocArray(_type.getArraySize()).ref); + (_store.template allocator<DataType>(_typeId).allocArray().ref); int buffer_id = iRef.bufferId(); if (buffers.empty() || buffers.back() != buffer_id) { buffers.push_back(buffer_id); @@ -143,10 +143,10 @@ void assertMemStats(const MemoryStats &exp, const MemoryStats &act) { - EXPECT_EQ(exp._allocElems, act._allocElems); - EXPECT_EQ(exp._usedElems, act._usedElems); - EXPECT_EQ(exp._deadElems, act._deadElems); - EXPECT_EQ(exp._holdElems, act._holdElems); + EXPECT_EQ(exp._alloc_entries, act._alloc_entries); + EXPECT_EQ(exp._used_entries, act._used_entries); + EXPECT_EQ(exp._dead_entries, act._dead_entries); + EXPECT_EQ(exp._hold_entries, act._hold_entries); EXPECT_EQ(exp._freeBuffers, act._freeBuffers); EXPECT_EQ(exp._activeBuffers, act._activeBuffers); EXPECT_EQ(exp._holdBuffers, act._holdBuffers); @@ -304,13 +304,13 @@ TEST(DataStoreTest, require_that_we_can_hold_and_trim_elements) { MyStore s; MyRef r1 = s.addEntry(1); - s.holdElem(r1, 1); + s.hold_entry(r1); s.assign_generation(10); MyRef r2 = s.addEntry(2); - s.holdElem(r2, 1); + s.hold_entry(r2); s.assign_generation(20); MyRef r3 = s.addEntry(3); - s.holdElem(r3, 1); + s.hold_entry(r3); s.assign_generation(30); EXPECT_EQ(1, s.getEntry(r1)); EXPECT_EQ(2, s.getEntry(r2)); @@ -358,11 +358,11 @@ TEST(DataStoreTest, require_that_we_can_use_free_lists) MyStore s; s.enableFreeLists(); auto r1 = s.addEntry(1); - s.holdElem(r1, 1); + s.hold_entry(r1); s.assign_generation(10); auto r2 = s.addEntry(2); expect_successive_refs(r1, r2); - s.holdElem(r2, 1); + s.hold_entry(r2); s.assign_generation(20); s.reclaim_entry_refs(11); auto r3 = s.addEntry(3); // reuse r1 @@ -389,21 +389,21 @@ TEST(DataStoreTest, require_that_we_can_use_free_lists_with_raw_allocator) s.enableFreeLists(); auto allocator = s.freeListRawAllocator<int>(grow_store.typeId()); - auto h1 = allocator.alloc(3); - auto h2 = allocator.alloc(3); + auto h1 = allocator.alloc(1); + auto h2 = allocator.alloc(1); expect_successive_handles(h1, h2); - s.holdElem(h1.ref, 3); - s.holdElem(h2.ref, 3); + s.hold_entry(h1.ref); + s.hold_entry(h2.ref); s.assign_generation(10); s.reclaim_entry_refs(11); - auto h3 = allocator.alloc(3); // reuse h2.ref from free list + auto h3 = allocator.alloc(1); // reuse h2.ref from free list EXPECT_EQ(h2, h3); - auto h4 = allocator.alloc(3); // reuse h1.ref from free list + auto h4 = allocator.alloc(1); // reuse h1.ref from free list EXPECT_EQ(h1, h4); - auto h5 = allocator.alloc(3); + auto h5 = allocator.alloc(1); expect_successive_handles(h2, h5); expect_successive_handles(h3, h5); } @@ -412,10 +412,10 @@ TEST(DataStoreTest, require_that_memory_stats_are_calculated) { MyStore s; MemoryStats m; - m._allocElems = MyRef::offsetSize(); - m._usedElems = 1; // ref = 0 is reserved - m._deadElems = 1; // ref = 0 is reserved - m._holdElems = 0; + m._alloc_entries = MyRef::offsetSize(); + m._used_entries = 1; // ref = 0 is reserved + m._dead_entries = 1; // ref = 0 is reserved + m._hold_entries = 0; m._activeBuffers = 1; m._freeBuffers = MyRef::numBuffers() - 1; m._holdBuffers = 0; @@ -423,7 +423,7 @@ TEST(DataStoreTest, require_that_memory_stats_are_calculated) // add entry MyRef r = s.addEntry(10); - m._usedElems++; + m._used_entries++; assertMemStats(m, s.getMemStats()); // hold buffer @@ -431,9 +431,9 @@ TEST(DataStoreTest, require_that_memory_stats_are_calculated) s.addEntry(30); s.holdBuffer(r.bufferId()); s.assign_generation(100); - m._usedElems += 2; - m._holdElems = m._usedElems; - m._deadElems = 0; + m._used_entries += 2; + m._hold_entries = m._used_entries; + m._dead_entries = 0; m._activeBuffers--; m._holdBuffers++; assertMemStats(m, s.getMemStats()); @@ -441,17 +441,17 @@ TEST(DataStoreTest, require_that_memory_stats_are_calculated) // new active buffer s.switch_primary_buffer(); s.addEntry(40); - m._allocElems += MyRef::offsetSize(); - m._usedElems++; + m._alloc_entries += MyRef::offsetSize(); + m._used_entries++; m._activeBuffers++; m._freeBuffers--; // trim hold buffer s.reclaim_memory(101); - m._allocElems -= MyRef::offsetSize(); - m._usedElems = 1; - m._deadElems = 0; - m._holdElems = 0; + m._alloc_entries -= MyRef::offsetSize(); + m._used_entries = 1; + m._dead_entries = 0; + m._hold_entries = 0; m._freeBuffers = MyRef::numBuffers() - 1; m._holdBuffers = 0; assertMemStats(m, s.getMemStats()); @@ -466,7 +466,7 @@ TEST(DataStoreTest, require_that_memory_stats_are_calculated) { // increase extra hold bytes auto prev_stats = s.getMemStats(); - s.get_active_buffer_state().hold_elems(0, 30); + s.get_active_buffer_state().hold_entries(0, 30); auto curr_stats = s.getMemStats(); EXPECT_EQ(prev_stats._holdBytes + 30, curr_stats._holdBytes); } @@ -475,7 +475,7 @@ TEST(DataStoreTest, require_that_memory_stats_are_calculated) TEST(DataStoreTest, require_that_memory_usage_is_calculated) { constexpr size_t BASE_ALLOCATED = 4228; - constexpr size_t BASE_USED = 308; + constexpr size_t BASE_USED = 284; MyStore s; MyRef r = s.addEntry(10); s.addEntry(20); @@ -494,7 +494,7 @@ TEST(DataStoreTest, require_that_memory_usage_is_calculated) TEST(DataStoreTest, require_that_we_can_disable_elemement_hold_list) { constexpr size_t BASE_ALLOCATED = 4228; - constexpr size_t BASE_USED = 308; + constexpr size_t BASE_USED = 284; MyStore s; MyRef r1 = s.addEntry(10); MyRef r2 = s.addEntry(20); @@ -505,14 +505,14 @@ TEST(DataStoreTest, require_that_we_can_disable_elemement_hold_list) EXPECT_EQ(4 * sizeof(int) + BASE_USED, m.usedBytes()); EXPECT_EQ(1 * sizeof(int), m.deadBytes()); EXPECT_EQ(0 * sizeof(int), m.allocatedBytesOnHold()); - s.holdElem(r1, 1); + s.hold_entry(r1); m = s.getMemoryUsage(); EXPECT_EQ(MyRef::offsetSize() * sizeof(int) + BASE_ALLOCATED, m.allocatedBytes()); EXPECT_EQ(4 * sizeof(int) + BASE_USED, m.usedBytes()); EXPECT_EQ(1 * sizeof(int), m.deadBytes()); EXPECT_EQ(1 * sizeof(int), m.allocatedBytesOnHold()); - s.disableElemHoldList(); - s.holdElem(r2, 1); + s.disable_entry_hold_list(); + s.hold_entry(r2); m = s.getMemoryUsage(); EXPECT_EQ(MyRef::offsetSize() * sizeof(int) + BASE_ALLOCATED, m.allocatedBytes()); EXPECT_EQ(4 * sizeof(int) + BASE_USED, m.usedBytes()); @@ -529,11 +529,11 @@ namespace { void assertGrowStats(GrowthStats expSizes, GrowthStats expFirstBufSizes, size_t expInitMemUsage, - size_t minArrays, size_t numArraysForNewBuffer, size_t maxArrays = 128) + size_t min_entries, size_t num_entries_for_new_buffer, size_t max_entries = 128) { - EXPECT_EQ(expSizes, IntGrowStore(1, minArrays, maxArrays, numArraysForNewBuffer).getGrowthStats(expSizes.size())); - EXPECT_EQ(expFirstBufSizes, IntGrowStore(1, minArrays, maxArrays, numArraysForNewBuffer).getFirstBufGrowStats()); - EXPECT_EQ(expInitMemUsage, IntGrowStore(1, minArrays, maxArrays, numArraysForNewBuffer).getMemoryUsage().allocatedBytes()); + EXPECT_EQ(expSizes, IntGrowStore(1, min_entries, max_entries, num_entries_for_new_buffer).getGrowthStats(expSizes.size())); + EXPECT_EQ(expFirstBufSizes, IntGrowStore(1, min_entries, max_entries, num_entries_for_new_buffer).getFirstBufGrowStats()); + EXPECT_EQ(expInitMemUsage, IntGrowStore(1, min_entries, max_entries, num_entries_for_new_buffer).getMemoryUsage().allocatedBytes()); } } @@ -574,10 +574,10 @@ namespace { template <typename DataType> void assertGrowStats(GrowthStats expSizes, uint32_t arraySize) { - uint32_t minArrays = 2048; - uint32_t maxArrays = RefType15::offsetSize(); - uint32_t numArraysForNewBuffer = 2048; - GrowStore<DataType, RefType15> store(arraySize, minArrays, maxArrays, numArraysForNewBuffer); + uint32_t min_entries = 2048; + uint32_t max_entries = RefType15::offsetSize(); + uint32_t num_entries_for_new_buffer = 2048; + GrowStore<DataType, RefType15> store(arraySize, min_entries, max_entries, num_entries_for_new_buffer); EXPECT_EQ(expSizes, store.getGrowthStats(expSizes.size())); } @@ -594,14 +594,14 @@ TEST(DataStoreTest, require_that_offset_in_EntryRefT_is_within_bounds_when_alloc * 3) Round up bytes to alloc to match the underlying allocator (power of 2 if less than huge page size): * After this we might end up with more bytes than the offset in EntryRef can handle. In this case this is 32768. * 4) Cap bytes to alloc to the max offset EntryRef can handle. - * The max bytes to alloc is: maxArrays * arraySize * elementSize. + * The max bytes to alloc is: max_entries * arraySize * elementSize. */ - assertGrowStats<uint8_t>({8192,16384,16384,65536,65536,98304,98304,98304,98304,98304,98304,98304}, 3); - assertGrowStats<uint8_t>({16384,16384,65536,65536,131072,131072,163840,163840,163840,163840,163840,163840}, 5); - assertGrowStats<uint8_t>({16384,32768,32768,131072,131072,229376,229376,229376,229376,229376,229376,229376}, 7); - assertGrowStats<uint32_t>({8192,16384,16384,65536,65536,98304,98304,98304,98304,98304,98304,98304}, 3); - assertGrowStats<uint32_t>({16384,16384,65536,65536,131072,131072,163840,163840,163840,163840,163840,163840}, 5); - assertGrowStats<uint32_t>({16384,32768,32768,131072,131072,229376,229376,229376,229376,229376,229376,229376}, 7); + assertGrowStats<uint8_t>({2730,5461,5461,21845,21845,32768,32768,32768,32768,32768,32768,32768}, 3); + assertGrowStats<uint8_t>({3276,3276,13107,13107,26214,26214,32768,32768,32768,32768,32768,32768}, 5); + assertGrowStats<uint8_t>({2340,4681,4681,18724,18724,32768,32768,32768,32768,32768,32768,32768}, 7); + assertGrowStats<uint32_t>({2730,5461,5461,21845,21845,32768,32768,32768,32768,32768,32768,32768}, 3); + assertGrowStats<uint32_t>({3276,3276,13107,13107,26214,26214,32768,32768,32768,32768,32768,32768}, 5); + assertGrowStats<uint32_t>({2340,4681,4681,18724,18724,32768,32768,32768,32768,32768,32768,32768}, 7); } namespace { @@ -667,9 +667,9 @@ TEST(DataStoreTest, can_reuse_active_buffer_as_primary_buffer) TEST(DataStoreTest, control_static_sizes) { EXPECT_EQ(88, sizeof(BufferTypeBase)); EXPECT_EQ(24, sizeof(FreeList)); - EXPECT_EQ(56, sizeof(BufferFreeList)); + EXPECT_EQ(48, sizeof(BufferFreeList)); EXPECT_EQ(1, sizeof(BufferState::State)); - EXPECT_EQ(144, sizeof(BufferState)); + EXPECT_EQ(120, sizeof(BufferState)); BufferState bs; EXPECT_EQ(0, bs.size()); } @@ -685,11 +685,11 @@ void test_free_element_to_held_buffer(bool before_hold_buffer) EXPECT_EQ(1u, s.primary_buffer_id()); if (before_hold_buffer) { - s.holdElem(ref, 1); + s.hold_entry(ref); } s.holdBuffer(0); // hold last buffer if (!before_hold_buffer) { - ASSERT_DEATH({ s.holdElem(ref, 1); }, "isActive\\(\\)"); + ASSERT_DEATH({ s.hold_entry(ref); }, "isActive\\(\\)"); } s.assign_generation(100); s.reclaim_memory(101); diff --git a/vespalib/src/tests/datastore/free_list/free_list_test.cpp b/vespalib/src/tests/datastore/free_list/free_list_test.cpp index 44e11b2316b..ec14d0dd28c 100644 --- a/vespalib/src/tests/datastore/free_list/free_list_test.cpp +++ b/vespalib/src/tests/datastore/free_list/free_list_test.cpp @@ -8,20 +8,18 @@ using namespace vespalib::datastore; using MyEntryRef = EntryRefT<8, 4>; -constexpr uint32_t array_size = 6; struct FreeListTest : public testing::Test { FreeList list; - std::atomic<ElemCount> dead_elems; + std::atomic<EntryCount> dead_entries; std::vector<BufferFreeList> bufs; FreeListTest() : list(), bufs() { for (size_t i = 0; i < 3; ++i) { - bufs.emplace_back(dead_elems); - bufs.back().set_array_size(array_size); + bufs.emplace_back(dead_entries); } } void TearDown() override { @@ -126,13 +124,13 @@ TEST_F(FreeListTest, buffer_free_list_can_be_disabled_and_detached_when_not_curr EXPECT_TRUE(list.empty()); } -TEST_F(FreeListTest, dead_elems_count_is_updated_when_popping_an_entry) +TEST_F(FreeListTest, dead_entries_count_is_updated_when_popping_an_entry) { enable(0); push_entry({10, 0}); - dead_elems.store(18, std::memory_order_relaxed); + dead_entries.store(18, std::memory_order_relaxed); pop_entry(); - EXPECT_EQ(18 - array_size, dead_elems.load(std::memory_order_relaxed)); + EXPECT_EQ(17, dead_entries.load(std::memory_order_relaxed)); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp index 5ccf9a8908c..a09a7213bf5 100644 --- a/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp +++ b/vespalib/src/tests/datastore/unique_store/unique_store_test.cpp @@ -96,8 +96,8 @@ struct TestBase : public ::testing::Test { } void assertBufferState(EntryRef ref, const TestBufferStats expStats) const { EXPECT_EQ(expStats._used, store.bufferState(ref).size()); - EXPECT_EQ(expStats._hold, store.bufferState(ref).stats().hold_elems()); - EXPECT_EQ(expStats._dead, store.bufferState(ref).stats().dead_elems()); + EXPECT_EQ(expStats._hold, store.bufferState(ref).stats().hold_entries()); + EXPECT_EQ(expStats._dead, store.bufferState(ref).stats().dead_entries()); } void assertStoreContent() const { for (const auto &elem : refStore) { @@ -147,10 +147,7 @@ struct TestBase : public ::testing::Test { auto getBuilder(uint32_t uniqueValuesHint) { return store.getBuilder(uniqueValuesHint); } auto getEnumerator(bool sort_unique_values) { return store.getEnumerator(sort_unique_values); } size_t get_reserved(EntryRef ref) { - return store.bufferState(ref).getTypeHandler()->getReservedElements(getBufferId(ref)); - } - size_t get_array_size(EntryRef ref) { - return store.bufferState(ref).getArraySize(); + return store.bufferState(ref).getTypeHandler()->get_reserved_entries(getBufferId(ref)); } }; @@ -309,29 +306,27 @@ TYPED_TEST(TestBase, can_add_and_get_values) } } -TYPED_TEST(TestBase, elements_are_put_on_hold_when_value_is_removed) +TYPED_TEST(TestBase, entries_are_put_on_hold_when_value_is_removed) { EntryRef ref = this->add(this->values()[0]); size_t reserved = this->get_reserved(ref); - size_t array_size = this->get_array_size(ref); - this->assertBufferState(ref, TestBufferStats().used(array_size + reserved).hold(0).dead(reserved)); + this->assertBufferState(ref, TestBufferStats().used(1 + reserved).hold(0).dead(reserved)); this->store.remove(ref); - this->assertBufferState(ref, TestBufferStats().used(array_size + reserved).hold(array_size).dead(reserved)); + this->assertBufferState(ref, TestBufferStats().used(1 + reserved).hold(1).dead(reserved)); } -TYPED_TEST(TestBase, elements_are_reference_counted) +TYPED_TEST(TestBase, entries_are_reference_counted) { EntryRef ref = this->add(this->values()[0]); EntryRef ref2 = this->add(this->values()[0]); EXPECT_EQ(ref.ref(), ref2.ref()); - // Note: The first buffer have the first element reserved -> we expect 2 elements used here. + // Note: The first buffer have the first entry reserved -> we expect 2 entries used here. size_t reserved = this->get_reserved(ref); - size_t array_size = this->get_array_size(ref); - this->assertBufferState(ref, TestBufferStats().used(array_size + reserved).hold(0).dead(reserved)); + this->assertBufferState(ref, TestBufferStats().used(1 + reserved).hold(0).dead(reserved)); this->store.remove(ref); - this->assertBufferState(ref, TestBufferStats().used(array_size + reserved).hold(0).dead(reserved)); + this->assertBufferState(ref, TestBufferStats().used(1 + reserved).hold(0).dead(reserved)); this->store.remove(ref); - this->assertBufferState(ref, TestBufferStats().used(array_size + reserved).hold(array_size).dead(reserved)); + this->assertBufferState(ref, TestBufferStats().used(1 + reserved).hold(1).dead(reserved)); } TEST_F(SmallOffsetNumberTest, new_underlying_buffer_is_allocated_when_current_is_full) @@ -360,8 +355,7 @@ TYPED_TEST(TestBase, store_can_be_compacted) this->remove(this->add(this->values()[2])); this->reclaim_memory(); size_t reserved = this->get_reserved(val0Ref); - size_t array_size = this->get_array_size(val0Ref); - this->assertBufferState(val0Ref, TestBufferStats().used(reserved + 3 * array_size).dead(reserved + array_size)); + this->assertBufferState(val0Ref, TestBufferStats().used(reserved + 3).dead(reserved + 1)); uint32_t val1BufferId = this->getBufferId(val0Ref); EXPECT_EQ(2u, this->refStore.size()); @@ -389,8 +383,7 @@ TYPED_TEST(TestBase, store_can_be_instantiated_with_builder) EntryRef val0Ref = builder.mapEnumValueToEntryRef(1); EntryRef val1Ref = builder.mapEnumValueToEntryRef(2); size_t reserved = this->get_reserved(val0Ref); - size_t array_size = this->get_array_size(val0Ref); - this->assertBufferState(val0Ref, TestBufferStats().used(2 * array_size + reserved).dead(reserved)); // Note: First element is reserved + this->assertBufferState(val0Ref, TestBufferStats().used(2 + reserved).dead(reserved)); // Note: First entry is reserved EXPECT_TRUE(val0Ref.valid()); EXPECT_TRUE(val1Ref.valid()); EXPECT_NE(val0Ref.ref(), val1Ref.ref()); @@ -472,13 +465,13 @@ TEST_F(DoubleTest, nan_is_handled) TEST_F(DoubleTest, control_memory_usage) { static constexpr size_t sizeof_deque = vespalib::datastore::DataStoreBase::sizeof_entry_ref_hold_list_deque; EXPECT_EQ(368u + sizeof_deque, sizeof(store)); - EXPECT_EQ(144u, sizeof(BufferState)); + EXPECT_EQ(120u, sizeof(BufferState)); EXPECT_EQ(28740u, store.get_values_memory_usage().allocatedBytes()); - EXPECT_EQ(24804u, store.get_values_memory_usage().usedBytes()); + EXPECT_EQ(24780u, store.get_values_memory_usage().usedBytes()); EXPECT_EQ(126952u, store.get_dictionary_memory_usage().allocatedBytes()); - EXPECT_EQ(25248u, store.get_dictionary_memory_usage().usedBytes()); + EXPECT_EQ(25200u, store.get_dictionary_memory_usage().usedBytes()); EXPECT_EQ(155692u, store.getMemoryUsage().allocatedBytes()); - EXPECT_EQ(50052, store.getMemoryUsage().usedBytes()); + EXPECT_EQ(49980u, store.getMemoryUsage().usedBytes()); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/datastore/unique_store_string_allocator/unique_store_string_allocator_test.cpp b/vespalib/src/tests/datastore/unique_store_string_allocator/unique_store_string_allocator_test.cpp index 7d4451556c8..8ea7f807f56 100644 --- a/vespalib/src/tests/datastore/unique_store_string_allocator/unique_store_string_allocator_test.cpp +++ b/vespalib/src/tests/datastore/unique_store_string_allocator/unique_store_string_allocator_test.cpp @@ -62,8 +62,8 @@ struct TestBase : public ::testing::Test { void assert_buffer_state(EntryRef ref, const TestBufferStats expStats) { auto & stats = buffer_state(ref).stats(); EXPECT_EQ(expStats._used, buffer_state(ref).size()); - EXPECT_EQ(expStats._hold, stats.hold_elems()); - EXPECT_EQ(expStats._dead, stats.dead_elems()); + EXPECT_EQ(expStats._hold, stats.hold_entries()); + EXPECT_EQ(expStats._dead, stats.dead_entries()); EXPECT_EQ(expStats._extra_used, stats.extra_used_bytes()); EXPECT_EQ(expStats._extra_hold, stats.extra_hold_bytes()); } @@ -83,14 +83,14 @@ TEST_F(StringTest, can_add_and_get_values) assert_add(spaces1000.c_str()); } -TEST_F(StringTest, elements_are_put_on_hold_when_value_is_removed) +TEST_F(StringTest, entries_are_put_on_hold_when_value_is_removed) { EntryRef ref = add(small.c_str()); - assert_buffer_state(ref, TestBufferStats().used(16).hold(0).dead(0)); + assert_buffer_state(ref, TestBufferStats().used(1).hold(0).dead(0)); remove(ref); - assert_buffer_state(ref, TestBufferStats().used(16).hold(16).dead(0)); + assert_buffer_state(ref, TestBufferStats().used(1).hold(1).dead(0)); reclaim_memory(); - assert_buffer_state(ref, TestBufferStats().used(16).hold(0).dead(16)); + assert_buffer_state(ref, TestBufferStats().used(1).hold(0).dead(1)); } TEST_F(StringTest, extra_bytes_used_is_tracked) @@ -139,7 +139,7 @@ TEST_F(StringTest, free_list_is_used_when_enabled) EntryRef ref4 = add(spaces1000.c_str()); EXPECT_EQ(ref1, ref3); EXPECT_EQ(ref2, ref4); - assert_buffer_state(ref1, TestBufferStats().used(16).hold(0).dead(0)); + assert_buffer_state(ref1, TestBufferStats().used(1).hold(0).dead(0)); assert_buffer_state(ref2, TestBufferStats().used(2).hold(0).dead(1).extra_used(1001)); } @@ -155,7 +155,7 @@ TEST_F(StringTest, free_list_is_not_used_when_disabled) EntryRef ref4 = add(spaces1000.c_str()); EXPECT_NE(ref1, ref3); EXPECT_NE(ref2, ref4); - assert_buffer_state(ref1, TestBufferStats().used(32).hold(0).dead(16)); + assert_buffer_state(ref1, TestBufferStats().used(2).hold(0).dead(1)); assert_buffer_state(ref2, TestBufferStats().used(3).hold(0).dead(2).extra_used(1001)); } @@ -173,7 +173,7 @@ TEST_F(StringTest, free_list_is_never_used_for_move_on_compact) EntryRef ref6 = move_on_compact(ref2); EXPECT_NE(ref5, ref3); EXPECT_NE(ref6, ref4); - assert_buffer_state(ref1, TestBufferStats().used(48).hold(0).dead(16)); + assert_buffer_state(ref1, TestBufferStats().used(3).hold(0).dead(1)); assert_buffer_state(ref2, TestBufferStats().used(4).hold(0).dead(2).extra_used(2002)); } diff --git a/vespalib/src/tests/signalhandler/CMakeLists.txt b/vespalib/src/tests/signalhandler/CMakeLists.txt index 4f78eb2e82d..88be14f994f 100644 --- a/vespalib/src/tests/signalhandler/CMakeLists.txt +++ b/vespalib/src/tests/signalhandler/CMakeLists.txt @@ -5,6 +5,11 @@ vespa_add_library(vespalib_signalhandler_test_my_shared_library TEST DEPENDS vespalib ) + +# Don't convert call to jump when returning a value from a function with +# a compatible stack. +set_source_files_properties(my_shared_library.cpp PROPERTIES COMPILE_OPTIONS "-fno-optimize-sibling-calls") + vespa_add_executable(vespalib_signalhandler_test_app TEST SOURCES signalhandler_test.cpp diff --git a/vespalib/src/vespa/vespalib/btree/btree.h b/vespalib/src/vespa/vespalib/btree/btree.h index 32b538b65ec..c2f5aac01b7 100644 --- a/vespalib/src/vespa/vespalib/btree/btree.h +++ b/vespalib/src/vespa/vespalib/btree/btree.h @@ -61,9 +61,9 @@ public: } void - disableElemHoldList() + disable_entry_hold_list() { - _alloc.disableElemHoldList(); + _alloc.disable_entry_hold_list(); } // Inherit doc from BTreeRoot diff --git a/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h b/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h index 784e95e3817..b537602c703 100644 --- a/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h +++ b/vespalib/src/vespa/vespalib/btree/btreenodeallocator.h @@ -60,8 +60,8 @@ public: _nodeStore.disableFreeLists(); } - void disableElemHoldList() { - _nodeStore.disableElemHoldList(); + void disable_entry_hold_list() { + _nodeStore.disable_entry_hold_list(); } /** diff --git a/vespalib/src/vespa/vespalib/btree/btreenodeallocator.hpp b/vespalib/src/vespa/vespalib/btree/btreenodeallocator.hpp index a38b68afe73..d23c8fc2054 100644 --- a/vespalib/src/vespa/vespalib/btree/btreenodeallocator.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreenodeallocator.hpp @@ -162,7 +162,7 @@ holdNode(BTreeNode::Ref nodeRef, InternalNodeType *node) { if (node->getFrozen()) { - _nodeStore.holdElem(nodeRef); + _nodeStore.hold_entry(nodeRef); } else { node->clean(); _internalHoldUntilFreeze.push_back(nodeRef); @@ -178,7 +178,7 @@ holdNode(BTreeNode::Ref nodeRef, LeafNodeType *node) { if (node->getFrozen()) { - _nodeStore.holdElem(nodeRef); + _nodeStore.hold_entry(nodeRef); } else { node->clean(); _leafHoldUntilFreeze.push_back(nodeRef); @@ -235,7 +235,7 @@ freeze() InternalNodeType *inode = mapInternalRef(i); (void) inode; assert(inode->getFrozen()); - _nodeStore.holdElem(i); + _nodeStore.hold_entry(i); } _internalHoldUntilFreeze.clear(); } @@ -245,7 +245,7 @@ freeze() LeafNodeType *lnode = mapLeafRef(i); (void) lnode; assert(lnode->getFrozen()); - _nodeStore.holdElem(i); + _nodeStore.hold_entry(i); } _leafHoldUntilFreeze.clear(); } diff --git a/vespalib/src/vespa/vespalib/btree/btreenodestore.h b/vespalib/src/vespa/vespalib/btree/btreenodestore.h index c59092bf75c..38bf4e5ed4e 100644 --- a/vespalib/src/vespa/vespalib/btree/btreenodestore.h +++ b/vespalib/src/vespa/vespalib/btree/btreenodestore.h @@ -29,16 +29,16 @@ class BTreeNodeBufferType : public datastore::BufferType<EntryType, FrozenBtreeN using ParentType = datastore::BufferType<EntryType, FrozenBtreeNode<EntryType>>; using ParentType::empty_entry; using ParentType::_arraySize; - using ElemCount = typename ParentType::ElemCount; + using EntryCount = typename ParentType::EntryCount; using CleanContext = typename ParentType::CleanContext; public: - BTreeNodeBufferType(uint32_t minArrays, uint32_t maxArrays) - : ParentType(1, minArrays, maxArrays) + BTreeNodeBufferType(uint32_t min_entries, uint32_t max_entries) + : ParentType(1, min_entries, max_entries) { } - void initializeReservedElements(void *buffer, ElemCount reservedElements) override; + void initialize_reserved_entries(void *buffer, EntryCount reserved_entries) override; - void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override; + void clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx) override; }; @@ -79,7 +79,7 @@ public: ~BTreeNodeStore(); void disableFreeLists() { _store.disableFreeLists(); } - void disableElemHoldList() { _store.disableElemHoldList(); } + void disable_entry_hold_list() { _store.disable_entry_hold_list(); } static bool isValidRef(EntryRef ref) { return ref.valid(); } @@ -152,8 +152,8 @@ public: return _store.freeListAllocator<InternalNodeType, BTreeNodeReclaimer>(NODETYPE_INTERNAL).alloc(rhs); } - void holdElem(EntryRef ref) { - _store.holdElem(ref, 1); + void hold_entry(EntryRef ref) { + _store.hold_entry(ref); } std::unique_ptr<vespalib::datastore::CompactingBuffers> start_compact_worst(const CompactionStrategy& compaction_strategy); diff --git a/vespalib/src/vespa/vespalib/btree/btreenodestore.hpp b/vespalib/src/vespa/vespalib/btree/btreenodestore.hpp index a1ffb4d445d..99054f35d61 100644 --- a/vespalib/src/vespa/vespalib/btree/btreenodestore.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreenodestore.hpp @@ -11,11 +11,11 @@ namespace vespalib::btree { template <typename EntryType> void -BTreeNodeBufferType<EntryType>::initializeReservedElements(void *buffer, ElemCount reservedElements) +BTreeNodeBufferType<EntryType>::initialize_reserved_entries(void *buffer, EntryCount reserved_entries) { - ParentType::initializeReservedElements(buffer, reservedElements); + ParentType::initialize_reserved_entries(buffer, reserved_entries); EntryType *e = static_cast<EntryType *>(buffer); - for (size_t j = reservedElements; j != 0; --j) { + for (size_t j = reserved_entries; j != 0; --j) { e->freeze(); ++e; } @@ -24,10 +24,10 @@ BTreeNodeBufferType<EntryType>::initializeReservedElements(void *buffer, ElemCou template <typename EntryType> void -BTreeNodeBufferType<EntryType>::cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext) +BTreeNodeBufferType<EntryType>::clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext) { EntryType *e = static_cast<EntryType *>(buffer) + offset; - for (size_t j = numElems; j != 0; --j) { + for (size_t j = num_entries; j != 0; --j) { e->cleanFrozen(); ++e; } diff --git a/vespalib/src/vespa/vespalib/btree/btreestore.h b/vespalib/src/vespa/vespalib/btree/btreestore.h index 9fdade850f1..7dd839f1529 100644 --- a/vespalib/src/vespa/vespalib/btree/btreestore.h +++ b/vespalib/src/vespa/vespalib/btree/btreestore.h @@ -50,8 +50,8 @@ public: using CompactionSpec = datastore::CompactionSpec; using CompactionStrategy = datastore::CompactionStrategy; using EntryRef = datastore::EntryRef; - template <typename EntryType> - using BufferType = datastore::BufferType<EntryType>; + template <typename ElemT> + using BufferType = datastore::BufferType<ElemT>; using BufferState = datastore::BufferState; static constexpr uint32_t clusterLimit = 8; @@ -105,9 +105,9 @@ public: _allocator.disableFreeLists(); } - void disableElemHoldList() { - _store.disableElemHoldList(); - _allocator.disableElemHoldList(); + void disable_entry_hold_list() { + _store.disable_entry_hold_list(); + _allocator.disable_entry_hold_list(); } BTreeTypeRefPair allocNewBTree() { diff --git a/vespalib/src/vespa/vespalib/btree/btreestore.hpp b/vespalib/src/vespa/vespalib/btree/btreestore.hpp index a19d0b34aa6..90c302af5e4 100644 --- a/vespalib/src/vespa/vespalib/btree/btreestore.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreestore.hpp @@ -74,7 +74,7 @@ allocNewKeyData(uint32_t clusterSize) { assert(clusterSize >= 1 && clusterSize <= clusterLimit); uint32_t typeId = clusterSize - 1; - return _store.allocator<KeyDataType>(typeId).allocArray(clusterSize); + return _store.allocator<KeyDataType>(typeId).allocArray(); } @@ -87,7 +87,7 @@ allocKeyData(uint32_t clusterSize) { assert(clusterSize >= 1 && clusterSize <= clusterLimit); uint32_t typeId = clusterSize - 1; - return _store.freeListAllocator<KeyDataType, datastore::DefaultReclaimer<KeyDataType>>(typeId).allocArray(clusterSize); + return _store.freeListAllocator<KeyDataType, datastore::DefaultReclaimer<KeyDataType>>(typeId).allocArray(); } @@ -156,7 +156,7 @@ makeTree(EntryRef &ref, lNode->freeze(); BTreeTypeRefPair tPair(allocBTree()); tPair.data->setRoots(lPair.ref); - _store.holdElem(ref, clusterSize); + _store.hold_entry(ref); ref = tPair.ref; } @@ -176,7 +176,7 @@ makeArray(EntryRef &ref, EntryRef root, LeafNodeType *leafNode) kd->setData(leafNode->getData(idx)); } assert(kd == kPair.data + clusterSize); - _store.holdElem(ref, 1); + _store.hold_entry(ref); if (!leafNode->getFrozen()) { leafNode->freeze(); } @@ -255,7 +255,7 @@ insert(EntryRef &ref, kd->setData(i->getData()); } assert(kd == kPair.data + clusterSize + 1); - _store.holdElem(ref, clusterSize); + _store.hold_entry(ref); ref = kPair.ref; return true; } @@ -284,7 +284,7 @@ insert(EntryRef &ref, lNode->freeze(); BTreeTypeRefPair tPair(allocBTree()); tPair.data->setRoots(lPair.ref); // allow immediate access to readers - _store.holdElem(ref, clusterSize); + _store.hold_entry(ref); ref = tPair.ref; return true; #endif @@ -339,7 +339,7 @@ remove(EntryRef &ref, if (oldi == olde || comp(key, oldi->_key)) return false; // not found if (clusterSize == 1) { - _store.holdElem(ref, 1); + _store.hold_entry(ref); ref = EntryRef(); return true; } @@ -357,7 +357,7 @@ remove(EntryRef &ref, kd->setData(i->getData()); } assert(kd == kPair.data + clusterSize - 1); - _store.holdElem(ref, clusterSize); + _store.hold_entry(ref); ref = kPair.ref; return true; } @@ -670,7 +670,7 @@ applyCluster(EntryRef &ref, if (newSizeMin <= clusterLimit) { uint32_t newSize = getNewClusterSize(ob, oe, a, ae, r, re, comp); if (newSize == 0) { - _store.holdElem(ref, clusterSize); + _store.hold_entry(ref); ref = EntryRef(); return true; } @@ -678,7 +678,7 @@ applyCluster(EntryRef &ref, KeyDataTypeRefPair kPair(allocKeyData(newSize)); applyCluster(ob, oe, kPair.data, kPair.data + newSize, a, ae, r, re, comp); - _store.holdElem(ref, clusterSize); + _store.hold_entry(ref); ref = kPair.ref; return true; } @@ -735,7 +735,7 @@ normalizeTree(EntryRef &ref, { EntryRef root = tree->getRoot(); if (!NodeAllocatorType::isValidRef(root)) { - _store.holdElem(ref, 1); + _store.hold_entry(ref); ref = EntryRef(); return; } @@ -798,10 +798,8 @@ clear(const EntryRef ref) if (clusterSize == 0) { BTreeType *tree = getWTreeEntry(iRef); tree->clear(_allocator); - _store.holdElem(ref, 1); - } else { - _store.holdElem(ref, clusterSize); } + _store.hold_entry(ref); } diff --git a/vespalib/src/vespa/vespalib/datastore/allocator.h b/vespalib/src/vespa/vespalib/datastore/allocator.h index 297270af0f5..30938bdc1c1 100644 --- a/vespalib/src/vespa/vespalib/datastore/allocator.h +++ b/vespalib/src/vespa/vespalib/datastore/allocator.h @@ -30,7 +30,7 @@ public: HandleType alloc(Args && ... args); HandleType allocArray(ConstArrayRef array); - HandleType allocArray(size_t size); + HandleType allocArray(); }; } diff --git a/vespalib/src/vespa/vespalib/datastore/allocator.hpp b/vespalib/src/vespa/vespalib/datastore/allocator.hpp index 85f0e842519..fa97ef9a5f5 100644 --- a/vespalib/src/vespa/vespalib/datastore/allocator.hpp +++ b/vespalib/src/vespa/vespalib/datastore/allocator.hpp @@ -20,7 +20,7 @@ template <typename ... Args> typename Allocator<EntryT, RefT>::HandleType Allocator<EntryT, RefT>::alloc(Args && ... args) { - _store.ensureBufferCapacity(_typeId, 1); + _store.ensure_buffer_capacity(_typeId, 1); uint32_t buffer_id = _store.primary_buffer_id(_typeId); BufferState &state = _store.getBufferState(buffer_id); assert(state.isActive()); @@ -36,39 +36,35 @@ template <typename EntryT, typename RefT> typename Allocator<EntryT, RefT>::HandleType Allocator<EntryT, RefT>::allocArray(ConstArrayRef array) { - _store.ensureBufferCapacity(_typeId, array.size()); + _store.ensure_buffer_capacity(_typeId, 1); uint32_t buffer_id = _store.primary_buffer_id(_typeId); BufferState &state = _store.getBufferState(buffer_id); assert(state.isActive()); assert(state.getArraySize() == array.size()); - size_t oldBufferSize = state.size(); - assert((oldBufferSize % array.size()) == 0); - RefT ref((oldBufferSize / array.size()), buffer_id); + RefT ref(state.size(), buffer_id); EntryT *buf = _store.template getEntryArray<EntryT>(ref, array.size()); for (size_t i = 0; i < array.size(); ++i) { new (static_cast<void *>(buf + i)) EntryT(array[i]); } - state.stats().pushed_back(array.size()); + state.stats().pushed_back(1); return HandleType(ref, buf); } template <typename EntryT, typename RefT> typename Allocator<EntryT, RefT>::HandleType -Allocator<EntryT, RefT>::allocArray(size_t size) +Allocator<EntryT, RefT>::allocArray() { - _store.ensureBufferCapacity(_typeId, size); + _store.ensure_buffer_capacity(_typeId, 1); uint32_t buffer_id = _store.primary_buffer_id(_typeId); BufferState &state = _store.getBufferState(buffer_id); assert(state.isActive()); - assert(state.getArraySize() == size); - size_t oldBufferSize = state.size(); - assert((oldBufferSize % size) == 0); - RefT ref((oldBufferSize / size), buffer_id); - EntryT *buf = _store.template getEntryArray<EntryT>(ref, size); - for (size_t i = 0; i < size; ++i) { + RefT ref(state.size(), buffer_id); + auto array_size = state.getArraySize(); + EntryT *buf = _store.template getEntryArray<EntryT>(ref, array_size); + for (size_t i = 0; i < array_size; ++i) { new (static_cast<void *>(buf + i)) EntryT(); } - state.stats().pushed_back(size); + state.stats().pushed_back(1); return HandleType(ref, buf); } diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.h b/vespalib/src/vespa/vespalib/datastore/array_store.h index c2b65d72f03..809ac10f6e3 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.h +++ b/vespalib/src/vespa/vespalib/datastore/array_store.h @@ -19,7 +19,7 @@ namespace vespalib::datastore { /** - * Datastore for storing arrays of type EntryT that is accessed via a 32-bit EntryRef. + * Datastore for storing arrays of type ElemT that is accessed via a 32-bit EntryRef. * * The default EntryRef type uses 19 bits for offset (524288 values) and 13 bits for buffer id (8192 buffers). * @@ -29,16 +29,18 @@ namespace vespalib::datastore { * * The max value of maxSmallArrayTypeId is (2^bufferBits - 1). */ -template <typename EntryT, typename RefT = EntryRefT<19>, typename TypeMapperT = ArrayStoreSimpleTypeMapper<EntryT> > +template <typename ElemT, typename RefT = EntryRefT<19>, typename TypeMapperT = ArrayStoreSimpleTypeMapper<ElemT> > class ArrayStore : public ICompactable { public: using AllocSpec = ArrayStoreConfig::AllocSpec; - using ArrayRef = vespalib::ArrayRef<EntryT>; - using ConstArrayRef = vespalib::ConstArrayRef<EntryT>; + using ArrayRef = vespalib::ArrayRef<ElemT>; + using ConstArrayRef = vespalib::ConstArrayRef<ElemT>; using DataStoreType = DataStoreT<RefT>; - using LargeArray = vespalib::Array<EntryT>; + using ElemType = ElemT; + using LargeArray = vespalib::Array<ElemT>; using LargeBufferType = typename TypeMapperT::LargeBufferType; + using RefType = RefT; using SmallBufferType = typename TypeMapperT::SmallBufferType; using TypeMapper = TypeMapperT; private: @@ -58,7 +60,7 @@ private: EntryRef addLargeArray(const ConstArrayRef &array); EntryRef allocate_large_array(size_t array_size); ConstArrayRef getSmallArray(RefT ref, size_t arraySize) const { - const EntryT *buf = _store.template getEntryArray<EntryT>(ref, arraySize); + const ElemT *buf = _store.template getEntryArray<ElemT>(ref, arraySize); return ConstArrayRef(buf, arraySize); } ConstArrayRef getLargeArray(RefT ref) const { @@ -85,7 +87,7 @@ public: } /** - * Allocate an array of the given size without any instantiation of EntryT elements. + * Allocate an array of the given size without any instantiation of ElemT elements. * * Use get_writable() to get a reference to the array for writing. * @@ -148,14 +150,14 @@ public: static ArrayStoreConfig optimizedConfigForHugePage(uint32_t maxSmallArrayTypeId, size_t hugePageSize, size_t smallPageSize, - size_t minNumArraysForNewBuffer, + size_t min_num_entries_for_new_buffer, float allocGrowFactor); static ArrayStoreConfig optimizedConfigForHugePage(uint32_t maxSmallArrayTypeId, const TypeMapper& mapper, size_t hugePageSize, size_t smallPageSize, - size_t minNumArraysForNewBuffer, + size_t min_num_entries_for_new_buffer, float allocGrowFactor); }; diff --git a/vespalib/src/vespa/vespalib/datastore/array_store.hpp b/vespalib/src/vespa/vespalib/datastore/array_store.hpp index 301cff1e414..8e9fe779ba9 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store.hpp +++ b/vespalib/src/vespa/vespalib/datastore/array_store.hpp @@ -15,9 +15,9 @@ namespace vespalib::datastore { -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> void -ArrayStore<EntryT, RefT, TypeMapperT>::initArrayTypes(const ArrayStoreConfig &cfg, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) +ArrayStore<ElemT, RefT, TypeMapperT>::initArrayTypes(const ArrayStoreConfig &cfg, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) { _largeArrayTypeId = _store.addType(&_largeArrayType); assert(_largeArrayTypeId == 0); @@ -31,14 +31,14 @@ ArrayStore<EntryT, RefT, TypeMapperT>::initArrayTypes(const ArrayStoreConfig &cf } } -template <typename EntryT, typename RefT, typename TypeMapperT> -ArrayStore<EntryT, RefT, TypeMapperT>::ArrayStore(const ArrayStoreConfig &cfg, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) +template <typename ElemT, typename RefT, typename TypeMapperT> +ArrayStore<ElemT, RefT, TypeMapperT>::ArrayStore(const ArrayStoreConfig &cfg, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) : ArrayStore(cfg, memory_allocator, TypeMapper()) { } -template <typename EntryT, typename RefT, typename TypeMapperT> -ArrayStore<EntryT, RefT, TypeMapperT>::ArrayStore(const ArrayStoreConfig &cfg, std::shared_ptr<alloc::MemoryAllocator> memory_allocator, +template <typename ElemT, typename RefT, typename TypeMapperT> +ArrayStore<ElemT, RefT, TypeMapperT>::ArrayStore(const ArrayStoreConfig &cfg, std::shared_ptr<alloc::MemoryAllocator> memory_allocator, TypeMapper&& mapper) : _largeArrayTypeId(0), _maxSmallArrayTypeId(cfg.maxSmallArrayTypeId()), @@ -56,25 +56,25 @@ ArrayStore<EntryT, RefT, TypeMapperT>::ArrayStore(const ArrayStoreConfig &cfg, s } } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> vespalib::MemoryUsage -ArrayStore<EntryT, RefT, TypeMapperT>::getMemoryUsage() const { +ArrayStore<ElemT, RefT, TypeMapperT>::getMemoryUsage() const { vespalib::MemoryUsage usage = _store.getMemoryUsage(); usage.incAllocatedBytes(_smallArrayTypes.capacity() * sizeof(SmallBufferType)); usage.incUsedBytes(_smallArrayTypes.size() * sizeof(SmallBufferType)); return usage; } -template <typename EntryT, typename RefT, typename TypeMapperT> -ArrayStore<EntryT, RefT, TypeMapperT>::~ArrayStore() +template <typename ElemT, typename RefT, typename TypeMapperT> +ArrayStore<ElemT, RefT, TypeMapperT>::~ArrayStore() { _store.reclaim_all_memory(); _store.dropBuffers(); } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> EntryRef -ArrayStore<EntryT, RefT, TypeMapperT>::add(const ConstArrayRef &array) +ArrayStore<ElemT, RefT, TypeMapperT>::add(const ConstArrayRef &array) { if (array.size() == 0) { return EntryRef(); @@ -86,9 +86,9 @@ ArrayStore<EntryT, RefT, TypeMapperT>::add(const ConstArrayRef &array) } } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> EntryRef -ArrayStore<EntryT, RefT, TypeMapperT>::allocate(size_t array_size) +ArrayStore<ElemT, RefT, TypeMapperT>::allocate(size_t array_size) { if (array_size == 0) { return EntryRef(); @@ -100,94 +100,93 @@ ArrayStore<EntryT, RefT, TypeMapperT>::allocate(size_t array_size) } } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> EntryRef -ArrayStore<EntryT, RefT, TypeMapperT>::addSmallArray(const ConstArrayRef &array) +ArrayStore<ElemT, RefT, TypeMapperT>::addSmallArray(const ConstArrayRef &array) { uint32_t typeId = _mapper.get_type_id(array.size()); - using NoOpReclaimer = DefaultReclaimer<EntryT>; - return _store.template freeListAllocator<EntryT, NoOpReclaimer>(typeId).allocArray(array).ref; + using NoOpReclaimer = DefaultReclaimer<ElemT>; + return _store.template freeListAllocator<ElemT, NoOpReclaimer>(typeId).allocArray(array).ref; } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> EntryRef -ArrayStore<EntryT, RefT, TypeMapperT>::allocate_small_array(size_t array_size) +ArrayStore<ElemT, RefT, TypeMapperT>::allocate_small_array(size_t array_size) { uint32_t type_id = _mapper.get_type_id(array_size); - return _store.template freeListRawAllocator<EntryT>(type_id).alloc(array_size).ref; + return _store.template freeListRawAllocator<ElemT>(type_id).alloc(1).ref; } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> EntryRef -ArrayStore<EntryT, RefT, TypeMapperT>::addLargeArray(const ConstArrayRef &array) +ArrayStore<ElemT, RefT, TypeMapperT>::addLargeArray(const ConstArrayRef &array) { using NoOpReclaimer = DefaultReclaimer<LargeArray>; auto handle = _store.template freeListAllocator<LargeArray, NoOpReclaimer>(_largeArrayTypeId) .alloc(array.cbegin(), array.cend()); auto& state = _store.getBufferState(RefT(handle.ref).bufferId()); - state.stats().inc_extra_used_bytes(sizeof(EntryT) * array.size()); + state.stats().inc_extra_used_bytes(sizeof(ElemT) * array.size()); return handle.ref; } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> EntryRef -ArrayStore<EntryT, RefT, TypeMapperT>::allocate_large_array(size_t array_size) +ArrayStore<ElemT, RefT, TypeMapperT>::allocate_large_array(size_t array_size) { using NoOpReclaimer = DefaultReclaimer<LargeArray>; auto handle = _store.template freeListAllocator<LargeArray, NoOpReclaimer>(_largeArrayTypeId).alloc(array_size); auto& state = _store.getBufferState(RefT(handle.ref).bufferId()); - state.stats().inc_extra_used_bytes(sizeof(EntryT) * array_size); + state.stats().inc_extra_used_bytes(sizeof(ElemT) * array_size); return handle.ref; } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> void -ArrayStore<EntryT, RefT, TypeMapperT>::remove(EntryRef ref) +ArrayStore<ElemT, RefT, TypeMapperT>::remove(EntryRef ref) { if (ref.valid()) { RefT internalRef(ref); uint32_t typeId = _store.getTypeId(internalRef.bufferId()); if (typeId != _largeArrayTypeId) { - size_t arraySize = _mapper.get_array_size(typeId); - _store.holdElem(ref, arraySize); + _store.hold_entry(ref); } else { - _store.holdElem(ref, 1, sizeof(EntryT) * get(ref).size()); + _store.hold_entry(ref, sizeof(ElemT) * get(ref).size()); } } } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> EntryRef -ArrayStore<EntryT, RefT, TypeMapperT>::move_on_compact(EntryRef ref) +ArrayStore<ElemT, RefT, TypeMapperT>::move_on_compact(EntryRef ref) { return add(get(ref)); } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> ICompactionContext::UP -ArrayStore<EntryT, RefT, TypeMapperT>::compact_worst(const CompactionStrategy &compaction_strategy) +ArrayStore<ElemT, RefT, TypeMapperT>::compact_worst(const CompactionStrategy &compaction_strategy) { auto compacting_buffers = _store.start_compact_worst_buffers(_compaction_spec, compaction_strategy); return std::make_unique<CompactionContext>(*this, std::move(compacting_buffers)); } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> std::unique_ptr<CompactingBuffers> -ArrayStore<EntryT, RefT, TypeMapperT>::start_compact_worst_buffers(const CompactionStrategy &compaction_strategy) +ArrayStore<ElemT, RefT, TypeMapperT>::start_compact_worst_buffers(const CompactionStrategy &compaction_strategy) { return _store.start_compact_worst_buffers(_compaction_spec, compaction_strategy); } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> vespalib::AddressSpace -ArrayStore<EntryT, RefT, TypeMapperT>::addressSpaceUsage() const +ArrayStore<ElemT, RefT, TypeMapperT>::addressSpaceUsage() const { return _store.getAddressSpaceUsage(); } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> vespalib::MemoryUsage -ArrayStore<EntryT, RefT, TypeMapperT>::update_stat(const CompactionStrategy& compaction_strategy) +ArrayStore<ElemT, RefT, TypeMapperT>::update_stat(const CompactionStrategy& compaction_strategy) { auto address_space_usage = _store.getAddressSpaceUsage(); auto memory_usage = getMemoryUsage(); @@ -195,20 +194,20 @@ ArrayStore<EntryT, RefT, TypeMapperT>::update_stat(const CompactionStrategy& com return memory_usage; } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> const BufferState & -ArrayStore<EntryT, RefT, TypeMapperT>::bufferState(EntryRef ref) +ArrayStore<ElemT, RefT, TypeMapperT>::bufferState(EntryRef ref) { RefT internalRef(ref); return _store.getBufferState(internalRef.bufferId()); } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> ArrayStoreConfig -ArrayStore<EntryT, RefT, TypeMapperT>::optimizedConfigForHugePage(uint32_t maxSmallArrayTypeId, +ArrayStore<ElemT, RefT, TypeMapperT>::optimizedConfigForHugePage(uint32_t maxSmallArrayTypeId, size_t hugePageSize, size_t smallPageSize, - size_t minNumArraysForNewBuffer, + size_t min_num_entries_for_new_buffer, float allocGrowFactor) { TypeMapper mapper; @@ -216,26 +215,26 @@ ArrayStore<EntryT, RefT, TypeMapperT>::optimizedConfigForHugePage(uint32_t maxSm mapper, hugePageSize, smallPageSize, - minNumArraysForNewBuffer, + min_num_entries_for_new_buffer, allocGrowFactor); } -template <typename EntryT, typename RefT, typename TypeMapperT> +template <typename ElemT, typename RefT, typename TypeMapperT> ArrayStoreConfig -ArrayStore<EntryT, RefT, TypeMapperT>::optimizedConfigForHugePage(uint32_t maxSmallArrayTypeId, +ArrayStore<ElemT, RefT, TypeMapperT>::optimizedConfigForHugePage(uint32_t maxSmallArrayTypeId, const TypeMapper& mapper, size_t hugePageSize, size_t smallPageSize, - size_t minNumArraysForNewBuffer, + size_t min_num_entries_for_new_buffer, float allocGrowFactor) { return ArrayStoreConfig::optimizeForHugePage(mapper.get_max_small_array_type_id(maxSmallArrayTypeId), [&](uint32_t type_id) noexcept { return mapper.get_array_size(type_id); }, hugePageSize, smallPageSize, - sizeof(EntryT), + sizeof(ElemT), RefT::offsetSize(), - minNumArraysForNewBuffer, + min_num_entries_for_new_buffer, allocGrowFactor); } diff --git a/vespalib/src/vespa/vespalib/datastore/array_store_config.cpp b/vespalib/src/vespa/vespalib/datastore/array_store_config.cpp index d5587841745..1df9354cd6c 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store_config.cpp +++ b/vespalib/src/vespa/vespalib/datastore/array_store_config.cpp @@ -49,19 +49,19 @@ ArrayStoreConfig::optimizeForHugePage(uint32_t maxSmallArrayTypeId, std::function<size_t(uint32_t)> type_id_to_array_size, size_t hugePageSize, size_t smallPageSize, - size_t entrySize, + size_t elem_size, size_t maxEntryRefOffset, - size_t minNumArraysForNewBuffer, + size_t min_num_entries_for_new_buffer, float allocGrowFactor) { AllocSpecVector allocSpecs; - allocSpecs.emplace_back(0, maxEntryRefOffset, minNumArraysForNewBuffer, allocGrowFactor); // large array spec; + allocSpecs.emplace_back(0, maxEntryRefOffset, min_num_entries_for_new_buffer, allocGrowFactor); // large array spec; for (uint32_t type_id = 1; type_id <= maxSmallArrayTypeId; ++type_id) { size_t arraySize = type_id_to_array_size(type_id); - size_t numArraysForNewBuffer = hugePageSize / (entrySize * arraySize); - numArraysForNewBuffer = capToLimits(numArraysForNewBuffer, minNumArraysForNewBuffer, maxEntryRefOffset); - numArraysForNewBuffer = alignToSmallPageSize(numArraysForNewBuffer, minNumArraysForNewBuffer, smallPageSize); - allocSpecs.emplace_back(0, maxEntryRefOffset, numArraysForNewBuffer, allocGrowFactor); + size_t num_entries_for_new_buffer = hugePageSize / (elem_size * arraySize); + num_entries_for_new_buffer = capToLimits(num_entries_for_new_buffer, min_num_entries_for_new_buffer, maxEntryRefOffset); + num_entries_for_new_buffer = alignToSmallPageSize(num_entries_for_new_buffer, min_num_entries_for_new_buffer, smallPageSize); + allocSpecs.emplace_back(0, maxEntryRefOffset, num_entries_for_new_buffer, allocGrowFactor); } return ArrayStoreConfig(allocSpecs); } diff --git a/vespalib/src/vespa/vespalib/datastore/array_store_config.h b/vespalib/src/vespa/vespalib/datastore/array_store_config.h index a326c00d042..cae241dba10 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store_config.h +++ b/vespalib/src/vespa/vespalib/datastore/array_store_config.h @@ -19,21 +19,21 @@ public: * Specification of buffer allocation strategy for arrays of a given size. */ struct AllocSpec { - // Minimum number of arrays to allocate in a buffer. - size_t minArraysInBuffer; - // Maximum number of arrays to allocate in a buffer. - size_t maxArraysInBuffer; - // Number of arrays needed before allocating a new buffer instead of just resizing the first one. - size_t numArraysForNewBuffer; + // Minimum number of entries to allocate in a buffer. + size_t min_entries_in_buffer; + // Maximum number of entries to allocate in a buffer. + size_t max_entries_in_buffer; + // Number of entries needed before allocating a new buffer instead of just resizing the first one. + size_t num_entries_for_new_buffer; // Grow factor used when allocating a new buffer. float allocGrowFactor; - AllocSpec(size_t minArraysInBuffer_, - size_t maxArraysInBuffer_, - size_t numArraysForNewBuffer_, + AllocSpec(size_t min_entries_in_buffer_, + size_t max_entries_in_buffer_, + size_t num_entries_for_new_buffer_, float allocGrowFactor_) noexcept - : minArraysInBuffer(minArraysInBuffer_), - maxArraysInBuffer(maxArraysInBuffer_), - numArraysForNewBuffer(numArraysForNewBuffer_), + : min_entries_in_buffer(min_entries_in_buffer_), + max_entries_in_buffer(max_entries_in_buffer_), + num_entries_for_new_buffer(num_entries_for_new_buffer_), allocGrowFactor(allocGrowFactor_) {} }; @@ -76,9 +76,9 @@ public: std::function<size_t(uint32_t)> type_id_to_array_size, size_t hugePageSize, size_t smallPageSize, - size_t entrySize, + size_t elem_size, size_t maxEntryRefOffset, - size_t minNumArraysForNewBuffer, + size_t min_num_entries_for_new_buffer, float allocGrowFactor); }; diff --git a/vespalib/src/vespa/vespalib/datastore/array_store_simple_type_mapper.h b/vespalib/src/vespa/vespalib/datastore/array_store_simple_type_mapper.h index a0cd7827b2d..5f9728a43cc 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store_simple_type_mapper.h +++ b/vespalib/src/vespa/vespalib/datastore/array_store_simple_type_mapper.h @@ -15,11 +15,11 @@ namespace vespalib::datastore { * * A more complex mapping can be used by creating a custom mapper and BufferType implementations. */ -template <typename EntryT> +template <typename ElemT> class ArrayStoreSimpleTypeMapper { public: - using SmallBufferType = SmallArrayBufferType<EntryT>; - using LargeBufferType = LargeArrayBufferType<EntryT>; + using SmallBufferType = SmallArrayBufferType<ElemT>; + using LargeBufferType = LargeArrayBufferType<ElemT>; uint32_t get_type_id(size_t array_size) const { return array_size; } size_t get_array_size(uint32_t type_id) const { return type_id; } diff --git a/vespalib/src/vespa/vespalib/datastore/array_store_type_mapper.h b/vespalib/src/vespa/vespalib/datastore/array_store_type_mapper.h index e707627de19..2a406a39bf9 100644 --- a/vespalib/src/vespa/vespalib/datastore/array_store_type_mapper.h +++ b/vespalib/src/vespa/vespalib/datastore/array_store_type_mapper.h @@ -12,7 +12,7 @@ namespace vespalib::datastore { * This class provides mapping between type ids and array sizes needed for * storing a value with size smaller than or equal to the array size. * - * The array sizes vector is a monotic increasing sequence that might end + * The array sizes vector is a monotonic strictly increasing sequence that might end * with exponential growth. */ class ArrayStoreTypeMapper diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_free_list.cpp b/vespalib/src/vespa/vespalib/datastore/buffer_free_list.cpp index 224ed4b0c8f..851db0222c2 100644 --- a/vespalib/src/vespa/vespalib/datastore/buffer_free_list.cpp +++ b/vespalib/src/vespa/vespalib/datastore/buffer_free_list.cpp @@ -20,9 +20,8 @@ BufferFreeList::detach() _free_list->detach(*this); } -BufferFreeList::BufferFreeList(std::atomic<ElemCount>& dead_elems) - : _dead_elems(dead_elems), - _array_size(0), +BufferFreeList::BufferFreeList(std::atomic<EntryCount>& dead_entries) + : _dead_entries(dead_entries), _free_list(), _free_refs() { @@ -66,7 +65,7 @@ BufferFreeList::pop_entry() { if (empty()) { detach(); } - _dead_elems.store(_dead_elems.load(std::memory_order_relaxed) - _array_size, std::memory_order_relaxed); + _dead_entries.store(_dead_entries.load(std::memory_order_relaxed) - 1, std::memory_order_relaxed); return ret; } diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_free_list.h b/vespalib/src/vespa/vespalib/datastore/buffer_free_list.h index 148ddd8db88..4348a41af04 100644 --- a/vespalib/src/vespa/vespalib/datastore/buffer_free_list.h +++ b/vespalib/src/vespa/vespalib/datastore/buffer_free_list.h @@ -19,8 +19,7 @@ class BufferFreeList { private: using EntryRefArray = vespalib::Array<EntryRef>; - std::atomic<ElemCount>& _dead_elems; - uint32_t _array_size; + std::atomic<EntryCount>& _dead_entries; FreeList* _free_list; EntryRefArray _free_refs; @@ -28,7 +27,7 @@ private: void detach(); public: - BufferFreeList(std::atomic<ElemCount>& dead_elems); + BufferFreeList(std::atomic<EntryCount>& dead_entrie); ~BufferFreeList(); BufferFreeList(BufferFreeList&&) = default; // Needed for emplace_back() during setup. BufferFreeList(const BufferFreeList&) = delete; @@ -37,10 +36,8 @@ public: void enable(FreeList& free_list); void disable(); - void set_array_size(uint32_t value) { _array_size = value; } bool enabled() const { return _free_list != nullptr; } bool empty() const { return _free_refs.empty(); } - uint32_t array_size() const { return _array_size; } void push_entry(EntryRef ref); EntryRef pop_entry(); }; diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_stats.cpp b/vespalib/src/vespa/vespalib/datastore/buffer_stats.cpp index 8d97414626e..0d96e3f6d47 100644 --- a/vespalib/src/vespa/vespalib/datastore/buffer_stats.cpp +++ b/vespalib/src/vespa/vespalib/datastore/buffer_stats.cpp @@ -6,27 +6,27 @@ namespace vespalib::datastore { BufferStats::BufferStats() - : _alloc_elems(0), - _used_elems(0), - _hold_elems(0), - _dead_elems(0), + : _alloc_entries(0), + _used_entries(0), + _hold_entries(0), + _dead_entries(0), _extra_used_bytes(0), _extra_hold_bytes(0) { } void -BufferStats::add_to_mem_stats(size_t element_size, MemoryStats& stats) const +BufferStats::add_to_mem_stats(size_t entry_size, MemoryStats& stats) const { size_t extra_used = extra_used_bytes(); - stats._allocElems += capacity(); - stats._usedElems += size(); - stats._deadElems += dead_elems(); - stats._holdElems += hold_elems(); - stats._allocBytes += (capacity() * element_size) + extra_used; - stats._usedBytes += (size() * element_size) + extra_used; - stats._deadBytes += dead_elems() * element_size; - stats._holdBytes += (hold_elems() * element_size) + extra_hold_bytes(); + stats._alloc_entries += capacity(); + stats._used_entries += size(); + stats._dead_entries += dead_entries(); + stats._hold_entries += hold_entries(); + stats._allocBytes += (capacity() * entry_size) + extra_used; + stats._usedBytes += (size() * entry_size) + extra_used; + stats._deadBytes += dead_entries() * entry_size; + stats._holdBytes += (hold_entries() * entry_size) + extra_hold_bytes(); } InternalBufferStats::InternalBufferStats() @@ -37,20 +37,20 @@ InternalBufferStats::InternalBufferStats() void InternalBufferStats::clear() { - _alloc_elems.store(0, std::memory_order_relaxed); - _used_elems.store(0, std::memory_order_relaxed); - _hold_elems.store(0, std::memory_order_relaxed); - _dead_elems.store(0, std::memory_order_relaxed); + _alloc_entries.store(0, std::memory_order_relaxed); + _used_entries.store(0, std::memory_order_relaxed); + _hold_entries.store(0, std::memory_order_relaxed); + _dead_entries.store(0, std::memory_order_relaxed); _extra_used_bytes.store(0, std::memory_order_relaxed); _extra_hold_bytes.store(0, std::memory_order_relaxed); } void -InternalBufferStats::dec_hold_elems(size_t value) +InternalBufferStats::dec_hold_entries(size_t value) { - ElemCount elems = hold_elems(); + EntryCount elems = hold_entries(); assert(elems >= value); - _hold_elems.store(elems - value, std::memory_order_relaxed); + _hold_entries.store(elems - value, std::memory_order_relaxed); } } diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_stats.h b/vespalib/src/vespa/vespalib/datastore/buffer_stats.h index 66f8b532c41..1974efa97ec 100644 --- a/vespalib/src/vespa/vespalib/datastore/buffer_stats.h +++ b/vespalib/src/vespa/vespalib/datastore/buffer_stats.h @@ -13,16 +13,16 @@ namespace vespalib::datastore { */ class BufferStats { protected: - // The number of elements that are allocated in the buffer. - std::atomic<ElemCount> _alloc_elems; - // The number of elements (of the allocated) that are used: _used_elems <= _alloc_elems. - std::atomic<ElemCount> _used_elems; - // The number of elements (of the used) that are on hold: _hold_elems <= _used_elems. - // "On hold" is a transitionary state used when removing elements. - std::atomic<ElemCount> _hold_elems; - // The number of elements (of the used) that are dead: _dead_elems <= _used_elems. - // A dead element was first on hold, and is now available for reuse in the free list (if enabled). - std::atomic<ElemCount> _dead_elems; + // The number of entries that are allocated in the buffer. + std::atomic<EntryCount> _alloc_entries; + // The number of entries (of the allocated) that are used: _used_entries <= _alloc_entries. + std::atomic<EntryCount> _used_entries; + // The number of entries (of the used) that are on hold: _hold_entries <= _used_entries. + // "On hold" is a transitionary state used when removing entries. + std::atomic<EntryCount> _hold_entries; + // The number of entries (of the used) that are dead: _dead_entries <= _used_entries. + // A dead entry was first on hold, and is now available for reuse in the free list (if enabled). + std::atomic<EntryCount> _dead_entries; // Number of bytes that are heap allocated (and used) by elements that are stored in this buffer. // For simple types this is always 0. @@ -34,22 +34,22 @@ protected: public: BufferStats(); - size_t size() const { return _used_elems.load(std::memory_order_relaxed); } - size_t capacity() const { return _alloc_elems.load(std::memory_order_relaxed); } + size_t size() const { return _used_entries.load(std::memory_order_relaxed); } + size_t capacity() const { return _alloc_entries.load(std::memory_order_relaxed); } size_t remaining() const { return capacity() - size(); } - void pushed_back(size_t num_elems) { - _used_elems.store(size() + num_elems, std::memory_order_relaxed); + void pushed_back(size_t num_entries) { + _used_entries.store(size() + num_entries, std::memory_order_relaxed); } - size_t dead_elems() const { return _dead_elems.load(std::memory_order_relaxed); } - size_t hold_elems() const { return _hold_elems.load(std::memory_order_relaxed); } + size_t dead_entries() const { return _dead_entries.load(std::memory_order_relaxed); } + size_t hold_entries() const { return _hold_entries.load(std::memory_order_relaxed); } size_t extra_used_bytes() const { return _extra_used_bytes.load(std::memory_order_relaxed); } size_t extra_hold_bytes() const { return _extra_hold_bytes.load(std::memory_order_relaxed); } void inc_extra_used_bytes(size_t value) { _extra_used_bytes.store(extra_used_bytes() + value, std::memory_order_relaxed); } - void add_to_mem_stats(size_t element_size, MemoryStats& stats) const; + void add_to_mem_stats(size_t entry_size, MemoryStats& stats) const; }; /** @@ -59,15 +59,15 @@ class InternalBufferStats : public BufferStats { public: InternalBufferStats(); void clear(); - void set_alloc_elems(size_t value) { _alloc_elems.store(value, std::memory_order_relaxed); } - void set_dead_elems(size_t value) { _dead_elems.store(value, std::memory_order_relaxed); } - void set_hold_elems(size_t value) { _hold_elems.store(value, std::memory_order_relaxed); } - void inc_dead_elems(size_t value) { _dead_elems.store(dead_elems() + value, std::memory_order_relaxed); } - void inc_hold_elems(size_t value) { _hold_elems.store(hold_elems() + value, std::memory_order_relaxed); } - void dec_hold_elems(size_t value); + void set_alloc_entries(size_t value) { _alloc_entries.store(value, std::memory_order_relaxed); } + void set_dead_entries(size_t value) { _dead_entries.store(value, std::memory_order_relaxed); } + void set_hold_entries(size_t value) { _hold_entries.store(value, std::memory_order_relaxed); } + void inc_dead_entries(size_t value) { _dead_entries.store(dead_entries() + value, std::memory_order_relaxed); } + void inc_hold_entries(size_t value) { _hold_entries.store(hold_entries() + value, std::memory_order_relaxed); } + void dec_hold_entries(size_t value); void inc_extra_hold_bytes(size_t value) { _extra_hold_bytes.store(extra_hold_bytes() + value, std::memory_order_relaxed); } - std::atomic<ElemCount>& used_elems_ref() { return _used_elems; } - std::atomic<ElemCount>& dead_elems_ref() { return _dead_elems; } + std::atomic<EntryCount>& used_entries_ref() { return _used_entries; } + std::atomic<EntryCount>& dead_entries_ref() { return _dead_entries; } std::atomic<size_t>& extra_used_bytes_ref() { return _extra_used_bytes; } std::atomic<size_t>& extra_hold_bytes_ref() { return _extra_hold_bytes; } }; diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp b/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp index 4a9ba2d33a8..0d43ede9e62 100644 --- a/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp +++ b/vespalib/src/vespa/vespalib/datastore/buffer_type.cpp @@ -27,85 +27,85 @@ BufferTypeBase::CleanContext::extraBytesCleaned(size_t value) } BufferTypeBase::BufferTypeBase(uint32_t arraySize, - uint32_t minArrays, - uint32_t maxArrays, - uint32_t numArraysForNewBuffer, + uint32_t min_entries, + uint32_t max_entries, + uint32_t num_entries_for_new_buffer, float allocGrowFactor) noexcept : _arraySize(arraySize), - _minArrays(std::min(minArrays, maxArrays)), - _maxArrays(maxArrays), - _numArraysForNewBuffer(std::min(numArraysForNewBuffer, maxArrays)), + _min_entries(std::min(min_entries, max_entries)), + _max_entries(max_entries), + _num_entries_for_new_buffer(std::min(num_entries_for_new_buffer, max_entries)), _allocGrowFactor(allocGrowFactor), _holdBuffers(0), - _holdUsedElems(0), + _hold_used_entries(0), _aggr_counts(), _active_buffers() { } BufferTypeBase::BufferTypeBase(uint32_t arraySize, - uint32_t minArrays, - uint32_t maxArrays) noexcept - : BufferTypeBase(arraySize, minArrays, maxArrays, 0u, DEFAULT_ALLOC_GROW_FACTOR) + uint32_t min_entries, + uint32_t max_entries) noexcept + : BufferTypeBase(arraySize, min_entries, max_entries, 0u, DEFAULT_ALLOC_GROW_FACTOR) { } BufferTypeBase::~BufferTypeBase() { assert(_holdBuffers == 0); - assert(_holdUsedElems == 0); + assert(_hold_used_entries == 0); assert(_aggr_counts.empty()); assert(_active_buffers.empty()); } -ElemCount -BufferTypeBase::getReservedElements(uint32_t bufferId) const +EntryCount +BufferTypeBase::get_reserved_entries(uint32_t bufferId) const { - return bufferId == 0 ? _arraySize : 0u; + return bufferId == 0 ? 1u : 0u; } void -BufferTypeBase::onActive(uint32_t bufferId, std::atomic<ElemCount>* usedElems, std::atomic<ElemCount>* deadElems, void* buffer) +BufferTypeBase::on_active(uint32_t bufferId, std::atomic<EntryCount>* used_entries, std::atomic<EntryCount>* dead_entries, void* buffer) { - _aggr_counts.add_buffer(usedElems, deadElems); + _aggr_counts.add_buffer(used_entries, dead_entries); assert(std::find(_active_buffers.begin(), _active_buffers.end(), bufferId) == _active_buffers.end()); _active_buffers.emplace_back(bufferId); - size_t reservedElems = getReservedElements(bufferId); - if (reservedElems != 0u) { - initializeReservedElements(buffer, reservedElems); - *usedElems = reservedElems; - *deadElems = reservedElems; + auto reserved_entries = get_reserved_entries(bufferId); + if (reserved_entries != 0u) { + initialize_reserved_entries(buffer, reserved_entries); + *used_entries = reserved_entries; + *dead_entries = reserved_entries; } } void -BufferTypeBase::onHold(uint32_t buffer_id, const std::atomic<ElemCount>* usedElems, const std::atomic<ElemCount>* deadElems) +BufferTypeBase::on_hold(uint32_t buffer_id, const std::atomic<EntryCount>* used_entries, const std::atomic<EntryCount>* dead_entries) { ++_holdBuffers; auto itr = std::find(_active_buffers.begin(), _active_buffers.end(), buffer_id); assert(itr != _active_buffers.end()); _active_buffers.erase(itr); - _aggr_counts.remove_buffer(usedElems, deadElems); - _holdUsedElems += *usedElems; + _aggr_counts.remove_buffer(used_entries, dead_entries); + _hold_used_entries += *used_entries; } void -BufferTypeBase::onFree(ElemCount usedElems) +BufferTypeBase::on_free(EntryCount used_entries) { --_holdBuffers; - assert(_holdUsedElems >= usedElems); - _holdUsedElems -= usedElems; + assert(_hold_used_entries >= used_entries); + _hold_used_entries -= used_entries; } void -BufferTypeBase::resume_primary_buffer(uint32_t buffer_id, std::atomic<ElemCount>* used_elems, std::atomic<ElemCount>* dead_elems) +BufferTypeBase::resume_primary_buffer(uint32_t buffer_id, std::atomic<EntryCount>* used_entries, std::atomic<EntryCount>* dead_entries) { auto itr = std::find(_active_buffers.begin(), _active_buffers.end(), buffer_id); assert(itr != _active_buffers.end()); _active_buffers.erase(itr); _active_buffers.emplace_back(buffer_id); - _aggr_counts.remove_buffer(used_elems, dead_elems); - _aggr_counts.add_buffer(used_elems, dead_elems); + _aggr_counts.remove_buffer(used_entries, dead_entries); + _aggr_counts.add_buffer(used_entries, dead_entries); } const alloc::MemoryAllocator* @@ -115,17 +115,17 @@ BufferTypeBase::get_memory_allocator() const } void -BufferTypeBase::clampMaxArrays(uint32_t maxArrays) +BufferTypeBase::clamp_max_entries(uint32_t max_entries) { - _maxArrays = std::min(_maxArrays, maxArrays); - _minArrays = std::min(_minArrays, _maxArrays); - _numArraysForNewBuffer = std::min(_numArraysForNewBuffer, _maxArrays); + _max_entries = std::min(_max_entries, max_entries); + _min_entries = std::min(_min_entries, _max_entries); + _num_entries_for_new_buffer = std::min(_num_entries_for_new_buffer, _max_entries); } size_t -BufferTypeBase::calcArraysToAlloc(uint32_t bufferId, ElemCount elemsNeeded, bool resizing) const +BufferTypeBase::calc_entries_to_alloc(uint32_t bufferId, EntryCount free_entries_needed, bool resizing) const { - size_t reservedElems = getReservedElements(bufferId); + size_t reserved_entries = get_reserved_entries(bufferId); BufferCounts last_bc; BufferCounts bc; if (resizing) { @@ -134,56 +134,53 @@ BufferTypeBase::calcArraysToAlloc(uint32_t bufferId, ElemCount elemsNeeded, bool } } bc = _aggr_counts.all_buffers(); - assert((bc.used_elems % _arraySize) == 0); - assert((bc.dead_elems % _arraySize) == 0); - assert(bc.used_elems >= bc.dead_elems); - size_t neededArrays = (elemsNeeded + (resizing ? last_bc.used_elems : reservedElems) + _arraySize - 1) / _arraySize; - - size_t liveArrays = (bc.used_elems - bc.dead_elems) / _arraySize; - size_t growArrays = (liveArrays * _allocGrowFactor); - size_t usedArrays = last_bc.used_elems / _arraySize; - size_t wantedArrays = std::max((resizing ? usedArrays : 0u) + growArrays, - static_cast<size_t>(_minArrays)); - - size_t result = wantedArrays; - if (result < neededArrays) { - result = neededArrays; + assert(bc.used_entries >= bc.dead_entries); + size_t needed_entries = static_cast<size_t>(free_entries_needed) + (resizing ? last_bc.used_entries : reserved_entries); + size_t live_entries = (bc.used_entries - bc.dead_entries); + size_t grow_entries = (live_entries * _allocGrowFactor); + size_t used_entries = last_bc.used_entries; + size_t wanted_entries = std::max((resizing ? used_entries : 0u) + grow_entries, + static_cast<size_t>(_min_entries)); + + size_t result = wanted_entries; + if (result < needed_entries) { + result = needed_entries; } - if (result > _maxArrays) { - result = _maxArrays; + if (result > _max_entries) { + result = _max_entries; } - if (result < neededArrays) { + if (result < needed_entries) { vespalib::asciistream s; s << "BufferTypeBase::calcArraysToAlloc(" << "bufferId=" << bufferId << - ",elemsNeeeded=" << elemsNeeded << + ",free_entries_needed=" << free_entries_needed << ",resizing=" << (resizing ? "true" : "false") << ")" << - " wantedArrays=" << wantedArrays << + " wanted_entries=" << wanted_entries << ", _arraySize=" << _arraySize << - ", _maxArrays=" << _maxArrays << - ", reservedElems=" << reservedElems << - ", liveArrays=" << liveArrays << - ", growArrays=" << growArrays << - ", usedArrays=" << usedArrays << + ", _max_entries=" << _max_entries << + ", reserved_entries=" << reserved_entries << + ", live_entries=" << live_entries << + ", grow_entries=" << grow_entries << + ", used_entries=" << used_entries << ", typeid(*this).name=\"" << typeid(*this).name() << "\"" << - ", newArrays=" << result << - " < neededArrays=" << neededArrays;; + ", new_entries=" << result << + " < needed_entries=" << needed_entries; throw vespalib::OverflowException(s.c_str()); } return result; } uint32_t -BufferTypeBase::get_scaled_num_arrays_for_new_buffer() const +BufferTypeBase::get_scaled_num_entries_for_new_buffer() const { uint32_t active_buffers_count = get_active_buffers_count(); - if (active_buffers_count <= 1u || _numArraysForNewBuffer == 0u) { - return _numArraysForNewBuffer; + if (active_buffers_count <= 1u || _num_entries_for_new_buffer == 0u) { + return _num_entries_for_new_buffer; } double scale_factor = std::pow(1.0 + _allocGrowFactor, active_buffers_count - 1); - double scaled_result = _numArraysForNewBuffer * scale_factor; - if (scaled_result >= _maxArrays) { - return _maxArrays; + double scaled_result = _num_entries_for_new_buffer * scale_factor; + if (scaled_result >= _max_entries) { + return _max_entries; } return scaled_result; } @@ -194,22 +191,22 @@ BufferTypeBase::AggregatedBufferCounts::AggregatedBufferCounts() } void -BufferTypeBase::AggregatedBufferCounts::add_buffer(const std::atomic<ElemCount>* used_elems, const std::atomic<ElemCount>* dead_elems) +BufferTypeBase::AggregatedBufferCounts::add_buffer(const std::atomic<EntryCount>* used_entries, const std::atomic<EntryCount>* dead_entries) { for (const auto& elem : _counts) { - assert(elem.used_ptr != used_elems); - assert(elem.dead_ptr != dead_elems); + assert(elem.used_ptr != used_entries); + assert(elem.dead_ptr != dead_entries); } - _counts.emplace_back(used_elems, dead_elems); + _counts.emplace_back(used_entries, dead_entries); } void -BufferTypeBase::AggregatedBufferCounts::remove_buffer(const std::atomic<ElemCount>* used_elems, const std::atomic<ElemCount>* dead_elems) +BufferTypeBase::AggregatedBufferCounts::remove_buffer(const std::atomic<EntryCount>* used_entries, const std::atomic<EntryCount>* dead_entries) { auto itr = std::find_if(_counts.begin(), _counts.end(), - [=](const auto& elem){ return elem.used_ptr == used_elems; }); + [=](const auto& elem){ return elem.used_ptr == used_entries; }); assert(itr != _counts.end()); - assert(itr->dead_ptr == dead_elems); + assert(itr->dead_ptr == dead_entries); _counts.erase(itr); } @@ -219,8 +216,8 @@ BufferTypeBase::AggregatedBufferCounts::last_buffer() const BufferCounts result; assert(!_counts.empty()); const auto& last = _counts.back(); - result.used_elems += last.used_ptr->load(std::memory_order_relaxed); - result.dead_elems += last.dead_ptr->load(std::memory_order_relaxed); + result.used_entries += last.used_ptr->load(std::memory_order_relaxed); + result.dead_entries += last.dead_ptr->load(std::memory_order_relaxed); return result; } @@ -229,8 +226,8 @@ BufferTypeBase::AggregatedBufferCounts::all_buffers() const { BufferCounts result; for (const auto& elem : _counts) { - result.used_elems += elem.used_ptr->load(std::memory_order_relaxed); - result.dead_elems += elem.dead_ptr->load(std::memory_order_relaxed); + result.used_entries += elem.used_ptr->load(std::memory_order_relaxed); + result.dead_entries += elem.dead_ptr->load(std::memory_order_relaxed); } return result; } diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_type.h b/vespalib/src/vespa/vespalib/datastore/buffer_type.h index bedbb2c984e..ea52b026228 100644 --- a/vespalib/src/vespa/vespalib/datastore/buffer_type.h +++ b/vespalib/src/vespa/vespalib/datastore/buffer_type.h @@ -10,7 +10,7 @@ namespace vespalib::alloc { class MemoryAllocator; } namespace vespalib::datastore { -using ElemCount = uint64_t; +using EntryCount = uint32_t; /** * Abstract class used to manage allocation and de-allocation of a specific data type in underlying memory buffers in a data store. @@ -22,7 +22,7 @@ using ElemCount = uint64_t; class BufferTypeBase { public: - using ElemCount = vespalib::datastore::ElemCount; + using EntryCount = vespalib::datastore::EntryCount; class CleanContext { private: std::atomic<size_t> &_extraUsedBytes; @@ -39,53 +39,52 @@ public: BufferTypeBase & operator=(const BufferTypeBase &rhs) = delete; BufferTypeBase(BufferTypeBase &&rhs) noexcept = default; BufferTypeBase & operator=(BufferTypeBase &&rhs) noexcept = default; - BufferTypeBase(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays) noexcept; - BufferTypeBase(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays, - uint32_t numArraysForNewBuffer, float allocGrowFactor) noexcept; + BufferTypeBase(uint32_t arraySize, uint32_t min_entries, uint32_t max_entries) noexcept; + BufferTypeBase(uint32_t arraySize, uint32_t min_entries, uint32_t max_entries, + uint32_t num_entries_for_new_buffer, float allocGrowFactor) noexcept; virtual ~BufferTypeBase(); - virtual void destroyElements(void *buffer, ElemCount numElems) = 0; - virtual void fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems) = 0; + virtual void destroy_entries(void *buffer, EntryCount num_entries) = 0; + virtual void fallback_copy(void *newBuffer, const void *oldBuffer, EntryCount num_entries) = 0; /** - * Return number of reserved elements at start of buffer, to avoid - * invalid reference and handle data at negative offset (alignment - * hacks) as used by dense tensor store. + * Return number of reserved entries at start of buffer, to avoid + * invalid reference. */ - virtual ElemCount getReservedElements(uint32_t bufferId) const; + virtual EntryCount get_reserved_entries(uint32_t bufferId) const; /** * Initialize reserved elements at start of buffer. */ - virtual void initializeReservedElements(void *buffer, ElemCount reservedElements) = 0; - virtual size_t elementSize() const = 0; - virtual void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) = 0; + virtual void initialize_reserved_entries(void *buffer, EntryCount reserved_entries) = 0; + virtual size_t entry_size() const = 0; // Size of entry measured in bytes + virtual void clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx) = 0; size_t getArraySize() const { return _arraySize; } - virtual void onActive(uint32_t bufferId, std::atomic<ElemCount>* usedElems, std::atomic<ElemCount>* deadElems, void* buffer); - void onHold(uint32_t buffer_id, const std::atomic<ElemCount>* usedElems, const std::atomic<ElemCount>* deadElems); - virtual void onFree(ElemCount usedElems); - void resume_primary_buffer(uint32_t buffer_id, std::atomic<ElemCount>* used_elems, std::atomic<ElemCount>* dead_elems); + virtual void on_active(uint32_t bufferId, std::atomic<EntryCount>* used_entries, std::atomic<EntryCount>* dead_entries, void* buffer); + void on_hold(uint32_t buffer_id, const std::atomic<EntryCount>* used_entries, const std::atomic<EntryCount>* dead_entries); + virtual void on_free(EntryCount used_entries); + void resume_primary_buffer(uint32_t buffer_id, std::atomic<EntryCount>* used_entries, std::atomic<EntryCount>* dead_entries); virtual const alloc::MemoryAllocator* get_memory_allocator() const; /** - * Calculate number of arrays to allocate for new buffer given how many elements are needed. + * Calculate number of entries to allocate for new buffer given how many free entries are needed. */ - virtual size_t calcArraysToAlloc(uint32_t bufferId, ElemCount elementsNeeded, bool resizing) const; + virtual size_t calc_entries_to_alloc(uint32_t bufferId, EntryCount free_entries_needed, bool resizing) const; - void clampMaxArrays(uint32_t maxArrays); + void clamp_max_entries(uint32_t max_entries); uint32_t get_active_buffers_count() const { return _active_buffers.size(); } const std::vector<uint32_t>& get_active_buffers() const noexcept { return _active_buffers; } - size_t getMaxArrays() const { return _maxArrays; } - uint32_t get_scaled_num_arrays_for_new_buffer() const; - uint32_t get_num_arrays_for_new_buffer() const noexcept { return _numArraysForNewBuffer; } + size_t get_max_entries() const { return _max_entries; } + uint32_t get_scaled_num_entries_for_new_buffer() const; + uint32_t get_num_entries_for_new_buffer() const noexcept { return _num_entries_for_new_buffer; } protected: struct BufferCounts { - ElemCount used_elems; - ElemCount dead_elems; - BufferCounts() : used_elems(0), dead_elems(0) {} - BufferCounts(ElemCount used_elems_in, ElemCount dead_elems_in) - : used_elems(used_elems_in), dead_elems(dead_elems_in) + EntryCount used_entries; + EntryCount dead_entries; + BufferCounts() : used_entries(0), dead_entries(0) {} + BufferCounts(EntryCount used_entries_in, EntryCount dead_entries_in) + : used_entries(used_entries_in), dead_entries(dead_entries_in) {} }; @@ -94,45 +93,48 @@ protected: */ class AggregatedBufferCounts { private: - struct Element { - const std::atomic<ElemCount>* used_ptr; - const std::atomic<ElemCount>* dead_ptr; - Element() noexcept : used_ptr(nullptr), dead_ptr(nullptr) {} - Element(const std::atomic<ElemCount>* used_ptr_in, const std::atomic<ElemCount>* dead_ptr_in) noexcept + struct ActiveBufferCounts { + const std::atomic<EntryCount>* used_ptr; + const std::atomic<EntryCount>* dead_ptr; + ActiveBufferCounts() noexcept : used_ptr(nullptr), dead_ptr(nullptr) {} + ActiveBufferCounts(const std::atomic<EntryCount>* used_ptr_in, const std::atomic<EntryCount>* dead_ptr_in) noexcept : used_ptr(used_ptr_in), dead_ptr(dead_ptr_in) {} }; - std::vector<Element> _counts; + std::vector<ActiveBufferCounts> _counts; public: AggregatedBufferCounts(); - void add_buffer(const std::atomic<ElemCount>* used_elems, const std::atomic<ElemCount>* dead_elems); - void remove_buffer(const std::atomic<ElemCount>* used_elems, const std::atomic<ElemCount>* dead_elems); + void add_buffer(const std::atomic<EntryCount>* used_entries, const std::atomic<EntryCount>* dead_entries); + void remove_buffer(const std::atomic<EntryCount>* used_entries, const std::atomic<EntryCount>* dead_entries); BufferCounts last_buffer() const; BufferCounts all_buffers() const; bool empty() const { return _counts.empty(); } }; uint32_t _arraySize; // Number of elements in an allocation unit - uint32_t _minArrays; // Minimum number of arrays to allocate in a buffer - uint32_t _maxArrays; // Maximum number of arrays to allocate in a buffer - // Number of arrays needed before allocating a new buffer instead of just resizing the first one - uint32_t _numArraysForNewBuffer; + uint32_t _min_entries; // Minimum number of entries to allocate in a buffer + uint32_t _max_entries; // Maximum number of entries to allocate in a buffer + // Number of entries needed before allocating a new buffer instead of just resizing the first one + uint32_t _num_entries_for_new_buffer; float _allocGrowFactor; uint32_t _holdBuffers; - size_t _holdUsedElems; // Number of used elements in all held buffers for this type. + size_t _hold_used_entries; // Number of used entries in all held buffers for this type. AggregatedBufferCounts _aggr_counts; std::vector<uint32_t> _active_buffers; }; /** - * Concrete class used to manage allocation and de-allocation of elements of type EntryType in data store buffers. + * Concrete class used to manage allocation and de-allocation of elements of type ElemType in data store buffers. */ -template <typename EntryType, typename EmptyType = EntryType> +template <typename ElemT, typename EmptyT = ElemT> class BufferType : public BufferTypeBase { +public: + using ElemType = ElemT; + using EmptyType = EmptyT; protected: - static const EntryType& empty_entry() noexcept; + static const ElemType& empty_entry() noexcept; public: BufferType() noexcept : BufferType(1,1,1) {} @@ -140,15 +142,15 @@ public: BufferType & operator=(const BufferType &rhs) = delete; BufferType(BufferType && rhs) noexcept = default; BufferType & operator=(BufferType && rhs) noexcept = default; - BufferType(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays) noexcept; - BufferType(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays, - uint32_t numArraysForNewBuffer, float allocGrowFactor) noexcept; + BufferType(uint32_t arraySize, uint32_t min_entries, uint32_t max_entries) noexcept; + BufferType(uint32_t arraySize, uint32_t min_entries, uint32_t max_entries, + uint32_t num_entries_for_new_buffer, float allocGrowFactor) noexcept; ~BufferType() override; - void destroyElements(void *buffer, ElemCount numElems) override; - void fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems) override; - void initializeReservedElements(void *buffer, ElemCount reservedElements) override; - void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCxt) override; - size_t elementSize() const override { return sizeof(EntryType); } + void destroy_entries(void *buffer, EntryCount num_entries) override; + void fallback_copy(void *newBuffer, const void *oldBuffer, EntryCount num_entries) override; + void initialize_reserved_entries(void *buffer, EntryCount reserved_entries) override; + void clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext cleanCxt) override; + size_t entry_size() const override { return sizeof(ElemType) * _arraySize; } }; extern template class BufferType<char>; diff --git a/vespalib/src/vespa/vespalib/datastore/buffer_type.hpp b/vespalib/src/vespa/vespalib/datastore/buffer_type.hpp index 72c8f574a70..60acca5ff39 100644 --- a/vespalib/src/vespa/vespalib/datastore/buffer_type.hpp +++ b/vespalib/src/vespa/vespalib/datastore/buffer_type.hpp @@ -6,78 +6,80 @@ namespace vespalib::datastore { -template <typename EntryType, typename EmptyType> -BufferType<EntryType, EmptyType>::BufferType(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays) noexcept - : BufferTypeBase(arraySize, minArrays, maxArrays) +template <typename ElemT, typename EmptyT> +BufferType<ElemT, EmptyT>::BufferType(uint32_t arraySize, uint32_t min_entries, uint32_t max_entries) noexcept + : BufferTypeBase(arraySize, min_entries, max_entries) { } -template <typename EntryType, typename EmptyType> -BufferType<EntryType, EmptyType>::BufferType(uint32_t arraySize, uint32_t minArrays, uint32_t maxArrays, - uint32_t numArraysForNewBuffer, float allocGrowFactor) noexcept - : BufferTypeBase(arraySize, minArrays, maxArrays, numArraysForNewBuffer, allocGrowFactor) +template <typename ElemT, typename EmptyT> +BufferType<ElemT, EmptyT>::BufferType(uint32_t arraySize, uint32_t min_entries, uint32_t max_entries, + uint32_t num_entries_for_new_buffer, float allocGrowFactor) noexcept + : BufferTypeBase(arraySize, min_entries, max_entries, num_entries_for_new_buffer, allocGrowFactor) { } -template <typename EntryType, typename EmptyType> -BufferType<EntryType, EmptyType>::~BufferType() = default; +template <typename ElemT, typename EmptyT> +BufferType<ElemT, EmptyT>::~BufferType() = default; -template <typename EntryType, typename EmptyType> +template <typename ElemT, typename EmptyT> void -BufferType<EntryType, EmptyType>::destroyElements(void *buffer, ElemCount numElems) +BufferType<ElemT, EmptyT>::destroy_entries(void *buffer, EntryCount num_entries) { - EntryType *e = static_cast<EntryType *>(buffer); - for (size_t j = numElems; j != 0; --j) { - e->~EntryType(); + auto num_elems = num_entries * getArraySize(); + ElemType *e = static_cast<ElemType *>(buffer); + for (size_t j = num_elems; j != 0; --j) { + e->~ElemType(); ++e; } } -template <typename EntryType, typename EmptyType> +template <typename ElemT, typename EmptyT> void -BufferType<EntryType, EmptyType>::fallbackCopy(void *newBuffer, - const void *oldBuffer, - ElemCount numElems) +BufferType<ElemT, EmptyT>::fallback_copy(void *newBuffer, const void *oldBuffer, EntryCount num_entries) { - EntryType *d = static_cast<EntryType *>(newBuffer); - const EntryType *s = static_cast<const EntryType *>(oldBuffer); - for (size_t j = numElems; j != 0; --j) { - new (static_cast<void *>(d)) EntryType(*s); + auto num_elems = num_entries * getArraySize(); + ElemType *d = static_cast<ElemType *>(newBuffer); + const ElemType *s = static_cast<const ElemType *>(oldBuffer); + for (size_t j = num_elems; j != 0; --j) { + new (static_cast<void *>(d)) ElemType(*s); ++s; ++d; } } -template <typename EntryType, typename EmptyType> +template <typename ElemT, typename EmptyT> void -BufferType<EntryType, EmptyType>::initializeReservedElements(void *buffer, ElemCount reservedElems) +BufferType<ElemT, EmptyT>::initialize_reserved_entries(void *buffer, EntryCount reserved_entries) { - EntryType *e = static_cast<EntryType *>(buffer); + auto reserved_elems = reserved_entries * getArraySize(); + ElemType *e = static_cast<ElemType *>(buffer); const auto& empty = empty_entry(); - for (size_t j = reservedElems; j != 0; --j) { - new (static_cast<void *>(e)) EntryType(empty); + for (size_t j = reserved_elems; j != 0; --j) { + new (static_cast<void *>(e)) ElemType(empty); ++e; } } -template <typename EntryType, typename EmptyType> +template <typename ElemT, typename EmptyT> void -BufferType<EntryType, EmptyType>::cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext) +BufferType<ElemT, EmptyT>::clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext) { - EntryType *e = static_cast<EntryType *>(buffer) + offset; + auto num_elems = num_entries * getArraySize(); + ElemType *e = static_cast<ElemType *>(buffer) + offset * getArraySize(); const auto& empty = empty_entry(); - for (size_t j = numElems; j != 0; --j) { + for (size_t j = num_elems; j != 0; --j) { *e = empty; ++e; } } -template <typename EntryType, typename EmptyType> -const EntryType& -BufferType<EntryType, EmptyType>::empty_entry() noexcept +template <typename ElemT, typename EmptyT> +const ElemT& +BufferType<ElemT, EmptyT>::empty_entry() noexcept { - // It's possible for EntryType to wrap e.g. an Alloc instance, which has a transitive + // It's possible for ElemType to wrap e.g. an Alloc instance, which has a transitive // dependency on globally constructed allocator object(s). To avoid issues with global // construction order, initialize the sentinel on the first access. - static EntryType empty = EmptyType(); + static ElemType empty = EmptyType(); return empty; } diff --git a/vespalib/src/vespa/vespalib/datastore/bufferstate.cpp b/vespalib/src/vespa/vespalib/datastore/bufferstate.cpp index 47fba1ef697..f312596d6f7 100644 --- a/vespalib/src/vespa/vespalib/datastore/bufferstate.cpp +++ b/vespalib/src/vespa/vespalib/datastore/bufferstate.cpp @@ -12,13 +12,13 @@ namespace vespalib::datastore { BufferState::BufferState() : _stats(), - _free_list(_stats.dead_elems_ref()), + _free_list(_stats.dead_entries_ref()), _typeHandler(nullptr), _buffer(Alloc::alloc(0, MemoryAllocator::HUGEPAGE_SIZE)), _arraySize(0), _typeId(0), _state(State::FREE), - _disableElemHoldList(false), + _disable_entry_hold_list(false), _compacting(false) { } @@ -28,15 +28,15 @@ BufferState::~BufferState() assert(getState() == State::FREE); assert(!_free_list.enabled()); assert(_free_list.empty()); - assert(_stats.hold_elems() == 0); + assert(_stats.hold_entries() == 0); } namespace { struct AllocResult { - size_t elements; + size_t entries; size_t bytes; - AllocResult(size_t elements_, size_t bytes_) : elements(elements_), bytes(bytes_) {} + AllocResult(size_t entries_, size_t bytes_) : entries(entries_), bytes(bytes_) {} }; size_t @@ -57,30 +57,30 @@ roundUpToMatchAllocator(size_t sz) } AllocResult -calcAllocation(uint32_t bufferId, - BufferTypeBase &typeHandler, - size_t elementsNeeded, - bool resizing) +calc_allocation(uint32_t bufferId, + BufferTypeBase &typeHandler, + size_t free_entries_needed, + bool resizing) { - size_t allocArrays = typeHandler.calcArraysToAlloc(bufferId, elementsNeeded, resizing); - size_t allocElements = allocArrays * typeHandler.getArraySize(); - size_t allocBytes = roundUpToMatchAllocator(allocElements * typeHandler.elementSize()); - size_t maxAllocBytes = typeHandler.getMaxArrays() * typeHandler.getArraySize() * typeHandler.elementSize(); + size_t alloc_entries = typeHandler.calc_entries_to_alloc(bufferId, free_entries_needed, resizing); + size_t entry_size = typeHandler.entry_size(); + size_t allocBytes = roundUpToMatchAllocator(alloc_entries * entry_size); + size_t maxAllocBytes = typeHandler.get_max_entries() * entry_size; if (allocBytes > maxAllocBytes) { // Ensure that allocated bytes does not exceed the maximum handled by this type. allocBytes = maxAllocBytes; } - size_t adjustedAllocElements = (allocBytes / typeHandler.elementSize()); - return AllocResult(adjustedAllocElements, allocBytes); + size_t adjusted_alloc_entries = allocBytes / entry_size; + return AllocResult(adjusted_alloc_entries, allocBytes); } } void -BufferState::onActive(uint32_t bufferId, uint32_t typeId, - BufferTypeBase *typeHandler, - size_t elementsNeeded, - std::atomic<void*>& buffer) +BufferState::on_active(uint32_t bufferId, uint32_t typeId, + BufferTypeBase *typeHandler, + size_t free_entries_needed, + std::atomic<void*>& buffer) { assert(buffer.load(std::memory_order_relaxed) == nullptr); assert(_buffer.get() == nullptr); @@ -88,30 +88,29 @@ BufferState::onActive(uint32_t bufferId, uint32_t typeId, assert(_typeHandler == nullptr); assert(capacity() == 0); assert(size() == 0); - assert(_stats.dead_elems() == 0u); - assert(_stats.hold_elems() == 0); + assert(_stats.dead_entries() == 0u); + assert(_stats.hold_entries() == 0); assert(_stats.extra_used_bytes() == 0); assert(_stats.extra_hold_bytes() == 0); assert(_free_list.empty()); - size_t reservedElements = typeHandler->getReservedElements(bufferId); - (void) reservedElements; - AllocResult alloc = calcAllocation(bufferId, *typeHandler, elementsNeeded, false); - assert(alloc.elements >= reservedElements + elementsNeeded); + size_t reserved_entries = typeHandler->get_reserved_entries(bufferId); + (void) reserved_entries; + AllocResult alloc = calc_allocation(bufferId, *typeHandler, free_entries_needed, false); + assert(alloc.entries >= reserved_entries + free_entries_needed); auto allocator = typeHandler->get_memory_allocator(); _buffer = (allocator != nullptr) ? Alloc::alloc_with_allocator(allocator) : Alloc::alloc(0, MemoryAllocator::HUGEPAGE_SIZE); _buffer.create(alloc.bytes).swap(_buffer); - assert(_buffer.get() != nullptr || alloc.elements == 0u); + assert(_buffer.get() != nullptr || alloc.entries == 0u); buffer.store(_buffer.get(), std::memory_order_release); - _stats.set_alloc_elems(alloc.elements); + _stats.set_alloc_entries(alloc.entries); _typeHandler.store(typeHandler, std::memory_order_release); assert(typeId <= std::numeric_limits<uint16_t>::max()); _typeId = typeId; _arraySize = typeHandler->getArraySize(); - _free_list.set_array_size(_arraySize); _state.store(State::ACTIVE, std::memory_order_release); - typeHandler->onActive(bufferId, &_stats.used_elems_ref(), &_stats.dead_elems_ref(), - buffer.load(std::memory_order::relaxed)); + typeHandler->on_active(bufferId, &_stats.used_entries_ref(), &_stats.dead_entries_ref(), + buffer.load(std::memory_order::relaxed)); } void @@ -121,11 +120,11 @@ BufferState::onHold(uint32_t buffer_id) assert(getTypeHandler() != nullptr); _state.store(State::HOLD, std::memory_order_release); _compacting = false; - assert(_stats.dead_elems() <= size()); - assert(_stats.hold_elems() <= (size() - _stats.dead_elems())); - _stats.set_dead_elems(0); - _stats.set_hold_elems(size()); - getTypeHandler()->onHold(buffer_id, &_stats.used_elems_ref(), &_stats.dead_elems_ref()); + assert(_stats.dead_entries() <= size()); + assert(_stats.hold_entries() <= (size() - _stats.dead_entries())); + _stats.set_dead_entries(0); + _stats.set_hold_entries(size()); + getTypeHandler()->on_hold(buffer_id, &_stats.used_entries_ref(), &_stats.dead_entries_ref()); _free_list.disable(); } @@ -135,20 +134,19 @@ BufferState::onFree(std::atomic<void*>& buffer) assert(buffer.load(std::memory_order_relaxed) == _buffer.get()); assert(getState() == State::HOLD); assert(_typeHandler != nullptr); - assert(_stats.dead_elems() <= size()); - assert(_stats.hold_elems() == (size() - _stats.dead_elems())); - getTypeHandler()->destroyElements(buffer, size()); + assert(_stats.dead_entries() <= size()); + assert(_stats.hold_entries() == (size() - _stats.dead_entries())); + getTypeHandler()->destroy_entries(buffer, size()); Alloc::alloc().swap(_buffer); - getTypeHandler()->onFree(size()); + getTypeHandler()->on_free(size()); buffer.store(nullptr, std::memory_order_release); _stats.clear(); _state.store(State::FREE, std::memory_order_release); _typeHandler = nullptr; _arraySize = 0; - _free_list.set_array_size(_arraySize); assert(!_free_list.enabled()); assert(_free_list.empty()); - _disableElemHoldList = false; + _disable_entry_hold_list = false; } @@ -171,67 +169,67 @@ BufferState::dropBuffer(uint32_t buffer_id, std::atomic<void*>& buffer) } void -BufferState::disable_elem_hold_list() +BufferState::disable_entry_hold_list() { - _disableElemHoldList = true; + _disable_entry_hold_list = true; } bool -BufferState::hold_elems(size_t num_elems, size_t extra_bytes) +BufferState::hold_entries(size_t num_entries, size_t extra_bytes) { assert(isActive()); - if (_disableElemHoldList) { + if (_disable_entry_hold_list) { // The elements are directly marked as dead as they are not put on hold. - _stats.inc_dead_elems(num_elems); + _stats.inc_dead_entries(num_entries); return true; } - _stats.inc_hold_elems(num_elems); + _stats.inc_hold_entries(num_entries); _stats.inc_extra_hold_bytes(extra_bytes); return false; } void -BufferState::free_elems(EntryRef ref, size_t num_elems, size_t ref_offset) +BufferState::free_entries(EntryRef ref, size_t num_entries, size_t ref_offset) { if (isActive()) { - if (_free_list.enabled() && (num_elems == getArraySize())) { + if (_free_list.enabled() && (num_entries == 1)) { _free_list.push_entry(ref); } } else { assert(isOnHold()); } - _stats.inc_dead_elems(num_elems); - _stats.dec_hold_elems(num_elems); - getTypeHandler()->cleanHold(_buffer.get(), (ref_offset * _arraySize), num_elems, - BufferTypeBase::CleanContext(_stats.extra_used_bytes_ref(), - _stats.extra_hold_bytes_ref())); + _stats.inc_dead_entries(num_entries); + _stats.dec_hold_entries(num_entries); + getTypeHandler()->clean_hold(_buffer.get(), ref_offset, num_entries, + BufferTypeBase::CleanContext(_stats.extra_used_bytes_ref(), + _stats.extra_hold_bytes_ref())); } void -BufferState::fallbackResize(uint32_t bufferId, - size_t elementsNeeded, +BufferState::fallback_resize(uint32_t bufferId, + size_t free_entries_needed, std::atomic<void*>& buffer, Alloc &holdBuffer) { assert(getState() == State::ACTIVE); assert(_typeHandler != nullptr); assert(holdBuffer.get() == nullptr); - AllocResult alloc = calcAllocation(bufferId, *_typeHandler, elementsNeeded, true); - assert(alloc.elements >= size() + elementsNeeded); - assert(alloc.elements > capacity()); + AllocResult alloc = calc_allocation(bufferId, *_typeHandler, free_entries_needed, true); + assert(alloc.entries >= size() + free_entries_needed); + assert(alloc.entries > capacity()); Alloc newBuffer = _buffer.create(alloc.bytes); - getTypeHandler()->fallbackCopy(newBuffer.get(), buffer.load(std::memory_order_relaxed), size()); + getTypeHandler()->fallback_copy(newBuffer.get(), buffer.load(std::memory_order_relaxed), size()); holdBuffer.swap(_buffer); std::atomic_thread_fence(std::memory_order_release); _buffer = std::move(newBuffer); buffer.store(_buffer.get(), std::memory_order_release); - _stats.set_alloc_elems(alloc.elements); + _stats.set_alloc_entries(alloc.entries); } void BufferState::resume_primary_buffer(uint32_t buffer_id) { - getTypeHandler()->resume_primary_buffer(buffer_id, &_stats.used_elems_ref(), &_stats.dead_elems_ref()); + getTypeHandler()->resume_primary_buffer(buffer_id, &_stats.used_entries_ref(), &_stats.dead_entries_ref()); } } diff --git a/vespalib/src/vespa/vespalib/datastore/bufferstate.h b/vespalib/src/vespa/vespalib/datastore/bufferstate.h index 5b98099ed69..f714f8e24d5 100644 --- a/vespalib/src/vespa/vespalib/datastore/bufferstate.h +++ b/vespalib/src/vespa/vespalib/datastore/bufferstate.h @@ -23,8 +23,8 @@ namespace vespalib::datastore { * It is kept in this state until all reader threads are no longer accessing the buffer. * Finally, it transitions back to FREE via onFree() and memory is de-allocated. * - * This class also supports use of free lists, where previously allocated elements in the buffer can be re-used. - * First the element is put on hold, then on the free list (counted as dead) to be re-used. + * This class also supports use of free lists, where previously allocated entries in the buffer can be re-used. + * First the entry is put on hold, then on the free list (counted as dead) to be re-used. */ class BufferState { @@ -45,7 +45,7 @@ private: uint32_t _arraySize; uint16_t _typeId; std::atomic<State> _state; - bool _disableElemHoldList : 1; + bool _disable_entry_hold_list : 1; bool _compacting : 1; public: @@ -62,14 +62,14 @@ public: /** * Transition from FREE to ACTIVE state. * - * @param bufferId Id of buffer to be active. - * @param typeId Registered data type id for buffer. - * @param typeHandler Type handler for registered data type. - * @param elementsNeeded Number of elements needed to be free in the memory allocated. - * @param buffer Start of allocated buffer return value. + * @param bufferId Id of buffer to be active. + * @param typeId Registered data type id for buffer. + * @param typeHandler Type handler for registered data type. + * @param free_entries_needed Number of entries needed to be free in the memory allocated. + * @param buffer Start of allocated buffer return value. */ - void onActive(uint32_t bufferId, uint32_t typeId, BufferTypeBase *typeHandler, - size_t elementsNeeded, std::atomic<void*>& buffer); + void on_active(uint32_t bufferId, uint32_t typeId, BufferTypeBase *typeHandler, + size_t free_entries_needed, std::atomic<void*>& buffer); /** * Transition from ACTIVE to HOLD state. @@ -82,24 +82,24 @@ public: void onFree(std::atomic<void*>& buffer); /** - * Disable hold of elements, just mark elements as dead without cleanup. + * Disable hold of entries, just mark entries as dead without cleanup. * Typically used when tearing down data structure in a controlled manner. */ - void disable_elem_hold_list(); + void disable_entry_hold_list(); /** - * Update stats to reflect that the given elements are put on hold. - * Returns true if element hold list is disabled for this buffer. + * Update stats to reflect that the given entries are put on hold. + * Returns true if entry hold list is disabled for this buffer. */ - bool hold_elems(size_t num_elems, size_t extra_bytes); + bool hold_entries(size_t num_entries, size_t extra_bytes); /** - * Free the given elements and update stats accordingly. + * Free the given entries and update stats accordingly. * * The given entry ref is put on the free list (if enabled). - * Hold cleaning of elements is executed on the buffer type. + * Hold cleaning of entries is executed on the buffer type. */ - void free_elems(EntryRef ref, size_t num_elems, size_t ref_offset); + void free_entries(EntryRef ref, size_t num_entries, size_t ref_offset); BufferStats& stats() { return _stats; } const BufferStats& stats() const { return _stats; } @@ -115,8 +115,7 @@ public: uint32_t getArraySize() const { return _arraySize; } bool getCompacting() const { return _compacting; } void setCompacting() { _compacting = true; } - uint32_t get_used_arrays() const noexcept { return size() / _arraySize; } - void fallbackResize(uint32_t bufferId, size_t elementsNeeded, std::atomic<void*>& buffer, Alloc &holdBuffer); + void fallback_resize(uint32_t bufferId, size_t free_entries_needed, std::atomic<void*>& buffer, Alloc &holdBuffer); bool isActive(uint32_t typeId) const { return (isActive() && (_typeId == typeId)); diff --git a/vespalib/src/vespa/vespalib/datastore/datastore.h b/vespalib/src/vespa/vespalib/datastore/datastore.h index 01b81d0fa58..f81348ce287 100644 --- a/vespalib/src/vespa/vespalib/datastore/datastore.h +++ b/vespalib/src/vespa/vespalib/datastore/datastore.h @@ -27,7 +27,7 @@ template <typename RefT = EntryRefT<22> > class DataStoreT : public DataStoreBase { private: - void free_elem_internal(EntryRef ref, size_t numElems); + void free_entry_internal(EntryRef ref, size_t num_entries); public: using RefType = RefT; @@ -38,12 +38,12 @@ public: ~DataStoreT() override; /** - * Hold element(s). + * Hold entries. */ - void holdElem(EntryRef ref, size_t numElems) { - holdElem(ref, numElems, 0); - } - void holdElem(EntryRef ref, size_t numElems, size_t extraBytes); + void hold_entry(EntryRef ref) { hold_entries(ref, 1, 0); } + void hold_entry(EntryRef ref, size_t extra_bytes) { hold_entries(ref, 1, extra_bytes); } + void hold_entries(EntryRef ref, size_t num_entries) { hold_entries(ref, num_entries, 0); } + void hold_entries(EntryRef ref, size_t num_entries, size_t extraBytes); void reclaim_entry_refs(generation_t oldest_used_gen) override; @@ -75,7 +75,7 @@ class DataStore : public DataStoreT<RefT> { protected: using ParentType = DataStoreT<RefT>; - using ParentType::ensureBufferCapacity; + using ParentType::ensure_buffer_capacity; using ParentType::getEntry; using ParentType::dropBuffers; using ParentType::init_primary_buffers; diff --git a/vespalib/src/vespa/vespalib/datastore/datastore.hpp b/vespalib/src/vespa/vespalib/datastore/datastore.hpp index bfb63954875..b21a5954eee 100644 --- a/vespalib/src/vespa/vespalib/datastore/datastore.hpp +++ b/vespalib/src/vespa/vespalib/datastore/datastore.hpp @@ -22,21 +22,21 @@ DataStoreT<RefT>::~DataStoreT() = default; template <typename RefT> void -DataStoreT<RefT>::free_elem_internal(EntryRef ref, size_t numElems) +DataStoreT<RefT>::free_entry_internal(EntryRef ref, size_t num_entries) { RefType intRef(ref); BufferState &state = getBufferState(intRef.bufferId()); - state.free_elems(ref, numElems, intRef.offset()); + state.free_entries(ref, num_entries, intRef.offset()); } template <typename RefT> void -DataStoreT<RefT>::holdElem(EntryRef ref, size_t numElems, size_t extraBytes) +DataStoreT<RefT>::hold_entries(EntryRef ref, size_t num_entries, size_t extraBytes) { RefType intRef(ref); BufferState &state = getBufferState(intRef.bufferId()); - if (!state.hold_elems(numElems, extraBytes)) { - _entry_ref_hold_list.insert({ref, numElems}); + if (!state.hold_entries(num_entries, extraBytes)) { + _entry_ref_hold_list.insert({ref, num_entries}); } } @@ -45,7 +45,7 @@ void DataStoreT<RefT>::reclaim_entry_refs(generation_t oldest_used_gen) { _entry_ref_hold_list.reclaim(oldest_used_gen, [this](const auto& elem) { - free_elem_internal(elem.ref, elem.num_elems); + free_entry_internal(elem.ref, elem.num_entries); }); } @@ -54,7 +54,7 @@ void DataStoreT<RefT>::reclaim_all_entry_refs() { _entry_ref_hold_list.reclaim_all([this](const auto& elem) { - free_elem_internal(elem.ref, elem.num_elems); + free_entry_internal(elem.ref, elem.num_entries); }); } diff --git a/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp b/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp index a40aa713bca..75ffe855a32 100644 --- a/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp +++ b/vespalib/src/vespa/vespalib/datastore/datastorebase.cpp @@ -40,18 +40,18 @@ constexpr size_t TOO_DEAD_SLACK = 0x4000u; bool primary_buffer_too_dead(const BufferState &state) { - size_t deadElems = state.stats().dead_elems(); - size_t deadBytes = deadElems * state.getArraySize(); - return ((deadBytes >= TOO_DEAD_SLACK) && (deadElems * 2 >= state.size())); + size_t dead_entries = state.stats().dead_entries(); + size_t deadBytes = dead_entries * state.getTypeHandler()->entry_size(); + return ((deadBytes >= TOO_DEAD_SLACK) && (dead_entries * 2 >= state.size())); } } -DataStoreBase::FallbackHold::FallbackHold(size_t bytesSize, BufferState::Alloc &&buffer, size_t usedElems, +DataStoreBase::FallbackHold::FallbackHold(size_t bytesSize, BufferState::Alloc &&buffer, size_t used_entries, BufferTypeBase *typeHandler, uint32_t typeId) : GenerationHeldBase(bytesSize), _buffer(std::move(buffer)), - _usedElems(usedElems), + _used_entries(used_entries), _typeHandler(typeHandler), _typeId(typeId) { @@ -59,7 +59,7 @@ DataStoreBase::FallbackHold::FallbackHold(size_t bytesSize, BufferState::Alloc & DataStoreBase::FallbackHold::~FallbackHold() { - _typeHandler->destroyElements(_buffer.get(), _usedElems); + _typeHandler->destroy_entries(_buffer.get(), _used_entries); } class DataStoreBase::BufferHold : public GenerationHeldBase { @@ -80,7 +80,7 @@ public: } }; -DataStoreBase::DataStoreBase(uint32_t numBuffers, uint32_t offset_bits, size_t maxArrays) +DataStoreBase::DataStoreBase(uint32_t numBuffers, uint32_t offset_bits, size_t max_entries) : _entry_ref_hold_list(), _buffers(numBuffers), _primary_buffer_ids(), @@ -89,12 +89,12 @@ DataStoreBase::DataStoreBase(uint32_t numBuffers, uint32_t offset_bits, size_t m _free_lists(), _compaction_count(0u), _genHolder(), - _maxArrays(maxArrays), + _max_entries(max_entries), _bufferIdLimit(0u), _hold_buffer_count(0u), _offset_bits(offset_bits), _freeListsEnabled(false), - _disableElemHoldList(false), + _disable_entry_hold_list(false), _initializing(false) { } @@ -105,19 +105,19 @@ DataStoreBase::~DataStoreBase() } void -DataStoreBase::switch_primary_buffer(uint32_t typeId, size_t elemsNeeded) +DataStoreBase::switch_primary_buffer(uint32_t typeId, size_t entries_needed) { size_t buffer_id = getFirstFreeBufferId(); if (buffer_id >= getMaxNumBuffers()) { LOG_ABORT(vespalib::make_string("switch_primary_buffer(%u, %zu): did not find a free buffer", - typeId, elemsNeeded).c_str()); + typeId, entries_needed).c_str()); } - onActive(buffer_id, typeId, elemsNeeded); + on_active(buffer_id, typeId, entries_needed); _primary_buffer_ids[typeId] = buffer_id; } bool -DataStoreBase::consider_grow_active_buffer(uint32_t type_id, size_t elems_needed) +DataStoreBase::consider_grow_active_buffer(uint32_t type_id, size_t entries_needed) { auto type_handler = _typeHandlers[type_id]; uint32_t buffer_id = primary_buffer_id(type_id); @@ -126,7 +126,7 @@ DataStoreBase::consider_grow_active_buffer(uint32_t type_id, size_t elems_needed if (active_buffers_count < min_active_buffers) { return false; } - if (type_handler->get_num_arrays_for_new_buffer() == 0u) { + if (type_handler->get_num_entries_for_new_buffer() == 0u) { return false; } assert(!getBufferState(buffer_id).getCompacting()); @@ -146,8 +146,7 @@ DataStoreBase::consider_grow_active_buffer(uint32_t type_id, size_t elems_needed if (checked_active_buffers < min_active_buffers) { return false; } - auto array_size = type_handler->getArraySize(); - if (elems_needed + min_used > type_handler->getMaxArrays() * array_size) { + if (entries_needed + min_used > type_handler->get_max_entries()) { return false; } if (min_buffer_id != buffer_id) { @@ -181,24 +180,22 @@ DataStoreBase::getBufferState(uint32_t buffer_id) noexcept { } void -DataStoreBase::switch_or_grow_primary_buffer(uint32_t typeId, size_t elemsNeeded) +DataStoreBase::switch_or_grow_primary_buffer(uint32_t typeId, size_t entries_needed) { auto typeHandler = _typeHandlers[typeId]; - uint32_t arraySize = typeHandler->getArraySize(); - size_t numArraysForNewBuffer = typeHandler->get_scaled_num_arrays_for_new_buffer(); - size_t numEntriesForNewBuffer = numArraysForNewBuffer * arraySize; + size_t num_entries_for_new_buffer = typeHandler->get_scaled_num_entries_for_new_buffer(); uint32_t bufferId = primary_buffer_id(typeId); - if (elemsNeeded + getBufferState(bufferId).size() >= numEntriesForNewBuffer) { - if (consider_grow_active_buffer(typeId, elemsNeeded)) { + if (entries_needed + getBufferState(bufferId).size() >= num_entries_for_new_buffer) { + if (consider_grow_active_buffer(typeId, entries_needed)) { bufferId = primary_buffer_id(typeId); - if (elemsNeeded > getBufferState(bufferId).remaining()) { - fallbackResize(bufferId, elemsNeeded); + if (entries_needed > getBufferState(bufferId).remaining()) { + fallback_resize(bufferId, entries_needed); } } else { - switch_primary_buffer(typeId, elemsNeeded); + switch_primary_buffer(typeId, entries_needed); } } else { - fallbackResize(bufferId, elemsNeeded); + fallback_resize(bufferId, entries_needed); } } @@ -209,7 +206,7 @@ DataStoreBase::init_primary_buffers() for (uint32_t typeId = 0; typeId < numTypes; ++typeId) { size_t buffer_id = getFirstFreeBufferId(); assert(buffer_id <= get_bufferid_limit_relaxed()); - onActive(buffer_id, typeId, 0u); + on_active(buffer_id, typeId, 0u); _primary_buffer_ids[typeId] = buffer_id; } } @@ -219,7 +216,7 @@ DataStoreBase::addType(BufferTypeBase *typeHandler) { uint32_t typeId = _primary_buffer_ids.size(); assert(typeId == _typeHandlers.size()); - typeHandler->clampMaxArrays(_maxArrays); + typeHandler->clamp_max_entries(_max_entries); _primary_buffer_ids.push_back(0); _typeHandlers.push_back(typeHandler); _free_lists.emplace_back(); @@ -328,12 +325,12 @@ DataStoreBase::disableFreeLists() } void -DataStoreBase::disableElemHoldList() +DataStoreBase::disable_entry_hold_list() { for_each_buffer([](BufferState & state) { - if (!state.isFree()) state.disable_elem_hold_list(); + if (!state.isFree()) state.disable_entry_hold_list(); }); - _disableElemHoldList = true; + _disable_entry_hold_list = true; } MemoryStats @@ -351,13 +348,13 @@ DataStoreBase::getMemStats() const if ((state == BufferState::State::FREE) || (typeHandler == nullptr)) { ++stats._freeBuffers; } else if (state == BufferState::State::ACTIVE) { - size_t elementSize = typeHandler->elementSize(); + size_t entry_size = typeHandler->entry_size(); ++stats._activeBuffers; - bState->stats().add_to_mem_stats(elementSize, stats); + bState->stats().add_to_mem_stats(entry_size, stats); } else if (state == BufferState::State::HOLD) { - size_t elementSize = typeHandler->elementSize(); + size_t entry_size = typeHandler->entry_size(); ++stats._holdBuffers; - bState->stats().add_to_mem_stats(elementSize, stats); + bState->stats().add_to_mem_stats(entry_size, stats); } else { LOG_ABORT("should not be reached"); } @@ -373,32 +370,30 @@ vespalib::AddressSpace DataStoreBase::getAddressSpaceUsage() const { uint32_t buffer_id_limit = get_bufferid_limit_acquire(); - size_t usedArrays = 0; - size_t deadArrays = 0; - size_t limitArrays = size_t(_maxArrays) * (getMaxNumBuffers() - buffer_id_limit); + size_t used_entries = 0; + size_t dead_entries = 0; + size_t limit_entries = size_t(_max_entries) * (getMaxNumBuffers() - buffer_id_limit); for (uint32_t bufferId = 0; bufferId < buffer_id_limit; ++bufferId) { const BufferState * bState = _buffers[bufferId].get_state_acquire(); assert(bState != nullptr); if (bState->isFree()) { - limitArrays += _maxArrays; + limit_entries += _max_entries; } else if (bState->isActive()) { - uint32_t arraySize = bState->getArraySize(); - usedArrays += bState->size() / arraySize; - deadArrays += bState->stats().dead_elems() / arraySize; - limitArrays += bState->capacity() / arraySize; + used_entries += bState->size(); + dead_entries += bState->stats().dead_entries(); + limit_entries += bState->capacity(); } else if (bState->isOnHold()) { - uint32_t arraySize = bState->getArraySize(); - usedArrays += bState->size() / arraySize; - limitArrays += bState->capacity() / arraySize; + used_entries += bState->size(); + limit_entries += bState->capacity(); } else { LOG_ABORT("should not be reached"); } } - return {usedArrays, deadArrays, limitArrays}; + return {used_entries, dead_entries, limit_entries}; } void -DataStoreBase::onActive(uint32_t bufferId, uint32_t typeId, size_t elemsNeeded) +DataStoreBase::on_active(uint32_t bufferId, uint32_t typeId, size_t entries_needed) { assert(typeId < _typeHandlers.size()); assert(bufferId <= _bufferIdLimit); @@ -407,8 +402,8 @@ DataStoreBase::onActive(uint32_t bufferId, uint32_t typeId, size_t elemsNeeded) BufferState *state = bufferMeta.get_state_relaxed(); if (state == nullptr) { BufferState & newState = _stash.create<BufferState>(); - if (_disableElemHoldList) { - newState.disable_elem_hold_list(); + if (_disable_entry_hold_list) { + newState.disable_entry_hold_list(); } if ( ! _freeListsEnabled) { newState.disable_free_list(); @@ -418,7 +413,7 @@ DataStoreBase::onActive(uint32_t bufferId, uint32_t typeId, size_t elemsNeeded) _bufferIdLimit.store(bufferId + 1, std::memory_order_release); } assert(state->isFree()); - state->onActive(bufferId, typeId, _typeHandlers[typeId], elemsNeeded, bufferMeta.get_atomic_buffer()); + state->on_active(bufferId, typeId, _typeHandlers[typeId], entries_needed, bufferMeta.get_atomic_buffer()); bufferMeta.setTypeId(typeId); bufferMeta.setArraySize(state->getArraySize()); if (_freeListsEnabled && state->isActive() && !state->getCompacting()) { @@ -436,17 +431,17 @@ DataStoreBase::finishCompact(const std::vector<uint32_t> &toHold) } void -DataStoreBase::fallbackResize(uint32_t bufferId, size_t elemsNeeded) +DataStoreBase::fallback_resize(uint32_t bufferId, size_t entries_needed) { BufferState &state = getBufferState(bufferId); BufferState::Alloc toHoldBuffer; - size_t oldUsedElems = state.size(); - size_t oldAllocElems = state.capacity(); - size_t elementSize = state.getTypeHandler()->elementSize(); - state.fallbackResize(bufferId, elemsNeeded, _buffers[bufferId].get_atomic_buffer(), toHoldBuffer); - auto hold = std::make_unique<FallbackHold>(oldAllocElems * elementSize, + size_t old_used_entries = state.size(); + size_t old_alloc_entries = state.capacity(); + size_t entry_size = state.getTypeHandler()->entry_size(); + state.fallback_resize(bufferId, entries_needed, _buffers[bufferId].get_atomic_buffer(), toHoldBuffer); + auto hold = std::make_unique<FallbackHold>(old_alloc_entries * entry_size, std::move(toHoldBuffer), - oldUsedElems, + old_used_entries, state.getTypeHandler(), state.getTypeId()); if (!_initializing) { @@ -465,7 +460,7 @@ DataStoreBase::markCompacting(uint32_t bufferId) } assert(!state.getCompacting()); state.setCompacting(); - state.disable_elem_hold_list(); + state.disable_entry_hold_list(); state.disable_free_list(); inc_compaction_count(); } @@ -492,15 +487,15 @@ DataStoreBase::start_compact_worst_buffers(CompactionSpec compaction_spec, const free_buffers++; } else if (state->isActive()) { auto typeHandler = state->getTypeHandler(); - uint32_t arraySize = typeHandler->getArraySize(); - uint32_t reservedElements = typeHandler->getReservedElements(bufferId); - size_t used_elems = state->size(); - size_t deadElems = state->stats().dead_elems() - reservedElements; + uint32_t reserved_entries = typeHandler->get_reserved_entries(bufferId); + size_t used_entries = state->size(); + size_t dead_entries = state->stats().dead_entries() - reserved_entries; + size_t entry_size = typeHandler->entry_size(); if (compaction_spec.compact_memory()) { - elem_buffers.add(bufferId, used_elems, deadElems); + elem_buffers.add(bufferId, used_entries * entry_size, dead_entries * entry_size); } if (compaction_spec.compact_address_space()) { - array_buffers.add(bufferId, used_elems / arraySize, deadElems / arraySize); + array_buffers.add(bufferId, used_entries, dead_entries); } } } diff --git a/vespalib/src/vespa/vespalib/datastore/datastorebase.h b/vespalib/src/vespa/vespalib/datastore/datastorebase.h index 9cab7a2e375..e5a38e3fd41 100644 --- a/vespalib/src/vespa/vespalib/datastore/datastorebase.h +++ b/vespalib/src/vespa/vespalib/datastore/datastorebase.h @@ -35,15 +35,16 @@ public: void init_primary_buffers(); /** - * Ensure that the primary buffer for the given type has a given number of elements free at end. + * Ensure that the primary buffer for the given type has a given number of entries free at end. * Switch to new buffer if current buffer is too full. * - * @param typeId Registered data type for buffer. - * @param elemsNeeded Number of elements needed to be free. + * @param typeId Registered data type for buffer. + * @param entries_needed Number of entries needed to be free. */ - void ensureBufferCapacity(uint32_t typeId, size_t elemsNeeded) { - if (elemsNeeded > getBufferState(primary_buffer_id(typeId)).remaining()) [[unlikely]] { - switch_or_grow_primary_buffer(typeId, elemsNeeded); + void ensure_buffer_capacity(uint32_t typeId, size_t entries_needed) { + auto &state = getBufferState(primary_buffer_id(typeId)); + if (entries_needed > state.remaining()) [[unlikely]] { + switch_or_grow_primary_buffer(typeId, entries_needed); } } @@ -58,10 +59,10 @@ public: * Switch to a new primary buffer, typically in preparation for compaction * or when the current primary buffer no longer has free space. * - * @param typeId Registered data type for buffer. - * @param elemsNeeded Number of elements needed to be free. + * @param typeId Registered data type for buffer. + * @param entries_needed Number of entries needed to be free. */ - void switch_primary_buffer(uint32_t typeId, size_t elemsNeeded); + void switch_primary_buffer(uint32_t typeId, size_t entries_needed); vespalib::MemoryUsage getMemoryUsage() const; vespalib::MemoryUsage getDynamicMemoryUsage() const; @@ -127,7 +128,7 @@ public: /** * Enable free list management. - * This only works for fixed size elements. + * This only works for fixed size entries. */ void enableFreeLists(); @@ -135,7 +136,7 @@ public: * Disable free list management. */ void disableFreeLists(); - void disableElemHoldList(); + void disable_entry_hold_list(); bool has_free_lists_enabled() const { return _freeListsEnabled; } @@ -177,25 +178,25 @@ public: bool has_held_buffers() const noexcept { return _hold_buffer_count != 0u; } /** - * Trim elem hold list, freeing elements that no longer needs to be held. + * Trim entry hold list, freeing entries that no longer needs to be held. * * @param oldest_used_gen the oldest generation that is still used. */ virtual void reclaim_entry_refs(generation_t oldest_used_gen) = 0; protected: - DataStoreBase(uint32_t numBuffers, uint32_t offset_bits, size_t maxArrays); + DataStoreBase(uint32_t numBuffers, uint32_t offset_bits, size_t max_entries); virtual ~DataStoreBase(); void* getBuffer(uint32_t bufferId) { return _buffers[bufferId].get_buffer_relaxed(); } struct EntryRefHoldElem { EntryRef ref; - size_t num_elems; + size_t num_entries; - EntryRefHoldElem(EntryRef ref_in, size_t num_elems_in) + EntryRefHoldElem(EntryRef ref_in, size_t num_entries_in) : ref(ref_in), - num_elems(num_elems_in) + num_entries(num_entries_in) {} }; @@ -215,11 +216,11 @@ private: { public: BufferState::Alloc _buffer; - size_t _usedElems; + size_t _used_entries; BufferTypeBase *_typeHandler; uint32_t _typeId; - FallbackHold(size_t bytesSize, BufferState::Alloc &&buffer, size_t usedElems, + FallbackHold(size_t bytesSize, BufferState::Alloc &&buffer, size_t used_entries, BufferTypeBase *typeHandler, uint32_t typeId); ~FallbackHold() override; @@ -227,8 +228,8 @@ private: class BufferHold; - bool consider_grow_active_buffer(uint32_t type_id, size_t elems_needed); - void switch_or_grow_primary_buffer(uint32_t typeId, size_t elemsNeeded); + bool consider_grow_active_buffer(uint32_t type_id, size_t entries_needed); + void switch_or_grow_primary_buffer(uint32_t typeId, size_t entries_needed); void markCompacting(uint32_t bufferId); /** * Hold of buffer has ended. @@ -238,14 +239,14 @@ private: /** * Switch buffer state to active for the given buffer. * - * @param bufferId Id of buffer to be active. - * @param typeId Registered data type for buffer. - * @param elemsNeeded Number of elements needed to be free. + * @param bufferId Id of buffer to be active. + * @param typeId Registered data type for buffer. + * @param entries_needed Number of entries needed to be free. */ - void onActive(uint32_t bufferId, uint32_t typeId, size_t elemsNeeded); + void on_active(uint32_t bufferId, uint32_t typeId, size_t entries_needed); void inc_hold_buffer_count(); - void fallbackResize(uint32_t bufferId, size_t elementsNeeded); + void fallback_resize(uint32_t bufferId, size_t entries_needed); uint32_t getFirstFreeBufferId(); template<typename FuncType> @@ -261,7 +262,7 @@ private: std::vector<BufferAndMeta> _buffers; // For fast mapping with known types // Provides a mapping from typeId -> primary buffer for that type. - // The primary buffer is used for allocations of new element(s) if no available slots are found in free lists. + // The primary buffer is used for allocations of new entries if no available slots are found in free lists. std::vector<uint32_t> _primary_buffer_ids; Stash _stash; @@ -269,12 +270,12 @@ private: std::vector<FreeList> _free_lists; mutable std::atomic<uint64_t> _compaction_count; vespalib::GenerationHolder _genHolder; - const uint32_t _maxArrays; + const uint32_t _max_entries; std::atomic<uint32_t> _bufferIdLimit; uint32_t _hold_buffer_count; const uint8_t _offset_bits; bool _freeListsEnabled; - bool _disableElemHoldList; + bool _disable_entry_hold_list; bool _initializing; }; diff --git a/vespalib/src/vespa/vespalib/datastore/free_list.h b/vespalib/src/vespa/vespalib/datastore/free_list.h index 20d4a6b96df..cd475af3104 100644 --- a/vespalib/src/vespa/vespalib/datastore/free_list.h +++ b/vespalib/src/vespa/vespalib/datastore/free_list.h @@ -30,7 +30,6 @@ public: bool empty() const { return _free_lists.empty(); } size_t size() const { return _free_lists.size(); } - uint32_t array_size() const { return _free_lists.back()->array_size(); } EntryRef pop_entry() { return _free_lists.back()->pop_entry(); } diff --git a/vespalib/src/vespa/vespalib/datastore/free_list_allocator.h b/vespalib/src/vespa/vespalib/datastore/free_list_allocator.h index cf899a76712..dc2d1ea3c34 100644 --- a/vespalib/src/vespa/vespalib/datastore/free_list_allocator.h +++ b/vespalib/src/vespa/vespalib/datastore/free_list_allocator.h @@ -29,7 +29,7 @@ public: HandleType alloc(Args && ... args); HandleType allocArray(ConstArrayRef array); - HandleType allocArray(size_t size); + HandleType allocArray(); }; } diff --git a/vespalib/src/vespa/vespalib/datastore/free_list_allocator.hpp b/vespalib/src/vespa/vespalib/datastore/free_list_allocator.hpp index b793e4f77a2..4e69db08a3c 100644 --- a/vespalib/src/vespa/vespalib/datastore/free_list_allocator.hpp +++ b/vespalib/src/vespa/vespalib/datastore/free_list_allocator.hpp @@ -71,8 +71,9 @@ FreeListAllocator<EntryT, RefT, ReclaimerT>::allocArray(ConstArrayRef array) if (free_list.empty()) { return ParentType::allocArray(array); } - assert(free_list.array_size() == array.size()); RefT ref = free_list.pop_entry(); + auto& state = _store.getBufferState(ref.bufferId()); + assert(state.getArraySize() == array.size()); EntryT *buf = _store.template getEntryArray<EntryT>(ref, array.size()); for (size_t i = 0; i < array.size(); ++i) { *(buf + i) = array[i]; @@ -82,14 +83,15 @@ FreeListAllocator<EntryT, RefT, ReclaimerT>::allocArray(ConstArrayRef array) template <typename EntryT, typename RefT, typename ReclaimerT> typename Allocator<EntryT, RefT>::HandleType -FreeListAllocator<EntryT, RefT, ReclaimerT>::allocArray(size_t size) +FreeListAllocator<EntryT, RefT, ReclaimerT>::allocArray() { auto& free_list = _store.getFreeList(_typeId); if (free_list.empty()) { - return ParentType::allocArray(size); + return ParentType::allocArray(); } - assert(free_list.array_size() == size); RefT ref = free_list.pop_entry(); + auto& state = _store.getBufferState(ref.bufferId()); + auto size = state.getArraySize(); EntryT *buf = _store.template getEntryArray<EntryT>(ref, size); return HandleType(ref, buf); } diff --git a/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.h b/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.h index 1b71c22f0ce..29684267546 100644 --- a/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.h +++ b/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.h @@ -27,7 +27,7 @@ private: public: FreeListRawAllocator(DataStoreBase &store, uint32_t typeId); - HandleType alloc(size_t numElems); + HandleType alloc(size_t num_entries); }; } diff --git a/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.hpp b/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.hpp index af832955cb7..7680cd8a9a5 100644 --- a/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.hpp +++ b/vespalib/src/vespa/vespalib/datastore/free_list_raw_allocator.hpp @@ -14,16 +14,18 @@ FreeListRawAllocator<EntryT, RefT>::FreeListRawAllocator(DataStoreBase &store, u template <typename EntryT, typename RefT> typename FreeListRawAllocator<EntryT, RefT>::HandleType -FreeListRawAllocator<EntryT, RefT>::alloc(size_t numElems) +FreeListRawAllocator<EntryT, RefT>::alloc(size_t num_entries) { auto& free_list = _store.getFreeList(_typeId); if (free_list.empty()) { - return ParentType::alloc(numElems); + return ParentType::alloc(num_entries); } - assert(free_list.array_size() == numElems); + assert(num_entries == 1); RefT ref = free_list.pop_entry(); + auto& state = _store.getBufferState(ref.bufferId()); + auto array_size = state.getArraySize(); // We must scale the offset according to array size as it was divided when the entry ref was created. - EntryT *entry = _store.template getEntryArray<EntryT>(ref, numElems); + EntryT *entry = _store.template getEntryArray<EntryT>(ref, array_size); return HandleType(ref, entry); } diff --git a/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.h b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.h index 600925969a3..e2718b94cd2 100644 --- a/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.h +++ b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.h @@ -14,11 +14,11 @@ namespace vespalib::datastore { /* * Class representing buffer type for large arrays in ArrayStore */ -template <typename EntryT> -class LargeArrayBufferType : public BufferType<Array<EntryT>> +template <typename ElemT> +class LargeArrayBufferType : public BufferType<Array<ElemT>> { using AllocSpec = ArrayStoreConfig::AllocSpec; - using ArrayType = Array<EntryT>; + using ArrayType = Array<ElemT>; using ParentType = BufferType<ArrayType>; using ParentType::empty_entry; using CleanContext = typename ParentType::CleanContext; @@ -31,7 +31,7 @@ public: { } ~LargeArrayBufferType() override; - void cleanHold(void* buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override; + void clean_hold(void* buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx) override; const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override; }; diff --git a/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.hpp b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.hpp index 3042bbff73f..72a2662991b 100644 --- a/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.hpp +++ b/vespalib/src/vespa/vespalib/datastore/large_array_buffer_type.hpp @@ -7,32 +7,32 @@ namespace vespalib::datastore { -template <typename EntryT> -LargeArrayBufferType<EntryT>::LargeArrayBufferType(const AllocSpec& spec, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) noexcept - : BufferType<Array<EntryT>>(1u, spec.minArraysInBuffer, spec.maxArraysInBuffer, spec.numArraysForNewBuffer, spec.allocGrowFactor), +template <typename ElemT> +LargeArrayBufferType<ElemT>::LargeArrayBufferType(const AllocSpec& spec, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) noexcept + : BufferType<Array<ElemT>>(1u, spec.min_entries_in_buffer, spec.max_entries_in_buffer, spec.num_entries_for_new_buffer, spec.allocGrowFactor), _memory_allocator(std::move(memory_allocator)) { } -template <typename EntryT> -LargeArrayBufferType<EntryT>::~LargeArrayBufferType() = default; +template <typename ElemT> +LargeArrayBufferType<ElemT>::~LargeArrayBufferType() = default; -template <typename EntryT> +template <typename ElemT> void -LargeArrayBufferType<EntryT>::cleanHold(void* buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) +LargeArrayBufferType<ElemT>::clean_hold(void* buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx) { ArrayType* elem = static_cast<ArrayType*>(buffer) + offset; const auto& empty = empty_entry(); - for (size_t i = 0; i < numElems; ++i) { - cleanCtx.extraBytesCleaned(sizeof(EntryT) * elem->size()); + for (size_t i = 0; i < num_entries; ++i) { + cleanCtx.extraBytesCleaned(sizeof(ElemT) * elem->size()); *elem = empty; ++elem; } } -template <typename EntryT> +template <typename ElemT> const vespalib::alloc::MemoryAllocator* -LargeArrayBufferType<EntryT>::get_memory_allocator() const +LargeArrayBufferType<ElemT>::get_memory_allocator() const { return _memory_allocator.get(); } diff --git a/vespalib/src/vespa/vespalib/datastore/memory_stats.cpp b/vespalib/src/vespa/vespalib/datastore/memory_stats.cpp index 8e060b4cfb4..5cb04796c5b 100644 --- a/vespalib/src/vespa/vespalib/datastore/memory_stats.cpp +++ b/vespalib/src/vespa/vespalib/datastore/memory_stats.cpp @@ -5,10 +5,10 @@ namespace vespalib::datastore { MemoryStats::MemoryStats() - : _allocElems(0), - _usedElems(0), - _deadElems(0), - _holdElems(0), + : _alloc_entries(0), + _used_entries(0), + _dead_entries(0), + _hold_entries(0), _allocBytes(0), _usedBytes(0), _deadBytes(0), @@ -22,10 +22,10 @@ MemoryStats::MemoryStats() MemoryStats& MemoryStats::operator+=(const MemoryStats& rhs) { - _allocElems += rhs._allocElems; - _usedElems += rhs._usedElems; - _deadElems += rhs._deadElems; - _holdElems += rhs._holdElems; + _alloc_entries += rhs._alloc_entries; + _used_entries += rhs._used_entries; + _dead_entries += rhs._dead_entries; + _hold_entries += rhs._hold_entries; _allocBytes += rhs._allocBytes; _usedBytes += rhs._usedBytes; _deadBytes += rhs._deadBytes; diff --git a/vespalib/src/vespa/vespalib/datastore/memory_stats.h b/vespalib/src/vespa/vespalib/datastore/memory_stats.h index 18d7dd77559..72a570dd625 100644 --- a/vespalib/src/vespa/vespalib/datastore/memory_stats.h +++ b/vespalib/src/vespa/vespalib/datastore/memory_stats.h @@ -13,10 +13,10 @@ namespace vespalib::datastore { class MemoryStats { public: - size_t _allocElems; - size_t _usedElems; - size_t _deadElems; - size_t _holdElems; + size_t _alloc_entries; + size_t _used_entries; + size_t _dead_entries; + size_t _hold_entries; size_t _allocBytes; size_t _usedBytes; size_t _deadBytes; diff --git a/vespalib/src/vespa/vespalib/datastore/raw_allocator.h b/vespalib/src/vespa/vespalib/datastore/raw_allocator.h index c10c8152e72..e7a59fadcf8 100644 --- a/vespalib/src/vespa/vespalib/datastore/raw_allocator.h +++ b/vespalib/src/vespa/vespalib/datastore/raw_allocator.h @@ -25,10 +25,10 @@ protected: public: RawAllocator(DataStoreBase &store, uint32_t typeId); - HandleType alloc(size_t numElems) { - return alloc(numElems, 0); + HandleType alloc(size_t num_entries) { + return alloc(num_entries, 0); } - HandleType alloc(size_t numElems, size_t extraElems); + HandleType alloc(size_t num_entries, size_t extra_entries); }; } diff --git a/vespalib/src/vespa/vespalib/datastore/raw_allocator.hpp b/vespalib/src/vespa/vespalib/datastore/raw_allocator.hpp index 04d99588218..9de361a8b19 100644 --- a/vespalib/src/vespa/vespalib/datastore/raw_allocator.hpp +++ b/vespalib/src/vespa/vespalib/datastore/raw_allocator.hpp @@ -16,19 +16,15 @@ RawAllocator<EntryT, RefT>::RawAllocator(DataStoreBase &store, uint32_t typeId) template <typename EntryT, typename RefT> typename RawAllocator<EntryT, RefT>::HandleType -RawAllocator<EntryT, RefT>::alloc(size_t numElems, size_t extraElems) +RawAllocator<EntryT, RefT>::alloc(size_t num_entries, size_t extra_entries) { - _store.ensureBufferCapacity(_typeId, numElems + extraElems); + _store.ensure_buffer_capacity(_typeId, num_entries + extra_entries); uint32_t buffer_id = _store.primary_buffer_id(_typeId); BufferState &state = _store.getBufferState(buffer_id); assert(state.isActive()); - size_t oldBufferSize = state.size(); - // Must perform scaling ourselves, according to array size - size_t arraySize = state.getArraySize(); - assert((numElems % arraySize) == 0u); - RefT ref((oldBufferSize / arraySize), buffer_id); - EntryT *buffer = _store.getEntryArray<EntryT>(ref, arraySize); - state.stats().pushed_back(numElems); + RefT ref(state.size(), buffer_id); + EntryT *buffer = _store.getEntryArray<EntryT>(ref, state.getArraySize()); + state.stats().pushed_back(num_entries); return HandleType(ref, buffer); } diff --git a/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.h b/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.h index 676a9d3790f..6bb6601839d 100644 --- a/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.h +++ b/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.h @@ -13,8 +13,8 @@ namespace vespalib::datastore { /* * Class representing buffer type for small arrays in ArrayStore */ -template <typename EntryT> -class SmallArrayBufferType : public BufferType<EntryT> +template <typename ElemT> +class SmallArrayBufferType : public BufferType<ElemT> { using AllocSpec = ArrayStoreConfig::AllocSpec; std::shared_ptr<alloc::MemoryAllocator> _memory_allocator; diff --git a/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.hpp b/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.hpp index 414804417eb..c9033936bd6 100644 --- a/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.hpp +++ b/vespalib/src/vespa/vespalib/datastore/small_array_buffer_type.hpp @@ -6,19 +6,19 @@ namespace vespalib::datastore { -template <typename EntryT> -SmallArrayBufferType<EntryT>::SmallArrayBufferType(uint32_t array_size, const AllocSpec& spec, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) noexcept - : BufferType<EntryT>(array_size, spec.minArraysInBuffer, spec.maxArraysInBuffer, spec.numArraysForNewBuffer, spec.allocGrowFactor), +template <typename ElemT> +SmallArrayBufferType<ElemT>::SmallArrayBufferType(uint32_t array_size, const AllocSpec& spec, std::shared_ptr<alloc::MemoryAllocator> memory_allocator) noexcept + : BufferType<ElemT>(array_size, spec.min_entries_in_buffer, spec.max_entries_in_buffer, spec.num_entries_for_new_buffer, spec.allocGrowFactor), _memory_allocator(std::move(memory_allocator)) { } -template <typename EntryT> -SmallArrayBufferType<EntryT>::~SmallArrayBufferType() = default; +template <typename ElemT> +SmallArrayBufferType<ElemT>::~SmallArrayBufferType() = default; -template <typename EntryT> +template <typename ElemT> const vespalib::alloc::MemoryAllocator* -SmallArrayBufferType<EntryT>::get_memory_allocator() const +SmallArrayBufferType<ElemT>::get_memory_allocator() const { return _memory_allocator.get(); } diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp index 52b0798543f..0efaf04b26e 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store.hpp @@ -106,7 +106,7 @@ private: _mapping.resize(data_store.get_bufferid_limit_relaxed()); for (const auto bufferId : _compacting_buffers->get_buffer_ids()) { BufferState &state = data_store.getBufferState(bufferId); - _mapping[bufferId].resize(state.get_used_arrays()); + _mapping[bufferId].resize(state.size()); } } diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.hpp index 8ad11b18218..49d2631e73f 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_allocator.hpp @@ -43,7 +43,7 @@ template <typename EntryT, typename RefT> void UniqueStoreAllocator<EntryT, RefT>::hold(EntryRef ref) { - _store.holdElem(ref, 1); + _store.hold_entry(ref); } template <typename EntryT, typename RefT> diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp index 32513d09c72..4a517521d77 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_enumerator.hpp @@ -44,7 +44,7 @@ UniqueStoreEnumerator<RefT>::allocate_enum_values(DataStoreBase & store) { _enumValues.resize(store.get_bufferid_limit_relaxed()); store.for_each_active_buffer([this](uint32_t buffer_id, const BufferState & state) { - _enumValues[buffer_id].resize(state.get_used_arrays()); + _enumValues[buffer_id].resize(state.size()); }); } diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.cpp b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.cpp index 1d3ba27d6bf..9f2105b7f08 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.cpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.cpp @@ -42,28 +42,28 @@ UniqueStoreSmallStringBufferType::UniqueStoreSmallStringBufferType(uint32_t arra UniqueStoreSmallStringBufferType::~UniqueStoreSmallStringBufferType() = default; void -UniqueStoreSmallStringBufferType::destroyElements(void *, ElemCount) +UniqueStoreSmallStringBufferType::destroy_entries(void *, EntryCount) { static_assert(std::is_trivially_destructible<UniqueStoreSmallStringEntry>::value, "UniqueStoreSmallStringEntry must be trivially destructable"); } void -UniqueStoreSmallStringBufferType::fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems) +UniqueStoreSmallStringBufferType::fallback_copy(void *newBuffer, const void *oldBuffer, EntryCount num_entries) { static_assert(std::is_trivially_copyable<UniqueStoreSmallStringEntry>::value, "UniqueStoreSmallStringEntry must be trivially copyable"); - if (numElems > 0) { - memcpy(newBuffer, oldBuffer, numElems); + if (num_entries > 0) { + memcpy(newBuffer, oldBuffer, num_entries * getArraySize()); } } void -UniqueStoreSmallStringBufferType::cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext) +UniqueStoreSmallStringBufferType::clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext) { - void *e = static_cast<char *>(buffer) + offset; - void *e_end = static_cast<char *>(e) + numElems; size_t array_size = getArraySize(); + void *e = static_cast<char *>(buffer) + offset * array_size; + void *e_end = static_cast<char *>(e) + num_entries * array_size; while (e < e_end) { static_cast<UniqueStoreSmallStringEntry *>(e)->clean_hold(array_size); e = static_cast<char *>(e) + array_size; @@ -86,10 +86,10 @@ UniqueStoreExternalStringBufferType::UniqueStoreExternalStringBufferType(uint32_ UniqueStoreExternalStringBufferType::~UniqueStoreExternalStringBufferType() = default; void -UniqueStoreExternalStringBufferType::cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) +UniqueStoreExternalStringBufferType::clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx) { UniqueStoreEntry<std::string> *elem = static_cast<UniqueStoreEntry<std::string> *>(buffer) + offset; - for (size_t i = 0; i < numElems; ++i) { + for (size_t i = 0; i < num_entries; ++i) { cleanCtx.extraBytesCleaned(elem->value().size() + 1); std::string().swap(elem->value()); ++elem; diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.h b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.h index a85b73f423d..d3348950891 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.h +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.h @@ -62,9 +62,9 @@ class UniqueStoreSmallStringBufferType : public BufferType<char> { public: UniqueStoreSmallStringBufferType(uint32_t array_size, uint32_t max_arrays, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator); ~UniqueStoreSmallStringBufferType() override; - void destroyElements(void *, ElemCount) override; - void fallbackCopy(void *newBuffer, const void *oldBuffer, ElemCount numElems) override; - void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext) override; + void destroy_entries(void *, EntryCount) override; + void fallback_copy(void *newBuffer, const void *oldBuffer, EntryCount numElems) override; + void clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext) override; const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override; }; @@ -76,7 +76,7 @@ class UniqueStoreExternalStringBufferType : public BufferType<UniqueStoreEntry<s public: UniqueStoreExternalStringBufferType(uint32_t array_size, uint32_t max_arrays, std::shared_ptr<vespalib::alloc::MemoryAllocator> memory_allocator); ~UniqueStoreExternalStringBufferType() override; - void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override; + void clean_hold(void *buffer, size_t offset, EntryCount num_entries, CleanContext cleanCtx) override; const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override; }; diff --git a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.hpp b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.hpp index 65cab4850ba..4ff8e1e1ab4 100644 --- a/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.hpp +++ b/vespalib/src/vespa/vespalib/datastore/unique_store_string_allocator.hpp @@ -42,7 +42,7 @@ UniqueStoreStringAllocator<RefT>::allocate(const char *value) uint32_t type_id = string_allocator::get_type_id(value_len); if (type_id != 0) { size_t array_size = string_allocator::array_sizes[type_id - 1]; - auto handle = _store.template freeListRawAllocator<char>(type_id).alloc(array_size); + auto handle = _store.template freeListRawAllocator<char>(type_id).alloc(1); new (static_cast<void *>(handle.data)) UniqueStoreSmallStringEntry(value, value_len, array_size); return handle.ref; } else { @@ -61,11 +61,10 @@ UniqueStoreStringAllocator<RefT>::hold(EntryRef ref) RefT iRef(ref); uint32_t type_id = _store.getTypeId(iRef.bufferId()); if (type_id != 0) { - size_t array_size = string_allocator::array_sizes[type_id - 1]; - _store.holdElem(ref, array_size); + _store.hold_entry(ref); } else { auto &value = _store.template getEntry<WrappedExternalEntryType>(iRef)->value(); - _store.holdElem(ref, 1, value.size() + 1); + _store.hold_entry(ref, value.size() + 1); } } @@ -79,7 +78,7 @@ UniqueStoreStringAllocator<RefT>::move_on_compact(EntryRef ref) static_assert(std::is_trivially_copyable<UniqueStoreSmallStringEntry>::value, "UniqueStoreSmallStringEntry must be trivially copyable"); size_t array_size = string_allocator::array_sizes[type_id - 1]; - auto handle = _store.template rawAllocator<char>(type_id).alloc(array_size); + auto handle = _store.template rawAllocator<char>(type_id).alloc(1); auto orig = _store.template getEntryArray<char>(iRef, array_size); memcpy(handle.data, orig, array_size); return handle.ref; diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index 663b3b65638..8ee3957af32 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -100,3 +100,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT zstdcompressor.cpp DEPENDS ) + +# Don't convert call to jump when returning a value from a function with +# a compatible stack. +set_source_files_properties(signalhandler.cpp PROPERTIES COMPILE_OPTIONS "-fno-optimize-sibling-calls") diff --git a/vespalib/src/vespa/vespalib/util/time.cpp b/vespalib/src/vespa/vespalib/util/time.cpp index cba26f24059..dd4972b1c21 100644 --- a/vespalib/src/vespa/vespalib/util/time.cpp +++ b/vespalib/src/vespa/vespalib/util/time.cpp @@ -93,7 +93,7 @@ Timer::waitAtLeast(duration dur, bool busyWait) { } -#if (defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 160000) || (!defined(_LIBCPP_VERSION) && defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 12) +#if (defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 170000) || (!defined(_LIBCPP_VERSION) && defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 12) // Temporary workaround until libc++ supports stream operators for duration // Temporary workaround while using libstdc++ 11 diff --git a/vespalib/src/vespa/vespalib/util/time.h b/vespalib/src/vespa/vespalib/util/time.h index b893661832f..10077a0b49a 100644 --- a/vespalib/src/vespa/vespalib/util/time.h +++ b/vespalib/src/vespa/vespalib/util/time.h @@ -101,7 +101,7 @@ duration adjustTimeoutByHz(duration timeout, long hz); } -#if (defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 160000) || (!defined(_LIBCPP_VERSION) && defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 12) +#if (defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 170000) || (!defined(_LIBCPP_VERSION) && defined(_GLIBCXX_RELEASE) && _GLIBCXX_RELEASE < 12) // Temporary workaround until libc++ supports stream operators for duration // Temporary workaround while using libstdc++ 11 diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java index af42e30422b..0b2595b6af8 100644 --- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java +++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Configurator.java @@ -2,20 +2,28 @@ package com.yahoo.vespa.zookeeper; import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.cloud.config.ZookeeperServerConfig.Server; import com.yahoo.security.tls.ConfigFileBasedTlsContext; import com.yahoo.security.tls.MixedMode; import com.yahoo.security.tls.TlsContext; import com.yahoo.security.tls.TransportSecurityUtils; + import java.io.FileWriter; import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.logging.Level; import java.util.stream.Collectors; +import static com.yahoo.stream.CustomCollectors.toLinkedMap; import static com.yahoo.vespa.defaults.Defaults.getDefaults; public class Configurator { @@ -64,7 +72,6 @@ public class Configurator { // override of Vespa TLS config for unit testing void writeConfigToDisk(VespaTlsConfig vespaTlsConfig) { configFilePath.toFile().getParentFile().mkdirs(); - try { writeZooKeeperConfigFile(zookeeperServerConfig, vespaTlsConfig); writeMyIdFile(zookeeperServerConfig); @@ -75,36 +82,75 @@ public class Configurator { private void writeZooKeeperConfigFile(ZookeeperServerConfig config, VespaTlsConfig vespaTlsConfig) throws IOException { + String dynamicConfigPath = config.dynamicReconfiguration() ? parseConfigFile(configFilePath).get("dynamicConfigFile") : null; + Map<String, String> dynamicConfig = dynamicConfigPath != null ? parseConfigFile(Paths.get(dynamicConfigPath)) : Map.of(); try (FileWriter writer = new FileWriter(configFilePath.toFile())) { - writer.write(transformConfigToString(config, vespaTlsConfig)); + writer.write(transformConfigToString(config, vespaTlsConfig, dynamicConfig)); } } - private String transformConfigToString(ZookeeperServerConfig config, VespaTlsConfig vespaTlsConfig) { - StringBuilder sb = new StringBuilder(); - sb.append("tickTime=").append(config.tickTime()).append("\n"); - sb.append("initLimit=").append(config.initLimit()).append("\n"); - sb.append("syncLimit=").append(config.syncLimit()).append("\n"); - sb.append("maxClientCnxns=").append(config.maxClientConnections()).append("\n"); - sb.append("snapCount=").append(config.snapshotCount()).append("\n"); - sb.append("dataDir=").append(getDefaults().underVespaHome(config.dataDir())).append("\n"); - sb.append("autopurge.purgeInterval=").append(config.autopurge().purgeInterval()).append("\n"); - sb.append("autopurge.snapRetainCount=").append(config.autopurge().snapRetainCount()).append("\n"); + private String transformConfigToString(ZookeeperServerConfig config, VespaTlsConfig vespaTlsConfig, Map<String, String> dynamicConfig) { + Map<String, String> configEntries = new LinkedHashMap<>(); + configEntries.put("tickTime", Integer.toString(config.tickTime())); + configEntries.put("initLimit", Integer.toString(config.initLimit())); + configEntries.put("syncLimit", Integer.toString(config.syncLimit())); + configEntries.put("maxClientCnxns", Integer.toString(config.maxClientConnections())); + configEntries.put("snapCount", Integer.toString(config.snapshotCount())); + configEntries.put("dataDir", getDefaults().underVespaHome(config.dataDir())); + configEntries.put("autopurge.purgeInterval", Integer.toString(config.autopurge().purgeInterval())); + configEntries.put("autopurge.snapRetainCount", Integer.toString(config.autopurge().snapRetainCount())); // See http://zookeeper.apache.org/doc/r3.6.3/zookeeperAdmin.html#sc_zkCommands // Includes all available commands in 3.6, except 'wchc' and 'wchp' - sb.append("4lw.commands.whitelist=conf,cons,crst,dirs,dump,envi,mntr,ruok,srst,srvr,stat,wchs").append("\n"); - sb.append("admin.enableServer=false").append("\n"); + configEntries.put("4lw.commands.whitelist", "conf,cons,crst,dirs,dump,envi,mntr,ruok,srst,srvr,stat,wchs"); + configEntries.put("admin.enableServer", "false"); // Use custom connection factory for TLS on client port - see class' Javadoc for rationale - sb.append("serverCnxnFactory=org.apache.zookeeper.server.VespaNettyServerCnxnFactory").append("\n"); - sb.append("quorumListenOnAllIPs=true").append("\n"); - sb.append("standaloneEnabled=false").append("\n"); - sb.append("reconfigEnabled=").append(config.dynamicReconfiguration()).append("\n"); - sb.append("skipACL=yes").append("\n"); - ensureThisServerIsRepresented(config.myid(), config.server()); - config.server().forEach(server -> sb.append(serverSpec(server, server.joining())).append("\n")); - sb.append(new TlsQuorumConfig().createConfig(vespaTlsConfig)); - sb.append(new TlsClientServerConfig().createConfig(vespaTlsConfig)); - return sb.toString(); + configEntries.put("serverCnxnFactory", "org.apache.zookeeper.server.VespaNettyServerCnxnFactory"); + configEntries.put("quorumListenOnAllIPs", "true"); + configEntries.put("standaloneEnabled", "false"); + configEntries.put("reconfigEnabled", Boolean.toString(config.dynamicReconfiguration())); + configEntries.put("skipACL", "yes"); + + addServerSpecs(configEntries, config, dynamicConfig); + + new TlsQuorumConfig().createConfig(configEntries, vespaTlsConfig); + new TlsClientServerConfig().createConfig(configEntries, vespaTlsConfig); + return transformConfigToString(configEntries); + } + + void addServerSpecs(Map<String, String> configEntries, ZookeeperServerConfig config, Map<String, String> dynamicConfig) { + int myIndex = ensureThisServerIsRepresented(config.myid(), config.server()); + + // If dynamic config refers to servers that are not in the current config, we must ignore it. + Set<String> currentServers = config.server().stream().map(Server::hostname).collect(Collectors.toSet()); + if (dynamicConfig.values().stream().anyMatch(spec -> ! currentServers.contains(spec.split(":", 2)[0]))) { + log.log(Level.WARNING, "Existing dynamic config refers to unknown servers, ignoring it"); + dynamicConfig = Map.of(); + } + + // If we have no existing, valid, dynamic config, we use all known servers as a starting point. + if (dynamicConfig.isEmpty()) { + configEntries.putAll(getServerConfig(config.server(), config.server(myIndex).joining() ? config.myid() : -1)); + } + // Otherwise, we use the existing, dynamic config as a starting point, and add this as a joiner if not present. + else { + Map.Entry<String, String> thisAsAJoiner = getServerConfig(config.server().subList(myIndex, myIndex + 1), config.myid()).entrySet().iterator().next(); + dynamicConfig.putIfAbsent(thisAsAJoiner.getKey(), thisAsAJoiner.getValue()); + configEntries.putAll(dynamicConfig); + } + + } + static Map<String, String> getServerConfig(List<ZookeeperServerConfig.Server> serversConfig, int joinerId) { + Map<String, String> configEntries = new LinkedHashMap<>(); + for (Server server : serversConfig) { + configEntries.put("server." + server.id(), serverSpec(server, server.id() == joinerId)); + } + return configEntries; + } + + static String transformConfigToString(Map<String, String> config) { + return config.entrySet().stream() + .map(entry -> entry.getKey() + "=" + entry.getValue()) + .collect(Collectors.joining("\n", "", "\n")); } private void writeMyIdFile(ZookeeperServerConfig config) throws IOException { @@ -113,25 +159,17 @@ public class Configurator { } } - private void ensureThisServerIsRepresented(int myid, List<ZookeeperServerConfig.Server> servers) { - boolean found = false; - for (ZookeeperServerConfig.Server server : servers) { - if (myid == server.id()) { - found = true; - break; - } - } - if (!found) { - throw new RuntimeException("No id in zookeeper server list that corresponds to my id (" + myid + ")"); + private static int ensureThisServerIsRepresented(int myid, List<ZookeeperServerConfig.Server> servers) { + for (int i = 0; i < servers.size(); i++) { + Server server = servers.get(i); + if (myid == server.id()) return i; } + throw new RuntimeException("No id in zookeeper server list that corresponds to my id (" + myid + ")"); } static String serverSpec(ZookeeperServerConfig.Server server, boolean joining) { StringBuilder sb = new StringBuilder(); - sb.append("server.") - .append(server.id()) - .append("=") - .append(server.hostname()) + sb.append(server.hostname()) .append(":") .append(server.quorumPort()) .append(":") @@ -150,6 +188,19 @@ public class Configurator { return sb.toString(); } + static Map<String, String> parseConfigFile(Path configFilePath) { + try { + return Files.exists(configFilePath) ? Files.readAllLines(configFilePath).stream() + .filter(line -> ! line.startsWith("#")) + .map(line -> line.split("=", 2)) + .collect(toLinkedMap(parts -> parts[0], parts -> parts[1])) + : Map.of(); + } + catch (IOException e) { + throw new UncheckedIOException("error reading zookeeper config", e); + } + } + static List<String> zookeeperServerHostnames(ZookeeperServerConfig zookeeperServerConfig) { return zookeeperServerConfig.server().stream() .map(ZookeeperServerConfig.Server::hostname) @@ -165,15 +216,15 @@ public class Configurator { private interface TlsConfig { String configFieldPrefix(); - default void appendSharedTlsConfig(StringBuilder builder, VespaTlsConfig vespaTlsConfig) { + default void appendSharedTlsConfig(Map<String, String> configEntries, VespaTlsConfig vespaTlsConfig) { vespaTlsConfig.context().ifPresent(ctx -> { VespaSslContextProvider.set(ctx); - builder.append(configFieldPrefix()).append(".context.supplier.class=").append(VespaSslContextProvider.class.getName()).append("\n"); + configEntries.put(configFieldPrefix() + ".context.supplier.class", VespaSslContextProvider.class.getName()); String enabledCiphers = Arrays.stream(ctx.parameters().getCipherSuites()).sorted().collect(Collectors.joining(",")); - builder.append(configFieldPrefix()).append(".ciphersuites=").append(enabledCiphers).append("\n"); + configEntries.put(configFieldPrefix() + ".ciphersuites", enabledCiphers); String enabledProtocols = Arrays.stream(ctx.parameters().getProtocols()).sorted().collect(Collectors.joining(",")); - builder.append(configFieldPrefix()).append(".enabledProtocols=").append(enabledProtocols).append("\n"); - builder.append(configFieldPrefix()).append(".clientAuth=NEED\n"); + configEntries.put(configFieldPrefix() + ".enabledProtocols", enabledProtocols); + configEntries.put(configFieldPrefix() + ".clientAuth", "NEED"); }); } @@ -185,16 +236,13 @@ public class Configurator { static class TlsClientServerConfig implements TlsConfig { - public String createConfig(VespaTlsConfig vespaTlsConfig) { - StringBuilder sb = new StringBuilder() - .append("client.portUnification=").append(enablePortUnification(vespaTlsConfig)).append("\n"); + public void createConfig(Map<String, String> configEntries, VespaTlsConfig vespaTlsConfig) { + configEntries.put("client.portUnification", String.valueOf(enablePortUnification(vespaTlsConfig))); // ZooKeeper Dynamic Reconfiguration requires the "non-secure" client port to exist // This is a hack to override the secure parameter through our connection factory wrapper // https://issues.apache.org/jira/browse/ZOOKEEPER-3577 VespaNettyServerCnxnFactory_isSecure = vespaTlsConfig.tlsEnabled() && vespaTlsConfig.mixedMode() == MixedMode.DISABLED; - appendSharedTlsConfig(sb, vespaTlsConfig); - - return sb.toString(); + appendSharedTlsConfig(configEntries, vespaTlsConfig); } @Override @@ -205,12 +253,10 @@ public class Configurator { static class TlsQuorumConfig implements TlsConfig { - public String createConfig(VespaTlsConfig vespaTlsConfig) { - StringBuilder sb = new StringBuilder() - .append("sslQuorum=").append(vespaTlsConfig.tlsEnabled()).append("\n") - .append("portUnification=").append(enablePortUnification(vespaTlsConfig)).append("\n"); - appendSharedTlsConfig(sb, vespaTlsConfig); - return sb.toString(); + public void createConfig(Map<String, String> configEntries, VespaTlsConfig vespaTlsConfig) { + configEntries.put("sslQuorum", String.valueOf(vespaTlsConfig.tlsEnabled())); + configEntries.put("portUnification", String.valueOf(enablePortUnification(vespaTlsConfig))); + appendSharedTlsConfig(configEntries, vespaTlsConfig); } @Override diff --git a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java index eb84b13d4d6..15431550d82 100644 --- a/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java +++ b/zookeeper-server/zookeeper-server-common/src/main/java/com/yahoo/vespa/zookeeper/Reconfigurer.java @@ -6,15 +6,15 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.component.annotation.Inject; import com.yahoo.protect.Process; import com.yahoo.yolean.Exceptions; + import java.time.Duration; import java.time.Instant; -import java.util.List; import java.util.Objects; import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; -import static com.yahoo.vespa.zookeeper.Configurator.serverSpec; +import static java.util.stream.Collectors.joining; /** * Starts zookeeper server and supports reconfiguring zookeeper cluster. Keep this as a component @@ -26,9 +26,11 @@ public class Reconfigurer extends AbstractComponent { private static final Logger log = java.util.logging.Logger.getLogger(Reconfigurer.class.getName()); - private static final Duration TIMEOUT = Duration.ofMinutes(15); + static final Duration TIMEOUT = Duration.ofMinutes(15); private final ExponentialBackoff backoff = new ExponentialBackoff(Duration.ofMillis(50), Duration.ofSeconds(10)); + private final Duration timeout; + private final boolean haltOnFailure; private final VespaZooKeeperAdmin vespaZooKeeperAdmin; private final Sleeper sleeper; @@ -38,12 +40,14 @@ public class Reconfigurer extends AbstractComponent { @Inject public Reconfigurer(VespaZooKeeperAdmin vespaZooKeeperAdmin) { - this(vespaZooKeeperAdmin, new Sleeper()); + this(vespaZooKeeperAdmin, new Sleeper(), true, TIMEOUT); } - Reconfigurer(VespaZooKeeperAdmin vespaZooKeeperAdmin, Sleeper sleeper) { + public Reconfigurer(VespaZooKeeperAdmin vespaZooKeeperAdmin, Sleeper sleeper, boolean haltOnFailure, Duration timeout) { this.vespaZooKeeperAdmin = Objects.requireNonNull(vespaZooKeeperAdmin); this.sleeper = Objects.requireNonNull(sleeper); + this.haltOnFailure = haltOnFailure; + this.timeout = timeout; } @Override @@ -86,14 +90,15 @@ public class Reconfigurer extends AbstractComponent { // TODO jonmv: wrap Curator in Provider, for Curator shutdown private void reconfigure(ZookeeperServerConfig newConfig) { Instant reconfigTriggered = Instant.now(); - String newServers = String.join(",", servers(newConfig)); + String newServers = servers(newConfig); log.log(Level.INFO, "Will reconfigure ZooKeeper cluster." + "\nServers in active config:" + servers(activeConfig) + - "\nServers in new config:" + servers(newConfig)); + "\nServers in new config:" + newServers); String connectionSpec = vespaZooKeeperAdmin.localConnectionSpec(activeConfig); Instant now = Instant.now(); - Duration reconfigTimeout = reconfigTimeout(); - Instant end = now.plus(reconfigTimeout); + // For reconfig to succeed, the current and resulting ensembles must have a majority. When an ensemble grows and + // the joining servers outnumber the existing ones, we have to wait for enough of them to start to have a majority. + Instant end = now.plus(timeout); // Loop reconfiguring since we might need to wait until another reconfiguration is finished before we can succeed for (int attempt = 1; ; attempt++) { try { @@ -116,29 +121,20 @@ public class Reconfigurer extends AbstractComponent { } else { log.log(Level.SEVERE, "Reconfiguration attempt " + attempt + " failed, and was failing for " + - reconfigTimeout + "; giving up now: " + Exceptions.toMessageString(e)); - shutdownAndDie(reconfigTimeout); + timeout + "; giving up now: " + Exceptions.toMessageString(e)); + shutdown(); + if (haltOnFailure) + Process.logAndDie("Reconfiguration did not complete within timeout " + timeout + ". Forcing container shutdown."); + else + throw e; } } } } - private void shutdownAndDie(Duration reconfigTimeout) { - shutdown(); - Process.logAndDie("Reconfiguration did not complete within timeout " + reconfigTimeout + ". Forcing container shutdown."); - } - - private static Duration reconfigTimeout() { - // For reconfig to succeed, the current and resulting ensembles must have a majority. When an ensemble grows and - // the joining servers outnumber the existing ones, we have to wait for enough of them to start to have a majority. - return TIMEOUT; - } - - private static List<String> servers(ZookeeperServerConfig config) { - return config.server().stream() - .filter(server -> ! server.retired()) - .map(server -> serverSpec(server, false)) - .toList(); + private static String servers(ZookeeperServerConfig config) { + return Configurator.getServerConfig(config.server().stream().filter(server -> ! server.retired()).toList(), -1) + .entrySet().stream().map(entry -> entry.getKey() + "=" + entry.getValue()).collect(joining(",")); } } diff --git a/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ConfiguratorTest.java b/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ConfiguratorTest.java index 5d0031d5b55..08acbf2b838 100644 --- a/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ConfiguratorTest.java +++ b/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ConfiguratorTest.java @@ -24,7 +24,10 @@ import java.math.BigInteger; import java.nio.file.Files; import java.security.KeyPair; import java.security.cert.X509Certificate; +import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.Set; import static com.yahoo.security.KeyAlgorithm.EC; @@ -62,12 +65,12 @@ public class ConfiguratorTest { } @Test - public void config_is_written_correctly_with_multiple_servers() { + public void config_is_written_correctly_with_multiple_servers() throws IOException { three_config_servers(false); } @Test - public void config_is_written_correctly_with_multiple_servers_on_hosted_vespa() { + public void config_is_written_correctly_with_multiple_servers_on_hosted_vespa() throws IOException { three_config_servers(true); } @@ -117,13 +120,49 @@ public class ConfiguratorTest { assertEquals("" + max_buffer, System.getProperty(ZOOKEEPER_JUTE_MAX_BUFFER)); } - private void three_config_servers(boolean hosted) { + @Test + public void test_parsing_config() throws IOException { ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder(); builder.zooKeeperConfigFile(cfgFile.getAbsolutePath()); builder.server(newServer(0, "foo", 123, 321, false)); builder.server(newServer(1, "bar", 234, 432, false)); builder.server(newServer(2, "baz", 345, 543, true)); builder.myidFile(idFile.getAbsolutePath()); + builder.myid(2); + builder.tickTime(1234); + builder.dynamicReconfiguration(true); + Configurator configurator = new Configurator(builder.build()); + configurator.writeConfigToDisk(VespaTlsConfig.tlsDisabled()); + validateIdFile(idFile, "2\n"); + + assertEquals(Files.readString(cfgFile.toPath()), + Configurator.transformConfigToString(Configurator.parseConfigFile(cfgFile.toPath()))); + + Map<String, String> originalConfig = Configurator.parseConfigFile(cfgFile.toPath()); + Map<String, String> staticConfig = new LinkedHashMap<>(originalConfig); + // Dynamic config says this is not a joiner. + Map<String, String> dynamicConfig = Configurator.getServerConfig(builder.build().server(), -1); + staticConfig.keySet().removeAll(dynamicConfig.keySet()); + assertEquals(originalConfig.size(), dynamicConfig.size() + staticConfig.size()); + File dynFile = folder.newFile(); + staticConfig.put("dynamicConfigFile", dynFile.getAbsolutePath()); + Files.write(cfgFile.toPath(), Configurator.transformConfigToString(staticConfig).getBytes()); + Files.write(dynFile.toPath(), Configurator.transformConfigToString(dynamicConfig).getBytes()); + + configurator.writeConfigToDisk(VespaTlsConfig.tlsDisabled()); + // Next generation of config should not mark this as a joiner either. + originalConfig.putAll(Configurator.getServerConfig(builder.build().server().subList(2, 3), -1)); + assertEquals(Configurator.transformConfigToString(originalConfig), + Files.readString(cfgFile.toPath())); + } + + private void three_config_servers(boolean hosted) throws IOException { + ZookeeperServerConfig.Builder builder = new ZookeeperServerConfig.Builder(); + builder.zooKeeperConfigFile(cfgFile.getAbsolutePath()); + builder.server(newServer(0, "foo", 123, 321, false)); + builder.server(newServer(1, "bar", 234, 432, true)); + builder.server(newServer(2, "baz", 345, 543, false)); + builder.myidFile(idFile.getAbsolutePath()); builder.myid(1); builder.tickTime(1234); builder.dynamicReconfiguration(hosted); @@ -205,8 +244,8 @@ public class ConfiguratorTest { String expected = commonConfig(hosted) + "server.0=foo:321:123;2181\n" + - "server.1=bar:432:234;2181\n" + - "server.2=baz:543:345:observer;2181\n" + + "server.1=bar:432:234:observer;2181\n" + + "server.2=baz:543:345;2181\n" + "sslQuorum=false\n" + "portUnification=false\n" + "client.portUnification=false\n"; diff --git a/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java b/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java index 697fba3b4c4..ea8dcac945c 100644 --- a/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java +++ b/zookeeper-server/zookeeper-server-common/src/test/java/com/yahoo/vespa/zookeeper/ReconfigurerTest.java @@ -144,11 +144,13 @@ public class ReconfigurerTest { TestableReconfigurer(TestableVespaZooKeeperAdmin zooKeeperAdmin) { super(zooKeeperAdmin, new Sleeper() { - @Override - public void sleep(Duration duration) { - // Do nothing - } - }); + @Override + public void sleep(Duration duration) { + // Do nothing + } + }, + false, + Reconfigurer.TIMEOUT); this.zooKeeperAdmin = zooKeeperAdmin; HostName.setHostNameForTestingOnly("node1"); } |