diff options
187 files changed, 3154 insertions, 1582 deletions
diff --git a/application-model/src/main/java/com/yahoo/vespa/archive/ArchiveStreamReader.java b/application-model/src/main/java/com/yahoo/vespa/archive/ArchiveStreamReader.java index 2f8b73839da..b7ad4ac6279 100644 --- a/application-model/src/main/java/com/yahoo/vespa/archive/ArchiveStreamReader.java +++ b/application-model/src/main/java/com/yahoo/vespa/archive/ArchiveStreamReader.java @@ -15,7 +15,6 @@ import java.io.InputStream; import java.io.OutputStream; import java.io.UncheckedIOException; import java.util.Objects; -import java.util.OptionalLong; import java.util.function.Predicate; import java.util.zip.GZIPInputStream; @@ -26,13 +25,13 @@ import java.util.zip.GZIPInputStream; */ public class ArchiveStreamReader implements AutoCloseable { - private final ArchiveInputStream archiveInputStream; + private final ArchiveInputStream<? extends ArchiveEntry> archiveInputStream; private final Options options; private long totalRead = 0; private long entriesRead = 0; - private ArchiveStreamReader(ArchiveInputStream archiveInputStream, Options options) { + private <T extends ArchiveEntry> ArchiveStreamReader(ArchiveInputStream<T> archiveInputStream, Options options) { this.archiveInputStream = Objects.requireNonNull(archiveInputStream); this.options = Objects.requireNonNull(options); } diff --git a/application/abi-spec.json b/application/abi-spec.json index 95a9d2a524a..27b01d02b5f 100644 --- a/application/abi-spec.json +++ b/application/abi-spec.json @@ -324,6 +324,7 @@ "public com.yahoo.application.container.Processing processing()", "public com.yahoo.application.container.DocumentProcessing documentProcessing()", "public com.yahoo.component.provider.ComponentRegistry components()", + "public com.yahoo.component.provider.ComponentRegistry handlers()", "public com.yahoo.application.container.handler.Response handleRequest(com.yahoo.application.container.handler.Request)", "public void close()" ], @@ -450,11 +451,13 @@ "public void <init>(java.lang.String, java.lang.String)", "public void <init>(java.lang.String, byte[], com.yahoo.application.container.handler.Request$Method)", "public void <init>(java.lang.String, byte[], com.yahoo.application.container.handler.Request$Method, java.security.Principal)", + "public void <init>(java.lang.String, byte[], com.yahoo.application.container.handler.Request$Method, java.security.Principal, java.net.SocketAddress)", "public void <init>(java.lang.String, java.lang.String, com.yahoo.application.container.handler.Request$Method)", "public com.yahoo.application.container.handler.Headers getHeaders()", "public byte[] getBody()", "public java.lang.String getUri()", "public java.util.Map getAttributes()", + "public java.util.Optional remoteAddress()", "public java.lang.String toString()", "public com.yahoo.application.container.handler.Request$Method getMethod()", "public java.util.Optional getUserPrincipal()" diff --git a/application/src/main/java/com/yahoo/application/container/JDisc.java b/application/src/main/java/com/yahoo/application/container/JDisc.java index 162a5f343a1..3223c8d31c9 100644 --- a/application/src/main/java/com/yahoo/application/container/JDisc.java +++ b/application/src/main/java/com/yahoo/application/container/JDisc.java @@ -169,6 +169,9 @@ public final class JDisc implements AutoCloseable { return container.getComponentRegistry(); } + /** @return registry of all request handlers configured */ + public ComponentRegistry<RequestHandler> handlers() { return container.getRequestHandlerRegistry(); } + /** * Handles the given {@link com.yahoo.application.container.handler.Request} by passing it to the {@link RequestHandler} * that is bound to the request's URI. diff --git a/application/src/main/java/com/yahoo/application/container/SynchronousRequestResponseHandler.java b/application/src/main/java/com/yahoo/application/container/SynchronousRequestResponseHandler.java index c54b3f60cf9..1b4862c75c0 100644 --- a/application/src/main/java/com/yahoo/application/container/SynchronousRequestResponseHandler.java +++ b/application/src/main/java/com/yahoo/application/container/SynchronousRequestResponseHandler.java @@ -90,6 +90,7 @@ final class SynchronousRequestResponseHandler { URI.create(request.getUri()), com.yahoo.jdisc.http.HttpRequest.Method.valueOf(request.getMethod().name())); request.getUserPrincipal().ifPresent(httpRequest::setUserPrincipal); + request.remoteAddress().ifPresent(httpRequest::setRemoteAddress); discRequest = httpRequest; } else { discRequest = new com.yahoo.jdisc.Request(currentContainer, URI.create(request.getUri())); diff --git a/application/src/main/java/com/yahoo/application/container/handler/Request.java b/application/src/main/java/com/yahoo/application/container/handler/Request.java index b6dff44269b..d877258cb15 100644 --- a/application/src/main/java/com/yahoo/application/container/handler/Request.java +++ b/application/src/main/java/com/yahoo/application/container/handler/Request.java @@ -3,6 +3,7 @@ package com.yahoo.application.container.handler; import com.yahoo.api.annotations.Beta; +import java.net.SocketAddress; import java.nio.charset.StandardCharsets; import java.security.Principal; import java.util.Map; @@ -24,6 +25,7 @@ public class Request { private final Method method; private final Map<String, Object> attributes = new ConcurrentHashMap<>(); private final Principal userPrincipal; + private final SocketAddress remoteAddress; /** * Creates a Request with an empty body. @@ -73,10 +75,24 @@ public class Request { * @param principal the user principal of the request */ public Request(String uri, byte[] body, Method method, Principal principal) { + this(uri, body, method, principal, null); + } + + /** + * Creates a Request with a message body, method and user principal. + * + * @param uri the URI of the request + * @param body the body of the request + * @param method the method of the request + * @param principal the user principal of the request + * @param remoteAddress the remote address of the request + */ + public Request(String uri, byte[] body, Method method, Principal principal, SocketAddress remoteAddress) { this.uri = uri; this.body = body; this.method = method; this.userPrincipal = principal; + this.remoteAddress = remoteAddress; } /** @@ -122,6 +138,8 @@ public class Request { return attributes; } + public Optional<SocketAddress> remoteAddress() { return Optional.ofNullable(remoteAddress); } + @Override public String toString() { String bodyStr = (body == null || body.length == 0) ? "[empty]" : "[omitted]"; diff --git a/client/go/go.mod b/client/go/go.mod index 0dab1722b0d..a122508b1b9 100644 --- a/client/go/go.mod +++ b/client/go/go.mod @@ -8,7 +8,7 @@ require ( github.com/fatih/color v1.16.0 // This is the most recent version compatible with Go 1.19. Upgrade when we upgrade our Go version github.com/go-json-experiment/json v0.0.0-20230216065249-540f01442424 - github.com/klauspost/compress v1.17.2 + github.com/klauspost/compress v1.17.3 github.com/mattn/go-colorable v0.1.13 github.com/mattn/go-isatty v0.0.20 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 diff --git a/client/go/go.sum b/client/go/go.sum index f4c69d92d9e..92e96c4e813 100644 --- a/client/go/go.sum +++ b/client/go/go.sum @@ -35,6 +35,8 @@ github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2g github.com/klauspost/compress v1.17.1/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.3 h1:qkRjuerhUU1EmXLYGkSH6EZL+vPSxIrYjLNAK4slzwA= +github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= diff --git a/client/go/internal/cli/cmd/cert.go b/client/go/internal/cli/cmd/cert.go index 1cc50b1faea..9668e78bd1c 100644 --- a/client/go/internal/cli/cmd/cert.go +++ b/client/go/internal/cli/cmd/cert.go @@ -153,7 +153,7 @@ func doCertAdd(cli *CLI, overwriteCertificate bool, args []string) error { if err != nil { return err } - pkg, err := cli.applicationPackageFrom(args, false) + pkg, err := cli.applicationPackageFrom(args, vespa.PackageOptions{}) if err != nil { return err } diff --git a/client/go/internal/cli/cmd/config.go b/client/go/internal/cli/cmd/config.go index cfadc6d32c5..a656279549a 100644 --- a/client/go/internal/cli/cmd/config.go +++ b/client/go/internal/cli/cmd/config.go @@ -152,7 +152,7 @@ $ vespa config set --local wait 600 config := cli.config if localArg { // Need an application package in working directory to allow local configuration - if _, err := cli.applicationPackageFrom(nil, false); err != nil { + if _, err := cli.applicationPackageFrom(nil, vespa.PackageOptions{}); err != nil { return fmt.Errorf("failed to write local configuration: %w", err) } config = cli.config.local @@ -188,7 +188,7 @@ $ vespa config unset --local application RunE: func(cmd *cobra.Command, args []string) error { config := cli.config if localArg { - if _, err := cli.applicationPackageFrom(nil, false); err != nil { + if _, err := cli.applicationPackageFrom(nil, vespa.PackageOptions{}); err != nil { return fmt.Errorf("failed to write local configuration: %w", err) } config = cli.config.local diff --git a/client/go/internal/cli/cmd/deploy.go b/client/go/internal/cli/cmd/deploy.go index aee26975901..8806a21c9fc 100644 --- a/client/go/internal/cli/cmd/deploy.go +++ b/client/go/internal/cli/cmd/deploy.go @@ -51,7 +51,7 @@ $ vespa deploy -t cloud -z perf.aws-us-east-1c`, DisableAutoGenTag: true, SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { - pkg, err := cli.applicationPackageFrom(args, true) + pkg, err := cli.applicationPackageFrom(args, vespa.PackageOptions{Compiled: true}) if err != nil { return err } @@ -113,7 +113,7 @@ func newPrepareCmd(cli *CLI) *cobra.Command { DisableAutoGenTag: true, SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { - pkg, err := cli.applicationPackageFrom(args, true) + pkg, err := cli.applicationPackageFrom(args, vespa.PackageOptions{Compiled: true}) if err != nil { return fmt.Errorf("could not find application package: %w", err) } diff --git a/client/go/internal/cli/cmd/feed.go b/client/go/internal/cli/cmd/feed.go index 1a32ac7110d..89e13a4673c 100644 --- a/client/go/internal/cli/cmd/feed.go +++ b/client/go/internal/cli/cmd/feed.go @@ -64,10 +64,36 @@ func newFeedCmd(cli *CLI) *cobra.Command { This command can be used to feed large amounts of documents to a Vespa cluster efficiently. -The contents of JSON-FILE must be either a JSON array or JSON objects separated by +The contents of json-file must be either a JSON array or JSON objects separated by newline (JSONL). -If JSON-FILE is a single dash ('-'), documents will be read from standard input. +If json-file is a single dash ('-'), documents will be read from standard input. + +Once feeding completes, metrics of the feed session are printed to standard out +in a JSON format: + +- feeder.operation.count: Number of operations passed to the feeder by the user, + not counting retries. +- feeder.seconds: Total time spent feeding. +- feeder.ok.count: Number of successful operations. +- feeder.ok.rate: Number of successful operations per second. +- feeder.error.count: Number of network errors (transport layer). +- feeder.inflight.count: Number of operations currently being sent. +- http.request.count: Number of HTTP requests made, including retries. +- http.request.bytes: Number of bytes sent. +- http.request.MBps: Request throughput measured in MB/s. This is the raw + operation throughput, and not the network throughput, + I.e. using compression does not affect this number. +- http.exception.count: Same as feeder.error.count. Present for compatiblity + with vespa-feed-client. +- http.response.count: Number of HTTP responses received. +- http.response.bytes: Number of bytes received. +- http.response.MBps: Response throughput measured in MB/s. +- http.response.error.count: Number of non-OK HTTP responses received. +- http.response.latency.millis.min: Lowest latency of a successful operation. +- http.response.latency.millis.avg: Average latency of successful operations. +- http.response.latency.millis.max: Highest latency of a successful operation. +- http.response.code.counts: Number of responses grouped by their HTTP code. `, Example: `$ vespa feed docs.jsonl moredocs.json $ cat docs.jsonl | vespa feed -`, @@ -244,6 +270,7 @@ type number float32 func (n number) MarshalJSON() ([]byte, error) { return []byte(fmt.Sprintf("%.3f", n)), nil } type feedSummary struct { + Operations int64 `json:"feeder.operation.count"` Seconds number `json:"feeder.seconds"` SuccessCount int64 `json:"feeder.ok.count"` SuccessRate number `json:"feeder.ok.rate"` @@ -272,6 +299,7 @@ func mbps(bytes int64, duration time.Duration) float64 { func writeSummaryJSON(w io.Writer, stats document.Stats, duration time.Duration) error { summary := feedSummary{ + Operations: stats.Operations, Seconds: number(duration.Seconds()), SuccessCount: stats.Successful(), SuccessRate: number(float64(stats.Successful()) / math.Max(1, duration.Seconds())), diff --git a/client/go/internal/cli/cmd/feed_test.go b/client/go/internal/cli/cmd/feed_test.go index 1cf9a6aba3c..daf649a0fd1 100644 --- a/client/go/internal/cli/cmd/feed_test.go +++ b/client/go/internal/cli/cmd/feed_test.go @@ -48,6 +48,7 @@ func TestFeed(t *testing.T) { assert.Equal(t, "", stderr.String()) want := `{ + "feeder.operation.count": 2, "feeder.seconds": 5.000, "feeder.ok.count": 2, "feeder.ok.rate": 0.400, @@ -122,6 +123,7 @@ func TestFeedInvalid(t *testing.T) { require.NotNil(t, cli.Run("feed", "-t", "http://127.0.0.1:8080", jsonFile)) want := `{ + "feeder.operation.count": 1, "feeder.seconds": 3.000, "feeder.ok.count": 1, "feeder.ok.rate": 0.333, diff --git a/client/go/internal/cli/cmd/prod.go b/client/go/internal/cli/cmd/prod.go index 9e78b299e4b..74e4b4c4a1c 100644 --- a/client/go/internal/cli/cmd/prod.go +++ b/client/go/internal/cli/cmd/prod.go @@ -57,7 +57,7 @@ https://cloud.vespa.ai/en/reference/deployment`, if err != nil { return err } - pkg, err := cli.applicationPackageFrom(args, false) + pkg, err := cli.applicationPackageFrom(args, vespa.PackageOptions{SourceOnly: true}) if err != nil { return err } @@ -143,7 +143,7 @@ $ vespa prod deploy`, // TODO: Add support for hosted return fmt.Errorf("prod deploy does not support %s target", target.Type()) } - pkg, err := cli.applicationPackageFrom(args, true) + pkg, err := cli.applicationPackageFrom(args, vespa.PackageOptions{Compiled: true}) if err != nil { return err } diff --git a/client/go/internal/cli/cmd/root.go b/client/go/internal/cli/cmd/root.go index 068a7ed90b6..4d1a7cf6f89 100644 --- a/client/go/internal/cli/cmd/root.go +++ b/client/go/internal/cli/cmd/root.go @@ -586,7 +586,7 @@ func isTerminal(w io.Writer) bool { // applicationPackageFrom returns an application loaded from args. If args is empty, the application package is loaded // from the working directory. If requirePackaging is true, the application package is required to be packaged with mvn // package. -func (c *CLI) applicationPackageFrom(args []string, requirePackaging bool) (vespa.ApplicationPackage, error) { +func (c *CLI) applicationPackageFrom(args []string, options vespa.PackageOptions) (vespa.ApplicationPackage, error) { path := "." if len(args) == 1 { path = args[0] @@ -603,5 +603,5 @@ func (c *CLI) applicationPackageFrom(args []string, requirePackaging bool) (vesp } else if len(args) > 1 { return vespa.ApplicationPackage{}, fmt.Errorf("expected 0 or 1 arguments, got %d", len(args)) } - return vespa.FindApplicationPackage(path, requirePackaging) + return vespa.FindApplicationPackage(path, options) } diff --git a/client/go/internal/vespa/application.go b/client/go/internal/vespa/application.go index cb43578af32..6d28b24100f 100644 --- a/client/go/internal/vespa/application.go +++ b/client/go/internal/vespa/application.go @@ -234,6 +234,14 @@ func copyFile(src *zip.File, dst string) error { return err } +type PackageOptions struct { + // If true, a Maven-based Vespa application package is required to be compiled + Compiled bool + + // If true, only consider the source directores of the application package + SourceOnly bool +} + // FindApplicationPackage finds the path to an application package from the zip file or directory zipOrDir. If // requirePackaging is true, the application package is required to be packaged with mvn package. // @@ -242,8 +250,8 @@ func copyFile(src *zip.File, dst string) error { // 2. target/application // 3. src/main/application // 4. Given path, if it contains services.xml -func FindApplicationPackage(zipOrDir string, requirePackaging bool) (ApplicationPackage, error) { - pkg, err := findApplicationPackage(zipOrDir, requirePackaging) +func FindApplicationPackage(zipOrDir string, options PackageOptions) (ApplicationPackage, error) { + pkg, err := findApplicationPackage(zipOrDir, options) if err != nil { return ApplicationPackage{}, err } @@ -253,20 +261,20 @@ func FindApplicationPackage(zipOrDir string, requirePackaging bool) (Application return pkg, nil } -func findApplicationPackage(zipOrDir string, requirePackaging bool) (ApplicationPackage, error) { +func findApplicationPackage(zipOrDir string, options PackageOptions) (ApplicationPackage, error) { if isZip(zipOrDir) { return ApplicationPackage{Path: zipOrDir}, nil } // Pre-packaged application. We prefer the uncompressed application because this allows us to add // security/clients.pem to the package on-demand hasPOM := util.PathExists(filepath.Join(zipOrDir, "pom.xml")) - if hasPOM { + if hasPOM && !options.SourceOnly { path := filepath.Join(zipOrDir, "target", "application") if util.PathExists(path) { testPath := existingPath(filepath.Join(zipOrDir, "target", "application-test")) return ApplicationPackage{Path: path, TestPath: testPath}, nil } - if requirePackaging { + if options.Compiled { return ApplicationPackage{}, fmt.Errorf("found pom.xml, but %s does not exist: run 'mvn package' first", path) } } diff --git a/client/go/internal/vespa/deploy_test.go b/client/go/internal/vespa/deploy_test.go index d1dffe0f6d6..9dfdc47d8e6 100644 --- a/client/go/internal/vespa/deploy_test.go +++ b/client/go/internal/vespa/deploy_test.go @@ -146,9 +146,9 @@ func TestFindApplicationPackage(t *testing.T) { existingFiles: []string{filepath.Join(dir, "pom.xml"), filepath.Join(dir, "src/test/application/tests/foo.json")}, }) assertFindApplicationPackage(t, dir, pkgFixture{ - existingFile: filepath.Join(dir, "pom.xml"), - requirePackaging: true, - fail: true, + existingFile: filepath.Join(dir, "pom.xml"), + compiled: true, + fail: true, }) assertFindApplicationPackage(t, dir, pkgFixture{ expectedPath: filepath.Join(dir, "target", "application"), @@ -159,6 +159,12 @@ func TestFindApplicationPackage(t *testing.T) { expectedTestPath: filepath.Join(dir, "target", "application-test"), existingFiles: []string{filepath.Join(dir, "target", "application"), filepath.Join(dir, "target", "application-test")}, }) + assertFindApplicationPackage(t, dir, pkgFixture{ + expectedPath: filepath.Join(dir, "src", "main", "application"), + expectedTestPath: filepath.Join(dir, "src", "test", "application"), + existingFiles: []string{filepath.Join(dir, "target", "application"), filepath.Join(dir, "target", "application-test")}, + sourceOnly: true, + }) zip := filepath.Join(dir, "myapp.zip") assertFindApplicationPackage(t, zip, pkgFixture{ expectedPath: zip, @@ -195,7 +201,8 @@ type pkgFixture struct { expectedTestPath string existingFile string existingFiles []string - requirePackaging bool + compiled bool + sourceOnly bool fail bool } @@ -207,7 +214,7 @@ func assertFindApplicationPackage(t *testing.T, zipOrDir string, fixture pkgFixt for _, f := range fixture.existingFiles { writeFile(t, f) } - pkg, err := FindApplicationPackage(zipOrDir, fixture.requirePackaging) + pkg, err := FindApplicationPackage(zipOrDir, PackageOptions{Compiled: fixture.compiled, SourceOnly: fixture.sourceOnly}) assert.Equal(t, err != nil, fixture.fail, "Expected error for "+zipOrDir) assert.Equal(t, fixture.expectedPath, pkg.Path) assert.Equal(t, fixture.expectedTestPath, pkg.TestPath) diff --git a/client/go/internal/vespa/document/dispatcher.go b/client/go/internal/vespa/document/dispatcher.go index 153631a8f5e..786e7c332a4 100644 --- a/client/go/internal/vespa/document/dispatcher.go +++ b/client/go/internal/vespa/document/dispatcher.go @@ -156,7 +156,7 @@ func (d *Dispatcher) processResults() { defer d.wg.Done() for op := range d.results { d.statsMu.Lock() - d.stats.Add(op.result) + d.stats.Add(op.result, op.attempts > 1) d.statsMu.Unlock() retry := d.shouldRetry(op, op.result) d.logResult(op, retry) diff --git a/client/go/internal/vespa/document/http_test.go b/client/go/internal/vespa/document/http_test.go index 57c663d6f4c..b2c1139f95f 100644 --- a/client/go/internal/vespa/document/http_test.go +++ b/client/go/internal/vespa/document/http_test.go @@ -120,7 +120,7 @@ func TestClientSend(t *testing.T) { if !reflect.DeepEqual(res, wantRes) { t.Fatalf("#%d: got result %+v, want %+v", i, res, wantRes) } - stats.Add(res) + stats.Add(res, false) r := httpClient.LastRequest if r.Method != tt.method { t.Errorf("#%d: got r.Method = %q, want %q", i, r.Method, tt.method) @@ -139,8 +139,9 @@ func TestClientSend(t *testing.T) { } } want := Stats{ - Requests: 5, - Responses: 4, + Operations: 5, + Requests: 5, + Responses: 4, ResponsesByCode: map[int]int64{ 200: 3, 502: 1, diff --git a/client/go/internal/vespa/document/stats.go b/client/go/internal/vespa/document/stats.go index 82630b4af1c..9ce10f2f250 100644 --- a/client/go/internal/vespa/document/stats.go +++ b/client/go/internal/vespa/document/stats.go @@ -40,16 +40,29 @@ func (r Result) Success() bool { // Stats represents feeding operation statistics. type Stats struct { + // Number of operations passed to the feeder by the user, not counting retries. + Operations int64 + // Number of responses received, grouped by the HTTP status code. Requests that do not receive a response (i.e. no + // status code) are not counted. ResponsesByCode map[int]int64 - Requests int64 - Responses int64 - Errors int64 - Inflight int64 - TotalLatency time.Duration - MinLatency time.Duration - MaxLatency time.Duration - BytesSent int64 - BytesRecv int64 + // Number of requests made, including retries. + Requests int64 + // Number of responses received. + Responses int64 + // Number of transport layer errors. + Errors int64 + // Number of requests currently in-flight. + Inflight int64 + // Sum of response latency + TotalLatency time.Duration + // Lowest recorded response latency + MinLatency time.Duration + // Highest recorded response latency + MaxLatency time.Duration + // Total bytes sent + BytesSent int64 + // Total bytes received + BytesRecv int64 } // AvgLatency returns the average latency for a request. @@ -82,14 +95,16 @@ func (s Stats) Clone() Stats { } // Add statistics from result to this. -func (s *Stats) Add(result Result) { +func (s *Stats) Add(result Result, retry bool) { + if !retry { + s.Operations++ + } s.Requests++ if s.ResponsesByCode == nil { s.ResponsesByCode = make(map[int]int64) } if result.Err == nil { - responsesByCode := s.ResponsesByCode[result.HTTPStatus] - s.ResponsesByCode[result.HTTPStatus] = responsesByCode + 1 + s.ResponsesByCode[result.HTTPStatus]++ s.Responses++ } else { s.Errors++ diff --git a/client/go/internal/vespa/document/stats_test.go b/client/go/internal/vespa/document/stats_test.go index c9b80878b75..a08dc075850 100644 --- a/client/go/internal/vespa/document/stats_test.go +++ b/client/go/internal/vespa/document/stats_test.go @@ -9,17 +9,19 @@ import ( func TestStatsAdd(t *testing.T) { var stats Stats - stats.Add(Result{HTTPStatus: 200, Latency: 200 * time.Millisecond}) - stats.Add(Result{HTTPStatus: 200, Latency: 400 * time.Millisecond}) - stats.Add(Result{HTTPStatus: 200, Latency: 100 * time.Millisecond}) - stats.Add(Result{HTTPStatus: 200, Latency: 500 * time.Millisecond}) - stats.Add(Result{HTTPStatus: 200, Latency: 300 * time.Millisecond}) - stats.Add(Result{HTTPStatus: 500, Latency: 100 * time.Millisecond}) + stats.Add(Result{HTTPStatus: 200, Latency: 200 * time.Millisecond}, false) + stats.Add(Result{HTTPStatus: 200, Latency: 400 * time.Millisecond}, false) + stats.Add(Result{HTTPStatus: 200, Latency: 100 * time.Millisecond}, false) + stats.Add(Result{HTTPStatus: 200, Latency: 500 * time.Millisecond}, false) + stats.Add(Result{HTTPStatus: 200, Latency: 300 * time.Millisecond}, false) + stats.Add(Result{HTTPStatus: 500, Latency: 100 * time.Millisecond}, false) + stats.Add(Result{HTTPStatus: 200, Latency: 100 * time.Millisecond}, true) expected := Stats{ - Requests: 6, - Responses: 6, - ResponsesByCode: map[int]int64{200: 5, 500: 1}, - TotalLatency: 1600 * time.Millisecond, + Operations: 6, + Requests: 7, + Responses: 7, + ResponsesByCode: map[int]int64{200: 6, 500: 1}, + TotalLatency: 1700 * time.Millisecond, MinLatency: 100 * time.Millisecond, MaxLatency: 500 * time.Millisecond, } @@ -33,11 +35,11 @@ func TestStatsAdd(t *testing.T) { func TestStatsClone(t *testing.T) { var a Stats - a.Add(Result{HTTPStatus: 200}) + a.Add(Result{HTTPStatus: 200}, false) b := a.Clone() - a.Add(Result{HTTPStatus: 200}) + a.Add(Result{HTTPStatus: 200}, false) - want := Stats{Requests: 1, Responses: 1, ResponsesByCode: map[int]int64{200: 1}} + want := Stats{Operations: 1, Requests: 1, Responses: 1, ResponsesByCode: map[int]int64{200: 1}} if !reflect.DeepEqual(b, want) { t.Errorf("got %+v, want %+v", b, want) } diff --git a/client/go/internal/vespa/target.go b/client/go/internal/vespa/target.go index 8c3f5c9b7c3..f3a94f762ff 100644 --- a/client/go/internal/vespa/target.go +++ b/client/go/internal/vespa/target.go @@ -9,7 +9,6 @@ import ( "io" "net/http" "strings" - "sync" "time" "github.com/vespa-engine/vespa/client/go/internal/util" @@ -51,9 +50,9 @@ type Service struct { TLSOptions TLSOptions deployAPI bool - once sync.Once auth Authenticator httpClient util.HTTPClient + customClient bool retryInterval time.Duration } @@ -109,7 +108,10 @@ type LogOptions struct { // Do sends request to this service. Authentication of the request happens automatically. func (s *Service) Do(request *http.Request, timeout time.Duration) (*http.Response, error) { - util.ConfigureTLS(s.httpClient, s.TLSOptions.KeyPair, s.TLSOptions.CACertificate, s.TLSOptions.TrustAll) + if !s.customClient { + // Do not override TLS config if a custom client has been configured + util.ConfigureTLS(s.httpClient, s.TLSOptions.KeyPair, s.TLSOptions.CACertificate, s.TLSOptions.TrustAll) + } if s.auth != nil { if err := s.auth.Authenticate(request); err != nil { return nil, fmt.Errorf("%w: %s", errAuth, err) @@ -118,8 +120,11 @@ func (s *Service) Do(request *http.Request, timeout time.Duration) (*http.Respon return s.httpClient.Do(request, timeout) } -// SetClient sets the HTTP client that this service should use. -func (s *Service) SetClient(client util.HTTPClient) { s.httpClient = client } +// SetClient sets a custom HTTP client that this service should use. +func (s *Service) SetClient(client util.HTTPClient) { + s.httpClient = client + s.customClient = true +} // Wait polls the health check of this service until it succeeds or timeout passes. func (s *Service) Wait(timeout time.Duration) error { diff --git a/client/js/app/package.json b/client/js/app/package.json index 23ddccbe7e7..75b12aecefc 100644 --- a/client/js/app/package.json +++ b/client/js/app/package.json @@ -20,9 +20,9 @@ "@fortawesome/free-regular-svg-icons": "^6", "@fortawesome/free-solid-svg-icons": "^6", "@fortawesome/react-fontawesome": "^0", - "@mantine/core": "^6.0.0", - "@mantine/hooks": "^6.0.0", - "@mantine/notifications": "^6.0.0", + "@mantine/core": "^5", + "@mantine/hooks": "^5", + "@mantine/notifications": "^5", "@vitejs/plugin-react": "^4", "esbuild-jest": "^0", "eslint": "^8", @@ -39,7 +39,7 @@ "pretty-quick": "^3", "react-router-dom": "^6", "use-context-selector": "^1", - "vite": "^4" + "vite": "^5.0.0" }, "jest": { "testMatch": [ diff --git a/client/js/app/yarn.lock b/client/js/app/yarn.lock index 248e1e2ac4a..3f4d43813d9 100644 --- a/client/js/app/yarn.lock +++ b/client/js/app/yarn.lock @@ -24,9 +24,9 @@ chalk "^2.4.2" "@babel/compat-data@^7.22.9": - version "7.23.2" - resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.23.2.tgz#6a12ced93455827037bfb5ed8492820d60fc32cc" - integrity sha512-0S9TQMmDHlqAZ2ITT95irXKfxN9bncq8ZCoJhun3nHL/lLUxd2NKBJYoNGWH7S0hz6fRQwWlAWn/ILM0C70KZQ== + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.23.3.tgz#3febd552541e62b5e883a25eb3effd7c7379db11" + integrity sha512-BmR4bWbDIoFJmJ9z2cZ8Gmm2MXgEDgjdWgpKmKWUt54UGFJdlj31ECtbaDvCG/qVdG3AQ1SfpZEs01lUFbzLOQ== "@babel/core@^7.1.0", "@babel/core@^7.12.17": version "7.22.9" @@ -70,21 +70,21 @@ json5 "^2.2.3" semver "^6.3.1" -"@babel/core@^7.23.2": - version "7.23.2" - resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.23.2.tgz#ed10df0d580fff67c5f3ee70fd22e2e4c90a9f94" - integrity sha512-n7s51eWdaWZ3vGT2tD4T7J6eJs3QoBXydv7vkUM06Bf1cbVD2Kc2UrkzhiQwobfV7NwOnQXYL7UBJ5VPU+RGoQ== +"@babel/core@^7.23.3": + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/core/-/core-7.23.3.tgz#5ec09c8803b91f51cc887dedc2654a35852849c9" + integrity sha512-Jg+msLuNuCJDyBvFv5+OKOUjWMZgd85bKjbICd3zWrKAo+bJ49HJufi7CQE0q0uR8NGyO6xkCACScNqyjHSZew== dependencies: "@ampproject/remapping" "^2.2.0" "@babel/code-frame" "^7.22.13" - "@babel/generator" "^7.23.0" + "@babel/generator" "^7.23.3" "@babel/helper-compilation-targets" "^7.22.15" - "@babel/helper-module-transforms" "^7.23.0" + "@babel/helper-module-transforms" "^7.23.3" "@babel/helpers" "^7.23.2" - "@babel/parser" "^7.23.0" + "@babel/parser" "^7.23.3" "@babel/template" "^7.22.15" - "@babel/traverse" "^7.23.2" - "@babel/types" "^7.23.0" + "@babel/traverse" "^7.23.3" + "@babel/types" "^7.23.3" convert-source-map "^2.0.0" debug "^4.1.0" gensync "^1.0.0-beta.2" @@ -111,12 +111,12 @@ "@jridgewell/trace-mapping" "^0.3.17" jsesc "^2.5.1" -"@babel/generator@^7.23.0": - version "7.23.0" - resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.23.0.tgz#df5c386e2218be505b34837acbcb874d7a983420" - integrity sha512-lN85QRR+5IbYrMWM6Y4pE/noaQtg4pNiqeNGX60eqOfo6gtEj6uw/JagelB8vVztSd7R6M5n1+PQkDbHbBRU4g== +"@babel/generator@^7.23.0", "@babel/generator@^7.23.3": + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.23.3.tgz#86e6e83d95903fbe7613f448613b8b319f330a8e" + integrity sha512-keeZWAV4LU3tW0qRi19HRpabC/ilM0HRBBzf9/k8FFiG4KVpiv0FIy4hHfLfFQZNhziCTPTmd59zoyv6DNISzg== dependencies: - "@babel/types" "^7.23.0" + "@babel/types" "^7.23.3" "@jridgewell/gen-mapping" "^0.3.2" "@jridgewell/trace-mapping" "^0.3.17" jsesc "^2.5.1" @@ -199,10 +199,10 @@ "@babel/helper-split-export-declaration" "^7.22.6" "@babel/helper-validator-identifier" "^7.22.5" -"@babel/helper-module-transforms@^7.23.0": - version "7.23.0" - resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.23.0.tgz#3ec246457f6c842c0aee62a01f60739906f7047e" - integrity sha512-WhDWw1tdrlT0gMgUJSlX0IQvoO1eN279zrAUbVB+KpV2c3Tylz8+GnKOLllCS6Z/iZQEyVYxhZVUdPTqs2YYPw== +"@babel/helper-module-transforms@^7.23.3": + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/helper-module-transforms/-/helper-module-transforms-7.23.3.tgz#d7d12c3c5d30af5b3c0fcab2a6d5217773e2d0f1" + integrity sha512-7bBs4ED9OmswdfDzpz4MpWgSrV7FXlc3zIagvLFjS5H+Mk7Snr21vQ6QwrsoCGMfNC4e4LQPdoULEt4ykz0SRQ== dependencies: "@babel/helper-environment-visitor" "^7.22.20" "@babel/helper-module-imports" "^7.22.15" @@ -280,10 +280,10 @@ chalk "^2.4.2" js-tokens "^4.0.0" -"@babel/parser@^7.1.0", "@babel/parser@^7.20.7", "@babel/parser@^7.22.15", "@babel/parser@^7.23.0": - version "7.23.0" - resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.0.tgz#da950e622420bf96ca0d0f2909cdddac3acd8719" - integrity sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw== +"@babel/parser@^7.1.0", "@babel/parser@^7.20.7", "@babel/parser@^7.22.15", "@babel/parser@^7.23.0", "@babel/parser@^7.23.3": + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.3.tgz#0ce0be31a4ca4f1884b5786057cadcb6c3be58f9" + integrity sha512-uVsWNvlVsIninV2prNz/3lHCb+5CJ+e+IUBfbjToAHODtfGYLfCFuY4AU7TskI+dAKk+njsPiBjq1gKTvZOBaw== "@babel/parser@^7.14.7", "@babel/parser@^7.22.16": version "7.22.16" @@ -402,17 +402,17 @@ "@babel/helper-plugin-utils" "^7.22.5" "@babel/helper-simple-access" "^7.22.5" -"@babel/plugin-transform-react-jsx-self@^7.22.5": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.22.5.tgz#ca2fdc11bc20d4d46de01137318b13d04e481d8e" - integrity sha512-nTh2ogNUtxbiSbxaT4Ds6aXnXEipHweN9YRgOX/oNXdf0cCrGn/+2LozFa3lnPV5D90MkjhgckCPBrsoSc1a7g== +"@babel/plugin-transform-react-jsx-self@^7.23.3": + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.23.3.tgz#ed3e7dadde046cce761a8e3cf003a13d1a7972d9" + integrity sha512-qXRvbeKDSfwnlJnanVRp0SfuWE5DQhwQr5xtLBzp56Wabyo+4CMosF6Kfp+eOD/4FYpql64XVJ2W0pVLlJZxOQ== dependencies: "@babel/helper-plugin-utils" "^7.22.5" -"@babel/plugin-transform-react-jsx-source@^7.22.5": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.22.5.tgz#49af1615bfdf6ed9d3e9e43e425e0b2b65d15b6c" - integrity sha512-yIiRO6yobeEIaI0RTbIr8iAK9FcBHLtZq0S89ZPjDLQXBA4xvghaKqI0etp/tF3htTM0sazJKKLz9oEiGRtu7w== +"@babel/plugin-transform-react-jsx-source@^7.23.3": + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.23.3.tgz#03527006bdc8775247a78643c51d4e715fe39a3e" + integrity sha512-91RS0MDnAWDNvGC6Wio5XYkyWI39FMFO+JK9+4AlgaTH+yWwVTsw7/sn6LK0lH7c5F+TFkpv/3LfCJ1Ydwof/g== dependencies: "@babel/helper-plugin-utils" "^7.22.5" @@ -439,7 +439,7 @@ "@babel/parser" "^7.22.15" "@babel/types" "^7.22.15" -"@babel/traverse@^7.22.11", "@babel/traverse@^7.22.15", "@babel/traverse@^7.22.17", "@babel/traverse@^7.22.8", "@babel/traverse@^7.23.2": +"@babel/traverse@^7.22.11", "@babel/traverse@^7.22.15", "@babel/traverse@^7.22.17", "@babel/traverse@^7.22.8": version "7.23.2" resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.2.tgz#329c7a06735e144a506bdb2cad0268b7f46f4ad8" integrity sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw== @@ -455,10 +455,26 @@ debug "^4.1.0" globals "^11.1.0" -"@babel/types@^7.0.0", "@babel/types@^7.20.7", "@babel/types@^7.22.15", "@babel/types@^7.22.5", "@babel/types@^7.23.0": - version "7.23.0" - resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.0.tgz#8c1f020c9df0e737e4e247c0619f58c68458aaeb" - integrity sha512-0oIyUfKoI3mSqMvsxBdclDwxXKXAUA8v/apZbc+iSyARYou1o8ZGDxbUYyLFoW2arqS2jDGqJuZvv1d/io1axg== +"@babel/traverse@^7.23.2", "@babel/traverse@^7.23.3": + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.3.tgz#26ee5f252e725aa7aca3474aa5b324eaf7908b5b" + integrity sha512-+K0yF1/9yR0oHdE0StHuEj3uTPzwwbrLGfNOndVJVV2TqA5+j3oljJUb4nmB954FLGjNem976+B+eDuLIjesiQ== + dependencies: + "@babel/code-frame" "^7.22.13" + "@babel/generator" "^7.23.3" + "@babel/helper-environment-visitor" "^7.22.20" + "@babel/helper-function-name" "^7.23.0" + "@babel/helper-hoist-variables" "^7.22.5" + "@babel/helper-split-export-declaration" "^7.22.6" + "@babel/parser" "^7.23.3" + "@babel/types" "^7.23.3" + debug "^4.1.0" + globals "^11.1.0" + +"@babel/types@^7.0.0", "@babel/types@^7.20.7", "@babel/types@^7.22.15", "@babel/types@^7.22.5", "@babel/types@^7.23.0", "@babel/types@^7.23.3": + version "7.23.3" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.3.tgz#d5ea892c07f2ec371ac704420f4dcdb07b5f9598" + integrity sha512-OZnvoH2l8PK5eUvEcUyCt/sXgr/h+UWpVuBbOljwcrAgUl6lpchoQ++PHGyQy1AtYnVA6CEq3y5xeEI10brpXw== dependencies: "@babel/helper-string-parser" "^7.22.5" "@babel/helper-validator-identifier" "^7.22.20" @@ -583,115 +599,115 @@ resolved "https://registry.yarnpkg.com/@emotion/weak-memoize/-/weak-memoize-0.3.1.tgz#d0fce5d07b0620caa282b5131c297bb60f9d87e6" integrity sha512-EsBwpc7hBUJWAsNPBmJy4hxWx12v6bshQsldrVmjxJoc3isbxhOrF2IcCpaXxfvq03NwkI7sbsOLXbYuqF/8Ww== -"@esbuild/android-arm64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.18.20.tgz#984b4f9c8d0377443cc2dfcef266d02244593622" - integrity sha512-Nz4rJcchGDtENV0eMKUNa6L12zz2zBDXuhj/Vjh18zGqB44Bi7MBMSXjgunJgjRhCmKOjnPuZp4Mb6OKqtMHLQ== - -"@esbuild/android-arm@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.18.20.tgz#fedb265bc3a589c84cc11f810804f234947c3682" - integrity sha512-fyi7TDI/ijKKNZTUJAQqiG5T7YjJXgnzkURqmGj13C6dCqckZBLdl4h7bkhHt/t0WP+zO9/zwroDvANaOqO5Sw== - -"@esbuild/android-x64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.18.20.tgz#35cf419c4cfc8babe8893d296cd990e9e9f756f2" - integrity sha512-8GDdlePJA8D6zlZYJV/jnrRAi6rOiNaCC/JclcXpB+KIuvfBN4owLtgzY2bsxnx666XjJx2kDPUmnTtR8qKQUg== - -"@esbuild/darwin-arm64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.18.20.tgz#08172cbeccf95fbc383399a7f39cfbddaeb0d7c1" - integrity sha512-bxRHW5kHU38zS2lPTPOyuyTm+S+eobPUnTNkdJEfAddYgEcll4xkT8DB9d2008DtTbl7uJag2HuE5NZAZgnNEA== - -"@esbuild/darwin-x64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.18.20.tgz#d70d5790d8bf475556b67d0f8b7c5bdff053d85d" - integrity sha512-pc5gxlMDxzm513qPGbCbDukOdsGtKhfxD1zJKXjCCcU7ju50O7MeAZ8c4krSJcOIJGFR+qx21yMMVYwiQvyTyQ== - -"@esbuild/freebsd-arm64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.18.20.tgz#98755cd12707f93f210e2494d6a4b51b96977f54" - integrity sha512-yqDQHy4QHevpMAaxhhIwYPMv1NECwOvIpGCZkECn8w2WFHXjEwrBn3CeNIYsibZ/iZEUemj++M26W3cNR5h+Tw== - -"@esbuild/freebsd-x64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.18.20.tgz#c1eb2bff03915f87c29cece4c1a7fa1f423b066e" - integrity sha512-tgWRPPuQsd3RmBZwarGVHZQvtzfEBOreNuxEMKFcd5DaDn2PbBxfwLcj4+aenoh7ctXcbXmOQIn8HI6mCSw5MQ== - -"@esbuild/linux-arm64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.18.20.tgz#bad4238bd8f4fc25b5a021280c770ab5fc3a02a0" - integrity sha512-2YbscF+UL7SQAVIpnWvYwM+3LskyDmPhe31pE7/aoTMFKKzIc9lLbyGUpmmb8a8AixOL61sQ/mFh3jEjHYFvdA== - -"@esbuild/linux-arm@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.18.20.tgz#3e617c61f33508a27150ee417543c8ab5acc73b0" - integrity sha512-/5bHkMWnq1EgKr1V+Ybz3s1hWXok7mDFUMQ4cG10AfW3wL02PSZi5kFpYKrptDsgb2WAJIvRcDm+qIvXf/apvg== - -"@esbuild/linux-ia32@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.18.20.tgz#699391cccba9aee6019b7f9892eb99219f1570a7" - integrity sha512-P4etWwq6IsReT0E1KHU40bOnzMHoH73aXp96Fs8TIT6z9Hu8G6+0SHSw9i2isWrD2nbx2qo5yUqACgdfVGx7TA== - -"@esbuild/linux-loong64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.18.20.tgz#e6fccb7aac178dd2ffb9860465ac89d7f23b977d" - integrity sha512-nXW8nqBTrOpDLPgPY9uV+/1DjxoQ7DoB2N8eocyq8I9XuqJ7BiAMDMf9n1xZM9TgW0J8zrquIb/A7s3BJv7rjg== - -"@esbuild/linux-mips64el@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.18.20.tgz#eeff3a937de9c2310de30622a957ad1bd9183231" - integrity sha512-d5NeaXZcHp8PzYy5VnXV3VSd2D328Zb+9dEq5HE6bw6+N86JVPExrA6O68OPwobntbNJ0pzCpUFZTo3w0GyetQ== - -"@esbuild/linux-ppc64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.18.20.tgz#2f7156bde20b01527993e6881435ad79ba9599fb" - integrity sha512-WHPyeScRNcmANnLQkq6AfyXRFr5D6N2sKgkFo2FqguP44Nw2eyDlbTdZwd9GYk98DZG9QItIiTlFLHJHjxP3FA== - -"@esbuild/linux-riscv64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.18.20.tgz#6628389f210123d8b4743045af8caa7d4ddfc7a6" - integrity sha512-WSxo6h5ecI5XH34KC7w5veNnKkju3zBRLEQNY7mv5mtBmrP/MjNBCAlsM2u5hDBlS3NGcTQpoBvRzqBcRtpq1A== - -"@esbuild/linux-s390x@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.18.20.tgz#255e81fb289b101026131858ab99fba63dcf0071" - integrity sha512-+8231GMs3mAEth6Ja1iK0a1sQ3ohfcpzpRLH8uuc5/KVDFneH6jtAJLFGafpzpMRO6DzJ6AvXKze9LfFMrIHVQ== - -"@esbuild/linux-x64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.18.20.tgz#c7690b3417af318a9b6f96df3031a8865176d338" - integrity sha512-UYqiqemphJcNsFEskc73jQ7B9jgwjWrSayxawS6UVFZGWrAAtkzjxSqnoclCXxWtfwLdzU+vTpcNYhpn43uP1w== - -"@esbuild/netbsd-x64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.18.20.tgz#30e8cd8a3dded63975e2df2438ca109601ebe0d1" - integrity sha512-iO1c++VP6xUBUmltHZoMtCUdPlnPGdBom6IrO4gyKPFFVBKioIImVooR5I83nTew5UOYrk3gIJhbZh8X44y06A== - -"@esbuild/openbsd-x64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.18.20.tgz#7812af31b205055874c8082ea9cf9ab0da6217ae" - integrity sha512-e5e4YSsuQfX4cxcygw/UCPIEP6wbIL+se3sxPdCiMbFLBWu0eiZOJ7WoD+ptCLrmjZBK1Wk7I6D/I3NglUGOxg== - -"@esbuild/sunos-x64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.18.20.tgz#d5c275c3b4e73c9b0ecd38d1ca62c020f887ab9d" - integrity sha512-kDbFRFp0YpTQVVrqUd5FTYmWo45zGaXe0X8E1G/LKFC0v8x0vWrhOWSLITcCn63lmZIxfOMXtCfti/RxN/0wnQ== - -"@esbuild/win32-arm64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.18.20.tgz#73bc7f5a9f8a77805f357fab97f290d0e4820ac9" - integrity sha512-ddYFR6ItYgoaq4v4JmQQaAI5s7npztfV4Ag6NrhiaW0RrnOXqBkgwZLofVTlq1daVTQNhtI5oieTvkRPfZrePg== - -"@esbuild/win32-ia32@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.18.20.tgz#ec93cbf0ef1085cc12e71e0d661d20569ff42102" - integrity sha512-Wv7QBi3ID/rROT08SABTS7eV4hX26sVduqDOTe1MvGMjNd3EjOz4b7zeexIR62GTIEKrfJXKL9LFxTYgkyeu7g== - -"@esbuild/win32-x64@0.18.20": - version "0.18.20" - resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.18.20.tgz#786c5f41f043b07afb1af37683d7c33668858f6d" - integrity sha512-kTdfRcSiDfQca/y9QIkng02avJ+NCaQvrMejlsB3RRv5sE9rRoeBPISaZpKxHELzRxZyLvNts1P27W3wV+8geQ== +"@esbuild/android-arm64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/android-arm64/-/android-arm64-0.19.5.tgz#276c5f99604054d3dbb733577e09adae944baa90" + integrity sha512-5d1OkoJxnYQfmC+Zd8NBFjkhyCNYwM4n9ODrycTFY6Jk1IGiZ+tjVJDDSwDt77nK+tfpGP4T50iMtVi4dEGzhQ== + +"@esbuild/android-arm@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/android-arm/-/android-arm-0.19.5.tgz#4a3cbf14758166abaae8ba9c01a80e68342a4eec" + integrity sha512-bhvbzWFF3CwMs5tbjf3ObfGqbl/17ict2/uwOSfr3wmxDE6VdS2GqY/FuzIPe0q0bdhj65zQsvqfArI9MY6+AA== + +"@esbuild/android-x64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/android-x64/-/android-x64-0.19.5.tgz#21a3d11cd4613d2d3c5ccb9e746c254eb9265b0a" + integrity sha512-9t+28jHGL7uBdkBjL90QFxe7DVA+KGqWlHCF8ChTKyaKO//VLuoBricQCgwhOjA1/qOczsw843Fy4cbs4H3DVA== + +"@esbuild/darwin-arm64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/darwin-arm64/-/darwin-arm64-0.19.5.tgz#714cb839f467d6a67b151ee8255886498e2b9bf6" + integrity sha512-mvXGcKqqIqyKoxq26qEDPHJuBYUA5KizJncKOAf9eJQez+L9O+KfvNFu6nl7SCZ/gFb2QPaRqqmG0doSWlgkqw== + +"@esbuild/darwin-x64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/darwin-x64/-/darwin-x64-0.19.5.tgz#2c553e97a6d2b4ae76a884e35e6cbab85a990bbf" + integrity sha512-Ly8cn6fGLNet19s0X4unjcniX24I0RqjPv+kurpXabZYSXGM4Pwpmf85WHJN3lAgB8GSth7s5A0r856S+4DyiA== + +"@esbuild/freebsd-arm64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/freebsd-arm64/-/freebsd-arm64-0.19.5.tgz#d554f556718adb31917a0da24277bf84b6ee87f3" + integrity sha512-GGDNnPWTmWE+DMchq1W8Sd0mUkL+APvJg3b11klSGUDvRXh70JqLAO56tubmq1s2cgpVCSKYywEiKBfju8JztQ== + +"@esbuild/freebsd-x64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/freebsd-x64/-/freebsd-x64-0.19.5.tgz#288f7358a3bb15d99e73c65c9adaa3dabb497432" + integrity sha512-1CCwDHnSSoA0HNwdfoNY0jLfJpd7ygaLAp5EHFos3VWJCRX9DMwWODf96s9TSse39Br7oOTLryRVmBoFwXbuuQ== + +"@esbuild/linux-arm64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-arm64/-/linux-arm64-0.19.5.tgz#95933ae86325c93cb6b5e8333d22120ecfdc901b" + integrity sha512-o3vYippBmSrjjQUCEEiTZ2l+4yC0pVJD/Dl57WfPwwlvFkrxoSO7rmBZFii6kQB3Wrn/6GwJUPLU5t52eq2meA== + +"@esbuild/linux-arm@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-arm/-/linux-arm-0.19.5.tgz#0acef93aa3e0579e46d33b666627bddb06636664" + integrity sha512-lrWXLY/vJBzCPC51QN0HM71uWgIEpGSjSZZADQhq7DKhPcI6NH1IdzjfHkDQws2oNpJKpR13kv7/pFHBbDQDwQ== + +"@esbuild/linux-ia32@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-ia32/-/linux-ia32-0.19.5.tgz#b6e5c9e80b42131cbd6b1ddaa48c92835f1ed67f" + integrity sha512-MkjHXS03AXAkNp1KKkhSKPOCYztRtK+KXDNkBa6P78F8Bw0ynknCSClO/ztGszILZtyO/lVKpa7MolbBZ6oJtQ== + +"@esbuild/linux-loong64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-loong64/-/linux-loong64-0.19.5.tgz#e5f0cf95a180158b01ff5f417da796a1c09dfbea" + integrity sha512-42GwZMm5oYOD/JHqHska3Jg0r+XFb/fdZRX+WjADm3nLWLcIsN27YKtqxzQmGNJgu0AyXg4HtcSK9HuOk3v1Dw== + +"@esbuild/linux-mips64el@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-mips64el/-/linux-mips64el-0.19.5.tgz#ae36fb86c7d5f641f3a0c8472e83dcb6ea36a408" + integrity sha512-kcjndCSMitUuPJobWCnwQ9lLjiLZUR3QLQmlgaBfMX23UEa7ZOrtufnRds+6WZtIS9HdTXqND4yH8NLoVVIkcg== + +"@esbuild/linux-ppc64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-ppc64/-/linux-ppc64-0.19.5.tgz#7960cb1666f0340ddd9eef7b26dcea3835d472d0" + integrity sha512-yJAxJfHVm0ZbsiljbtFFP1BQKLc8kUF6+17tjQ78QjqjAQDnhULWiTA6u0FCDmYT1oOKS9PzZ2z0aBI+Mcyj7Q== + +"@esbuild/linux-riscv64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-riscv64/-/linux-riscv64-0.19.5.tgz#32207df26af60a3a9feea1783fc21b9817bade19" + integrity sha512-5u8cIR/t3gaD6ad3wNt1MNRstAZO+aNyBxu2We8X31bA8XUNyamTVQwLDA1SLoPCUehNCymhBhK3Qim1433Zag== + +"@esbuild/linux-s390x@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-s390x/-/linux-s390x-0.19.5.tgz#b38d5681db89a3723862dfa792812397b1510a7d" + integrity sha512-Z6JrMyEw/EmZBD/OFEFpb+gao9xJ59ATsoTNlj39jVBbXqoZm4Xntu6wVmGPB/OATi1uk/DB+yeDPv2E8PqZGw== + +"@esbuild/linux-x64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/linux-x64/-/linux-x64-0.19.5.tgz#46feba2ad041a241379d150f415b472fe3885075" + integrity sha512-psagl+2RlK1z8zWZOmVdImisMtrUxvwereIdyJTmtmHahJTKb64pAcqoPlx6CewPdvGvUKe2Jw+0Z/0qhSbG1A== + +"@esbuild/netbsd-x64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/netbsd-x64/-/netbsd-x64-0.19.5.tgz#3b5c1fb068f26bfc681d31f682adf1bea4ef0702" + integrity sha512-kL2l+xScnAy/E/3119OggX8SrWyBEcqAh8aOY1gr4gPvw76la2GlD4Ymf832UCVbmuWeTf2adkZDK+h0Z/fB4g== + +"@esbuild/openbsd-x64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/openbsd-x64/-/openbsd-x64-0.19.5.tgz#ca6830316ca68056c5c88a875f103ad3235e00db" + integrity sha512-sPOfhtzFufQfTBgRnE1DIJjzsXukKSvZxloZbkJDG383q0awVAq600pc1nfqBcl0ice/WN9p4qLc39WhBShRTA== + +"@esbuild/sunos-x64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/sunos-x64/-/sunos-x64-0.19.5.tgz#9efc4eb9539a7be7d5a05ada52ee43cda0d8e2dd" + integrity sha512-dGZkBXaafuKLpDSjKcB0ax0FL36YXCvJNnztjKV+6CO82tTYVDSH2lifitJ29jxRMoUhgkg9a+VA/B03WK5lcg== + +"@esbuild/win32-arm64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/win32-arm64/-/win32-arm64-0.19.5.tgz#29f8184afa7a02a956ebda4ed638099f4b8ff198" + integrity sha512-dWVjD9y03ilhdRQ6Xig1NWNgfLtf2o/STKTS+eZuF90fI2BhbwD6WlaiCGKptlqXlURVB5AUOxUj09LuwKGDTg== + +"@esbuild/win32-ia32@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/win32-ia32/-/win32-ia32-0.19.5.tgz#f3de07afb292ecad651ae4bb8727789de2d95b05" + integrity sha512-4liggWIA4oDgUxqpZwrDhmEfAH4d0iljanDOK7AnVU89T6CzHon/ony8C5LeOdfgx60x5cnQJFZwEydVlYx4iw== + +"@esbuild/win32-x64@0.19.5": + version "0.19.5" + resolved "https://registry.yarnpkg.com/@esbuild/win32-x64/-/win32-x64-0.19.5.tgz#faad84c41ba12e3a0acb52571df9bff37bee75f6" + integrity sha512-czTrygUsB/jlM8qEW5MD8bgYU2Xg14lo6kBDXW6HdxKjh8M5PzETGiSHaz9MtbXBYDloHNUAUW2tMiKW4KM9Mw== "@eslint-community/eslint-utils@^4.2.0": version "4.4.0" @@ -720,10 +736,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@eslint/js@8.53.0": - version "8.53.0" - resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.53.0.tgz#bea56f2ed2b5baea164348ff4d5a879f6f81f20d" - integrity sha512-Kn7K8dx/5U6+cT1yEhpX1w4PCSg0M+XyRILPgvwcEBjerFWCwQj5sbr3/VmxqV0JGHCBCzyd6LxypEuehypY1w== +"@eslint/js@8.54.0": + version "8.54.0" + resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.54.0.tgz#4fab9a2ff7860082c304f750e94acd644cf984cf" + integrity sha512-ut5V+D+fOoWPgGGNj83GGjnntO39xDy6DWxO0wb7Jp3DcMX0TfIqdzHF85VTQkerdyGmuuMD9AKAo5KiNlf/AQ== "@floating-ui/core@^1.4.2": version "1.5.0" @@ -1093,43 +1109,42 @@ "@jridgewell/resolve-uri" "^3.1.0" "@jridgewell/sourcemap-codec" "^1.4.14" -"@mantine/core@^6.0.0": - version "6.0.21" - resolved "https://registry.yarnpkg.com/@mantine/core/-/core-6.0.21.tgz#6e3a1b8d0f6869518a644d5f5e3d55a5db7e1e51" - integrity sha512-Kx4RrRfv0I+cOCIcsq/UA2aWcYLyXgW3aluAuW870OdXnbII6qg7RW28D+r9D76SHPxWFKwIKwmcucAG08Divg== +"@mantine/core@^5": + version "5.10.5" + resolved "https://registry.npm.ouryahoo.com:4443/npm-registry/api/npm/npm-registry/@mantine/core/-/core-5.10.5.tgz#071e14dcf8b94a36d0243f1f4b30305ac0074afd" + integrity sha512-F4tqHSEVM9D6/iSqHfPda+Xl5XgSEPHAAkT01Zwzj4Jnbd10qGrlqr/SFUop2CIcuKYnmra9XltUahUPXBC2BQ== dependencies: "@floating-ui/react" "^0.19.1" - "@mantine/styles" "6.0.21" - "@mantine/utils" "6.0.21" + "@mantine/styles" "5.10.5" + "@mantine/utils" "5.10.5" "@radix-ui/react-scroll-area" "1.0.2" - react-remove-scroll "^2.5.5" react-textarea-autosize "8.3.4" -"@mantine/hooks@^6.0.0": - version "6.0.21" - resolved "https://registry.yarnpkg.com/@mantine/hooks/-/hooks-6.0.21.tgz#bc009d8380ad18455b90f3ddaf484de16a13da95" - integrity sha512-sYwt5wai25W6VnqHbS5eamey30/HD5dNXaZuaVEAJ2i2bBv8C0cCiczygMDpAFiSYdXoSMRr/SZ2CrrPTzeNew== +"@mantine/hooks@^5": + version "5.10.5" + resolved "https://registry.npm.ouryahoo.com:4443/npm-registry/api/npm/npm-registry/@mantine/hooks/-/hooks-5.10.5.tgz#568586a0fa649be46f057ddc920bf98761017ffb" + integrity sha512-hFQp71QZDfivPzfIUOQZfMKLiOL/Cn2EnzacRlbUr55myteTfzYN8YMt+nzniE/6c4IRopFHEAdbKEtfyQc6kg== -"@mantine/notifications@^6.0.0": - version "6.0.21" - resolved "https://registry.yarnpkg.com/@mantine/notifications/-/notifications-6.0.21.tgz#bec53664abce13a2cc61a1be1840d82a746f62da" - integrity sha512-qsrqxuJHK8b67sf9Pfk+xyhvpf9jMsivW8vchfnJfjv7yz1lLvezjytMFp4fMDoYhjHnDPOEc/YFockK4muhOw== +"@mantine/notifications@^5": + version "5.10.5" + resolved "https://registry.npm.ouryahoo.com:4443/npm-registry/api/npm/npm-registry/@mantine/notifications/-/notifications-5.10.5.tgz#2f3f2d013ce4637e64e935aa5dd8c1df1a7acec0" + integrity sha512-IzTAXA7Zb9DcI94Mv5O2OinhLmI7fvs/VutDw9uCpp6OHtLuF/XN1d262jrsGhMZT0c4nuUsotSLFZF3GWZwXg== dependencies: - "@mantine/utils" "6.0.21" + "@mantine/utils" "5.10.5" react-transition-group "4.4.2" -"@mantine/styles@6.0.21": - version "6.0.21" - resolved "https://registry.yarnpkg.com/@mantine/styles/-/styles-6.0.21.tgz#8ea097fc76cbb3ed55f5cfd719d2f910aff5031b" - integrity sha512-PVtL7XHUiD/B5/kZ/QvZOZZQQOj12QcRs3Q6nPoqaoPcOX5+S7bMZLMH0iLtcGq5OODYk0uxlvuJkOZGoPj8Mg== +"@mantine/styles@5.10.5": + version "5.10.5" + resolved "https://registry.npm.ouryahoo.com:4443/npm-registry/api/npm/npm-registry/@mantine/styles/-/styles-5.10.5.tgz#ace82a71b4fe3d14ee14638f1735d5680d93d36d" + integrity sha512-0NXk8c/XGzuTUkZc6KceF2NaTCMEu5mHR4ru0x+ttb9DGnLpHuGWduTHjSfr4hl6eAJgedD0zauO+VAhDzO9zA== dependencies: clsx "1.1.1" csstype "3.0.9" -"@mantine/utils@6.0.21": - version "6.0.21" - resolved "https://registry.yarnpkg.com/@mantine/utils/-/utils-6.0.21.tgz#6185506e91cba3e308aaa8ea9ababc8e767995d6" - integrity sha512-33RVDRop5jiWFao3HKd3Yp7A9mEq4HAJxJPTuYm1NkdqX6aTKOQK7wT8v8itVodBp+sb4cJK6ZVdD1UurK/txQ== +"@mantine/utils@5.10.5": + version "5.10.5" + resolved "https://registry.npm.ouryahoo.com:4443/npm-registry/api/npm/npm-registry/@mantine/utils/-/utils-5.10.5.tgz#ad620d714e545c6efb7f69d94ce46e3fd2fe01fb" + integrity sha512-FGMq4dGs5HhDAtI0z46uzxzKKPmZ3h5uKUyKg1ZHoFR1mBtcUMbB6FylFmHqKFRWlJ5IXqX9dwmiVrLYUOfTmA== "@nodelib/fs.scandir@2.1.5": version "2.1.5" @@ -1254,10 +1269,70 @@ dependencies: "@babel/runtime" "^7.13.10" -"@remix-run/router@1.11.0": - version "1.11.0" - resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.11.0.tgz#e0e45ac3fff9d8a126916f166809825537e9f955" - integrity sha512-BHdhcWgeiudl91HvVa2wxqZjSHbheSgIiDvxrF1VjFzBzpTtuDPkOdOi3Iqvc08kXtFkLjhbS+ML9aM8mJS+wQ== +"@remix-run/router@1.12.0": + version "1.12.0" + resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.12.0.tgz#e89b64b6fa97a8a5b740a4c38c2904b80f1f229a" + integrity sha512-2hXv036Bux90e1GXTWSMfNzfDDK8LA8JYEWfyHxzvwdp6GyoWEovKc9cotb3KCKmkdwsIBuFGX7ScTWyiHv7Eg== + +"@rollup/rollup-android-arm-eabi@4.4.1": + version "4.4.1" + resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.4.1.tgz#f276b0fa322270aa42d1f56c982db6ef8d6a4393" + integrity sha512-Ss4suS/sd+6xLRu+MLCkED2mUrAyqHmmvZB+zpzZ9Znn9S8wCkTQCJaQ8P8aHofnvG5L16u9MVnJjCqioPErwQ== + +"@rollup/rollup-android-arm64@4.4.1": + version "4.4.1" + resolved "https://registry.yarnpkg.com/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.4.1.tgz#f0492f00d18e1067785f8e820e137c00528c5e62" + integrity sha512-sRSkGTvGsARwWd7TzC8LKRf8FiPn7257vd/edzmvG4RIr9x68KBN0/Ek48CkuUJ5Pj/Dp9vKWv6PEupjKWjTYA== + +"@rollup/rollup-darwin-arm64@4.4.1": + version "4.4.1" + resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.4.1.tgz#40443db7f4559171d797581e0618ec1a4c8dcee9" + integrity sha512-nz0AiGrrXyaWpsmBXUGOBiRDU0wyfSXbFuF98pPvIO8O6auQsPG6riWsfQqmCCC5FNd8zKQ4JhgugRNAkBJ8mQ== + +"@rollup/rollup-darwin-x64@4.4.1": + version "4.4.1" + resolved "https://registry.yarnpkg.com/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.4.1.tgz#2868f37a9f9c2c22c091b6209f6ce7454437edf9" + integrity sha512-Ogqvf4/Ve/faMaiPRvzsJEqajbqs00LO+8vtrPBVvLgdw4wBg6ZDXdkDAZO+4MLnrc8mhGV6VJAzYScZdPLtJg== + +"@rollup/rollup-linux-arm-gnueabihf@4.4.1": + version "4.4.1" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.4.1.tgz#d78d7ad358d24058166ab5599de3dcb5ab951add" + integrity sha512-9zc2tqlr6HfO+hx9+wktUlWTRdje7Ub15iJqKcqg5uJZ+iKqmd2CMxlgPpXi7+bU7bjfDIuvCvnGk7wewFEhCg== + +"@rollup/rollup-linux-arm64-gnu@4.4.1": + version "4.4.1" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.4.1.tgz#5d07588b40a04f5b6fbd9e0169c8dc32c1c2ed21" + integrity sha512-phLb1fN3rq2o1j1v+nKxXUTSJnAhzhU0hLrl7Qzb0fLpwkGMHDem+o6d+ZI8+/BlTXfMU4kVWGvy6g9k/B8L6Q== + +"@rollup/rollup-linux-arm64-musl@4.4.1": + version "4.4.1" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.4.1.tgz#d452e88a02755f449f6e98d4ce424d655ef42cfe" + integrity sha512-M2sDtw4tf57VPSjbTAN/lz1doWUqO2CbQuX3L9K6GWIR5uw9j+ROKCvvUNBY8WUbMxwaoc8mH9HmmBKsLht7+w== + +"@rollup/rollup-linux-x64-gnu@4.4.1": + version "4.4.1" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.4.1.tgz#e8e8e87ab098784383a5ced4aa4bbfa7b2c92a4e" + integrity sha512-mHIlRLX+hx+30cD6c4BaBOsSqdnCE4ok7/KDvjHYAHoSuveoMMxIisZFvcLhUnyZcPBXDGZTuBoalcuh43UfQQ== + +"@rollup/rollup-linux-x64-musl@4.4.1": + version "4.4.1" + resolved "https://registry.yarnpkg.com/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.4.1.tgz#3e5da42626672e2d620ed12746158b0cf6143b23" + integrity sha512-tB+RZuDi3zxFx7vDrjTNGVLu2KNyzYv+UY8jz7e4TMEoAj7iEt8Qk6xVu6mo3pgjnsHj6jnq3uuRsHp97DLwOA== + +"@rollup/rollup-win32-arm64-msvc@4.4.1": + version "4.4.1" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.4.1.tgz#0f0d0c6b75c53643fab8238c76889a95bca3b9cc" + integrity sha512-Hdn39PzOQowK/HZzYpCuZdJC91PE6EaGbTe2VCA9oq2u18evkisQfws0Smh9QQGNNRa/T7MOuGNQoLeXhhE3PQ== + +"@rollup/rollup-win32-ia32-msvc@4.4.1": + version "4.4.1" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.4.1.tgz#8bb9e8fbf0fdf96fe3bebcee23f5cfdbbd9a4a0a" + integrity sha512-tLpKb1Elm9fM8c5w3nl4N1eLTP4bCqTYw9tqUBxX8/hsxqHO3dxc2qPbZ9PNkdK4tg4iLEYn0pOUnVByRd2CbA== + +"@rollup/rollup-win32-x64-msvc@4.4.1": + version "4.4.1" + resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.4.1.tgz#8311b77e6cce322865ba12ada8c3779369610d18" + integrity sha512-eAhItDX9yQtZVM3yvXS/VR3qPqcnXvnLyx1pLXl4JzyNMBNO3KC986t/iAg2zcMzpAp9JSvxB5VZGnBiNoA98w== "@sinclair/typebox@^0.27.8": version "0.27.8" @@ -1289,10 +1364,10 @@ "@types/babel__template" "*" "@types/babel__traverse" "*" -"@types/babel__core@^7.20.3": - version "7.20.3" - resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.20.3.tgz#d5625a50b6f18244425a1359a858c73d70340778" - integrity sha512-54fjTSeSHwfan8AyHWrKbfBWiEUrNTZsUwPTDSNaaP1QDQIZbeNUg3a59E9D+375MzUw/x1vx2/0F5LBz+AeYA== +"@types/babel__core@^7.20.4": + version "7.20.4" + resolved "https://registry.yarnpkg.com/@types/babel__core/-/babel__core-7.20.4.tgz#26a87347e6c6f753b3668398e34496d6d9ac6ac0" + integrity sha512-mLnSC22IC4vcWiuObSRjrLd9XcBTGf59vUSoq2jkQDJ/QQ8PMI9rSuzE+aEV8karUMbskw07bKYoUJCKTUaygg== dependencies: "@babel/parser" "^7.20.7" "@babel/types" "^7.20.7" @@ -1301,24 +1376,24 @@ "@types/babel__traverse" "*" "@types/babel__generator@*": - version "7.6.6" - resolved "https://registry.yarnpkg.com/@types/babel__generator/-/babel__generator-7.6.6.tgz#676f89f67dc8ddaae923f70ebc5f1fa800c031a8" - integrity sha512-66BXMKb/sUWbMdBNdMvajU7i/44RkrA3z/Yt1c7R5xejt8qh84iU54yUWCtm0QwGJlDcf/gg4zd/x4mpLAlb/w== + version "7.6.7" + resolved "https://registry.yarnpkg.com/@types/babel__generator/-/babel__generator-7.6.7.tgz#a7aebf15c7bc0eb9abd638bdb5c0b8700399c9d0" + integrity sha512-6Sfsq+EaaLrw4RmdFWE9Onp63TOUue71AWb4Gpa6JxzgTYtimbM086WnYTy2U67AofR++QKCo08ZP6pwx8YFHQ== dependencies: "@babel/types" "^7.0.0" "@types/babel__template@*": - version "7.4.3" - resolved "https://registry.yarnpkg.com/@types/babel__template/-/babel__template-7.4.3.tgz#db9ac539a2fe05cfe9e168b24f360701bde41f5f" - integrity sha512-ciwyCLeuRfxboZ4isgdNZi/tkt06m8Tw6uGbBSBgWrnnZGNXiEyM27xc/PjXGQLqlZ6ylbgHMnm7ccF9tCkOeQ== + version "7.4.4" + resolved "https://registry.yarnpkg.com/@types/babel__template/-/babel__template-7.4.4.tgz#5672513701c1b2199bc6dad636a9d7491586766f" + integrity sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A== dependencies: "@babel/parser" "^7.1.0" "@babel/types" "^7.0.0" "@types/babel__traverse@*": - version "7.20.3" - resolved "https://registry.yarnpkg.com/@types/babel__traverse/-/babel__traverse-7.20.3.tgz#a971aa47441b28ef17884ff945d0551265a2d058" - integrity sha512-Lsh766rGEFbaxMIDH7Qa+Yha8cMVI3qAK6CHt3OR0YfxOIn5Z54iHiyDRycHrBqeIiqGa20Kpsv1cavfBKkRSw== + version "7.20.4" + resolved "https://registry.yarnpkg.com/@types/babel__traverse/-/babel__traverse-7.20.4.tgz#ec2c06fed6549df8bc0eb4615b683749a4a92e1b" + integrity sha512-mSM/iKUk5fDDrEV/e83qY+Cr3I1+Q3qqTuEn++HAWYjEa1+NxZr6CNrcJGf2ZTnq4HoFGC3zaTPZTobCzCFukA== dependencies: "@babel/types" "^7.20.7" @@ -1405,14 +1480,14 @@ integrity sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ== "@vitejs/plugin-react@^4": - version "4.1.1" - resolved "https://registry.yarnpkg.com/@vitejs/plugin-react/-/plugin-react-4.1.1.tgz#a10254dc76778027407d01b6ddbca53b23852a72" - integrity sha512-Jie2HERK+uh27e+ORXXwEP5h0Y2lS9T2PRGbfebiHGlwzDO0dEnd2aNtOR/qjBlPb1YgxwAONeblL1xqLikLag== + version "4.2.0" + resolved "https://registry.yarnpkg.com/@vitejs/plugin-react/-/plugin-react-4.2.0.tgz#d71352b1a443c09c7aae8f278dd071ab3d9d8490" + integrity sha512-+MHTH/e6H12kRp5HUkzOGqPMksezRMmW+TNzlh/QXfI8rRf6l2Z2yH/v12no1UvTwhZgEDMuQ7g7rrfMseU6FQ== dependencies: - "@babel/core" "^7.23.2" - "@babel/plugin-transform-react-jsx-self" "^7.22.5" - "@babel/plugin-transform-react-jsx-source" "^7.22.5" - "@types/babel__core" "^7.20.3" + "@babel/core" "^7.23.3" + "@babel/plugin-transform-react-jsx-self" "^7.23.3" + "@babel/plugin-transform-react-jsx-source" "^7.23.3" + "@types/babel__core" "^7.20.4" react-refresh "^0.14.0" acorn-jsx@^5.3.2: @@ -1894,9 +1969,9 @@ camelcase@^6.2.0: integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== caniuse-lite@^1.0.30001541: - version "1.0.30001559" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001559.tgz#95a982440d3d314c471db68d02664fb7536c5a30" - integrity sha512-cPiMKZgqgkg5LY3/ntGeLFUpi6tzddBNS58A4tnTgQw1zON7u2sZMU7SzOeVH4tj20++9ggL+V6FDOFMTaFFYA== + version "1.0.30001562" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001562.tgz#9d16c5fd7e9c592c4cd5e304bc0f75b0008b2759" + integrity sha512-kfte3Hym//51EdX4239i+Rmp20EsLIYGdPkERegTgU19hQWCRhsRFGKHTliUlsry53tv17K7n077Kqa0WJU4ng== capture-exit@^2.0.0: version "2.0.0" @@ -2204,11 +2279,6 @@ detect-newline@^3.0.0: resolved "https://registry.yarnpkg.com/detect-newline/-/detect-newline-3.1.0.tgz#576f5dfc63ae1a192ff192d8ad3af6308991b651" integrity sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA== -detect-node-es@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/detect-node-es/-/detect-node-es-1.1.0.tgz#163acdf643330caa0b4cd7c21e7ee7755d6fa493" - integrity sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ== - diff-sequences@^29.6.3: version "29.6.3" resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-29.6.3.tgz#4deaf894d11407c51efc8418012f9e70b84ea921" @@ -2237,9 +2307,9 @@ dom-helpers@^5.0.1: csstype "^3.0.2" electron-to-chromium@^1.4.535: - version "1.4.574" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.574.tgz#6de04d7c6e244e5ffcae76d2e2a33b02cab66781" - integrity sha512-bg1m8L0n02xRzx4LsTTMbBPiUd9yIR+74iPtS/Ao65CuXvhVZHP0ym1kSdDG3yHFDXqHQQBKujlN1AQ8qZnyFg== + version "1.4.586" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.586.tgz#68683163ed52a111213e2482ff847e76a5c6e891" + integrity sha512-qMa+E6yf1fNQbg3G66pHLXeJUP5CCCzNat1VPczOZOqgI2w4u+8y9sQnswMdGs5m4C1rOePq37EVBr/nsPQY7w== emittery@^0.13.1: version "0.13.1" @@ -2364,33 +2434,33 @@ esbuild-jest@^0: "@babel/plugin-transform-modules-commonjs" "^7.12.13" babel-jest "^26.6.3" -esbuild@^0.18.10: - version "0.18.20" - resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.18.20.tgz#4709f5a34801b43b799ab7d6d82f7284a9b7a7a6" - integrity sha512-ceqxoedUrcayh7Y7ZX6NdbbDzGROiyVBgC4PriJThBKSVPWnnFHZAkfI1lJT8QFkOwH4qOS2SJkS4wvpGl8BpA== +esbuild@^0.19.3: + version "0.19.5" + resolved "https://registry.yarnpkg.com/esbuild/-/esbuild-0.19.5.tgz#53a0e19dfbf61ba6c827d51a80813cf071239a8c" + integrity sha512-bUxalY7b1g8vNhQKdB24QDmHeY4V4tw/s6Ak5z+jJX9laP5MoQseTOMemAr0gxssjNcH0MCViG8ONI2kksvfFQ== optionalDependencies: - "@esbuild/android-arm" "0.18.20" - "@esbuild/android-arm64" "0.18.20" - "@esbuild/android-x64" "0.18.20" - "@esbuild/darwin-arm64" "0.18.20" - "@esbuild/darwin-x64" "0.18.20" - "@esbuild/freebsd-arm64" "0.18.20" - "@esbuild/freebsd-x64" "0.18.20" - "@esbuild/linux-arm" "0.18.20" - "@esbuild/linux-arm64" "0.18.20" - "@esbuild/linux-ia32" "0.18.20" - "@esbuild/linux-loong64" "0.18.20" - "@esbuild/linux-mips64el" "0.18.20" - "@esbuild/linux-ppc64" "0.18.20" - "@esbuild/linux-riscv64" "0.18.20" - "@esbuild/linux-s390x" "0.18.20" - "@esbuild/linux-x64" "0.18.20" - "@esbuild/netbsd-x64" "0.18.20" - "@esbuild/openbsd-x64" "0.18.20" - "@esbuild/sunos-x64" "0.18.20" - "@esbuild/win32-arm64" "0.18.20" - "@esbuild/win32-ia32" "0.18.20" - "@esbuild/win32-x64" "0.18.20" + "@esbuild/android-arm" "0.19.5" + "@esbuild/android-arm64" "0.19.5" + "@esbuild/android-x64" "0.19.5" + "@esbuild/darwin-arm64" "0.19.5" + "@esbuild/darwin-x64" "0.19.5" + "@esbuild/freebsd-arm64" "0.19.5" + "@esbuild/freebsd-x64" "0.19.5" + "@esbuild/linux-arm" "0.19.5" + "@esbuild/linux-arm64" "0.19.5" + "@esbuild/linux-ia32" "0.19.5" + "@esbuild/linux-loong64" "0.19.5" + "@esbuild/linux-mips64el" "0.19.5" + "@esbuild/linux-ppc64" "0.19.5" + "@esbuild/linux-riscv64" "0.19.5" + "@esbuild/linux-s390x" "0.19.5" + "@esbuild/linux-x64" "0.19.5" + "@esbuild/netbsd-x64" "0.19.5" + "@esbuild/openbsd-x64" "0.19.5" + "@esbuild/sunos-x64" "0.19.5" + "@esbuild/win32-arm64" "0.19.5" + "@esbuild/win32-ia32" "0.19.5" + "@esbuild/win32-x64" "0.19.5" escalade@^3.1.1: version "3.1.1" @@ -2517,14 +2587,14 @@ eslint-visitor-keys@^3.3.0, eslint-visitor-keys@^3.4.1, eslint-visitor-keys@^3.4 integrity sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag== eslint@^8: - version "8.53.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.53.0.tgz#14f2c8244298fcae1f46945459577413ba2697ce" - integrity sha512-N4VuiPjXDUa4xVeV/GC/RV3hQW9Nw+Y463lkWaKKXKYMvmRiRDAtfpuPFLN+E1/6ZhyR8J2ig+eVREnYgUsiag== + version "8.54.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.54.0.tgz#588e0dd4388af91a2e8fa37ea64924074c783537" + integrity sha512-NY0DfAkM8BIZDVl6PgSa1ttZbx3xHgJzSNJKYcQglem6CppHyMhRIQkBVSSMaSRnLhig3jsDbEzOjwCVt4AmmA== dependencies: "@eslint-community/eslint-utils" "^4.2.0" "@eslint-community/regexpp" "^4.6.1" "@eslint/eslintrc" "^2.1.3" - "@eslint/js" "8.53.0" + "@eslint/js" "8.54.0" "@humanwhocodes/config-array" "^0.11.13" "@humanwhocodes/module-importer" "^1.0.1" "@nodelib/fs.walk" "^1.2.8" @@ -2810,9 +2880,9 @@ find-up@^5.0.0: path-exists "^4.0.0" flat-cache@^3.0.4: - version "3.1.1" - resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-3.1.1.tgz#a02a15fdec25a8f844ff7cc658f03dd99eb4609b" - integrity sha512-/qM2b3LUIaIgviBQovTLvijfyOQXPtSRnRK26ksj2J7rzPIecePUIpJsZ4T02Qg+xiAEKIs5K8dsHEd+VaKa/Q== + version "3.2.0" + resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-3.2.0.tgz#2c0c2d5040c99b1632771a9d105725c0115363ee" + integrity sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw== dependencies: flatted "^3.2.9" keyv "^4.5.3" @@ -2852,7 +2922,7 @@ fsevents@^2.1.2: resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== -fsevents@^2.3.2, fsevents@~2.3.2: +fsevents@^2.3.2, fsevents@~2.3.2, fsevents@~2.3.3: version "2.3.3" resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.3.tgz#cac6407785d03675a2a5e1a5305c697b347d90d6" integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw== @@ -2897,11 +2967,6 @@ get-intrinsic@^1.0.2, get-intrinsic@^1.1.1, get-intrinsic@^1.1.3, get-intrinsic@ has-symbols "^1.0.3" hasown "^2.0.0" -get-nonce@^1.0.0: - version "1.0.1" - resolved "https://registry.yarnpkg.com/get-nonce/-/get-nonce-1.0.1.tgz#fdf3f0278073820d2ce9426c18f07481b1e0cdf3" - integrity sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q== - get-package-type@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/get-package-type/-/get-package-type-0.1.0.tgz#8de2d803cff44df3bc6c456e6668b36c3926e11a" @@ -3115,11 +3180,16 @@ husky@^8: resolved "https://registry.yarnpkg.com/husky/-/husky-8.0.3.tgz#4936d7212e46d1dea28fef29bb3a108872cd9184" integrity sha512-+dQSyqPh4x1hlO1swXBiNb2HzTDN1I2IGLQx1GrBuiqFJfoMrnZWwVmatvSiO+Iz8fBUnf+lekwNo4c2LlXItg== -ignore@^5.1.4, ignore@^5.2.0: +ignore@^5.1.4: version "5.2.4" resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.2.4.tgz#a291c0c6178ff1b960befe47fcdec301674a6324" integrity sha512-MAb38BcSbH0eHNBxn7ql2NH/kX33OkB3lZ1BNdh7ENeRChHTYsTvWrMubiIAMNS2llXEEgZ1MUOBtXChP3kaFQ== +ignore@^5.2.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.3.0.tgz#67418ae40d34d6999c95ff56016759c718c82f78" + integrity sha512-g7dmpshy+gD7mh88OC9NwSGTKoc3kyLAZQRU1mt53Aw/vnvfXnbC+F/7F7QoYVKbV+KNvJx8wArewKy1vXMtlg== + import-fresh@^3.2.1: version "3.3.0" resolved "https://registry.yarnpkg.com/import-fresh/-/import-fresh-3.3.0.tgz#37162c25fcb9ebaa2e6e53d5b4d88ce17d9e0c2b" @@ -3172,13 +3242,6 @@ internal-slot@^1.0.5: hasown "^2.0.0" side-channel "^1.0.4" -invariant@^2.2.4: - version "2.2.4" - resolved "https://registry.yarnpkg.com/invariant/-/invariant-2.2.4.tgz#610f3c92c9359ce1db616e538008d23ff35158e6" - integrity sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA== - dependencies: - loose-envify "^1.0.0" - is-accessor-descriptor@^0.1.6: version "0.1.6" resolved "https://registry.yarnpkg.com/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz#a9e12cb3ae8d876727eeef3843f8a0897b5c98d6" @@ -4147,7 +4210,7 @@ lodash@^4: resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== -loose-envify@^1.0.0, loose-envify@^1.1.0, loose-envify@^1.4.0: +loose-envify@^1.1.0, loose-envify@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf" integrity sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q== @@ -4634,7 +4697,7 @@ posix-character-classes@^0.1.0: resolved "https://registry.yarnpkg.com/posix-character-classes/-/posix-character-classes-0.1.1.tgz#01eac0fe3b5af71a2a6c02feabb8c1fef7e00eab" integrity sha512-xTgYBc3fuo7Yt7JbiuFxSYGToMoz8fLoE6TC9Wx1P/u+LfeThMOAqmuyECnlBaaJb+u1m9hHiXUEtwW4OzfUJg== -postcss@^8.4.27: +postcss@^8.4.31: version "8.4.31" resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.31.tgz#92b451050a9f914da6755af352bdc0192508656d" integrity sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ== @@ -4656,9 +4719,9 @@ prettier-linter-helpers@^1.0.0: fast-diff "^1.1.2" prettier@3: - version "3.0.3" - resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.0.3.tgz#432a51f7ba422d1469096c0fdc28e235db8f9643" - integrity sha512-L/4pUDMxcNa8R/EthV08Zt42WBO4h1rarVtK0K+QJG0X187OLo7l699jWw0GKuwzkPQ//jMFA/8Xm6Fh3J/DAg== + version "3.1.0" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.1.0.tgz#c6d16474a5f764ea1a4a373c593b779697744d5e" + integrity sha512-TQLvXjq5IAibjh8EpBIkNKxO749UEWABoiIZehEPiY4GNpVdhaFKqSTu+QrlU6D2dPAfubRmtJTi4K4YkQ5eXw== pretty-format@^29.7.0: version "29.7.0" @@ -4744,48 +4807,20 @@ react-refresh@^0.14.0: resolved "https://registry.yarnpkg.com/react-refresh/-/react-refresh-0.14.0.tgz#4e02825378a5f227079554d4284889354e5f553e" integrity sha512-wViHqhAd8OHeLS/IRMJjTSDHF3U9eWi62F/MledQGPdJGDhodXJ9PBLNGr6WWL7qlH12Mt3TyTpbS+hGXMjCzQ== -react-remove-scroll-bar@^2.3.4: - version "2.3.4" - resolved "https://registry.yarnpkg.com/react-remove-scroll-bar/-/react-remove-scroll-bar-2.3.4.tgz#53e272d7a5cb8242990c7f144c44d8bd8ab5afd9" - integrity sha512-63C4YQBUt0m6ALadE9XV56hV8BgJWDmmTPY758iIJjfQKt2nYwoUrPk0LXRXcB/yIj82T1/Ixfdpdk68LwIB0A== - dependencies: - react-style-singleton "^2.2.1" - tslib "^2.0.0" - -react-remove-scroll@^2.5.5: - version "2.5.6" - resolved "https://registry.yarnpkg.com/react-remove-scroll/-/react-remove-scroll-2.5.6.tgz#7510b8079e9c7eebe00e65a33daaa3aa29a10336" - integrity sha512-bO856ad1uDYLefgArk559IzUNeQ6SWH4QnrevIUjH+GczV56giDfl3h0Idptf2oIKxQmd1p9BN25jleKodTALg== - dependencies: - react-remove-scroll-bar "^2.3.4" - react-style-singleton "^2.2.1" - tslib "^2.1.0" - use-callback-ref "^1.3.0" - use-sidecar "^1.1.2" - react-router-dom@^6: - version "6.18.0" - resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.18.0.tgz#0a50c167209d6e7bd2ed9de200a6579ea4fb1dca" - integrity sha512-Ubrue4+Ercc/BoDkFQfc6og5zRQ4A8YxSO3Knsne+eRbZ+IepAsK249XBH/XaFuOYOYr3L3r13CXTLvYt5JDjw== + version "6.19.0" + resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.19.0.tgz#ee807e36ae7dd954db7a3f770e38b7cc026c66a8" + integrity sha512-N6dWlcgL2w0U5HZUUqU2wlmOrSb3ighJmtQ438SWbhB1yuLTXQ8yyTBMK3BSvVjp7gBtKurT554nCtMOgxCZmQ== dependencies: - "@remix-run/router" "1.11.0" - react-router "6.18.0" + "@remix-run/router" "1.12.0" + react-router "6.19.0" -react-router@6.18.0: - version "6.18.0" - resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.18.0.tgz#32e2bedc318e095a48763b5ed7758e54034cd36a" - integrity sha512-vk2y7Dsy8wI02eRRaRmOs9g2o+aE72YCx5q9VasT1N9v+lrdB79tIqrjMfByHiY5+6aYkH2rUa5X839nwWGPDg== +react-router@6.19.0: + version "6.19.0" + resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.19.0.tgz#6d5062fa33495859daca98d86292ab03b037a9ca" + integrity sha512-0W63PKCZ7+OuQd7Tm+RbkI8kCLmn4GPjDbX61tWljPxWgqTKlEpeQUwPkT1DRjYhF8KSihK0hQpmhU4uxVMcdw== dependencies: - "@remix-run/router" "1.11.0" - -react-style-singleton@^2.2.1: - version "2.2.1" - resolved "https://registry.yarnpkg.com/react-style-singleton/-/react-style-singleton-2.2.1.tgz#f99e420492b2d8f34d38308ff660b60d0b1205b4" - integrity sha512-ZWj0fHEMyWkHzKYUr2Bs/4zU6XLmq9HsgBURm7g5pAVfyn49DgUiNgY2d4lXRlYSiCif9YBGpQleewkcqddc7g== - dependencies: - get-nonce "^1.0.0" - invariant "^2.2.4" - tslib "^2.0.0" + "@remix-run/router" "1.12.0" react-textarea-autosize@8.3.4: version "8.3.4" @@ -4961,11 +4996,23 @@ rimraf@^3.0.2: dependencies: glob "^7.1.3" -rollup@^3.27.1: - version "3.29.4" - resolved "https://registry.yarnpkg.com/rollup/-/rollup-3.29.4.tgz#4d70c0f9834146df8705bfb69a9a19c9e1109981" - integrity sha512-oWzmBZwvYrU0iJHtDmhsm662rC15FRXmcjCk1xD771dFDx5jJ02ufAQQTn0etB2emNk4J9EZg/yWKpsn9BWGRw== +rollup@^4.2.0: + version "4.4.1" + resolved "https://registry.yarnpkg.com/rollup/-/rollup-4.4.1.tgz#2f85169f23d13dabb3d9b846d753965757353820" + integrity sha512-idZzrUpWSblPJX66i+GzrpjKE3vbYrlWirUHteoAbjKReZwa0cohAErOYA5efoMmNCdvG9yrJS+w9Kl6csaH4w== optionalDependencies: + "@rollup/rollup-android-arm-eabi" "4.4.1" + "@rollup/rollup-android-arm64" "4.4.1" + "@rollup/rollup-darwin-arm64" "4.4.1" + "@rollup/rollup-darwin-x64" "4.4.1" + "@rollup/rollup-linux-arm-gnueabihf" "4.4.1" + "@rollup/rollup-linux-arm64-gnu" "4.4.1" + "@rollup/rollup-linux-arm64-musl" "4.4.1" + "@rollup/rollup-linux-x64-gnu" "4.4.1" + "@rollup/rollup-linux-x64-musl" "4.4.1" + "@rollup/rollup-win32-arm64-msvc" "4.4.1" + "@rollup/rollup-win32-ia32-msvc" "4.4.1" + "@rollup/rollup-win32-x64-msvc" "4.4.1" fsevents "~2.3.2" rsvp@^4.8.4: @@ -5435,7 +5482,7 @@ tsconfig-paths@^3.14.2: minimist "^1.2.6" strip-bom "^3.0.0" -tslib@^2.0.0, tslib@^2.1.0, tslib@^2.5.0, tslib@^2.6.0: +tslib@^2.0.0, tslib@^2.5.0, tslib@^2.6.0: version "2.6.2" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== @@ -5561,13 +5608,6 @@ urix@^0.1.0: resolved "https://registry.yarnpkg.com/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72" integrity sha512-Am1ousAhSLBeB9cG/7k7r2R0zj50uDRlZHPGbazid5s9rlF1F/QKYObEKSIunSjIOkJZqwRRLpvewjEkM7pSqg== -use-callback-ref@^1.3.0: - version "1.3.0" - resolved "https://registry.yarnpkg.com/use-callback-ref/-/use-callback-ref-1.3.0.tgz#772199899b9c9a50526fedc4993fc7fa1f7e32d5" - integrity sha512-3FT9PRuRdbB9HfXhEq35u4oZkvpJ5kuYbpqhCfmiZyReuRgpnhDlbr2ZEnnuS0RrJAPn6l23xjFg9kpDM+Ms7w== - dependencies: - tslib "^2.0.0" - use-composed-ref@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/use-composed-ref/-/use-composed-ref-1.3.0.tgz#3d8104db34b7b264030a9d916c5e94fbe280dbda" @@ -5590,14 +5630,6 @@ use-latest@^1.2.1: dependencies: use-isomorphic-layout-effect "^1.1.1" -use-sidecar@^1.1.2: - version "1.1.2" - resolved "https://registry.yarnpkg.com/use-sidecar/-/use-sidecar-1.1.2.tgz#2f43126ba2d7d7e117aa5855e5d8f0276dfe73c2" - integrity sha512-epTbsLuzZ7lPClpz2TyryBfztm7m+28DlEv2ZCQ3MDr5ssiwyOwGH/e5F9CkfWjJ1t4clvI58yF822/GUkjjhw== - dependencies: - detect-node-es "^1.1.0" - tslib "^2.0.0" - use@^3.1.0: version "3.1.1" resolved "https://registry.yarnpkg.com/use/-/use-3.1.1.tgz#d50c8cac79a19fbc20f2911f56eb973f4e10070f" @@ -5612,16 +5644,16 @@ v8-to-istanbul@^9.0.1: "@types/istanbul-lib-coverage" "^2.0.1" convert-source-map "^1.6.0" -vite@^4: - version "4.5.0" - resolved "https://registry.yarnpkg.com/vite/-/vite-4.5.0.tgz#ec406295b4167ac3bc23e26f9c8ff559287cff26" - integrity sha512-ulr8rNLA6rkyFAlVWw2q5YJ91v098AFQ2R0PRFwPzREXOUJQPtFUG0t+/ZikhaOCDqFoDhN6/v8Sq0o4araFAw== +vite@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/vite/-/vite-5.0.0.tgz#3bfb65acda2a97127e4fa240156664a1f234ce08" + integrity sha512-ESJVM59mdyGpsiNAeHQOR/0fqNoOyWPYesFto8FFZugfmhdHx8Fzd8sF3Q/xkVhZsyOxHfdM7ieiVAorI9RjFw== dependencies: - esbuild "^0.18.10" - postcss "^8.4.27" - rollup "^3.27.1" + esbuild "^0.19.3" + postcss "^8.4.31" + rollup "^4.2.0" optionalDependencies: - fsevents "~2.3.2" + fsevents "~2.3.3" walker@^1.0.7, walker@^1.0.8, walker@~1.0.5: version "1.0.8" diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index fab30efd00d..78b32d8af7b 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -1276,18 +1276,18 @@ "public boolean useV8GeoPositions()", "public int maxCompactBuffers()", "public java.util.List ignoredHttpUserAgents()", - "public com.yahoo.config.provision.NodeResources$Architecture adminClusterArchitecture()", "public boolean enableProxyProtocolMixedMode()", "public java.lang.String logFileCompressionAlgorithm(java.lang.String)", "public boolean enableGlobalPhase()", "public java.lang.String summaryDecodePolicy()", "public boolean enableNestedMultivalueGrouping()", - "public boolean useReconfigurableDispatcher()", "public int contentLayerMetadataFeatureLevel()", "public boolean dynamicHeapSize()", "public java.lang.String unknownConfigDefinition()", "public int searchHandlerThreadpool()", - "public long mergingMaxMemoryUsagePerNode()" + "public long mergingMaxMemoryUsagePerNode()", + "public boolean usePerDocumentThrottledDeleteBucket()", + "public boolean alwaysMarkPhraseExpensive()" ], "fields" : [ ] }, @@ -1453,7 +1453,9 @@ "methods" : [ "public abstract long aggregatedModelCostInBytes()", "public abstract void registerModel(com.yahoo.config.application.api.ApplicationFile)", - "public abstract void registerModel(java.net.URI)" + "public abstract void registerModel(com.yahoo.config.application.api.ApplicationFile, com.yahoo.config.model.api.OnnxModelOptions)", + "public abstract void registerModel(java.net.URI)", + "public abstract void registerModel(java.net.URI, com.yahoo.config.model.api.OnnxModelOptions)" ], "fields" : [ ] }, @@ -1471,7 +1473,9 @@ "public com.yahoo.config.model.api.OnnxModelCost$Calculator newCalculator(com.yahoo.config.application.api.ApplicationPackage, com.yahoo.config.provision.ApplicationId)", "public long aggregatedModelCostInBytes()", "public void registerModel(com.yahoo.config.application.api.ApplicationFile)", - "public void registerModel(java.net.URI)" + "public void registerModel(com.yahoo.config.application.api.ApplicationFile, com.yahoo.config.model.api.OnnxModelOptions)", + "public void registerModel(java.net.URI)", + "public void registerModel(java.net.URI, com.yahoo.config.model.api.OnnxModelOptions)" ], "fields" : [ ] }, @@ -1484,12 +1488,56 @@ "abstract" ], "methods" : [ - "public com.yahoo.config.model.api.OnnxModelCost$Calculator newCalculator(com.yahoo.config.application.api.ApplicationPackage, com.yahoo.config.application.api.DeployLogger)", "public abstract com.yahoo.config.model.api.OnnxModelCost$Calculator newCalculator(com.yahoo.config.application.api.ApplicationPackage, com.yahoo.config.provision.ApplicationId)", "public static com.yahoo.config.model.api.OnnxModelCost disabled()" ], "fields" : [ ] }, + "com.yahoo.config.model.api.OnnxModelOptions$GpuDevice" : { + "superClass" : "java.lang.Record", + "interfaces" : [ ], + "attributes" : [ + "public", + "final", + "record" + ], + "methods" : [ + "public void <init>(int, boolean)", + "public void <init>(int)", + "public final java.lang.String toString()", + "public final int hashCode()", + "public final boolean equals(java.lang.Object)", + "public int deviceNumber()", + "public boolean required()" + ], + "fields" : [ ] + }, + "com.yahoo.config.model.api.OnnxModelOptions" : { + "superClass" : "java.lang.Record", + "interfaces" : [ ], + "attributes" : [ + "public", + "final", + "record" + ], + "methods" : [ + "public void <init>(java.lang.String, int, int, com.yahoo.config.model.api.OnnxModelOptions$GpuDevice)", + "public void <init>(java.util.Optional, java.util.Optional, java.util.Optional, java.util.Optional)", + "public static com.yahoo.config.model.api.OnnxModelOptions empty()", + "public com.yahoo.config.model.api.OnnxModelOptions withExecutionMode(java.lang.String)", + "public com.yahoo.config.model.api.OnnxModelOptions withInterOpThreads(java.lang.Integer)", + "public com.yahoo.config.model.api.OnnxModelOptions withIntraOpThreads(java.lang.Integer)", + "public com.yahoo.config.model.api.OnnxModelOptions withGpuDevice(com.yahoo.config.model.api.OnnxModelOptions$GpuDevice)", + "public final java.lang.String toString()", + "public final int hashCode()", + "public final boolean equals(java.lang.Object)", + "public java.util.Optional executionMode()", + "public java.util.Optional interOpThreads()", + "public java.util.Optional intraOpThreads()", + "public java.util.Optional gpuDevice()" + ], + "fields" : [ ] + }, "com.yahoo.config.model.api.PortInfo" : { "superClass" : "java.lang.Object", "interfaces" : [ ], diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 833e2f020bc..1687a3053fd 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -106,18 +106,18 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"arnej"}) default boolean useV8GeoPositions() { return false; } @ModelFeatureFlag(owners = {"baldersheim", "geirst", "toregge"}) default int maxCompactBuffers() { return 1; } @ModelFeatureFlag(owners = {"arnej", "andreer"}) default List<String> ignoredHttpUserAgents() { return List.of(); } - @ModelFeatureFlag(owners = {"hmusum"}, removeAfter = "8.250") default Architecture adminClusterArchitecture() { return Architecture.getDefault(); } @ModelFeatureFlag(owners = {"tokle"}) default boolean enableProxyProtocolMixedMode() { return true; } @ModelFeatureFlag(owners = {"arnej"}) default String logFileCompressionAlgorithm(String defVal) { return defVal; } @ModelFeatureFlag(owners = {"arnej, bjorncs"}) default boolean enableGlobalPhase() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Select summary decode type") default String summaryDecodePolicy() { return "eager"; } - @ModelFeatureFlag(owners = {"baldersheim"}) default boolean enableNestedMultivalueGrouping() { return false; } - @ModelFeatureFlag(owners = {"jonmv"}, removeAfter = "8.250") default boolean useReconfigurableDispatcher() { return true; } + @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter = "8.261") default boolean enableNestedMultivalueGrouping() { return true; } @ModelFeatureFlag(owners = {"vekterli"}) default int contentLayerMetadataFeatureLevel() { return 0; } @ModelFeatureFlag(owners = {"bjorncs"}) default boolean dynamicHeapSize() { return false; } @ModelFeatureFlag(owners = {"hmusum"}) default String unknownConfigDefinition() { return "warn"; } @ModelFeatureFlag(owners = {"hmusum"}) default int searchHandlerThreadpool() { return 2; } @ModelFeatureFlag(owners = {"vekterli"}) default long mergingMaxMemoryUsagePerNode() { return -1; } + @ModelFeatureFlag(owners = {"vekterli"}) default boolean usePerDocumentThrottledDeleteBucket() { return false; } + @ModelFeatureFlag(owners = {"baldersheim"}) default boolean alwaysMarkPhraseExpensive() { return false; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java index abfddfe40be..b98667457e4 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelCost.java @@ -1,10 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - package com.yahoo.config.model.api; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.ApplicationId; import java.net.URI; @@ -12,19 +10,17 @@ import java.net.URI; /** * @author bjorncs */ +// TODO: Rename public interface OnnxModelCost { - // TODO: Remove when 8.250 is oldest model in use - default Calculator newCalculator(ApplicationPackage appPkg, DeployLogger deployLogger) { - return newCalculator(appPkg, ApplicationId.defaultId()); - } - Calculator newCalculator(ApplicationPackage appPkg, ApplicationId applicationId); interface Calculator { long aggregatedModelCostInBytes(); void registerModel(ApplicationFile path); + void registerModel(ApplicationFile path, OnnxModelOptions onnxModelOptions); void registerModel(URI uri); + void registerModel(URI uri, OnnxModelOptions onnxModelOptions); } static OnnxModelCost disabled() { return new DisabledOnnxModelCost(); } @@ -33,7 +29,9 @@ public interface OnnxModelCost { @Override public Calculator newCalculator(ApplicationPackage appPkg, ApplicationId applicationId) { return this; } @Override public long aggregatedModelCostInBytes() {return 0;} @Override public void registerModel(ApplicationFile path) {} + @Override public void registerModel(ApplicationFile path, OnnxModelOptions onnxModelOptions) {} @Override public void registerModel(URI uri) {} + @Override public void registerModel(URI uri, OnnxModelOptions onnxModelOptions) {} } } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelOptions.java b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelOptions.java new file mode 100644 index 00000000000..92817baae3f --- /dev/null +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/OnnxModelOptions.java @@ -0,0 +1,49 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.model.api; + +import java.util.Optional; + +/** + * Onnx model options that are relevant when deciding if an Onnx model needs to be reloaded. If any of the + * values in this class change, reload is needed. + * + * @author hmusum + */ +public record OnnxModelOptions(Optional<String> executionMode, Optional<Integer> interOpThreads, + Optional<Integer> intraOpThreads, Optional<GpuDevice> gpuDevice) { + + public OnnxModelOptions(String executionMode, int interOpThreads, int intraOpThreads, GpuDevice gpuDevice) { + this(Optional.of(executionMode), Optional.of(interOpThreads), Optional.of(intraOpThreads), Optional.of(gpuDevice)); + } + + public static OnnxModelOptions empty() { + return new OnnxModelOptions(Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); + } + + public OnnxModelOptions withExecutionMode(String executionMode) { + return new OnnxModelOptions(Optional.ofNullable(executionMode), interOpThreads, intraOpThreads, gpuDevice); + } + + public OnnxModelOptions withInterOpThreads(Integer interOpThreads) { + return new OnnxModelOptions(executionMode, Optional.ofNullable(interOpThreads), intraOpThreads, gpuDevice); + } + + public OnnxModelOptions withIntraOpThreads(Integer intraOpThreads) { + return new OnnxModelOptions(executionMode, interOpThreads, Optional.ofNullable(intraOpThreads), gpuDevice); + } + + public OnnxModelOptions withGpuDevice(GpuDevice gpuDevice) { + return new OnnxModelOptions(executionMode, interOpThreads, intraOpThreads, Optional.ofNullable(gpuDevice)); + } + + public record GpuDevice(int deviceNumber, boolean required) { + public GpuDevice { + if (deviceNumber < 0) throw new IllegalArgumentException("deviceNumber cannot be negative, got " + deviceNumber); + } + + public GpuDevice(int deviceNumber) { + this(deviceNumber, false); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 1bda8a509f1..41df042284e 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -88,6 +88,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private int contentLayerMetadataFeatureLevel = 0; private boolean dynamicHeapSize = false; private long mergingMaxMemoryUsagePerNode = -1; + private boolean usePerDocumentThrottledDeleteBucket = false; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -128,7 +129,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public int maxCompactBuffers() { return maxCompactBuffers; } @Override public boolean useV8GeoPositions() { return useV8GeoPositions; } @Override public List<String> environmentVariables() { return environmentVariables; } - @Override public Architecture adminClusterArchitecture() { return adminClusterNodeResourcesArchitecture; } @Override public boolean sharedStringRepoNoReclaim() { return sharedStringRepoNoReclaim; } @Override public boolean loadCodeAsHugePages() { return loadCodeAsHugePages; } @Override public int mbusNetworkThreads() { return mbus_network_threads; } @@ -148,6 +148,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public int contentLayerMetadataFeatureLevel() { return contentLayerMetadataFeatureLevel; } @Override public boolean dynamicHeapSize() { return dynamicHeapSize; } @Override public long mergingMaxMemoryUsagePerNode() { return mergingMaxMemoryUsagePerNode; } + @Override public boolean usePerDocumentThrottledDeleteBucket() { return usePerDocumentThrottledDeleteBucket; } public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) { this.sharedStringRepoNoReclaim = sharedStringRepoNoReclaim; @@ -390,6 +391,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setUsePerDocumentThrottledDeleteBucket(boolean enableThrottling) { + this.usePerDocumentThrottledDeleteBucket = enableThrottling; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/schema/OnnxModel.java b/config-model/src/main/java/com/yahoo/schema/OnnxModel.java index f3f09150c1d..9456baafd57 100644 --- a/config-model/src/main/java/com/yahoo/schema/OnnxModel.java +++ b/config-model/src/main/java/com/yahoo/schema/OnnxModel.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.schema; +import com.yahoo.config.model.api.OnnxModelOptions; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.model.ml.OnnxModelInfo; @@ -27,10 +28,7 @@ public class OnnxModel extends DistributableResource implements Cloneable { private final Set<String> initializers = new HashSet<>(); // Runtime options - private String statelessExecutionMode = null; - private Integer statelessInterOpThreads = null; - private Integer statelessIntraOpThreads = null; - private GpuDevice gpuDevice = null; + private OnnxModelOptions onnxModelOptions = OnnxModelOptions.empty(); public OnnxModel(String name) { super(name); @@ -133,50 +131,46 @@ public class OnnxModel extends DistributableResource implements Cloneable { public void setStatelessExecutionMode(String executionMode) { if ("parallel".equalsIgnoreCase(executionMode)) { - this.statelessExecutionMode = "parallel"; + onnxModelOptions = onnxModelOptions.withExecutionMode("parallel"); } else if ("sequential".equalsIgnoreCase(executionMode)) { - this.statelessExecutionMode = "sequential"; + onnxModelOptions = onnxModelOptions.withExecutionMode("sequential"); } } public Optional<String> getStatelessExecutionMode() { - return Optional.ofNullable(statelessExecutionMode); + return onnxModelOptions.executionMode(); } public void setStatelessInterOpThreads(int interOpThreads) { if (interOpThreads >= 0) { - this.statelessInterOpThreads = interOpThreads; + onnxModelOptions = onnxModelOptions.withInterOpThreads(interOpThreads); } } public Optional<Integer> getStatelessInterOpThreads() { - return Optional.ofNullable(statelessInterOpThreads); + return onnxModelOptions.interOpThreads(); } public void setStatelessIntraOpThreads(int intraOpThreads) { if (intraOpThreads >= 0) { - this.statelessIntraOpThreads = intraOpThreads; + onnxModelOptions = onnxModelOptions.withIntraOpThreads(intraOpThreads); } } public Optional<Integer> getStatelessIntraOpThreads() { - return Optional.ofNullable(statelessIntraOpThreads); + return onnxModelOptions.intraOpThreads(); } public void setGpuDevice(int deviceNumber, boolean required) { if (deviceNumber >= 0) { - this.gpuDevice = new GpuDevice(deviceNumber, required); + onnxModelOptions = onnxModelOptions.withGpuDevice(new OnnxModelOptions.GpuDevice(deviceNumber, required)); } } - public Optional<GpuDevice> getGpuDevice() { - return Optional.ofNullable(gpuDevice); + public Optional<OnnxModelOptions.GpuDevice> getGpuDevice() { + return onnxModelOptions.gpuDevice(); } - public record GpuDevice(int deviceNumber, boolean required) { - public GpuDevice { - if (deviceNumber < 0) throw new IllegalArgumentException("deviceNumber cannot be negative, got " + deviceNumber); - } - } + public OnnxModelOptions onnxModelOptions() { return onnxModelOptions; } } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java index 87b79ddcdc3..388d2627224 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java @@ -171,7 +171,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private final OptionalDouble approximateThreshold; private final OptionalDouble targetHitsMaxAdjustmentFactor; private final double rankScoreDropLimit; - private final boolean enableNestedMultivalueGrouping; + private final boolean alwaysMarkPhraseExpensive; /** * The rank type definitions used to derive settings for the native rank features @@ -186,6 +186,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private RankingExpression secondPhaseRanking; private RankingExpression globalPhaseRanking; private final int globalPhaseRerankCount; + private final SerializationContext functionSerializationContext; /** * Creates a raw rank profile from the given rank profile @@ -212,7 +213,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { minHitsPerThread = compiled.getMinHitsPerThread(); numSearchPartitions = compiled.getNumSearchPartitions(); termwiseLimit = compiled.getTermwiseLimit().orElse(deployProperties.featureFlags().defaultTermwiseLimit()); - enableNestedMultivalueGrouping = deployProperties.featureFlags().enableNestedMultivalueGrouping(); + alwaysMarkPhraseExpensive = deployProperties.featureFlags().alwaysMarkPhraseExpensive(); postFilterThreshold = compiled.getPostFilterThreshold(); approximateThreshold = compiled.getApproximateThreshold(); targetHitsMaxAdjustmentFactor = compiled.getTargetHitsMaxAdjustmentFactor(); @@ -225,7 +226,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { List<ExpressionFunction> functionExpressions = functions.values().stream().map(RankProfile.RankingExpressionFunction::function).toList(); Map<String, String> functionProperties = new LinkedHashMap<>(); var typeContext = compiled.typeContext(queryProfiles); - SerializationContext functionSerializationContext = new SerializationContext(functionExpressions, Map.of(), typeContext); + this.functionSerializationContext = new SerializationContext(functionExpressions, Map.of(), typeContext); if (firstPhaseRanking != null) { functionProperties.putAll(firstPhaseRanking.getRankProperties(functionSerializationContext)); } @@ -265,8 +266,6 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private void derivePropertiesAndFeaturesFromFunctions(Map<String, RankProfile.RankingExpressionFunction> functions, Map<String, String> functionProperties, SerializationContext functionContext) { - if (functions.isEmpty()) return; - replaceFunctionFeatures(summaryFeatures, functionContext); replaceFunctionFeatures(matchFeatures, functionContext); @@ -465,8 +464,8 @@ public class RawRankProfile implements RankProfilesConfig.Producer { if (termwiseLimit < 1.0) { properties.add(new Pair<>("vespa.matching.termwise_limit", termwiseLimit + "")); } - if (enableNestedMultivalueGrouping) { - properties.add(new Pair<>("vespa.temporary.enable_nested_multivalue_grouping", String.valueOf(enableNestedMultivalueGrouping))); + if (alwaysMarkPhraseExpensive) { + properties.add(new Pair<>("vespa.matching.always_mark_phrase_expensive", String.valueOf(alwaysMarkPhraseExpensive))); } if (postFilterThreshold.isPresent()) { properties.add(new Pair<>("vespa.matching.global_filter.upper_limit", String.valueOf(postFilterThreshold.getAsDouble()))); @@ -556,11 +555,12 @@ public class RawRankProfile implements RankProfilesConfig.Producer { if ("".equals(name)) name = phase; + String expressionAsString = expression.getRoot().toString(functionSerializationContext).toString(); if (expression.getRoot() instanceof ReferenceNode) { - properties.add(new Pair<>("vespa.rank." + phase, expression.getRoot().toString())); + properties.add(new Pair<>("vespa.rank." + phase, expressionAsString)); } else { properties.add(new Pair<>("vespa.rank." + phase, wrapInRankingExpression(name))); - properties.add(new Pair<>(RankingExpression.propertyName(name), expression.getRoot().toString())); + properties.add(new Pair<>(RankingExpression.propertyName(name), expressionAsString)); } return properties; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/BertEmbedder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/BertEmbedder.java index a644382625b..67fb720b8c0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/BertEmbedder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/BertEmbedder.java @@ -1,13 +1,15 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - package com.yahoo.vespa.model.container.component; import com.yahoo.config.ModelReference; +import com.yahoo.config.model.api.OnnxModelOptions; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.embedding.BertBaseEmbedderConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import org.w3c.dom.Element; +import static com.yahoo.embedding.BertBaseEmbedderConfig.OnnxExecutionMode; +import static com.yahoo.embedding.BertBaseEmbedderConfig.PoolingStrategy; import static com.yahoo.text.XML.getChildValue; import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATION_BUNDLE_NAME; @@ -16,6 +18,7 @@ import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATI */ public class BertEmbedder extends TypedComponent implements BertBaseEmbedderConfig.Producer { + private final OnnxModelOptions onnxModelOptions; private final ModelReference modelRef; private final ModelReference vocabRef; private final Integer maxTokens; @@ -23,18 +26,18 @@ public class BertEmbedder extends TypedComponent implements BertBaseEmbedderConf private final String transformerAttentionMask; private final String transformerTokenTypeIds; private final String transformerOutput; - private final Integer tranformerStartSequenceToken; + private final Integer transformerStartSequenceToken; private final Integer transformerEndSequenceToken; private final String poolingStrategy; - private final String onnxExecutionMode; - private final Integer onnxInteropThreads; - private final Integer onnxIntraopThreads; - private final Integer onnxGpuDevice; - public BertEmbedder(ApplicationContainerCluster cluster, Element xml, DeployState state) { super("ai.vespa.embedding.BertBaseEmbedder", INTEGRATION_BUNDLE_NAME, xml); var model = Model.fromXml(state, xml, "transformer-model").orElseThrow(); + this.onnxModelOptions = new OnnxModelOptions( + getChildValue(xml, "onnx-execution-mode"), + getChildValue(xml, "onnx-interop-threads").map(Integer::parseInt), + getChildValue(xml, "onnx-intraop-threads").map(Integer::parseInt), + getChildValue(xml, "onnx-gpu-device").map(Integer::parseInt).map(OnnxModelOptions.GpuDevice::new)); modelRef = model.modelReference(); vocabRef = Model.fromXml(state, xml, "tokenizer-vocab").orElseThrow().modelReference(); maxTokens = getChildValue(xml, "max-tokens").map(Integer::parseInt).orElse(null); @@ -42,14 +45,10 @@ public class BertEmbedder extends TypedComponent implements BertBaseEmbedderConf transformerAttentionMask = getChildValue(xml, "transformer-attention-mask").orElse(null); transformerTokenTypeIds = getChildValue(xml, "transformer-token-type-ids").orElse(null); transformerOutput = getChildValue(xml, "transformer-output").orElse(null); - tranformerStartSequenceToken = getChildValue(xml, "transformer-start-sequence-token").map(Integer::parseInt).orElse(null); + transformerStartSequenceToken = getChildValue(xml, "transformer-start-sequence-token").map(Integer::parseInt).orElse(null); transformerEndSequenceToken = getChildValue(xml, "transformer-end-sequence-token").map(Integer::parseInt).orElse(null); poolingStrategy = getChildValue(xml, "pooling-strategy").orElse(null); - onnxExecutionMode = getChildValue(xml, "onnx-execution-mode").orElse(null); - onnxInteropThreads = getChildValue(xml, "onnx-interop-threads").map(Integer::parseInt).orElse(null); - onnxIntraopThreads = getChildValue(xml, "onnx-intraop-threads").map(Integer::parseInt).orElse(null); - onnxGpuDevice = getChildValue(xml, "onnx-gpu-device").map(Integer::parseInt).orElse(null); - model.registerOnnxModelCost(cluster); + model.registerOnnxModelCost(cluster, onnxModelOptions); } @Override @@ -60,12 +59,13 @@ public class BertEmbedder extends TypedComponent implements BertBaseEmbedderConf if (transformerAttentionMask != null) b.transformerAttentionMask(transformerAttentionMask); if (transformerTokenTypeIds != null) b.transformerTokenTypeIds(transformerTokenTypeIds); if (transformerOutput != null) b.transformerOutput(transformerOutput); - if (tranformerStartSequenceToken != null) b.transformerStartSequenceToken(tranformerStartSequenceToken); + if (transformerStartSequenceToken != null) b.transformerStartSequenceToken(transformerStartSequenceToken); if (transformerEndSequenceToken != null) b.transformerEndSequenceToken(transformerEndSequenceToken); - if (poolingStrategy != null) b.poolingStrategy(BertBaseEmbedderConfig.PoolingStrategy.Enum.valueOf(poolingStrategy)); - if (onnxExecutionMode != null) b.onnxExecutionMode(BertBaseEmbedderConfig.OnnxExecutionMode.Enum.valueOf(onnxExecutionMode)); - if (onnxInteropThreads != null) b.onnxInterOpThreads(onnxInteropThreads); - if (onnxIntraopThreads != null) b.onnxIntraOpThreads(onnxIntraopThreads); - if (onnxGpuDevice != null) b.onnxGpuDevice(onnxGpuDevice); + if (poolingStrategy != null) b.poolingStrategy(PoolingStrategy.Enum.valueOf(poolingStrategy)); + onnxModelOptions.executionMode().ifPresent(value -> b.onnxExecutionMode(OnnxExecutionMode.Enum.valueOf(value))); + onnxModelOptions.interOpThreads().ifPresent(b::onnxInterOpThreads); + onnxModelOptions.intraOpThreads().ifPresent(b::onnxIntraOpThreads); + onnxModelOptions.gpuDevice().ifPresent(value -> b.onnxGpuDevice(value.deviceNumber())); } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ColBertEmbedder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ColBertEmbedder.java index ed56579988d..d22e6afc3d1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ColBertEmbedder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ColBertEmbedder.java @@ -1,13 +1,14 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - package com.yahoo.vespa.model.container.component; import com.yahoo.config.ModelReference; +import com.yahoo.config.model.api.OnnxModelOptions; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.embedding.ColBertEmbedderConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import org.w3c.dom.Element; +import static com.yahoo.embedding.ColBertEmbedderConfig.TransformerExecutionMode; import static com.yahoo.text.XML.getChildValue; import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATION_BUNDLE_NAME; @@ -16,6 +17,8 @@ import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATI * @author bergum */ public class ColBertEmbedder extends TypedComponent implements ColBertEmbedderConfig.Producer { + + private final OnnxModelOptions onnxModelOptions; private final ModelReference modelRef; private final ModelReference vocabRef; @@ -31,14 +34,15 @@ public class ColBertEmbedder extends TypedComponent implements ColBertEmbedderCo private final String transformerAttentionMask; private final String transformerOutput; - private final String onnxExecutionMode; - private final Integer onnxInteropThreads; - private final Integer onnxIntraopThreads; - private final Integer onnxGpuDevice; public ColBertEmbedder(ApplicationContainerCluster cluster, Element xml, DeployState state) { super("ai.vespa.embedding.ColBertEmbedder", INTEGRATION_BUNDLE_NAME, xml); var model = Model.fromXml(state, xml, "transformer-model").orElseThrow(); + this.onnxModelOptions = new OnnxModelOptions( + getChildValue(xml, "onnx-execution-mode"), + getChildValue(xml, "onnx-interop-threads").map(Integer::parseInt), + getChildValue(xml, "onnx-intraop-threads").map(Integer::parseInt), + getChildValue(xml, "onnx-gpu-device").map(Integer::parseInt).map(OnnxModelOptions.GpuDevice::new)); modelRef = model.modelReference(); vocabRef = Model.fromXml(state, xml, "tokenizer-model") .map(Model::modelReference) @@ -52,11 +56,7 @@ public class ColBertEmbedder extends TypedComponent implements ColBertEmbedderCo transformerInputIds = getChildValue(xml, "transformer-input-ids").orElse(null); transformerAttentionMask = getChildValue(xml, "transformer-attention-mask").orElse(null); transformerOutput = getChildValue(xml, "transformer-output").orElse(null); - onnxExecutionMode = getChildValue(xml, "onnx-execution-mode").orElse(null); - onnxInteropThreads = getChildValue(xml, "onnx-interop-threads").map(Integer::parseInt).orElse(null); - onnxIntraopThreads = getChildValue(xml, "onnx-intraop-threads").map(Integer::parseInt).orElse(null); - onnxGpuDevice = getChildValue(xml, "onnx-gpu-device").map(Integer::parseInt).orElse(null); - model.registerOnnxModelCost(cluster); + model.registerOnnxModelCost(cluster, onnxModelOptions); } private static ModelReference resolveDefaultVocab(Model model, DeployState state) { @@ -79,10 +79,10 @@ public class ColBertEmbedder extends TypedComponent implements ColBertEmbedderCo if (transformerStartSequenceToken != null) b.transformerStartSequenceToken(transformerStartSequenceToken); if (transformerEndSequenceToken != null) b.transformerEndSequenceToken(transformerEndSequenceToken); if (transformerMaskToken != null) b.transformerMaskToken(transformerMaskToken); - if (onnxExecutionMode != null) b.transformerExecutionMode( - ColBertEmbedderConfig.TransformerExecutionMode.Enum.valueOf(onnxExecutionMode)); - if (onnxInteropThreads != null) b.transformerInterOpThreads(onnxInteropThreads); - if (onnxIntraopThreads != null) b.transformerIntraOpThreads(onnxIntraopThreads); - if (onnxGpuDevice != null) b.transformerGpuDevice(onnxGpuDevice); + onnxModelOptions.executionMode().ifPresent(value -> b.transformerExecutionMode(TransformerExecutionMode.Enum.valueOf(value))); + onnxModelOptions.interOpThreads().ifPresent(b::transformerInterOpThreads); + onnxModelOptions.intraOpThreads().ifPresent(b::transformerIntraOpThreads); + onnxModelOptions.gpuDevice().ifPresent(value -> b.transformerGpuDevice(value.deviceNumber())); } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceEmbedder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceEmbedder.java index 31b86142445..d98c72ab3a4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceEmbedder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/HuggingFaceEmbedder.java @@ -1,13 +1,15 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - package com.yahoo.vespa.model.container.component; import com.yahoo.config.ModelReference; +import com.yahoo.config.model.api.OnnxModelOptions; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.embedding.huggingface.HuggingFaceEmbedderConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import org.w3c.dom.Element; +import static com.yahoo.embedding.huggingface.HuggingFaceEmbedderConfig.PoolingStrategy; +import static com.yahoo.embedding.huggingface.HuggingFaceEmbedderConfig.TransformerExecutionMode; import static com.yahoo.text.XML.getChildValue; import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATION_BUNDLE_NAME; @@ -16,6 +18,8 @@ import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATI * @author bjorncs */ public class HuggingFaceEmbedder extends TypedComponent implements HuggingFaceEmbedderConfig.Producer { + + private final OnnxModelOptions onnxModelOptions; private final ModelReference modelRef; private final ModelReference vocabRef; private final Integer maxTokens; @@ -24,15 +28,16 @@ public class HuggingFaceEmbedder extends TypedComponent implements HuggingFaceEm private final String transformerTokenTypeIds; private final String transformerOutput; private final Boolean normalize; - private final String onnxExecutionMode; - private final Integer onnxInteropThreads; - private final Integer onnxIntraopThreads; - private final Integer onnxGpuDevice; private final String poolingStrategy; public HuggingFaceEmbedder(ApplicationContainerCluster cluster, Element xml, DeployState state) { super("ai.vespa.embedding.huggingface.HuggingFaceEmbedder", INTEGRATION_BUNDLE_NAME, xml); var model = Model.fromXml(state, xml, "transformer-model").orElseThrow(); + this.onnxModelOptions = new OnnxModelOptions( + getChildValue(xml, "onnx-execution-mode"), + getChildValue(xml, "onnx-interop-threads").map(Integer::parseInt), + getChildValue(xml, "onnx-intraop-threads").map(Integer::parseInt), + getChildValue(xml, "onnx-gpu-device").map(Integer::parseInt).map(OnnxModelOptions.GpuDevice::new)); modelRef = model.modelReference(); vocabRef = Model.fromXml(state, xml, "tokenizer-model") .map(Model::modelReference) @@ -43,12 +48,8 @@ public class HuggingFaceEmbedder extends TypedComponent implements HuggingFaceEm transformerTokenTypeIds = getChildValue(xml, "transformer-token-type-ids").orElse(null); transformerOutput = getChildValue(xml, "transformer-output").orElse(null); normalize = getChildValue(xml, "normalize").map(Boolean::parseBoolean).orElse(null); - onnxExecutionMode = getChildValue(xml, "onnx-execution-mode").orElse(null); - onnxInteropThreads = getChildValue(xml, "onnx-interop-threads").map(Integer::parseInt).orElse(null); - onnxIntraopThreads = getChildValue(xml, "onnx-intraop-threads").map(Integer::parseInt).orElse(null); - onnxGpuDevice = getChildValue(xml, "onnx-gpu-device").map(Integer::parseInt).orElse(null); poolingStrategy = getChildValue(xml, "pooling-strategy").orElse(null); - model.registerOnnxModelCost(cluster); + model.registerOnnxModelCost(cluster, onnxModelOptions); } private static ModelReference resolveDefaultVocab(Model model, DeployState state) { @@ -68,11 +69,11 @@ public class HuggingFaceEmbedder extends TypedComponent implements HuggingFaceEm if (transformerTokenTypeIds != null) b.transformerTokenTypeIds(transformerTokenTypeIds); if (transformerOutput != null) b.transformerOutput(transformerOutput); if (normalize != null) b.normalize(normalize); - if (onnxExecutionMode != null) b.transformerExecutionMode( - HuggingFaceEmbedderConfig.TransformerExecutionMode.Enum.valueOf(onnxExecutionMode)); - if (onnxInteropThreads != null) b.transformerInterOpThreads(onnxInteropThreads); - if (onnxIntraopThreads != null) b.transformerIntraOpThreads(onnxIntraopThreads); - if (onnxGpuDevice != null) b.transformerGpuDevice(onnxGpuDevice); - if (poolingStrategy != null) b.poolingStrategy(HuggingFaceEmbedderConfig.PoolingStrategy.Enum.valueOf(poolingStrategy)); + if (poolingStrategy != null) b.poolingStrategy(PoolingStrategy.Enum.valueOf(poolingStrategy)); + onnxModelOptions.executionMode().ifPresent(value -> b.transformerExecutionMode(TransformerExecutionMode.Enum.valueOf(value))); + onnxModelOptions.interOpThreads().ifPresent(b::transformerInterOpThreads); + onnxModelOptions.intraOpThreads().ifPresent(b::transformerIntraOpThreads); + onnxModelOptions.gpuDevice().ifPresent(value -> b.transformerGpuDevice(value.deviceNumber())); } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Model.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Model.java index c5daf23d6f8..0d350242fd0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/Model.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/Model.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.container.component; import com.yahoo.config.ModelReference; import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.model.api.OnnxModelOptions; import com.yahoo.config.model.builder.xml.XmlHelper; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.path.Path; @@ -54,10 +55,10 @@ class Model { return new Model(ds, model.getTagName(), modelId, url, path); } - void registerOnnxModelCost(ApplicationContainerCluster c) { + void registerOnnxModelCost(ApplicationContainerCluster c, OnnxModelOptions onnxModelOptions) { var resolvedUrl = resolvedUrl().orElse(null); - if (file != null) c.onnxModelCost().registerModel(file); - else if (resolvedUrl != null) c.onnxModelCost().registerModel(resolvedUrl); + if (file != null) c.onnxModelCost().registerModel(file, onnxModelOptions); + else if (resolvedUrl != null) c.onnxModelCost().registerModel(resolvedUrl, onnxModelOptions); } String name() { return paramName; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index d86d117f1d2..31468c05b99 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -52,11 +52,11 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> private final List<SearchCluster> searchClusters = new LinkedList<>(); private final Collection<String> schemasWithGlobalPhase; private final boolean globalPhase; + private final ApplicationPackage app; private QueryProfiles queryProfiles; private SemanticRules semanticRules; private PageTemplates pageTemplates; - private ApplicationPackage app; public ContainerSearch(DeployState deployState, ApplicationContainerCluster cluster, SearchChains chains) { super(chains); @@ -102,7 +102,7 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> if ( ! owningCluster.getComponentsMap().containsKey(factory.getComponentId())) { var onnxModels = documentDb.getDerivedConfiguration().getRankProfileList().getOnnxModels(); onnxModels.asMap().forEach( - (__, model) -> owningCluster.onnxModelCost().registerModel(app.getFile(model.getFilePath()))); + (__, model) -> owningCluster.onnxModelCost().registerModel(app.getFile(model.getFilePath()), model.onnxModelOptions())); owningCluster.addComponent(factory); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 18020f5df5d..5ffd34c6557 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -800,7 +800,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { !container.getHostResource().realResources().gpuResources().isZero()); onnxModel.setGpuDevice(gpuDevice, hasGpu); } - cluster.onnxModelCost().registerModel(context.getApplicationPackage().getFile(onnxModel.getFilePath())); + cluster.onnxModelCost().registerModel(context.getApplicationPackage().getFile(onnxModel.getFilePath()), onnxModel.onnxModelOptions()); } cluster.setModelEvaluation(new ContainerModelEvaluation(cluster, profiles, models)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java index c8f5be71f3c..18b9129cead 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java @@ -47,6 +47,7 @@ public class FileStorProducer implements StorFilestorConfig.Producer { private final int responseNumThreads; private final StorFilestorConfig.Response_sequencer_type.Enum responseSequencerType; private final boolean useAsyncMessageHandlingOnSchedule; + private final boolean usePerDocumentThrottledDeleteBucket; private static StorFilestorConfig.Response_sequencer_type.Enum convertResponseSequencerType(String sequencerType) { try { @@ -62,6 +63,7 @@ public class FileStorProducer implements StorFilestorConfig.Producer { this.responseNumThreads = featureFlags.defaultNumResponseThreads(); this.responseSequencerType = convertResponseSequencerType(featureFlags.responseSequencerType()); this.useAsyncMessageHandlingOnSchedule = featureFlags.useAsyncMessageHandlingOnSchedule(); + this.usePerDocumentThrottledDeleteBucket = featureFlags.usePerDocumentThrottledDeleteBucket(); } @Override @@ -73,6 +75,7 @@ public class FileStorProducer implements StorFilestorConfig.Producer { builder.num_response_threads(responseNumThreads); builder.response_sequencer_type(responseSequencerType); builder.use_async_message_handling_on_schedule(useAsyncMessageHandlingOnSchedule); + builder.use_per_document_throttled_delete_bucket(usePerDocumentThrottledDeleteBucket); var throttleBuilder = new StorFilestorConfig.Async_operation_throttler.Builder(); builder.async_operation_throttler(throttleBuilder); } diff --git a/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg b/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg index 4600884b615..b62d7fd239c 100644 --- a/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg +++ b/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg @@ -17,6 +17,8 @@ rankprofile[].fef.property[].value "true" rankprofile[].fef.property[].name "vespa.type.attribute.aa" rankprofile[].fef.property[].value "tensor(d1[3])" rankprofile[].name "simple" +rankprofile[].fef.property[].name "vespa.type.feature.attribute(aa)" +rankprofile[].fef.property[].value "tensor(d1[3])" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "rankingExpression(firstphase)" rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" diff --git a/config-model/src/test/derived/tensor/rank-profiles.cfg b/config-model/src/test/derived/tensor/rank-profiles.cfg index cd8375cb68d..a72e9dc27cd 100644 --- a/config-model/src/test/derived/tensor/rank-profiles.cfg +++ b/config-model/src/test/derived/tensor/rank-profiles.cfg @@ -47,7 +47,7 @@ rankprofile[].name "profile2" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "rankingExpression(firstphase)" rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" -rankprofile[].fef.property[].value "reduce(reduce(join(attribute(f4), tensor(x[10],y[10],z[3])((x==y)*(y==z)), f(a,b)(a * b)), sum, x), sum)" +rankprofile[].fef.property[].value "reduce(reduce(join(attribute(f4), tensor(x[10],y[10],z[3])(((x == y) * (y == z))), f(a,b)(a * b)), sum, x), sum)" rankprofile[].fef.property[].name "vespa.type.attribute.f7" rankprofile[].fef.property[].value "tensor<int8>(p{},x[5])" rankprofile[].fef.property[].name "vespa.type.attribute.f2" @@ -206,3 +206,22 @@ rankprofile[].fef.property[].name "vespa.type.query.para" rankprofile[].fef.property[].value "tensor<float>(p{})" rankprofile[].fef.property[].name "vespa.type.query.qvec" rankprofile[].fef.property[].value "tensor<float>(x[40])" +rankprofile[].name "with-just-unpack" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "rankingExpression(firstphase)" +rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" +rankprofile[].fef.property[].value "reduce(query(para) * map_subspaces(attribute(f7), f(denseSubspaceInput)(tensor(x[40])((bit(denseSubspaceInput{x:(x / 8)},(x % 8)))))) * query(qvec), sum)" +rankprofile[].fef.property[].name "vespa.type.attribute.f7" +rankprofile[].fef.property[].value "tensor<int8>(p{},x[5])" +rankprofile[].fef.property[].name "vespa.type.attribute.f2" +rankprofile[].fef.property[].value "tensor<float>(x[2],y[1])" +rankprofile[].fef.property[].name "vespa.type.attribute.f3" +rankprofile[].fef.property[].value "tensor(x{})" +rankprofile[].fef.property[].name "vespa.type.attribute.f4" +rankprofile[].fef.property[].value "tensor(x[10],y[10])" +rankprofile[].fef.property[].name "vespa.type.attribute.f5" +rankprofile[].fef.property[].value "tensor<float>(x[10])" +rankprofile[].fef.property[].name "vespa.type.query.para" +rankprofile[].fef.property[].value "tensor(p{})" +rankprofile[].fef.property[].name "vespa.type.query.qvec" +rankprofile[].fef.property[].value "tensor(x[40])" diff --git a/config-model/src/test/derived/tensor/tensor.sd b/config-model/src/test/derived/tensor/tensor.sd index 3a5fda3ac5d..a0f5cd92c56 100644 --- a/config-model/src/test/derived/tensor/tensor.sd +++ b/config-model/src/test/derived/tensor/tensor.sd @@ -135,4 +135,14 @@ schema tensor { } } + rank-profile with-just-unpack { + inputs { + query(para) tensor<double>(p{}) + query(qvec) tensor<double>(x[40]) + } + first-phase { + expression: sum(query(para)*unpack_bits(attribute(f7), double, little)*query(qvec)) + } + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java index 8531aff3b1a..9cadf5cffd8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java @@ -1,14 +1,13 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - package com.yahoo.vespa.model.application.validation; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.OnnxModelCost; +import com.yahoo.config.model.api.OnnxModelOptions; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.provision.InMemoryProvisioner; @@ -123,12 +122,20 @@ class JvmHeapSizeValidatorTest { @Override public Calculator newCalculator(ApplicationPackage appPkg, ApplicationId applicationId) { return this; } @Override public long aggregatedModelCostInBytes() { return totalCost.get(); } @Override public void registerModel(ApplicationFile path) {} + @Override public void registerModel(ApplicationFile path, OnnxModelOptions onnxModelOptions) {} @Override public void registerModel(URI uri) { assertEquals("https://my/url/model.onnx", uri.toString()); totalCost.addAndGet(modelCost); } + + @Override + public void registerModel(URI uri, OnnxModelOptions onnxModelOptions) { + assertEquals("https://my/url/model.onnx", uri.toString()); + totalCost.addAndGet(modelCost); + } + } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index e7b2c549fa5..bdd61d93136 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -355,6 +355,24 @@ public class StorageClusterTest { assertTrue(config.async_operation_throttler().throttle_individual_merge_feed_ops()); } + private void verifyUsePerDocumentThrottledDeleteBucket(boolean expected, Boolean enabled) { + var props = new TestProperties(); + if (enabled != null) { + props.setUsePerDocumentThrottledDeleteBucket(enabled); + } + var config = filestorConfigFromProducer(simpleCluster(props)); + assertEquals(expected, config.use_per_document_throttled_delete_bucket()); + } + + @Test + void delete_bucket_throttling_is_controlled_by_feature_flag() { + // TODO update default once rolled out and tested + verifyUsePerDocumentThrottledDeleteBucket(false, null); + + verifyUsePerDocumentThrottledDeleteBucket(false, false); + verifyUsePerDocumentThrottledDeleteBucket(true, true); + } + @Test void testCapacity() { String xml = joinLines( diff --git a/configd/src/apps/sentinel/CMakeLists.txt b/configd/src/apps/sentinel/CMakeLists.txt index 607aba2785c..d3cfcff4135 100644 --- a/configd/src/apps/sentinel/CMakeLists.txt +++ b/configd/src/apps/sentinel/CMakeLists.txt @@ -14,6 +14,7 @@ vespa_add_executable(configd_config-sentinel_app output-connection.cpp outward-check.cpp peer-check.cpp + platform-specific.cpp report-connectivity.cpp rpchooks.cpp rpcserver.cpp diff --git a/configd/src/apps/sentinel/platform-specific.cpp b/configd/src/apps/sentinel/platform-specific.cpp new file mode 100644 index 00000000000..d1c3682e857 --- /dev/null +++ b/configd/src/apps/sentinel/platform-specific.cpp @@ -0,0 +1,45 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "platform-specific.h" +#include <vespa/vespalib/util/error.h> +#include <cstdlib> +#include <string_view> +#ifdef __linux__ +#include <sys/prctl.h> +#endif + +#include <vespa/log/log.h> +LOG_SETUP(".sentinel.platform-specific"); + +using namespace std::string_view_literals; + +namespace config::platform_specific { + +namespace { + +[[maybe_unused]] [[nodiscard]] +bool is_env_toggled(const char* var_name) { + const char* maybe_toggled = getenv(var_name); + return (maybe_toggled && (maybe_toggled == "true"sv || maybe_toggled == "yes"sv)); +} + +} + +void pledge_no_new_privileges_if_env_configured() { +#ifdef __linux__ + if (is_env_toggled("VESPA_PR_SET_NO_NEW_PRIVS")) { + // One-way toggle to prevent any subprocess from possibly getting extra privileges via + // setuid/setgid executables (modulo exciting things like kernel bugs or a small, trained + // rat that climbs into your computer and pulls an adorably tiny lever labeled "root access"). + // Helps mitigate a certain class of vulnerabilities, and also allows processes to install + // their own seccomp filters. + // See https://www.kernel.org/doc/Documentation/prctl/no_new_privs.txt + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0) { + LOG(warning, "Failed to invoke prctl(PR_SET_NO_NEW_PRIVS): %s", vespalib::getErrorString(errno).c_str()); + } else { + LOG(debug, "Successfully invoked prctl(PR_SET_NO_NEW_PRIVS)"); + } + } +#endif +} + +} diff --git a/configd/src/apps/sentinel/platform-specific.h b/configd/src/apps/sentinel/platform-specific.h new file mode 100644 index 00000000000..d68d5f73768 --- /dev/null +++ b/configd/src/apps/sentinel/platform-specific.h @@ -0,0 +1,16 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +namespace config::platform_specific { + +/** + * If running on Linux, sets the `no_new_privs` process bit, which amongst other + * things prevents all launched sub-process(es) from acquiring more privileges + * through setuid/setgid executables. + * + * Only takes effect if the `VESPA_PR_SET_NO_NEW_PRIVS` environment variable is + * set to "true" or "yes". + */ +void pledge_no_new_privileges_if_env_configured(); + +} diff --git a/configd/src/apps/sentinel/sentinel.cpp b/configd/src/apps/sentinel/sentinel.cpp index 59c690275c3..4f1d6019065 100644 --- a/configd/src/apps/sentinel/sentinel.cpp +++ b/configd/src/apps/sentinel/sentinel.cpp @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "manager.h" +#include "platform-specific.h" #include <vespa/config/common/exceptions.h> #include <vespa/vespalib/util/signalhandler.h> #include <vespa/vespalib/util/exceptions.h> @@ -58,6 +59,8 @@ main(int argc, char **argv) } setlocale(LC_ALL, "C"); + platform_specific::pledge_no_new_privileges_if_env_configured(); // Affects all launched subprocesses + sentinel::Env environment; LOG(debug, "Reading configuration"); try { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java index c4fe8b42c25..07fee3c45d2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigServerBootstrap.java @@ -103,7 +103,7 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable @Override public void deconstruct() { - log.log(Level.INFO, "Stopping config server"); + log.log(Level.FINE, "Stopping config server"); down(); server.stop(); log.log(Level.FINE, "RPC server stopped"); @@ -220,13 +220,13 @@ public class ConfigServerBootstrap extends AbstractComponent implements Runnable // Keep track of deployment status per application Map<ApplicationId, Future<?>> deployments = new HashMap<>(); if (applicationIds.size() > 0) { - log.log(Level.INFO, () -> "Redeploying " + applicationIds.size() + " apps " + applicationIds + " with " + + log.log(Level.FINE, () -> "Redeploying " + applicationIds.size() + " apps " + applicationIds + " with " + configserverConfig.numRedeploymentThreads() + " threads"); applicationIds.forEach(appId -> deployments.put(appId, executor.submit(() -> { - log.log(Level.INFO, () -> "Starting redeployment of " + appId); + log.log(Level.FINE, () -> "Starting redeployment of " + appId); applicationRepository.deployFromLocalActive(appId, true /* bootstrap */) .ifPresent(Deployment::activate); - log.log(Level.INFO, () -> appId + " redeployed"); + log.log(Level.FINE, () -> appId + " redeployed"); }))); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 2564584a7df..a75e2e7c5aa 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -45,7 +45,6 @@ import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.function.Predicate; -import static com.yahoo.config.provision.NodeResources.Architecture; import static com.yahoo.vespa.config.server.ConfigServerSpec.fromConfig; import static com.yahoo.vespa.flags.FetchVector.Dimension.CLUSTER_TYPE; @@ -190,7 +189,6 @@ public class ModelContextImpl implements ModelContext { private final boolean useV8GeoPositions; private final int maxCompactBuffers; private final List<String> ignoredHttpUserAgents; - private final Architecture adminClusterArchitecture; private final boolean enableProxyProtocolMixedMode; private final boolean sharedStringRepoNoReclaim; private final String logFileCompressionAlgorithm; @@ -204,12 +202,13 @@ public class ModelContextImpl implements ModelContext { private final int heapPercentage; private final boolean enableGlobalPhase; private final String summaryDecodePolicy; - private final boolean enableNestedMultivalueGrouping; + private final boolean alwaysMarkPhraseExpensive; private final int contentLayerMetadataFeatureLevel; private final boolean dynamicHeapSize; private final String unknownConfigDefinition; private final int searchHandlerThreadpool; private final long mergingMaxMemoryUsagePerNode; + private final boolean usePerDocumentThrottledDeleteBucket; public FeatureFlags(FlagSource source, ApplicationId appId, Version version) { this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -234,7 +233,6 @@ public class ModelContextImpl implements ModelContext { this.useV8GeoPositions = flagValue(source, appId, version, Flags.USE_V8_GEO_POSITIONS); this.maxCompactBuffers = flagValue(source, appId, version, Flags.MAX_COMPACT_BUFFERS); this.ignoredHttpUserAgents = flagValue(source, appId, version, PermanentFlags.IGNORED_HTTP_USER_AGENTS); - this.adminClusterArchitecture = Architecture.valueOf(flagValue(source, appId, version, PermanentFlags.ADMIN_CLUSTER_NODE_ARCHITECTURE)); this.enableProxyProtocolMixedMode = flagValue(source, appId, version, Flags.ENABLE_PROXY_PROTOCOL_MIXED_MODE); this.sharedStringRepoNoReclaim = flagValue(source, appId, version, Flags.SHARED_STRING_REPO_NO_RECLAIM); this.logFileCompressionAlgorithm = flagValue(source, appId, version, Flags.LOG_FILE_COMPRESSION_ALGORITHM); @@ -249,12 +247,13 @@ public class ModelContextImpl implements ModelContext { this.heapPercentage = flagValue(source, appId, version, PermanentFlags.HEAP_SIZE_PERCENTAGE); this.enableGlobalPhase = flagValue(source, appId, version, Flags.ENABLE_GLOBAL_PHASE); this.summaryDecodePolicy = flagValue(source, appId, version, Flags.SUMMARY_DECODE_POLICY); - this.enableNestedMultivalueGrouping = flagValue(source, appId, version, Flags.ENABLE_NESTED_MULTIVALUE_GROUPING); this.contentLayerMetadataFeatureLevel = flagValue(source, appId, version, Flags.CONTENT_LAYER_METADATA_FEATURE_LEVEL); this.dynamicHeapSize = flagValue(source, appId, version, Flags.DYNAMIC_HEAP_SIZE); this.unknownConfigDefinition = flagValue(source, appId, version, Flags.UNKNOWN_CONFIG_DEFINITION); this.searchHandlerThreadpool = flagValue(source, appId, version, Flags.SEARCH_HANDLER_THREADPOOL); this.mergingMaxMemoryUsagePerNode = flagValue(source, appId, version, Flags.MERGING_MAX_MEMORY_USAGE_PER_NODE); + this.usePerDocumentThrottledDeleteBucket = flagValue(source, appId, version, Flags.USE_PER_DOCUMENT_THROTTLED_DELETE_BUCKET); + this.alwaysMarkPhraseExpensive = flagValue(source, appId, version, Flags.ALWAYS_MARK_PHRASE_EXPENSIVE); } @Override public int heapSizePercentage() { return heapPercentage; } @@ -285,7 +284,6 @@ public class ModelContextImpl implements ModelContext { @Override public boolean useV8GeoPositions() { return useV8GeoPositions; } @Override public int maxCompactBuffers() { return maxCompactBuffers; } @Override public List<String> ignoredHttpUserAgents() { return ignoredHttpUserAgents; } - @Override public Architecture adminClusterArchitecture() { return adminClusterArchitecture; } @Override public boolean enableProxyProtocolMixedMode() { return enableProxyProtocolMixedMode; } @Override public boolean sharedStringRepoNoReclaim() { return sharedStringRepoNoReclaim; } @Override public int mbusJavaRpcNumTargets() { return mbus_java_num_targets; } @@ -302,12 +300,13 @@ public class ModelContextImpl implements ModelContext { return defVal; } @Override public boolean enableGlobalPhase() { return enableGlobalPhase; } - @Override public boolean enableNestedMultivalueGrouping() { return enableNestedMultivalueGrouping; } + @Override public boolean alwaysMarkPhraseExpensive() { return alwaysMarkPhraseExpensive; } @Override public int contentLayerMetadataFeatureLevel() { return contentLayerMetadataFeatureLevel; } @Override public boolean dynamicHeapSize() { return dynamicHeapSize; } @Override public String unknownConfigDefinition() { return unknownConfigDefinition; } @Override public int searchHandlerThreadpool() { return searchHandlerThreadpool; } @Override public long mergingMaxMemoryUsagePerNode() { return mergingMaxMemoryUsagePerNode; } + @Override public boolean usePerDocumentThrottledDeleteBucket() { return usePerDocumentThrottledDeleteBucket; } private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java index 5a244b4b4b4..93c41f1b087 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/CompressedApplicationInputStreamTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.config.server.application; import com.google.common.io.ByteStreams; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.yolean.Exceptions; +import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.ArchiveOutputStream; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; @@ -19,6 +20,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; import java.util.List; +import java.util.Objects; import java.util.zip.GZIPOutputStream; import static org.junit.Assert.assertEquals; @@ -33,13 +35,13 @@ public class CompressedApplicationInputStreamTest { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); - private static void writeFileToTar(ArchiveOutputStream taos, File file) throws IOException { + private static <T extends ArchiveEntry> void writeFileToTar(ArchiveOutputStream<T> taos, File file) throws IOException { taos.putArchiveEntry(taos.createArchiveEntry(file, file.getName())); ByteStreams.copy(new FileInputStream(file), taos); taos.closeArchiveEntry(); } - private static File createArchiveFile(ArchiveOutputStream taos, File outFile) throws IOException { + private static <T extends ArchiveEntry> File createArchiveFile(ArchiveOutputStream<T> taos, File outFile) throws IOException { File app = new File("src/test/resources/deploy/validapp"); writeFileToTar(taos, new File(app, "services.xml")); writeFileToTar(taos, new File(app, "hosts.xml")); @@ -50,13 +52,13 @@ public class CompressedApplicationInputStreamTest { public static File createTarFile(Path dir) throws IOException { File outFile = Files.createTempFile(dir, "testapp", ".tar.gz").toFile(); - ArchiveOutputStream archiveOutputStream = new TarArchiveOutputStream(new GZIPOutputStream(new FileOutputStream(outFile))); + var archiveOutputStream = new TarArchiveOutputStream(new GZIPOutputStream(new FileOutputStream(outFile))); return createArchiveFile(archiveOutputStream, outFile); } private File createZipFile(Path dir) throws IOException { File outFile = Files.createTempFile(dir, "testapp", ".tar.gz").toFile(); - ArchiveOutputStream archiveOutputStream = new ZipArchiveOutputStream(new FileOutputStream(outFile)); + var archiveOutputStream = new ZipArchiveOutputStream(new FileOutputStream(outFile)); return createArchiveFile(archiveOutputStream, outFile); } @@ -102,7 +104,7 @@ public class CompressedApplicationInputStreamTest { try (CompressedApplicationInputStream unpacked = streamFromTarGz(gzFile)) { outApp = unpacked.decompress(); } - List<File> files = Arrays.asList(outApp.listFiles()); + List<File> files = Arrays.asList(Objects.requireNonNull(outApp.listFiles())); assertEquals(5, files.size()); assertTrue(files.contains(new File(outApp, "services.xml"))); assertTrue(files.contains(new File(outApp, "hosts.xml"))); diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index aae5ea31fbb..d07d21ae71f 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -660,6 +660,10 @@ "public com.yahoo.container.jdisc.HttpRequestBuilder withRequestContent(java.io.InputStream)", "public com.yahoo.container.jdisc.HttpRequestBuilder withScheme(java.lang.String)", "public com.yahoo.container.jdisc.HttpRequestBuilder withHostname(java.lang.String)", + "public com.yahoo.container.jdisc.HttpRequestBuilder withPrincipal(java.security.Principal)", + "public com.yahoo.container.jdisc.HttpRequestBuilder withRemoteAddress(java.net.SocketAddress)", + "public com.yahoo.container.jdisc.HttpRequestBuilder withAttribute(java.lang.String, java.lang.Object)", + "public com.yahoo.container.jdisc.HttpRequestBuilder withPort(int)", "public com.yahoo.container.jdisc.HttpRequest build()" ], "fields" : [ ] @@ -1089,6 +1093,7 @@ "public com.yahoo.jdisc.http.ConnectorConfig$Builder maxContentSizeErrorMessageTemplate(java.lang.String)", "public com.yahoo.jdisc.http.ConnectorConfig$Builder reuseAddress(boolean)", "public com.yahoo.jdisc.http.ConnectorConfig$Builder idleTimeout(double)", + "public com.yahoo.jdisc.http.ConnectorConfig$Builder shutdownIdleTimeout(double)", "public com.yahoo.jdisc.http.ConnectorConfig$Builder tcpKeepAliveEnabled(boolean)", "public com.yahoo.jdisc.http.ConnectorConfig$Builder tcpNoDelay(boolean)", "public com.yahoo.jdisc.http.ConnectorConfig$Builder throttling(com.yahoo.jdisc.http.ConnectorConfig$Throttling$Builder)", @@ -1475,6 +1480,7 @@ "public java.lang.String maxContentSizeErrorMessageTemplate()", "public boolean reuseAddress()", "public double idleTimeout()", + "public double shutdownIdleTimeout()", "public boolean tcpKeepAliveEnabled()", "public boolean tcpNoDelay()", "public com.yahoo.jdisc.http.ConnectorConfig$Throttling throttling()", diff --git a/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequestBuilder.java b/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequestBuilder.java index a2d792e6ae0..147f388e08c 100644 --- a/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequestBuilder.java +++ b/container-core/src/main/java/com/yahoo/container/jdisc/HttpRequestBuilder.java @@ -4,6 +4,8 @@ package com.yahoo.container.jdisc; import com.yahoo.jdisc.http.HttpRequest.Method; import java.io.InputStream; +import java.net.SocketAddress; +import java.security.Principal; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -19,9 +21,13 @@ public class HttpRequestBuilder { private final String path; private final Map<String, List<String>> queryParameters = new TreeMap<>(); private final Map<String, String> headers = new TreeMap<>(); + private final Map<String, Object> attributes = new TreeMap<>(); private String scheme; private String hostname; private InputStream content; + private Principal principal; + private SocketAddress socketAddress; + private int port = -1; private HttpRequestBuilder(Method method, String path) { this.method = method; @@ -43,10 +49,20 @@ public class HttpRequestBuilder { public HttpRequestBuilder withHostname(String hostname) { this.hostname = hostname; return this; } + public HttpRequestBuilder withPrincipal(Principal p) { principal = p; return this; } + + public HttpRequestBuilder withRemoteAddress(SocketAddress sa) { socketAddress = sa; return this; } + + public HttpRequestBuilder withAttribute(String name, Object value) { attributes.put(name, value); return this; } + + public HttpRequestBuilder withPort(int port) { this.port = port; return this; } + public HttpRequest build() { String scheme = this.scheme != null ? this.scheme : "http"; String hostname = this.hostname != null ? this.hostname : "localhost"; - StringBuilder uriBuilder = new StringBuilder(scheme).append("://").append(hostname).append(path); + StringBuilder uriBuilder = new StringBuilder(scheme).append("://").append(hostname); + if (port > 0) uriBuilder.append(':').append(port); + uriBuilder.append(path); if (queryParameters.size() > 0) { uriBuilder.append('?'); queryParameters.forEach((name, values) -> { @@ -66,6 +82,9 @@ public class HttpRequestBuilder { request = HttpRequest.createTestRequest(uriBuilder.toString(), method); } headers.forEach((name, value) -> request.getJDiscRequest().headers().put(name, value)); + if (principal != null) request.getJDiscRequest().setUserPrincipal(principal); + if (socketAddress != null) request.getJDiscRequest().setRemoteAddress(socketAddress); + request.getJDiscRequest().context().putAll(attributes); return request; } } diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscServerConnector.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscServerConnector.java index 983adec034d..3159766981c 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscServerConnector.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/JDiscServerConnector.java @@ -51,7 +51,11 @@ class JDiscServerConnector extends ServerConnector { setName(config.name()); setAcceptQueueSize(config.acceptQueueSize()); setReuseAddress(config.reuseAddress()); - setIdleTimeout((long) (config.idleTimeout() * 1000)); + long idleTimeout = (long)(config.idleTimeout() * 1000); + setIdleTimeout(idleTimeout); + long shutdownIdleTimeout = (long) (config.shutdownIdleTimeout() * 1000); + // Ensure shutdown idle timeout is less than idle timeout and stop timeout + setShutdownIdleTimeout(Math.min(shutdownIdleTimeout, Math.min(idleTimeout, server.getStopTimeout()))); } public ConnectionStatistics getStatistics() { diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java index a022d208d05..9fee54dd1d4 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ServletRequestReader.java @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc.http.server.jetty; -import com.yahoo.jdisc.Response; import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; import com.yahoo.jdisc.http.ConnectorConfig; @@ -19,6 +18,8 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.logging.Level; import java.util.logging.Logger; +import static com.yahoo.jdisc.Response.Status.REQUEST_TOO_LONG; + /** * Finished when either * 1) There was an error @@ -111,7 +112,8 @@ class ServletRequestReader { long maxContentSize = resolveMaxContentSize(cfg); var msgTemplate = resolveMaxContentSizeErrorMessage(cfg); this.requestContentChannel = maxContentSize >= 0 - ? new ByteLimitedContentChannel(Objects.requireNonNull(requestContentChannel), maxContentSize, msgTemplate) + ? new ByteLimitedContentChannel( + Objects.requireNonNull(requestContentChannel), maxContentSize, msgTemplate, req.getContentLengthLong()) : Objects.requireNonNull(requestContentChannel); this.janitor = Objects.requireNonNull(janitor); this.metricReporter = Objects.requireNonNull(metricReporter); @@ -285,24 +287,29 @@ class ServletRequestReader { private static class ByteLimitedContentChannel implements ContentChannel { private final long maxContentSize; private final String messageTemplate; + private final long contentLengthHeader; private final AtomicLong bytesWritten = new AtomicLong(); private final ContentChannel delegate; - ByteLimitedContentChannel(ContentChannel delegate, long maxContentSize, String messageTemplate) { + ByteLimitedContentChannel(ContentChannel delegate, long maxContentSize, String messageTemplate, long contentLengthHeader) { this.delegate = delegate; this.maxContentSize = maxContentSize; this.messageTemplate = messageTemplate; + this.contentLengthHeader = contentLengthHeader; } @Override public void write(ByteBuffer buf, CompletionHandler handler) { long written = bytesWritten.addAndGet(buf.remaining()); - if (written > maxContentSize) { + if (contentLengthHeader != -1 && contentLengthHeader > maxContentSize) { handler.failed(new RequestException( - Response.Status.REQUEST_TOO_LONG, messageTemplate.formatted(written, maxContentSize))); - return; + REQUEST_TOO_LONG, messageTemplate.formatted(contentLengthHeader, maxContentSize))); + } else if (written > maxContentSize) { + handler.failed(new RequestException( + REQUEST_TOO_LONG, messageTemplate.formatted(written, maxContentSize))); + } else { + delegate.write(buf, handler); } - delegate.write(buf, handler); } @Override public void close(CompletionHandler h) { delegate.close(h); } diff --git a/container-core/src/main/java/com/yahoo/restapi/RestApi.java b/container-core/src/main/java/com/yahoo/restapi/RestApi.java index 18d8d8c49b4..ee5628988c9 100644 --- a/container-core/src/main/java/com/yahoo/restapi/RestApi.java +++ b/container-core/src/main/java/com/yahoo/restapi/RestApi.java @@ -15,6 +15,7 @@ import com.yahoo.security.tls.ConnectionAuthContext; import javax.net.ssl.SSLSession; import java.io.InputStream; +import java.net.InetSocketAddress; import java.security.Principal; import java.util.List; import java.util.Optional; @@ -153,6 +154,7 @@ public interface RestApi { Principal userPrincipalOrThrow(); Optional<SSLSession> sslSession(); Optional<ConnectionAuthContext> connectionAuthContext(); + InetSocketAddress remoteAddress(); interface Parameters { Optional<String> getString(String name); @@ -193,6 +195,7 @@ public interface RestApi { interface FilterContext { RequestContext requestContext(); String route(); + void setPrincipal(Principal principal); HttpResponse executeNext(); } } diff --git a/container-core/src/main/java/com/yahoo/restapi/RestApiException.java b/container-core/src/main/java/com/yahoo/restapi/RestApiException.java index e3acf4258f1..3a44bc4da5f 100644 --- a/container-core/src/main/java/com/yahoo/restapi/RestApiException.java +++ b/container-core/src/main/java/com/yahoo/restapi/RestApiException.java @@ -57,6 +57,7 @@ public class RestApiException extends RuntimeException { } public static class BadRequest extends RestApiException { + public BadRequest() { this("Bad request"); } public BadRequest(String message) { this(message, null); } public BadRequest(Throwable cause) { this(cause.getMessage(), cause); } public BadRequest(String message, Throwable cause) { super(ErrorResponse::badRequest, message, cause); } @@ -69,6 +70,7 @@ public class RestApiException extends RuntimeException { } public static class Forbidden extends RestApiException { + public Forbidden() { this("Forbidden"); } public Forbidden(String message) { super(ErrorResponse::forbidden, message, null); } public Forbidden(String message, Throwable cause) { super(ErrorResponse::forbidden, message, cause); } } diff --git a/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java b/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java index 39dd17d3563..090e06c221f 100644 --- a/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java +++ b/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java @@ -21,6 +21,7 @@ import com.yahoo.security.tls.TransportSecurityUtils; import javax.net.ssl.SSLSession; import java.io.InputStream; +import java.net.InetSocketAddress; import java.net.URI; import java.security.Principal; import java.util.ArrayList; @@ -76,7 +77,11 @@ class RestApiImpl implements RestApi { resolvedRoute, requestContext, filters, createFilterContextRecursive(resolvedRoute, requestContext, resolvedRoute.filters, null)); if (filterContext != null) { - return filterContext.executeFirst(); + try { + return filterContext.executeFirst(); + } catch (RuntimeException e) { + return mapException(requestContext, e); + } } else { return dispatchToRoute(resolvedRoute, requestContext); } @@ -487,7 +492,7 @@ class RestApiImpl implements RestApi { @Override public Optional<ConnectionAuthContext> connectionAuthContext() { return sslSession().flatMap(TransportSecurityUtils::getConnectionAuthContext); } - + @Override public InetSocketAddress remoteAddress() { return (InetSocketAddress) request.getJDiscRequest().getRemoteAddress(); } private class PathParametersImpl implements RestApi.RequestContext.PathParameters { @Override @@ -496,7 +501,7 @@ class RestApiImpl implements RestApi { } @Override public String getStringOrThrow(String name) { return getString(name) - .orElseThrow(() -> new RestApiException.BadRequest("Path parameter '" + name + "' is missing")); + .orElseThrow(() -> new RestApiException.NotFound("Path parameter '" + name + "' is missing")); } @Override public HttpURL.Path getFullPath() { return pathMatcher.getPath(); @@ -554,6 +559,7 @@ class RestApiImpl implements RestApi { @Override public RestApi.RequestContext requestContext() { return requestContext; } @Override public String route() { return route.name != null ? route.name : route.pathPattern; } + @Override public void setPrincipal(Principal p) { requestContext.request.getJDiscRequest().setUserPrincipal(p); } HttpResponse executeFirst() { return filter.filterRequest(this); } diff --git a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def index d2f081fe7d5..5e59d998e86 100644 --- a/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def +++ b/container-core/src/main/resources/configdefinitions/jdisc.http.jdisc.http.connector.def @@ -33,6 +33,9 @@ reuseAddress bool default=true # The maximum idle time for a connection, which roughly translates to the Socket.setSoTimeout(int). idleTimeout double default=180.0 +# The idle timeout that takes effect during graceful shutdown of Jetty +shutdownIdleTimeout double default=5.0 + # TODO Vespa 9 Remove # Has no effect since Jetty 11 upgrade tcpKeepAliveEnabled bool default=false diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java index 6e218a6ab66..adb35db8ebf 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java @@ -160,6 +160,20 @@ public class HttpServerTest { } @Test + void requireThatTooLargePayloadFailsWith413() throws Exception { + final JettyTestDriver driver = JettyTestDriver.newConfiguredInstance( + new EchoRequestHandler(), + new ServerConfig.Builder(), + new ConnectorConfig.Builder() + .maxContentSize(100)); + driver.client().newPost("/status.html") + .setBinaryContent(new byte[200]) + .execute() + .expectStatusCode(is(REQUEST_TOO_LONG)); + assertTrue(driver.close()); + } + + @Test void requireThatMultipleHostHeadersReturns400() throws Exception { var metricConsumer = new MetricConsumerMock(); JettyTestDriver driver = JettyTestDriver.newConfiguredInstance( diff --git a/container-core/src/test/java/com/yahoo/restapi/RestApiImplTest.java b/container-core/src/test/java/com/yahoo/restapi/RestApiImplTest.java index bf0ccb95887..d27e04bbd7a 100644 --- a/container-core/src/test/java/com/yahoo/restapi/RestApiImplTest.java +++ b/container-core/src/test/java/com/yahoo/restapi/RestApiImplTest.java @@ -158,6 +158,42 @@ class RestApiImplTest { assertRequiredCapability(restApi, Method.POST, "/api2", Capability.CONTENT__DOCUMENT_API); } + @Test + void maps_exception_for_filter_throwing() { + RestApi.Filter throwingFilter = (ctx) -> { + throw new RestApiException.Forbidden("forbidden"); + }; + var restApi = RestApi.builder() + .setDefaultRoute(route("{*}").defaultHandler(ctx -> "hello world")) + .addFilter(throwingFilter) + .build(); + verifyJsonResponse(restApi, Method.GET, "/", null, 403, "{\"error-code\":\"FORBIDDEN\",\"message\":\"forbidden\"}"); + } + + @Test + void missing_parameters_are_mapped_to_4xx_response() { + var restApi = RestApi.builder() + .addRoute(route("/missing-path-param").get(ctx -> ctx.pathParameters().getStringOrThrow("missing"))) + .addRoute(route("/missing-query-param").get(ctx -> ctx.queryParameters().getStringOrThrow("missing"))) + .build(); + verifyJsonResponse(restApi, Method.GET, "/missing-path-param", null, 404, + "{\"error-code\":\"NOT_FOUND\",\"message\":\"Path parameter 'missing' is missing\"}"); + verifyJsonResponse(restApi, Method.GET, "/missing-query-param", null, 400, + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Query parameter 'missing' is missing\"}"); + } + + @Test + void principal_from_filter_is_visible_to_handler() { + var restApi = RestApi.builder() + .addRoute(route("/api1").get(ctx -> ctx.userPrincipalOrThrow().getName())) + .addFilter(ctx -> { + ctx.setPrincipal(() -> "my-principal-name"); + return ctx.executeNext(); + }) + .build(); + verifyJsonResponse(restApi, Method.GET, "/api1", null, 200, "{\"message\":\"my-principal-name\"}"); + } + private static void verifyJsonResponse( RestApi restApi, Method method, String path, String requestContent, int expectedStatusCode, String expectedJson) { diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 31b4dd2c920..f2348c58d4d 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -643,6 +643,23 @@ "public static final java.lang.String HIGHLIGHTTERMS" ] }, + "com.yahoo.prelude.query.InItem" : { + "superClass" : "com.yahoo.prelude.query.Item", + "interfaces" : [ ], + "attributes" : [ + "public", + "abstract" + ], + "methods" : [ + "public void <init>(java.lang.String)", + "public void setIndexName(java.lang.String)", + "public java.lang.String getIndexName()", + "public java.lang.String getName()", + "public boolean equals(java.lang.Object)", + "public int hashCode()" + ], + "fields" : [ ] + }, "com.yahoo.prelude.query.IndexedItem" : { "superClass" : "java.lang.Object", "interfaces" : [ @@ -781,6 +798,8 @@ "public static final enum com.yahoo.prelude.query.Item$ItemType TRUE", "public static final enum com.yahoo.prelude.query.Item$ItemType FALSE", "public static final enum com.yahoo.prelude.query.Item$ItemType FUZZY", + "public static final enum com.yahoo.prelude.query.Item$ItemType STRING_IN", + "public static final enum com.yahoo.prelude.query.Item$ItemType NUMERIC_IN", "public final int code" ] }, @@ -1068,6 +1087,26 @@ ], "fields" : [ ] }, + "com.yahoo.prelude.query.NumericInItem" : { + "superClass" : "com.yahoo.prelude.query.InItem", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.lang.String)", + "public com.yahoo.prelude.query.Item$ItemType getItemType()", + "public int encode(java.nio.ByteBuffer)", + "protected void encodeThis(java.nio.ByteBuffer)", + "public int getTermCount()", + "protected void appendBodyString(java.lang.StringBuilder)", + "public void addToken(long)", + "public java.util.Collection getTokens()", + "public boolean equals(java.lang.Object)", + "public int hashCode()" + ], + "fields" : [ ] + }, "com.yahoo.prelude.query.ONearItem" : { "superClass" : "com.yahoo.prelude.query.NearItem", "interfaces" : [ ], @@ -1529,6 +1568,26 @@ ], "fields" : [ ] }, + "com.yahoo.prelude.query.StringInItem" : { + "superClass" : "com.yahoo.prelude.query.InItem", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.lang.String)", + "public com.yahoo.prelude.query.Item$ItemType getItemType()", + "public int encode(java.nio.ByteBuffer)", + "protected void encodeThis(java.nio.ByteBuffer)", + "public int getTermCount()", + "protected void appendBodyString(java.lang.StringBuilder)", + "public void addToken(java.lang.String)", + "public java.util.Collection getTokens()", + "public boolean equals(java.lang.Object)", + "public int hashCode()" + ], + "fields" : [ ] + }, "com.yahoo.prelude.query.Substring" : { "superClass" : "java.lang.Object", "interfaces" : [ ], diff --git a/container-search/src/main/java/com/yahoo/prelude/query/InItem.java b/container-search/src/main/java/com/yahoo/prelude/query/InItem.java new file mode 100644 index 00000000000..27213000e3a --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/query/InItem.java @@ -0,0 +1,46 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query; + +import java.util.Objects; + +import static java.util.Objects.requireNonNullElse; + +/* + * Abstract class representing an IN operator. + * + * @author toregge + */ +public abstract class InItem extends Item { + private String indexName; + public InItem(String indexName) { + this.indexName = requireNonNullElse(indexName, ""); + } + + @Override + public void setIndexName(String index) { + this.indexName = requireNonNullElse(index, ""); + } + public String getIndexName() { + return indexName; + } + + @Override + public String getName() { + return getItemType().name(); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! super.equals(o)) return false; + var other = (InItem)o; + if ( ! Objects.equals(this.indexName, other.indexName)) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), indexName); + } + +}; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/Item.java b/container-search/src/main/java/com/yahoo/prelude/query/Item.java index e38579be2df..f43b55424e6 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/Item.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/Item.java @@ -56,7 +56,9 @@ public abstract class Item implements Cloneable { GEO_LOCATION_TERM(27), TRUE(28), FALSE(29), - FUZZY(30); + FUZZY(30), + STRING_IN(31), + NUMERIC_IN(32); public final int code; @@ -241,8 +243,9 @@ public abstract class Item implements Cloneable { byte FEAT_UNIQUEID = 0b01000000; byte FEAT_FLAGS = -0b10000000; - byte type = (byte) (getCode() & CODE_MASK); - if (type != getCode()) + int code = getCode(); + byte type = code >= CODE_MASK ? CODE_MASK : (byte) code; + if (code >= 0x80 + CODE_MASK) throw new IllegalStateException("must increase number of bytes in serialization format for queries"); if (weight != DEFAULT_WEIGHT) { @@ -257,6 +260,10 @@ public abstract class Item implements Cloneable { } buffer.put(type); + if (code >= CODE_MASK) { + // This is an extension to the serialization to work around original 5 bits limit for code + buffer.put((byte) (code - CODE_MASK)); + } if ((type & FEAT_WEIGHT) != 0) { IntegerCompressor.putCompressedNumber(weight, buffer); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/NumericInItem.java b/container-search/src/main/java/com/yahoo/prelude/query/NumericInItem.java new file mode 100644 index 00000000000..9333173d898 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/query/NumericInItem.java @@ -0,0 +1,89 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query; + +import com.yahoo.compress.IntegerCompressor; + +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; + +/* + * Class representing an IN operator with a set of 64-bit + * integer values. + * + * @author toregge + */ +public class NumericInItem extends InItem { + private Set<Long> tokens; + + public NumericInItem(String indexName) { + super(indexName); + tokens = new HashSet<>(1000); + } + + @Override + public Item.ItemType getItemType() { + return Item.ItemType.NUMERIC_IN; + } + + @Override + public int encode(ByteBuffer buffer) { + encodeThis(buffer); + return 1; + } + + @Override + protected void encodeThis(ByteBuffer buffer) { + super.encodeThis(buffer); + IntegerCompressor.putCompressedPositiveNumber(tokens.size(), buffer); + putString(getIndexName(), buffer); + for (var token : tokens) { + buffer.putLong(token); + } + } + + @Override + public int getTermCount() { + return 1; + } + + @Override + protected void appendBodyString(StringBuilder buffer) { + buffer.append(getIndexName()); + buffer.append("{"); + for (var token : tokens) { + buffer.append(token.toString()); + if (token < Integer.MIN_VALUE || token > Integer.MAX_VALUE) { + buffer.append("L"); + } + buffer.append(","); + } + if (!tokens.isEmpty()) { + buffer.deleteCharAt(buffer.length() - 1); // remove extra "," + } + buffer.append("}"); + } + + public void addToken(long token) { + tokens.add(token); + } + + public Collection<Long> getTokens() { return Set.copyOf(tokens); } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! super.equals(o)) return false; + var other = (NumericInItem)o; + if ( ! Objects.equals(this.tokens, other.tokens)) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), tokens); + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/query/StringInItem.java b/container-search/src/main/java/com/yahoo/prelude/query/StringInItem.java new file mode 100644 index 00000000000..ebcf0de1a21 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/query/StringInItem.java @@ -0,0 +1,87 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query; + +import com.yahoo.compress.IntegerCompressor; + +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; + +/* + * Class representing an IN operator with a set of string values. + * + * @author toregge + */ +public class StringInItem extends InItem { + private Set<String> tokens; + + public StringInItem(String indexName) { + super(indexName); + tokens = new HashSet<>(1000); + } + + @Override + public ItemType getItemType() { + return ItemType.STRING_IN; + } + + @Override + public int encode(ByteBuffer buffer) { + encodeThis(buffer); + return 1; + } + + @Override + protected void encodeThis(ByteBuffer buffer) { + super.encodeThis(buffer); + IntegerCompressor.putCompressedPositiveNumber(tokens.size(), buffer); + putString(getIndexName(), buffer); + for (var entry : tokens) { + putString(entry, buffer); + } + } + + @Override + public int getTermCount() { + return 1; + } + + @Override + protected void appendBodyString(StringBuilder buffer) { + buffer.append(getIndexName()); + buffer.append("{"); + for (var entry : tokens) { + buffer.append("\""); + buffer.append(entry); + buffer.append("\","); + } + if (!tokens.isEmpty()) { + buffer.deleteCharAt(buffer.length() - 1); // remove extra "," + } + buffer.append("}"); + } + + public void addToken(String token) { + Objects.requireNonNull(token, "Token string must not be null"); + tokens.add(token); + } + + public Collection<String> getTokens() { return Set.copyOf(tokens); } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! super.equals(o)) return false; + var other = (StringInItem)o; + if ( ! Objects.equals(this.tokens, other.tokens)) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), tokens); + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/yql/ParameterListParser.java b/container-search/src/main/java/com/yahoo/search/yql/ParameterListParser.java index 1993871aa4c..397cb056ba2 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/ParameterListParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/ParameterListParser.java @@ -1,6 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.yql; +import com.yahoo.prelude.query.NumericInItem; +import com.yahoo.prelude.query.StringInItem; import com.yahoo.prelude.query.WeightedSetItem; import java.util.Arrays; @@ -61,6 +63,41 @@ class ParameterListParser { s.pass('}'); } + public static void addStringTokensFromString(String string, StringInItem out) { + if (string == null) { + return; + } + var s = new ParsableString(string); + while (!s.atEnd()) { + String token; + if (s.passOptional('\'')) { + token = s.stringTo(s.position('\'')); + s.pass('\''); + } + else if (s.passOptional('"')) { + token = s.stringTo(s.position('"')); + s.pass('"'); + } + else { + token = s.stringTo(s.positionOrEnd(',')).trim(); + } + out.addToken(token); + s.passOptional(','); + } + } + + public static void addNumericTokensFromString(String string, NumericInItem out) { + if (string == null) { + return; + } + var s = new ParsableString(string); + while (!s.atEnd()) { + long token = s.longTo(s.positionOrEnd(',')); + out.addToken(token); + s.passOptional(','); + } + } + private static class ParsableString { int position = 0; @@ -142,6 +179,17 @@ class ParameterListParser { throw new IllegalArgumentException("Expected one of " + Arrays.toString(characters) + " after " + position); } + int positionOrEnd(char ... characters) { + int localPosition = position; + while (localPosition < s.length()) { + char nextChar = s.charAt(localPosition); + for (char character : characters) + if (nextChar == character) return localPosition; + localPosition++; + } + return localPosition; + } + boolean atEnd() { return position >= s.length(); } diff --git a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java index 490fc7aa07f..634163bf0c2 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java +++ b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java @@ -84,6 +84,7 @@ import com.yahoo.prelude.query.NearItem; import com.yahoo.prelude.query.NearestNeighborItem; import com.yahoo.prelude.query.NotItem; import com.yahoo.prelude.query.NullItem; +import com.yahoo.prelude.query.NumericInItem; import com.yahoo.prelude.query.ONearItem; import com.yahoo.prelude.query.OrItem; import com.yahoo.prelude.query.PhraseItem; @@ -95,6 +96,7 @@ import com.yahoo.prelude.query.RankItem; import com.yahoo.prelude.query.RegExpItem; import com.yahoo.prelude.query.SameElementItem; import com.yahoo.prelude.query.SegmentingRule; +import com.yahoo.prelude.query.StringInItem; import com.yahoo.prelude.query.Substring; import com.yahoo.prelude.query.SubstringItem; import com.yahoo.prelude.query.SuffixItem; @@ -1077,6 +1079,51 @@ public class VespaSerializer { } + private static class StringInSerializer extends Serializer<StringInItem> { + @Override + void onExit(StringBuilder destination, StringInItem item) { + + } + + @Override + boolean serialize(StringBuilder destination, StringInItem item) { + destination.append(item.getIndexName()).append(" in ("); + int initLen = destination.length(); + List<String> tokens = new ArrayList<>(item.getTokens()); + Collections.sort(tokens); + for (var token : tokens) { + comma(destination, initLen); + destination.append('"'); + escape(token, destination); + destination.append("\""); + } + destination.append(")"); + return false; + } + } + + private static class NumericInSerializer extends Serializer<NumericInItem> { + @Override + void onExit(StringBuilder destination, NumericInItem item) { + } + + @Override + boolean serialize(StringBuilder destination, NumericInItem item) { + destination.append(item.getIndexName()).append(" in ("); + int initLen = destination.length(); + List<Long> tokens = new ArrayList<>(item.getTokens()); + Collections.sort(tokens); + for (var token : tokens) { + comma(destination, initLen); + destination.append(token.toString()); + if (token < Integer.MIN_VALUE || token > Integer.MAX_VALUE) + destination.append("L"); + } + destination.append(")"); + return false; + } + } + private static class WordSerializer extends Serializer<WordItem> { @Override @@ -1284,6 +1331,8 @@ public class VespaSerializer { dispatchBuilder.put(RegExpItem.class, new RegExpSerializer()); dispatchBuilder.put(UriItem.class, new UriSerializer()); dispatchBuilder.put(FuzzyItem.class, new FuzzySerializer()); + dispatchBuilder.put(StringInItem.class, new StringInSerializer()); + dispatchBuilder.put(NumericInItem.class, new NumericInSerializer()); dispatch = ImmutableMap.copyOf(dispatchBuilder); } diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index eaabdf2d2d4..5e1dfb99479 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -44,6 +44,7 @@ import com.yahoo.prelude.query.NearItem; import com.yahoo.prelude.query.NearestNeighborItem; import com.yahoo.prelude.query.NotItem; import com.yahoo.prelude.query.NullItem; +import com.yahoo.prelude.query.NumericInItem; import com.yahoo.prelude.query.ONearItem; import com.yahoo.prelude.query.OrItem; import com.yahoo.prelude.query.PhraseItem; @@ -56,6 +57,7 @@ import com.yahoo.prelude.query.RegExpItem; import com.yahoo.prelude.query.SameElementItem; import com.yahoo.prelude.query.SegmentItem; import com.yahoo.prelude.query.SegmentingRule; +import com.yahoo.prelude.query.StringInItem; import com.yahoo.prelude.query.Substring; import com.yahoo.prelude.query.SubstringItem; import com.yahoo.prelude.query.SuffixItem; @@ -353,10 +355,12 @@ public class YqlParser implements Parser { case CALL -> buildFunctionCall(ast); case LITERAL -> buildLiteral(ast); case NOT -> buildNot(ast); + case IN -> buildIn(ast); default -> throw newUnexpectedArgumentException(ast.getOperator(), ExpressionOperator.AND, ExpressionOperator.CALL, ExpressionOperator.CONTAINS, ExpressionOperator.EQ, ExpressionOperator.GT, ExpressionOperator.GTEQ, + ExpressionOperator.IN, ExpressionOperator.LT, ExpressionOperator.LTEQ, ExpressionOperator.OR); }; @@ -409,6 +413,18 @@ public class YqlParser implements Parser { return fillWeightedSet(ast, args.get(1), new DotProductItem(getIndex(args.get(0)))); } + private Item buildIn(OperatorNode<ExpressionOperator> ast) { + String field = getIndex(ast.getArgument(0)); + boolean stringField = indexFactsSession.getIndex(field).isString(); + Item item = null; + if (stringField) { + item = fillStringIn(ast, ast.getArgument(1), new StringInItem(field)); + } else { + item = fillNumericIn(ast, ast.getArgument(1), new NumericInItem(field)); + } + return item; + } + private ParsedDegree degreesFromArg(OperatorNode<ExpressionOperator> ast, boolean first) { Object arg = null; switch (ast.getOperator()) { @@ -591,6 +607,52 @@ public class YqlParser implements Parser { return leafStyleSettings(ast, out); } + private StringInItem fillStringIn(OperatorNode<ExpressionOperator> ast, + OperatorNode<ExpressionOperator> arg, + StringInItem out) { + assertHasOperator(arg, ExpressionOperator.ARRAY); + List<OperatorNode<ExpressionOperator>> values = arg.getArgument(0); + for (var value : values) { + switch (value.getOperator()) { + case LITERAL -> { + String tokenValue = value.getArgument(0, String.class); + out.addToken(tokenValue); + } + case VARREF -> { + Preconditions.checkState(userQuery != null, "Query properties are not available"); + String varRef = value.getArgument(0, String.class); + ParameterListParser.addStringTokensFromString(userQuery.properties().getString(varRef), out); + } + default -> throw newUnexpectedArgumentException(value.getOperator(), + ExpressionOperator.LITERAL, ExpressionOperator.VARREF); + } + } + return out; + } + + private NumericInItem fillNumericIn(OperatorNode<ExpressionOperator> ast, + OperatorNode<ExpressionOperator> arg, + NumericInItem out) { + assertHasOperator(arg, ExpressionOperator.ARRAY); + List<OperatorNode<ExpressionOperator>> values = arg.getArgument(0); + for (var value : values) { + switch (value.getOperator()) { + case LITERAL -> { + Long tokenValue = value.getArgument(0, Number.class).longValue(); + out.addToken(tokenValue); + } + case VARREF -> { + Preconditions.checkState(userQuery != null, "Query properties are not available"); + String varRef = value.getArgument(0, String.class); + ParameterListParser.addNumericTokensFromString(userQuery.properties().getString(varRef), out); + } + default -> throw newUnexpectedArgumentException(value.getOperator(), + ExpressionOperator.LITERAL, ExpressionOperator.VARREF); + } + } + return out; + } + private static class PrefixExpander extends IndexNameExpander { private final String prefix; public PrefixExpander(String prefix) { diff --git a/container-search/src/test/java/com/yahoo/prelude/query/test/ItemEncodingTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/test/ItemEncodingTestCase.java index af7cf2e356f..ffbde8c87e0 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/test/ItemEncodingTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/test/ItemEncodingTestCase.java @@ -5,16 +5,21 @@ import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.EquivItem; import com.yahoo.prelude.query.MarkerWordItem; import com.yahoo.prelude.query.NearItem; +import com.yahoo.prelude.query.NumericInItem; import com.yahoo.prelude.query.ONearItem; import com.yahoo.prelude.query.PureWeightedInteger; import com.yahoo.prelude.query.PureWeightedString; +import com.yahoo.prelude.query.StringInItem; import com.yahoo.prelude.query.WeakAndItem; import com.yahoo.prelude.query.WordItem; import org.junit.jupiter.api.Test; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * Item encoding tests @@ -23,10 +28,18 @@ import static org.junit.jupiter.api.Assertions.assertEquals; */ public class ItemEncodingTestCase { - private void assertType(ByteBuffer buffer, int etype, int features) { - byte type = buffer.get(); - assertEquals(etype, type & 0x1f, "Code"); - assertEquals(features, (type & 0xe0) >> 5, "Features"); + private void assertType(ByteBuffer buffer, int etype, int efeatures) { + byte CODE_MASK = 0b00011111; + byte features_and_type = buffer.get(); + int features = (features_and_type & 0xe0) >> 5; + int type = features_and_type & CODE_MASK; + if (type == CODE_MASK) { + byte type_extension = buffer.get(); + assertTrue(type_extension >= 0); + type += type_extension; + } + assertEquals(etype, type, "Code"); + assertEquals(efeatures, features, "Features"); } private void assertWeight(ByteBuffer buffer, int weight) { @@ -325,6 +338,44 @@ public class ItemEncodingTestCase { ; } + @Test + void testStringInItem() { + var a = new StringInItem("default"); + a.addToken("foo"); + ByteBuffer buffer = ByteBuffer.allocate(128); + int count = a.encode(buffer); + buffer.flip(); + // 2 bytes type, 1 byte item count, 1 byte string len, 7 bytes string content + // 1 byte string len, 3 bytes string content + assertEquals(15, buffer.remaining(), "Serialization size"); + assertType(buffer, 31, 0); + assertEquals(1, buffer.get()); // 1 item + assertString(buffer, "default"); + assertString(buffer, "foo"); + } + + @Test + void testNumericInItem() { + var a = new NumericInItem("default"); + a.addToken(42); + a.addToken(97000000000L); + ByteBuffer buffer = ByteBuffer.allocate(128); + int count = a.encode(buffer); + buffer.flip(); + // 2 bytes type, 1 byte item count, 1 byte string len, 7 bytes string content + // 16 bytes (2 64-bit integer value) + assertEquals(27, buffer.remaining(), "Serialization size"); + assertType(buffer, 32, 0); + assertEquals(2, buffer.get()); // 2 items + assertString(buffer, "default"); + var array = new ArrayList<Long>(); + array.add(buffer.getLong()); + array.add(buffer.getLong()); + Collections.sort(array); + assertEquals(42, array.get(0)); + assertEquals(97000000000L, array.get(1)); + } + private void assertString(ByteBuffer buffer, String word) { assertEquals(word.length(), buffer.get(), "Word length"); for (int i=0; i<word.length(); i++) { diff --git a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java index f3612c3f303..6aac2faa4e9 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java @@ -1,6 +1,10 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.yql; +import com.yahoo.prelude.Index; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.IndexModel; +import com.yahoo.prelude.SearchDefinition; import com.yahoo.prelude.query.SameElementItem; import com.yahoo.search.Query; import com.yahoo.search.grouping.Continuation; @@ -43,6 +47,15 @@ public class VespaSerializerTestCase { parser = null; } + static private IndexFacts createIndexFactsForInTest() { + SearchDefinition sd = new SearchDefinition("sourceA"); + sd.addIndex(new Index("field")); + Index stringIndex = new Index("string"); + stringIndex.setString(true); + sd.addIndex(stringIndex); + return new IndexFacts(new IndexModel(sd)); + } + @Test void requireThatGroupingRequestsAreSerialized() { Query query = new Query(); @@ -451,4 +464,11 @@ public class VespaSerializerTestCase { parseAndConfirm("foo contains ({maxEditDistance:3,prefixLength:5}fuzzy(\"a\"))"); } + @Test + void testIn() { + parser = new YqlParser(new ParserEnvironment().setIndexFacts(createIndexFactsForInTest())); + parseAndConfirm("field in (2, 3)"); + parseAndConfirm("field in (9000000000L, 12000000000L)"); + parseAndConfirm("string in (\"a\", \"b\")"); + } } diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index c1194cfc84b..bd29e2afd53 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -16,12 +16,14 @@ import com.yahoo.prelude.query.IndexedItem; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.MarkerWordItem; import com.yahoo.prelude.query.NearestNeighborItem; +import com.yahoo.prelude.query.NumericInItem; import com.yahoo.prelude.query.PhraseItem; import com.yahoo.prelude.query.PhraseSegmentItem; import com.yahoo.prelude.query.PrefixItem; import com.yahoo.prelude.query.QueryCanonicalizer; import com.yahoo.prelude.query.RegExpItem; import com.yahoo.prelude.query.SegmentingRule; +import com.yahoo.prelude.query.StringInItem; import com.yahoo.prelude.query.Substring; import com.yahoo.prelude.query.SubstringItem; import com.yahoo.prelude.query.SuffixItem; @@ -46,6 +48,8 @@ import com.yahoo.search.query.parser.Parsable; import com.yahoo.search.query.parser.ParserEnvironment; import com.yahoo.search.searchchain.Execution; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.util.ArrayList; @@ -63,7 +67,36 @@ import static org.junit.jupiter.api.Assertions.*; */ public class YqlParserTestCase { - private final YqlParser parser = new YqlParser(new ParserEnvironment()); + private YqlParser parser; + + @BeforeEach + public void setUp() throws Exception { + ParserEnvironment env = new ParserEnvironment(); + parser = new YqlParser(env); + } + + @AfterEach + public void tearDown() throws Exception { + parser = null; + } + + static private IndexFacts createIndexFactsForInTest() { + SearchDefinition sd = new SearchDefinition("default"); + sd.addIndex(new Index("field")); + Index stringIndex = new Index("string"); + stringIndex.setString(true); + sd.addIndex(stringIndex); + return new IndexFacts(new IndexModel(sd)); + } + + private static Query createUserQuery() { + var builder = new Query.Builder(); + var query = builder.build(); + // Following two properties are used by testing of IN operator (cf. testIn) + query.properties().set("foostring", "'this', \"might\", work "); + query.properties().set("foonumeric", "26, 25, -11, 24 "); + return query; + } @Test void failsGracefullyOnMissingQuoteEscapingAndSubsequentUnicodeCharacter() { @@ -1141,6 +1174,51 @@ public class YqlParserTestCase { parse("select * from sources * where (default contains ({stem: false}\"m\") AND default contains ({origin: {original: \"m\'s\", offset: 0, length: 3}, andSegmenting: true}phrase(\"m\", \"s\"))) timeout 472"); } + @Test + void testIn() { + parser = new YqlParser(new ParserEnvironment().setIndexFacts(createIndexFactsForInTest())); + parser.setUserQuery(createUserQuery()); + var query = parse("select * from sources * where field in (42, 22L, -7, @foonumeric)"); + assertNumericInItem("field", new long[]{-11, -7, 22, 24, 25, 26, 42}, query); + parser.setUserQuery(createUserQuery()); + query = parse("select * from sources * where string in ('a','b', @foostring)"); + assertStringInItem("string", new String[]{"a","b","might","this", "work"}, query); + parser.setUserQuery(createUserQuery()); + query = parse("select * from sources * where field in (29.9, -7.4)"); + assertNumericInItem("field", new long[]{-7, 29}, query); + parser.setUserQuery(null); + assertParseFail("select * from sources * where string in ('a', 25L)", + new ClassCastException("Cannot cast java.lang.Long to java.lang.String")); + assertParseFail("select * from sources * where field in ('a', 25L)", + new ClassCastException("Cannot cast java.lang.String to java.lang.Number")); + assertParseFail("select * from sources * where nofield in ('a', 25L)", + new IllegalArgumentException("Field 'nofield' does not exist.")); + assertParseFail("select * from sources * where field not in (25)", + new IllegalArgumentException("Expected AND, CALL, CONTAINS, EQ, GT, GTEQ, IN, LT, LTEQ or OR, got NOT_IN.")); + } + + private static void assertNumericInItem(String field, long[] values, QueryTree query) { + var exp = buildNumericInItem(field, values); + assertEquals(exp, query.getRoot()); + } + + private static void assertStringInItem(String field, String[] values, QueryTree query) { + var exp = buildStringInItem(field, values); + assertEquals(exp, query.getRoot()); + } + + private static NumericInItem buildNumericInItem(String field, long[] values) { + var item = new NumericInItem(field); + for (var value : values) item.addToken(value); + return item; + } + + private static StringInItem buildStringInItem(String field, String[] values) { + var item = new StringInItem(field); + for (var value : values) item.addToken(value); + return item; + } + private void assertUrlQuery(String field, Query query, boolean startAnchor, boolean endAnchor, boolean endAnchorIsDefault) { boolean startAnchorIsDefault = false; // Always diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index 3d77104dd5c..c04b733619e 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -67,7 +67,7 @@ <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> <athenz.vespa.version>1.11.46</athenz.vespa.version> - <aws-sdk.vespa.version>1.12.580</aws-sdk.vespa.version> + <aws-sdk.vespa.version>1.12.590</aws-sdk.vespa.version> <!-- Athenz END --> <!-- WARNING: If you change curator version, you also need to update @@ -77,7 +77,7 @@ xargs perl -pi -e 's/major = [0-9]+, minor = [0-9]+, micro = [0-9]+/major = 5, minor = 3, micro = 0/g' --> <bouncycastle.vespa.version>1.76</bouncycastle.vespa.version> - <byte-buddy.vespa.version>1.14.9</byte-buddy.vespa.version> + <byte-buddy.vespa.version>1.14.10</byte-buddy.vespa.version> <checker-qual.vespa.version>3.38.0</checker-qual.vespa.version> <commons-beanutils.vespa.version>1.9.4</commons-beanutils.vespa.version> <commons-codec.vespa.version>1.16.0</commons-codec.vespa.version> @@ -88,7 +88,7 @@ <commons-io.vespa.version>2.15.0</commons-io.vespa.version> <commons-lang3.vespa.version>3.13.0</commons-lang3.vespa.version> <commons.math3.vespa.version>3.6.1</commons.math3.vespa.version> - <commons-compress.vespa.version>1.24.0</commons-compress.vespa.version> + <commons-compress.vespa.version>1.25.0</commons-compress.vespa.version> <commons-cli.vespa.version>1.6.0</commons-cli.vespa.version> <curator.vespa.version>5.5.0</curator.vespa.version> <dropwizard.metrics.vespa.version>4.2.22</dropwizard.metrics.vespa.version> @@ -126,9 +126,9 @@ <org.json.vespa.version>20231013</org.json.vespa.version> <org.lz4.vespa.version>1.8.0</org.lz4.vespa.version> <prometheus.client.vespa.version>0.16.0</prometheus.client.vespa.version> - <protobuf.vespa.version>3.25.0</protobuf.vespa.version> + <protobuf.vespa.version>3.25.1</protobuf.vespa.version> <questdb.vespa.version>7.3.4</questdb.vespa.version> - <spifly.vespa.version>1.3.6</spifly.vespa.version> + <spifly.vespa.version>1.3.7</spifly.vespa.version> <snappy.vespa.version>1.1.10.5</snappy.vespa.version> <surefire.vespa.version>3.2.2</surefire.vespa.version> <velocity.vespa.version>2.3</velocity.vespa.version> @@ -167,7 +167,7 @@ <maven-site-plugin.vespa.version>3.12.1</maven-site-plugin.vespa.version> <maven-source-plugin.vespa.version>3.3.0</maven-source-plugin.vespa.version> <properties-maven-plugin.vespa.version>1.2.1</properties-maven-plugin.vespa.version> - <versions-maven-plugin.vespa.version>2.16.1</versions-maven-plugin.vespa.version> + <versions-maven-plugin.vespa.version>2.16.2</versions-maven-plugin.vespa.version> </properties> <profiles> diff --git a/document/src/main/java/com/yahoo/document/annotation/Annotation.java b/document/src/main/java/com/yahoo/document/annotation/Annotation.java index 3d9300550ff..237ca6db58b 100644 --- a/document/src/main/java/com/yahoo/document/annotation/Annotation.java +++ b/document/src/main/java/com/yahoo/document/annotation/Annotation.java @@ -129,7 +129,7 @@ public class Annotation implements Comparable<Annotation> { } /** - * WARNING! Should only be used by deserializers! Sets the span node that this annotation points to. + * WARNING! Should only be used by deserializers! Sets the span node that this annotation points to. * * @param spanNode the span node that this annotation shall point to. */ @@ -221,10 +221,9 @@ public class Annotation implements Comparable<Annotation> { @Override public String toString() { - String retval = "annotation of type " + type; - retval += ((value == null) ? " (no value)" : " (with value)"); - retval += ((spanNode == null) ? " (no span)" : (" with span "+spanNode)); - return retval; + return type + " annotation " + + ((value == null) ? " (no value)" : " (with value)") + + ((spanNode == null) ? " (no span)" : (" with span "+spanNode)); } diff --git a/document/src/main/java/com/yahoo/document/annotation/AnnotationContainer.java b/document/src/main/java/com/yahoo/document/annotation/AnnotationContainer.java index 6e2b986a478..ac2e6aefa1b 100644 --- a/document/src/main/java/com/yahoo/document/annotation/AnnotationContainer.java +++ b/document/src/main/java/com/yahoo/document/annotation/AnnotationContainer.java @@ -23,11 +23,7 @@ abstract class AnnotationContainer { */ abstract void annotate(Annotation annotation); - /** - * Returns a mutable collection of annotations. - * - * @return a mutable collection of annotations. - */ + /** Returns a mutable collection of the annotations in this. */ abstract Collection<Annotation> annotations(); /** diff --git a/document/src/main/java/com/yahoo/document/annotation/AnnotationType2AnnotationContainer.java b/document/src/main/java/com/yahoo/document/annotation/AnnotationType2AnnotationContainer.java index d8709baa3a1..01b8f990bb4 100644 --- a/document/src/main/java/com/yahoo/document/annotation/AnnotationType2AnnotationContainer.java +++ b/document/src/main/java/com/yahoo/document/annotation/AnnotationType2AnnotationContainer.java @@ -16,6 +16,7 @@ import java.util.NoSuchElementException; */ // TODO: Should this be removed? public class AnnotationType2AnnotationContainer extends IteratingAnnotationContainer { + private final Multimap<AnnotationType, Annotation> annotationType2Annotation = Multimaps.newMultimap(new IdentityHashMap<>(), ArrayList::new); @Override @@ -31,7 +32,6 @@ public class AnnotationType2AnnotationContainer extends IteratingAnnotationConta } @Override - @SuppressWarnings("unchecked") Collection<Annotation> annotations() { return annotationType2Annotation.values(); } @@ -56,12 +56,12 @@ public class AnnotationType2AnnotationContainer extends IteratingAnnotationConta } private class NonRecursiveIterator implements Iterator<Annotation> { + private final IdentityHashMap<SpanNode, SpanNode> nodes; private final Iterator<Annotation> annotationIt; private Annotation next = null; private boolean nextCalled; - @SuppressWarnings("unchecked") public NonRecursiveIterator(IdentityHashMap<SpanNode, SpanNode> nodes) { this.nodes = nodes; this.annotationIt = annotationType2Annotation.values().iterator(); @@ -106,4 +106,5 @@ public class AnnotationType2AnnotationContainer extends IteratingAnnotationConta nextCalled = false; } } + } diff --git a/document/src/main/java/com/yahoo/document/annotation/IteratingAnnotationContainer.java b/document/src/main/java/com/yahoo/document/annotation/IteratingAnnotationContainer.java index 54af80e6e43..c7c3545fc09 100644 --- a/document/src/main/java/com/yahoo/document/annotation/IteratingAnnotationContainer.java +++ b/document/src/main/java/com/yahoo/document/annotation/IteratingAnnotationContainer.java @@ -5,30 +5,29 @@ import java.util.IdentityHashMap; import java.util.Iterator; /** - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ abstract class IteratingAnnotationContainer extends AnnotationContainer { @Override Iterator<Annotation> iterator(SpanNode node) { - IdentityHashMap<SpanNode, SpanNode> nodes = new IdentityHashMap<SpanNode, SpanNode>(); + IdentityHashMap<SpanNode, SpanNode> nodes = new IdentityHashMap<>(); nodes.put(node, node); return iterator(nodes); } @Override Iterator<Annotation> iteratorRecursive(SpanNode node) { - IdentityHashMap<SpanNode, SpanNode> nodes = new IdentityHashMap<SpanNode, SpanNode>(); + IdentityHashMap<SpanNode, SpanNode> nodes = new IdentityHashMap<>(); nodes.put(node, node); - { - Iterator<SpanNode> childrenIt = node.childIteratorRecursive(); - while (childrenIt.hasNext()) { - SpanNode child = childrenIt.next(); - nodes.put(child, child); - } + Iterator<SpanNode> childrenIt = node.childIteratorRecursive(); + while (childrenIt.hasNext()) { + SpanNode child = childrenIt.next(); + nodes.put(child, child); } return iterator(nodes); } abstract Iterator<Annotation> iterator(IdentityHashMap<SpanNode, SpanNode> nodes); + } diff --git a/document/src/main/java/com/yahoo/document/annotation/ListAnnotationContainer.java b/document/src/main/java/com/yahoo/document/annotation/ListAnnotationContainer.java index c2c22558a32..b8d8cd692d8 100644 --- a/document/src/main/java/com/yahoo/document/annotation/ListAnnotationContainer.java +++ b/document/src/main/java/com/yahoo/document/annotation/ListAnnotationContainer.java @@ -13,6 +13,7 @@ import java.util.NoSuchElementException; * @author Einar M R Rosenvinge */ public class ListAnnotationContainer extends IteratingAnnotationContainer { + private final List<Annotation> annotations = new LinkedList<>(); @Override @@ -38,9 +39,8 @@ public class ListAnnotationContainer extends IteratingAnnotationContainer { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof ListAnnotationContainer)) return false; - ListAnnotationContainer that = (ListAnnotationContainer) o; - if (!annotations.equals(that.annotations)) return false; + if (!(o instanceof ListAnnotationContainer other)) return false; + if (!annotations.equals(other.annotations)) return false; return true; } @@ -50,8 +50,9 @@ public class ListAnnotationContainer extends IteratingAnnotationContainer { } private class AnnotationIterator implements Iterator<Annotation> { - private IdentityHashMap<SpanNode, SpanNode> nodes; - private PeekableListIterator<Annotation> base; + + private final IdentityHashMap<SpanNode, SpanNode> nodes; + private final PeekableListIterator<Annotation> base; private boolean nextCalled = false; AnnotationIterator(ListIterator<Annotation> baseIt, IdentityHashMap<SpanNode, SpanNode> nodes) { @@ -91,4 +92,5 @@ public class ListAnnotationContainer extends IteratingAnnotationContainer { nextCalled = false; } } + } diff --git a/document/src/main/java/com/yahoo/document/annotation/SpanList.java b/document/src/main/java/com/yahoo/document/annotation/SpanList.java index 9530689c3f1..04bdf5c892e 100644 --- a/document/src/main/java/com/yahoo/document/annotation/SpanList.java +++ b/document/src/main/java/com/yahoo/document/annotation/SpanList.java @@ -11,13 +11,14 @@ import java.util.ListIterator; /** * A node in a Span tree that can have child nodes. * - * @author <a href="mailto:einarmr@yahoo-inc.com">Einar M R Rosenvinge</a> + * @author Einar M R Rosenvinge */ public class SpanList extends SpanNode { + public static final byte ID = 2; private final List<SpanNode> children; - private int cachedFrom = Integer.MIN_VALUE; //triggers calculateFrom() - private int cachedTo = Integer.MIN_VALUE; //triggers calculateTo() + private int cachedFrom = Integer.MIN_VALUE; // triggers calculateFrom() + private int cachedTo = Integer.MIN_VALUE; // triggers calculateTo() /** Creates a new SpanList. */ public SpanList() { @@ -39,7 +40,7 @@ public class SpanList extends SpanNode { * @param other the SpanList to copy. */ public SpanList(SpanList other) { - this.children = new LinkedList<SpanNode>(); + this.children = new LinkedList<>(); for (SpanNode otherNode : other.children) { if (otherNode instanceof Span) { children.add(new Span((Span) otherNode)); @@ -86,15 +87,15 @@ public class SpanList extends SpanNode { /** Create a span, add it to this list and return it */ public Span span(int from, int length) { - Span span=new Span(from,length); + Span span = new Span(from, length); add(span); return span; } void setInvalid() { - //invalidate ourselves: + // invalidate ourselves: super.setInvalid(); - //invalidate all our children: + // invalidate all our children: for (SpanNode node : children()) { node.setInvalid(); } @@ -213,20 +214,12 @@ public class SpanList extends SpanNode { return this; } - /** - * Returns a <strong>modifiable</strong> list of the immediate children of this SpanList. - * - * @return a <strong>modifiable</strong> list of the immediate children of this SpanList. - */ + /** Returns a modifiable list of the immediate children of this SpanList. */ protected List<SpanNode> children() { return children; } - /** - * Returns the number of children this SpanList holds. - * - * @return the number of children this SpanList holds. - */ + /** Returns the number of children this SpanList holds. */ public int numChildren() { return children().size(); } @@ -394,11 +387,9 @@ public class SpanList extends SpanNode { @Override public boolean equals(Object o) { if (this == o) return true; - if (!(o instanceof SpanList)) return false; + if (!(o instanceof SpanList spanList)) return false; if (!super.equals(o)) return false; - SpanList spanList = (SpanList) o; - if (children() != null ? !children().equals(spanList.children()) : spanList.children() != null) return false; return true; @@ -415,4 +406,5 @@ public class SpanList extends SpanNode { public String toString() { return "SpanList with " + children().size() + " children"; } + } diff --git a/document/src/main/java/com/yahoo/document/annotation/SpanNode2AnnotationContainer.java b/document/src/main/java/com/yahoo/document/annotation/SpanNode2AnnotationContainer.java index a66639b3bfd..f563fd0d6cd 100644 --- a/document/src/main/java/com/yahoo/document/annotation/SpanNode2AnnotationContainer.java +++ b/document/src/main/java/com/yahoo/document/annotation/SpanNode2AnnotationContainer.java @@ -18,6 +18,7 @@ import java.util.List; * @author Einar M R Rosenvinge */ class SpanNode2AnnotationContainer extends AnnotationContainer { + private final Multimap<SpanNode, Annotation> spanNode2Annotation = Multimaps.newMultimap(new IdentityHashMap<>(), ArrayList::new); @Override diff --git a/document/src/main/java/com/yahoo/document/annotation/SpanTree.java b/document/src/main/java/com/yahoo/document/annotation/SpanTree.java index f785cf3b3ec..2faf17ce428 100644 --- a/document/src/main/java/com/yahoo/document/annotation/SpanTree.java +++ b/document/src/main/java/com/yahoo/document/annotation/SpanTree.java @@ -19,6 +19,7 @@ import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; /** * A SpanTree holds a root node of a tree of SpanNodes, and a List of Annotations pointing to these nodes @@ -431,7 +432,7 @@ public class SpanTree implements Iterable<Annotation>, SpanNodeParent, Comparabl } /** - * Adds an Annotation to the internal list of annotations for this SpanTree. Use this when + * Adds an Annotation to the internal list of annotations for this SpanTree. Use this when * adding an Annotation that shall annotate a SpanNode. Upon return, Annotation.getSpanNode() * returns the given node. * @@ -446,7 +447,7 @@ public class SpanTree implements Iterable<Annotation>, SpanNodeParent, Comparabl } /** - * Adds an Annotation to the internal list of annotations for this SpanTree. Use this when + * Adds an Annotation to the internal list of annotations for this SpanTree. Use this when * adding an Annotation that shall annotate a SpanNode. Upon return, Annotation.getSpanNode() * returns the given node. This one is unchecked and assumes that the SpanNode is valid and has * already been attached to the Annotation. @@ -538,10 +539,7 @@ public class SpanTree implements Iterable<Annotation>, SpanNodeParent, Comparabl } } - /** - * Returns an Iterator over all annotations in this tree. Note that the iteration order is non-deterministic. - * @return an Iterator over all annotations in this tree. - */ + /** Returns an Iterator over all annotations in this tree. Note that the iteration order is non-deterministic. */ public Iterator<Annotation> iterator() { return annotations.annotations().iterator(); } @@ -641,7 +639,9 @@ public class SpanTree implements Iterable<Annotation>, SpanNodeParent, Comparabl @Override public String toString() { - return "SpanTree '" + name + "'"; + return "SpanTree '" + name + "' with root: " + root + + ( annotations.annotations().size() > 5 ? "" : + ", annotations: " + annotations.annotations().stream().map(Annotation::toString).collect(Collectors.joining(", "))); } @Override diff --git a/document/src/main/java/com/yahoo/document/datatypes/Struct.java b/document/src/main/java/com/yahoo/document/datatypes/Struct.java index 0b1bbf5d3ca..bb54b41069b 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/Struct.java +++ b/document/src/main/java/com/yahoo/document/datatypes/Struct.java @@ -218,8 +218,7 @@ public class Struct extends StructuredFieldValue { StringBuilder retVal = new StringBuilder(); retVal.append("Struct (").append(getDataType()).append("): "); int [] increasing = getInOrder(); - for (int i = 0; i < increasing.length; i++) { - int id = increasing[i]; + for (int id : increasing) { retVal.append(getDataType().getField(id)).append("=").append(values.get(id)).append(", "); } return retVal.toString(); diff --git a/eval/src/apps/tensor_conformance/generate.cpp b/eval/src/apps/tensor_conformance/generate.cpp index 0ebfb772451..1c68a94a852 100644 --- a/eval/src/apps/tensor_conformance/generate.cpp +++ b/eval/src/apps/tensor_conformance/generate.cpp @@ -236,6 +236,28 @@ void generate_map(TestBuilder &dst) { //----------------------------------------------------------------------------- +void generate_map_subspaces(TestBuilder &dst) { + auto my_seq = Seq({-128, -43, 85, 127}); + auto scalar = GenSpec(7.0); + auto sparse = GenSpec().from_desc("x8_1").seq(my_seq); + auto mixed = GenSpec().from_desc("x4_1y4").seq(my_seq); + auto dense = GenSpec().from_desc("y4").seq(my_seq); + vespalib::string map_a("map_subspaces(a,f(a)(a*3+2))"); + vespalib::string unpack_a("map_subspaces(a,f(a)(tensor<int8>(y[8])(bit(a,7-y%8))))"); + vespalib::string unpack_y4("map_subspaces(a,f(a)(tensor<int8>(y[32])(bit(a{y:(y/8)},7-y%8))))"); + vespalib::string pack_y4("map_subspaces(a,f(a)(a{y:0}+a{y:1}-a{y:2}+a{y:3}))"); + generate(map_a, scalar, dst); + generate(map_a, sparse, dst); + generate(unpack_a, scalar, dst); + generate(unpack_a, sparse, dst); + generate(unpack_y4, mixed, dst); + generate(unpack_y4, dense, dst); + generate(pack_y4, mixed, dst); + generate(pack_y4, dense, dst); +} + +//----------------------------------------------------------------------------- + void generate_join_expr(const vespalib::string &expr, const Sequence &seq, TestBuilder &dst) { for (const auto &layouts: join_layouts) { GenSpec a = GenSpec::from_desc(layouts.first).seq(seq); @@ -560,6 +582,7 @@ Generator::generate(TestBuilder &dst) generate_inject(dst); generate_reduce(dst); generate_map(dst); + generate_map_subspaces(dst); generate_join(dst); generate_merge(dst); generate_concat(dst); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java index bcfeddacff3..080db640dfe 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceCompressor.java @@ -6,7 +6,6 @@ import com.yahoo.compress.ZstdOutputStream; import net.jpountz.lz4.LZ4BlockInputStream; import net.jpountz.lz4.LZ4BlockOutputStream; import org.apache.commons.compress.archivers.ArchiveEntry; -import org.apache.commons.compress.archivers.ArchiveInputStream; import org.apache.commons.compress.archivers.ArchiveOutputStream; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; @@ -18,7 +17,6 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.List; import java.util.Objects; import java.util.logging.Level; @@ -52,24 +50,23 @@ public class FileReferenceCompressor { } public File compress(File directory, File outputFile) throws IOException { - return compress(directory, - Files.find(Paths.get(directory.getAbsolutePath()), - recurseDepth, - (p, basicFileAttributes) -> basicFileAttributes.isRegularFile()) - .map(Path::toFile).toList(), - outputFile); + try (var paths = Files.find(Path.of(directory.getAbsolutePath()), recurseDepth, + (p, basicFileAttributes) -> basicFileAttributes.isRegularFile())) + { + return compress(directory, paths.map(Path::toFile).toList(), outputFile); + } } public void decompress(File inputFile, File outputDir) throws IOException { log.log(Level.FINEST, () -> "Decompressing '" + inputFile + "' into '" + outputDir + "'"); - try (ArchiveInputStream ais = new TarArchiveInputStream(decompressedInputStream(inputFile))) { + try (TarArchiveInputStream ais = new TarArchiveInputStream(decompressedInputStream(inputFile))) { decompress(ais, outputDir); } catch (IllegalArgumentException e) { throw new RuntimeException("Unable to decompress '" + inputFile.getAbsolutePath() + "': " + e.getMessage()); } } - private static void decompress(ArchiveInputStream archiveInputStream, File outputFile) throws IOException { + private static void decompress(TarArchiveInputStream archiveInputStream, File outputFile) throws IOException { int entries = 0; ArchiveEntry entry; while ((entry = archiveInputStream.getNextEntry()) != null) { @@ -99,7 +96,7 @@ public class FileReferenceCompressor { } } - private static void createArchiveFile(ArchiveOutputStream archiveOutputStream, File baseDir, List<File> inputFiles) throws IOException { + private static <T extends ArchiveEntry> void createArchiveFile(ArchiveOutputStream<T> archiveOutputStream, File baseDir, List<File> inputFiles) throws IOException { inputFiles.forEach(file -> { try { writeFileToTar(archiveOutputStream, baseDir, file); @@ -110,7 +107,7 @@ public class FileReferenceCompressor { archiveOutputStream.close(); } - private static void writeFileToTar(ArchiveOutputStream taos, File baseDir, File file) throws IOException { + private static <T extends ArchiveEntry> void writeFileToTar(ArchiveOutputStream<T> taos, File baseDir, File file) throws IOException { taos.putArchiveEntry(taos.createArchiveEntry(file, baseDir.toPath().relativize(file.toPath()).toString())); try (FileInputStream inputStream = new FileInputStream(file)) { inputStream.transferTo(taos); @@ -119,35 +116,33 @@ public class FileReferenceCompressor { } private OutputStream compressedOutputStream(File outputFile) throws IOException { - switch (type) { - case compressed: + return switch (type) { + case compressed -> { log.log(Level.FINEST, () -> "Compressing with compression type " + compressionType); - return switch (compressionType) { + yield switch (compressionType) { case gzip -> new GZIPOutputStream(new FileOutputStream(outputFile)); case lz4 -> new LZ4BlockOutputStream(new FileOutputStream(outputFile)); case zstd -> new ZstdOutputStream(new FileOutputStream(outputFile)); }; - case file: - return new FileOutputStream(outputFile); - default: - throw new RuntimeException("Unknown file reference type " + type); - } + } + case file -> new FileOutputStream(outputFile); + default -> throw new RuntimeException("Unknown file reference type " + type); + }; } private InputStream decompressedInputStream(File inputFile) throws IOException { - switch (type) { - case compressed: + return switch (type) { + case compressed -> { log.log(Level.FINEST, () -> "Decompressing with compression type " + compressionType); - return switch (compressionType) { + yield switch (compressionType) { case gzip -> new GZIPInputStream(new FileInputStream(inputFile)); case lz4 -> new LZ4BlockInputStream(new FileInputStream(inputFile)); case zstd -> new ZstdInputStream(new FileInputStream(inputFile)); }; - case file: - return new FileInputStream(inputFile); - default: - throw new RuntimeException("Unknown file reference type " + type); - } + } + case file -> new FileInputStream(inputFile); + default -> throw new RuntimeException("Unknown file reference type " + type); + }; } } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 491b7db3c13..64218dbc800 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -302,11 +302,6 @@ public class Flags { "Takes effect at redeployment", INSTANCE_ID); - public static final UnboundBooleanFlag ENABLE_CROWDSTRIKE = defineFeatureFlag( - "enable-crowdstrike", true, List.of("andreer"), "2023-04-13", "2023-11-14", - "Whether to enable CrowdStrike.", "Takes effect on next host admin tick", - HOSTNAME); - public static final UnboundBooleanFlag ENABLE_THE_ONE_THAT_SHOULD_NOT_BE_NAMED = defineFeatureFlag( "enable-the-one-that-should-not-be-named", false, List.of("hmusum"), "2023-05-08", "2023-12-01", "Whether to enable the one program that should not be named", @@ -318,25 +313,13 @@ public class Flags { "Where specified, CNAME records are used instead of the default ALIAS records, which have a default 60s TTL.", "Takes effect at redeployment from controller"); - public static final UnboundBooleanFlag ENABLE_NESTED_MULTIVALUE_GROUPING = defineFeatureFlag( - "enable-nested-multivalue-grouping", true, - List.of("baldersheim"), "2023-06-29", "2023-12-31", - "Should we enable proper nested multivalue grouping", + public static final UnboundBooleanFlag ALWAYS_MARK_PHRASE_EXPENSIVE = defineFeatureFlag( + "always-mark-phrase-expensive", false, + List.of("baldersheim"), "2023-11-20", "2023-12-31", + "If true all phrases will be marked expensive, independent of parents", "Takes effect at redeployment", INSTANCE_ID); - public static final UnboundBooleanFlag EXCLUSIVE_PROVISIONING = defineFeatureFlag( - "exclusive-provisioning", true, - List.of("hakonhall"), "2023-10-12", "2023-12-20", - "Unused, remove once Vespa >=8.257 has rolled out everywhere", - "no-op"); - - public static final UnboundBooleanFlag MAKE_EXCLUSIVE = defineFeatureFlag( - "make-exclusive", true, - List.of("hakonhall"), "2023-10-20", "2023-12-20", - "Unused, remove once Vespa >=8.257 has rolled out everywhere", - "no-op"); - public static final UnboundBooleanFlag WRITE_CONFIG_SERVER_SESSION_DATA_AS_ONE_BLOB = defineFeatureFlag( "write-config-server-session-data-as-blob", false, List.of("hmusum"), "2023-07-19", "2024-01-01", @@ -351,21 +334,21 @@ public class Flags { public static final UnboundBooleanFlag MORE_WIREGUARD = defineFeatureFlag( "more-wireguard", false, - List.of("andreer"), "2023-08-21", "2023-11-14", + List.of("andreer"), "2023-08-21", "2023-12-14", "Use wireguard in INternal enCLAVES", "Takes effect on next host-admin run", HOSTNAME, CLOUD_ACCOUNT); public static final UnboundBooleanFlag IPV6_AWS_TARGET_GROUPS = defineFeatureFlag( "ipv6-aws-target-groups", false, - List.of("andreer"), "2023-08-28", "2023-11-14", + List.of("andreer"), "2023-08-28", "2023-12-14", "Always use IPv6 target groups for load balancers in aws", "Takes effect on next load-balancer provisioning", HOSTNAME, CLOUD_ACCOUNT); public static final UnboundBooleanFlag PROVISION_IPV6_ONLY_AWS = defineFeatureFlag( "provision-ipv6-only", false, - List.of("andreer"), "2023-08-28", "2023-11-14", + List.of("andreer"), "2023-08-28", "2023-12-14", "Provision without private IPv4 addresses in INternal enCLAVES in AWS", "Takes effect on next host provisioning / run of host-admin", HOSTNAME, CLOUD_ACCOUNT); @@ -421,6 +404,16 @@ public class Flags { "Takes effect at redeployment", INSTANCE_ID); + public static final UnboundBooleanFlag USE_PER_DOCUMENT_THROTTLED_DELETE_BUCKET = defineFeatureFlag( + "use-per-document-throttled-delete-bucket", false, + List.of("vekterli"), "2023-11-13", "2024-03-01", + "If set, DeleteBucket operations are internally expanded to an individually persistence-" + + "throttled remove per document stored in the bucket. This makes the cost model of " + + "executing a DeleteBucket symmetrical with feeding the documents to the bucket in the " + + "first place.", + "Takes effect at redeployment", + INSTANCE_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/fnet/src/vespa/fnet/databuffer.h b/fnet/src/vespa/fnet/databuffer.h index c6e1cf2a4f0..3fce600985d 100644 --- a/fnet/src/vespa/fnet/databuffer.h +++ b/fnet/src/vespa/fnet/databuffer.h @@ -319,56 +319,6 @@ public: } /** - * Will write a positive number in compressed form. - * @param number to write - **/ - void writeCompressedPositive(uint64_t n) { - _freept += vespalib::compress::Integer::compressPositive(n, _freept); - } - - /** - * Will write a number in compressed form. - * @param number to write - **/ - void writeCompressed(int64_t n) { - _freept += vespalib::compress::Integer::compress(n, _freept); - } - - /** - * Will read a compressed positive integer. - * @return uncompressed number - **/ - uint64_t readCompressedPositiveInteger() { - uint64_t n; - _datapt += vespalib::compress::Integer::decompressPositive(n, _datapt); - return n; - } - - /** - * Will read a compressed integer. - * @return uncompressed number - **/ - int64_t readCompressedInteger() { - int64_t n; - _datapt += vespalib::compress::Integer::decompress(n, _datapt); - return n; - } - - /** - * @return Number of bytes the compressed positiv number will occupy. - **/ - static size_t getCompressedPositiveLength(uint64_t n) { - return vespalib::compress::Integer::compressedPositiveLength(n); - } - - /** - * @return Number of bytes the compressed positiv number will occupy. - **/ - static size_t getCompressedLength(int64_t n) { - return vespalib::compress::Integer::compressedLength(n); - } - - /** * Read an 8-bit unsigned integer from this buffer. * * @return the integer that has been read. diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java index 26058eeb8f3..fdfadf65400 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/NGramExpression.java @@ -15,6 +15,8 @@ import com.yahoo.vespa.indexinglanguage.linguistics.LinguisticsAnnotator; import java.util.Iterator; +import static com.yahoo.language.LinguisticsCase.toLowerCase; + /** * A filter which splits incoming text into n-grams. * @@ -68,8 +70,9 @@ public final class NGramExpression extends Expression { // annotate gram as a word term String gramString = gram.extractFrom(output.getString()); - typedSpan(gram.getStart(), gram.getCodePointCount(), TokenType.ALPHABETIC, spanList). - annotate(LinguisticsAnnotator.lowerCaseTermAnnotation(gramString, gramString)); + typedSpan(gram.getStart(), + gram.getCodePointCount(), + TokenType.ALPHABETIC, spanList).annotate(LinguisticsAnnotator.termAnnotation(toLowerCase(gramString), gramString)); lastPosition = gram.getStart() + gram.getCodePointCount(); } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java index 684bae3bf97..5c1bf0813c4 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/AnnotatorConfig.java @@ -13,7 +13,7 @@ public class AnnotatorConfig implements Cloneable { private Language language; private StemMode stemMode; private boolean removeAccents; - private int maxTermOccurences; + private int maxTermOccurrences; private int maxTokenizeLength; public static final int DEFAULT_MAX_TERM_OCCURRENCES; @@ -29,7 +29,7 @@ public class AnnotatorConfig implements Cloneable { language = Language.ENGLISH; stemMode = StemMode.NONE; removeAccents = false; - maxTermOccurences = DEFAULT_MAX_TERM_OCCURRENCES; + maxTermOccurrences = DEFAULT_MAX_TERM_OCCURRENCES; maxTokenizeLength = DEFAULT_MAX_TOKENIZE_LENGTH; } @@ -37,7 +37,7 @@ public class AnnotatorConfig implements Cloneable { language = rhs.language; stemMode = rhs.stemMode; removeAccents = rhs.removeAccents; - maxTermOccurences = rhs.maxTermOccurences; + maxTermOccurrences = rhs.maxTermOccurrences; maxTokenizeLength = rhs.maxTokenizeLength; } @@ -74,11 +74,11 @@ public class AnnotatorConfig implements Cloneable { } public int getMaxTermOccurrences() { - return maxTermOccurences; + return maxTermOccurrences; } public AnnotatorConfig setMaxTermOccurrences(int maxTermCount) { - this.maxTermOccurences = maxTermCount; + this.maxTermOccurrences = maxTermCount; return this; } @@ -109,7 +109,7 @@ public class AnnotatorConfig implements Cloneable { if (removeAccents != rhs.removeAccents) { return false; } - if (maxTermOccurences != rhs.maxTermOccurences) { + if (maxTermOccurrences != rhs.maxTermOccurrences) { return false; } if (maxTokenizeLength != rhs.maxTokenizeLength) { @@ -121,6 +121,7 @@ public class AnnotatorConfig implements Cloneable { @Override public int hashCode() { return getClass().hashCode() + language.hashCode() + stemMode.hashCode() + - Boolean.valueOf(removeAccents).hashCode() + maxTermOccurences + maxTokenizeLength; + Boolean.valueOf(removeAccents).hashCode() + maxTermOccurrences + maxTokenizeLength; } + } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java index 191d067effe..86d4e91a567 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotator.java @@ -34,8 +34,8 @@ public class LinguisticsAnnotator { final Map<String, Integer> termOccurrences = new HashMap<>(); final int maxOccurrences; - public TermOccurrences(int maxOccurences) { - this.maxOccurrences = maxOccurences; + public TermOccurrences(int maxOccurrences) { + this.maxOccurrences = maxOccurrences; } boolean termCountBelowLimit(String term) { @@ -86,24 +86,23 @@ public class LinguisticsAnnotator { } /** - * Creates a TERM annotation which has the lowercase value as annotation (only) if it is different from the + * Creates a TERM annotation which has the term as annotation (only) if it is different from the * original. * - * @param termToLowerCase the term to lower case - * @param origTerm the original term + * @param term the term + * @param origTerm the original term * @return the created TERM annotation */ - public static Annotation lowerCaseTermAnnotation(String termToLowerCase, String origTerm) { - String annotationValue = toLowerCase(termToLowerCase); - if (annotationValue.equals(origTerm)) { + public static Annotation termAnnotation(String term, String origTerm) { + if (term.equals(origTerm)) return new Annotation(AnnotationTypes.TERM); - } - return new Annotation(AnnotationTypes.TERM, new StringFieldValue(annotationValue)); + else + return new Annotation(AnnotationTypes.TERM, new StringFieldValue(term)); } private static void addAnnotation(Span here, String term, String orig, TermOccurrences termOccurrences) { if (termOccurrences.termCountBelowLimit(term)) { - here.annotate(lowerCaseTermAnnotation(term, orig)); + here.annotate(termAnnotation(term, orig)); } } @@ -127,29 +126,24 @@ public class LinguisticsAnnotator { } if (mode == StemMode.ALL) { Span where = parent.span((int)token.getOffset(), token.getOrig().length()); - String lowercasedOrig = toLowerCase(token.getOrig()); - addAnnotation(where, token.getOrig(), token.getOrig(), termOccurrences); - String lowercasedTerm = lowercasedOrig; + String lowercasedOrig = toLowerCase(token.getOrig()); String term = token.getTokenString(); if (term != null) { - lowercasedTerm = toLowerCase(term); - } - if (! lowercasedOrig.equals(lowercasedTerm)) { addAnnotation(where, term, token.getOrig(), termOccurrences); + if ( ! term.equals(lowercasedOrig)) + addAnnotation(where, lowercasedOrig, token.getOrig(), termOccurrences); } for (int i = 0; i < token.getNumStems(); i++) { String stem = token.getStem(i); - String lowercasedStem = toLowerCase(stem); - if (! (lowercasedOrig.equals(lowercasedStem) || lowercasedTerm.equals(lowercasedStem))) { + if (! (stem.equals(lowercasedOrig) || stem.equals(term))) addAnnotation(where, stem, token.getOrig(), termOccurrences); - } } } else { String term = token.getTokenString(); if (term == null || term.trim().isEmpty()) return; if (termOccurrences.termCountBelowLimit(term)) { - parent.span((int)token.getOffset(), token.getOrig().length()).annotate(lowerCaseTermAnnotation(term, token.getOrig())); + parent.span((int)token.getOffset(), token.getOrig().length()).annotate(termAnnotation(term, token.getOrig())); } } } diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NGramTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NGramTestCase.java index bcde8751de8..b4e266ab3eb 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NGramTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/expressions/NGramTestCase.java @@ -57,8 +57,8 @@ public class NGramTestCase { new NGramExpression(new SimpleLinguistics(), 3).execute(context); StringFieldValue value = (StringFieldValue)context.getValue(); - assertEquals("Grams are pure annotations - field value is unchanged", "en gul Bille sang... ", - value.getString()); + assertEquals("Grams are pure annotations - field value is unchanged", + "en gul Bille sang... ", value.getString()); SpanTree gramTree = value.getSpanTree(SpanTrees.LINGUISTICS); assertNotNull(gramTree); SpanList grams = (SpanList)gramTree.getRoot(); diff --git a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java index 67bff3843ee..0bdf98f2ae0 100644 --- a/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java +++ b/indexinglanguage/src/test/java/com/yahoo/vespa/indexinglanguage/linguistics/LinguisticsAnnotatorTestCase.java @@ -19,7 +19,6 @@ import org.junit.Test; import org.mockito.Mockito; import java.util.*; -import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -29,8 +28,6 @@ import static org.junit.Assert.assertTrue; */ public class LinguisticsAnnotatorTestCase { - private static final AnnotatorConfig CONFIG = new AnnotatorConfig(); - @Test public void requireThatAnnotateFailsWithZeroTokens() { assertAnnotations(null, "foo"); @@ -42,7 +39,7 @@ public class LinguisticsAnnotatorTestCase { if (type.isIndexable()) { continue; } - assertAnnotations(null, "foo", newToken("foo", "bar", type)); + assertAnnotations(null, "foo", token("foo", "bar", type)); } } @@ -54,7 +51,29 @@ public class LinguisticsAnnotatorTestCase { if (!type.isIndexable()) { continue; } - assertAnnotations(expected, "foo", newToken("foo", "bar", type)); + assertAnnotations(expected, "foo", token("foo", "bar", type)); + } + } + + @Test + public void requireThatIndexableTokenStringsAreAnnotatedWithModeALL() { + SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS); + var span1 = expected.spanList().span(0, 6); + span1.annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("tesla"))); + span1.annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("teslas"))); + var span2 = expected.spanList().span(0, 4); + span2.annotate(new Annotation(AnnotationTypes.TERM)); + span2.annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("car"))); + var span3 = expected.spanList().span(0, 8); + span3.annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("modelxes"))); + span3.annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("modelx"))); + span3.annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("mex"))); + for (TokenType type : TokenType.values()) { + if (!type.isIndexable()) continue; + assertAnnotations(expected, "Tesla cars", new AnnotatorConfig().setStemMode("ALL"), + token("Teslas", "tesla", type), + token("cars", "car", type), + SimpleToken.fromStems("ModelXes", List.of("modelxes", "modelx", "mex"))); } } @@ -63,7 +82,7 @@ public class LinguisticsAnnotatorTestCase { SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS); expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar"))); for (TokenType type : TokenType.values()) { - assertAnnotations(expected, "foo", newToken("foo", "bar", type, true)); + assertAnnotations(expected, "foo", token("foo", "bar", type, true)); } } @@ -76,21 +95,21 @@ public class LinguisticsAnnotatorTestCase { if (!specialToken && !type.isIndexable()) { continue; } - assertAnnotations(expected, "foo", newToken("foo", "foo", type, specialToken)); + assertAnnotations(expected, "foo", token("foo", "foo", type, specialToken)); } } } @Test - public void requireThatTermAnnotationsAreLowerCased() { + public void requireThatTermAnnotationsPreserveCasing() { SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS); - expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar"))); + expected.spanList().span(0, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("BaR"))); for (boolean specialToken : Arrays.asList(true, false)) { for (TokenType type : TokenType.values()) { if (!specialToken && !type.isIndexable()) { continue; } - assertAnnotations(expected, "foo", newToken("foo", "BAR", type, specialToken)); + assertAnnotations(expected, "foo", token("foo", "BaR", type, specialToken)); } } } @@ -102,11 +121,11 @@ public class LinguisticsAnnotatorTestCase { expected.spanList().span(3, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("bar"))); expected.spanList().span(6, 3).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("baz"))); - SimpleToken token = newToken("FOOBARBAZ", "foobarbaz", TokenType.ALPHABETIC) - .addComponent(newToken("FOO", "foo", TokenType.ALPHABETIC).setOffset(0)) - .addComponent(newToken("BARBAZ", "barbaz", TokenType.ALPHABETIC).setOffset(3) - .addComponent(newToken("BAR", "bar", TokenType.ALPHABETIC).setOffset(3)) - .addComponent(newToken("BAZ", "baz", TokenType.ALPHABETIC).setOffset(6))); + SimpleToken token = token("FOOBARBAZ", "foobarbaz", TokenType.ALPHABETIC) + .addComponent(token("FOO", "foo", TokenType.ALPHABETIC).setOffset(0)) + .addComponent(token("BARBAZ", "barbaz", TokenType.ALPHABETIC).setOffset(3) + .addComponent(token("BAR", "bar", TokenType.ALPHABETIC).setOffset(3)) + .addComponent(token("BAZ", "baz", TokenType.ALPHABETIC).setOffset(6))); assertAnnotations(expected, "foobarbaz", token); } @@ -116,11 +135,11 @@ public class LinguisticsAnnotatorTestCase { expected.spanList().span(0, 9).annotate(new Annotation(AnnotationTypes.TERM, new StringFieldValue("foobarbaz"))); - SimpleToken token = newToken("FOOBARBAZ", "foobarbaz", TokenType.ALPHABETIC).setSpecialToken(true) - .addComponent(newToken("FOO", "foo", TokenType.ALPHABETIC).setOffset(0)) - .addComponent(newToken("BARBAZ", "barbaz", TokenType.ALPHABETIC).setOffset(3) - .addComponent(newToken("BAR", "bar", TokenType.ALPHABETIC).setOffset(3)) - .addComponent(newToken("BAZ", "baz", TokenType.ALPHABETIC).setOffset(6))); + SimpleToken token = token("FOOBARBAZ", "foobarbaz", TokenType.ALPHABETIC).setSpecialToken(true) + .addComponent(token("FOO", "foo", TokenType.ALPHABETIC).setOffset(0)) + .addComponent(token("BARBAZ", "barbaz", TokenType.ALPHABETIC).setOffset(3) + .addComponent(token("BAR", "bar", TokenType.ALPHABETIC).setOffset(3)) + .addComponent(token("BAZ", "baz", TokenType.ALPHABETIC).setOffset(6))); assertAnnotations(expected, "foobarbaz", token); } @@ -140,7 +159,8 @@ public class LinguisticsAnnotatorTestCase { continue; } assertAnnotations(expected, "foo", - newLinguistics(List.of(newToken("foo", "foo", type, specialToken)), + new AnnotatorConfig(), + newLinguistics(List.of(token("foo", "foo", type, specialToken)), Collections.singletonMap("foo", "bar"))); } } @@ -154,11 +174,9 @@ public class LinguisticsAnnotatorTestCase { StringFieldValue val = new StringFieldValue("foo"); val.setSpanTree(spanTree); - Linguistics linguistics = newLinguistics(List.of(newToken("foo", "bar", TokenType.ALPHABETIC, false)), + Linguistics linguistics = newLinguistics(List.of(token("foo", "bar", TokenType.ALPHABETIC, false)), Collections.<String, String>emptyMap()); - new LinguisticsAnnotator(linguistics, CONFIG).annotate(val); - - assertTrue(new LinguisticsAnnotator(linguistics, CONFIG).annotate(val)); + assertTrue(new LinguisticsAnnotator(linguistics, new AnnotatorConfig()).annotate(val)); assertEquals(spanTree, val.getSpanTree(SpanTrees.LINGUISTICS)); } @@ -186,10 +204,9 @@ public class LinguisticsAnnotatorTestCase { } @Test - public void requireThatMaxTermOccurencesIsHonored() { + public void requireThatMaxTermOccurrencesIsHonored() { final String inputTerm = "foo"; - final String stemmedInputTerm = "bar"; // completely different from - // inputTerm for safer test + final String stemmedInputTerm = "bar"; // completely different from inputTerm for safer test final String paddedInputTerm = inputTerm + " "; final SpanTree expected = new SpanTree(SpanTrees.LINGUISTICS); final int inputTermOccurence = AnnotatorConfig.DEFAULT_MAX_TERM_OCCURRENCES * 2; @@ -204,7 +221,7 @@ public class LinguisticsAnnotatorTestCase { StringBuilder input = new StringBuilder(); Token[] tokens = new Token[inputTermOccurence]; for (int i = 0; i < inputTermOccurence; ++i) { - SimpleToken t = newToken(inputTerm, stemmedInputTerm, type); + SimpleToken t = token(inputTerm, stemmedInputTerm, type); t.setOffset(i * paddedInputTerm.length()); tokens[i] = t; input.append(paddedInputTerm); @@ -214,28 +231,29 @@ public class LinguisticsAnnotatorTestCase { } // -------------------------------------------------------------------------------- - // // Utilities - // - // -------------------------------------------------------------------------------- - private static SimpleToken newToken(String orig, String stem, TokenType type) { - return newToken(orig, stem, type, false); + private static SimpleToken token(String orig, String stem, TokenType type) { + return token(orig, stem, type, false); } - private static SimpleToken newToken(String orig, String stem, TokenType type, boolean specialToken) { + private static SimpleToken token(String orig, String stem, TokenType type, boolean specialToken) { return new SimpleToken(orig).setTokenString(stem) .setType(type) .setSpecialToken(specialToken); } private static void assertAnnotations(SpanTree expected, String value, Token... tokens) { - assertAnnotations(expected, value, newLinguistics(Arrays.asList(tokens), Collections.<String, String>emptyMap())); + assertAnnotations(expected, value, new AnnotatorConfig(), newLinguistics(Arrays.asList(tokens), Collections.emptyMap())); + } + + private static void assertAnnotations(SpanTree expected, String value, AnnotatorConfig config, Token... tokens) { + assertAnnotations(expected, value, config, newLinguistics(Arrays.asList(tokens), Collections.emptyMap())); } - private static void assertAnnotations(SpanTree expected, String str, Linguistics linguistics) { + private static void assertAnnotations(SpanTree expected, String str, AnnotatorConfig config, Linguistics linguistics) { StringFieldValue val = new StringFieldValue(str); - assertEquals(expected != null, new LinguisticsAnnotator(linguistics, CONFIG).annotate(val)); + assertEquals(expected != null, new LinguisticsAnnotator(linguistics, config).annotate(val)); assertEquals(expected, val.getSpanTree(SpanTrees.LINGUISTICS)); } diff --git a/jdisc-cloud-aws/pom.xml b/jdisc-cloud-aws/pom.xml index 3dfbf1e9154..af5c5c7cc8f 100644 --- a/jdisc-cloud-aws/pom.xml +++ b/jdisc-cloud-aws/pom.xml @@ -47,6 +47,23 @@ <groupId>com.amazonaws</groupId> <artifactId>aws-java-sdk-ssm</artifactId> </dependency> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-api</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-engine</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>testutil</artifactId> + <version>${project.version}</version> + <scope>test</scope> + </dependency> + </dependencies> <build> diff --git a/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/VespaAwsCredentialsProvider.java b/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/VespaAwsCredentialsProvider.java index fc9c03a824a..6de8b5c0142 100644 --- a/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/VespaAwsCredentialsProvider.java +++ b/jdisc-cloud-aws/src/main/java/com/yahoo/jdisc/cloud/aws/VespaAwsCredentialsProvider.java @@ -14,45 +14,87 @@ import java.io.IOException; import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Level; +import java.util.logging.Logger; public class VespaAwsCredentialsProvider implements AWSCredentialsProvider { + private static final Logger logger = Logger.getLogger(VespaAwsCredentialsProvider.class.getName()); private static final String DEFAULT_CREDENTIALS_PATH = "/opt/vespa/var/vespa/aws/credentials.json"; - - private final AtomicReference<AWSCredentials> credentials = new AtomicReference<>(); + private static final Duration REFRESH_INTERVAL = Duration.ofMinutes(30); + private final AtomicReference<Credentials> credentials = new AtomicReference<>(); private final Path credentialsPath; - + private final Clock clock; public VespaAwsCredentialsProvider() { - this.credentialsPath = Path.of(DEFAULT_CREDENTIALS_PATH); + this(Path.of(DEFAULT_CREDENTIALS_PATH), Clock.systemUTC()); + } + + VespaAwsCredentialsProvider(Path credentialsPath, Clock clock) { + this.credentialsPath = credentialsPath; + this.clock = clock; refresh(); } @Override public AWSCredentials getCredentials() { - return credentials.get(); + Credentials sessionCredentials = credentials.get(); + if (Duration.between(clock.instant(), sessionCredentials.expiry).abs().compareTo(REFRESH_INTERVAL)<0) { + refresh(); + sessionCredentials = credentials.get(); + } + return sessionCredentials; } @Override public void refresh() { try { + logger.log(Level.FINE, "Refreshing credentials from disk"); credentials.set(readCredentials()); } catch (Exception e) { throw new RuntimeException("Unable to get credentials. Please ensure cluster is configured as exclusive. See: https://cloud.vespa.ai/en/reference/services#nodes"); } } - private AWSSessionCredentials readCredentials() { + private Credentials readCredentials() { try { Slime slime = SlimeUtils.jsonToSlime(Files.readAllBytes(credentialsPath)); Cursor cursor = slime.get(); String accessKey = cursor.field("awsAccessKey").asString(); String secretKey = cursor.field("awsSecretKey").asString(); String sessionToken = cursor.field("sessionToken").asString(); - return new BasicSessionCredentials(accessKey, secretKey, sessionToken); + Instant defaultExpiry = Instant.now().plus(Duration.ofHours(1)); + Instant expiry; + try { + expiry = SlimeUtils.optionalString(cursor.field("expiry")).map(Instant::parse).orElse(defaultExpiry); + } catch (Exception e) { + expiry = defaultExpiry; + logger.warning("Unable to read expiry from credentials"); + } + return new Credentials(accessKey, secretKey, sessionToken, expiry); } catch (IOException e) { throw new UncheckedIOException(e); } } + + record Credentials (String awsAccessKey, String awsSecretKey, String sessionToken, Instant expiry) implements AWSSessionCredentials { + @Override + public String getSessionToken() { + return sessionToken; + } + + @Override + public String getAWSAccessKeyId() { + return awsAccessKey; + } + + @Override + public String getAWSSecretKey() { + return awsSecretKey; + } + } } diff --git a/jdisc-cloud-aws/src/test/java/com/yahoo/jdisc/cloud/aws/VespaAwsCredentialsProviderTest.java b/jdisc-cloud-aws/src/test/java/com/yahoo/jdisc/cloud/aws/VespaAwsCredentialsProviderTest.java new file mode 100644 index 00000000000..63cfd2f1eeb --- /dev/null +++ b/jdisc-cloud-aws/src/test/java/com/yahoo/jdisc/cloud/aws/VespaAwsCredentialsProviderTest.java @@ -0,0 +1,66 @@ +package com.yahoo.jdisc.cloud.aws; + +import com.amazonaws.auth.AWSCredentials; +import com.yahoo.test.ManualClock; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; + + +public class VespaAwsCredentialsProviderTest { + Path credentialsPath = TestFileSystem.create().getPath("/credentials.json"); + ManualClock clock = new ManualClock(Instant.now()); + + @Test + void refreshes_credentials() throws IOException { + Instant originalExpiry = clock.instant().plus(Duration.ofHours(12)); + writeCredentials(credentialsPath, originalExpiry); + VespaAwsCredentialsProvider credentialsProvider = new VespaAwsCredentialsProvider(credentialsPath, clock); + AWSCredentials credentials = credentialsProvider.getCredentials(); + assertExpiryEquals(originalExpiry, credentials); + + Instant updatedExpiry = clock.instant().plus(Duration.ofHours(24)); + writeCredentials(credentialsPath, updatedExpiry); + // File updated, but old credentials still valid + credentials = credentialsProvider.getCredentials(); + assertExpiryEquals(originalExpiry, credentials); + + // Credentials refreshes when it is < 30 minutes left until expiry + clock.advance(Duration.ofHours(11).plus(Duration.ofMinutes(31))); + credentials = credentialsProvider.getCredentials(); + assertExpiryEquals(updatedExpiry, credentials); + } + + @Test + void deserializes_credentials() throws IOException { + Instant originalExpiry = clock.instant().plus(Duration.ofHours(12)); + writeCredentials(credentialsPath, originalExpiry); + VespaAwsCredentialsProvider credentialsProvider = new VespaAwsCredentialsProvider(credentialsPath, clock); + AWSCredentials credentials = credentialsProvider.getCredentials(); + assertExpiryEquals(originalExpiry, credentials); + Assertions.assertEquals("awsAccessKey", credentials.getAWSAccessKeyId()); + Assertions.assertEquals("awsSecretKey", credentials.getAWSSecretKey()); + Assertions.assertEquals("sessionToken", ((VespaAwsCredentialsProvider.Credentials)credentials).getSessionToken()); + } + + private void writeCredentials(Path path, Instant expiry) throws IOException { + String content = """ + { + "awsAccessKey": "awsAccessKey", + "awsSecretKey": "awsSecretKey", + "sessionToken": "sessionToken", + "expiry": "%s" + }""".formatted(expiry.toString()); + Files.writeString(path, content); + } + + private void assertExpiryEquals(Instant expiry, AWSCredentials credentials) { + Assertions.assertEquals(expiry, ((VespaAwsCredentialsProvider.Credentials)credentials).expiry()); + } +} diff --git a/linguistics/src/main/java/com/yahoo/language/LinguisticsCase.java b/linguistics/src/main/java/com/yahoo/language/LinguisticsCase.java index 5ad6a382abd..f0439a21fec 100644 --- a/linguistics/src/main/java/com/yahoo/language/LinguisticsCase.java +++ b/linguistics/src/main/java/com/yahoo/language/LinguisticsCase.java @@ -26,6 +26,7 @@ public class LinguisticsCase { public static String toLowerCase(String in) { // def is picked from http://docs.oracle.com/javase/6/docs/api/java/lang/String.html#toLowerCase%28%29 // Also, at the time of writing, English is the default language for queries + if (in == null) return null; return Lowercase.toLowerCase(in); } diff --git a/linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java b/linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java index 33f5ee7e4bb..9178c2d7e09 100644 --- a/linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java +++ b/linguistics/src/main/java/com/yahoo/language/process/GramSplitter.java @@ -189,9 +189,8 @@ public class GramSplitter { @Override public boolean equals(Object o) { if (this == o) return true; - if ( ! (o instanceof Gram)) return false; + if ( ! (o instanceof Gram gram)) return false; - Gram gram = (Gram)o; if (codePointCount != gram.codePointCount) return false; if (start != gram.start) return false; return true; diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java index 6cc68c7ac14..809e9b8d133 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java @@ -15,35 +15,48 @@ import java.util.Objects; public class SimpleToken implements Token { private final List<Token> components = new ArrayList<>(); - private final String orig; + private final String original; private TokenType type = TokenType.UNKNOWN; private TokenScript script = TokenScript.UNKNOWN; private String tokenString; + private List<String> stems = null; // Any additional stems after tokenString private boolean specialToken = false; private long offset = 0; - public SimpleToken(String orig) { - this(orig, null); + public SimpleToken(String original) { + this(original, (String)null); } - public SimpleToken(String orig, String tokenString) { - this.orig = orig; + public SimpleToken(String original, String tokenString) { + this.original = original; this.tokenString = tokenString; } + /** Exposed as fromStems */ + private SimpleToken(String original, List<String> stems) { + this.type = TokenType.ALPHABETIC; // Only type which may have stems + this.original = original; + this.tokenString = stems.get(0); + this.stems = List.copyOf(stems.subList(1, stems.size())); + } + @Override public String getOrig() { - return orig; + return original; } @Override public int getNumStems() { - return tokenString != null ? 1 : 0; + return (tokenString != null ? 1 : 0) + (stems != null ? stems.size() : 0); } @Override public String getStem(int i) { - return tokenString; + if (i == 0) + return tokenString; + if (stems != null && i-1 < stems.size()) + return stems.get(i-1); + return tokenString; // TODO Vespa 9: throw new IllegalArgumentException() instead } @Override @@ -131,12 +144,12 @@ public class SimpleToken implements Token { @Override public int hashCode() { - return orig.hashCode(); + return original.hashCode(); } @Override public String toString() { - return "token '" + orig + "'"; + return "token '" + original + "'"; } public String toDetailString() { @@ -171,4 +184,8 @@ public class SimpleToken implements Token { return getType().isIndexable() && (getOrig().length() > 0); } + public static SimpleToken fromStems(String original, List<String> stems) { + return new SimpleToken(original, stems); + } + } diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java index 98a84a48095..b72d2bd6d37 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java @@ -106,7 +106,7 @@ public class SimpleTokenizer implements Tokenizer { String oldToken = token; token = stemmer.stem(token); String newToken = token; - log.log(Level.FINEST, () -> "stem '" + oldToken+"' to '" + newToken+"'"); + log.log(Level.FINEST, () -> "stem '" + oldToken + "' to '" + newToken + "'"); } String result = token; log.log(Level.FINEST, () -> "processed token is: " + result); diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandler.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandler.java index e8da690ea9b..d73561b5eff 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandler.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandler.java @@ -5,7 +5,10 @@ import ai.vespa.metricsproxy.core.MetricsConsumers; import ai.vespa.metricsproxy.core.MetricsManager; import ai.vespa.metricsproxy.http.TextResponse; import ai.vespa.metricsproxy.http.ValuesFetcher; +import ai.vespa.metricsproxy.metric.dimensions.ApplicationDimensions; +import ai.vespa.metricsproxy.metric.dimensions.NodeDimensions; import ai.vespa.metricsproxy.metric.model.MetricsPacket; +import ai.vespa.metricsproxy.node.NodeMetricGatherer; import ai.vespa.metricsproxy.service.VespaServices; import com.yahoo.component.annotation.Inject; import com.yahoo.container.handler.metrics.HttpHandlerBase; @@ -13,6 +16,7 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.restapi.Path; import java.net.URI; +import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.concurrent.Executor; @@ -31,14 +35,18 @@ public class PrometheusHandler extends HttpHandlerBase { static final String VALUES_PATH = V1_PATH + "/values"; private final ValuesFetcher valuesFetcher; + private final NodeMetricGatherer nodeMetricGatherer; @Inject public PrometheusHandler(Executor executor, MetricsManager metricsManager, VespaServices vespaServices, - MetricsConsumers metricsConsumers) { + MetricsConsumers metricsConsumers, + ApplicationDimensions applicationDimensions, + NodeDimensions nodeDimensions) { super(executor); valuesFetcher = new ValuesFetcher(metricsManager, vespaServices, metricsConsumers); + this.nodeMetricGatherer = new NodeMetricGatherer(metricsManager, applicationDimensions, nodeDimensions); } @Override @@ -50,7 +58,8 @@ public class PrometheusHandler extends HttpHandlerBase { private TextResponse valuesResponse(String consumer) { try { - List<MetricsPacket> metrics = valuesFetcher.fetch(consumer); + List<MetricsPacket> metrics = new ArrayList<>(valuesFetcher.fetch(consumer)); + metrics.addAll(nodeMetricGatherer.gatherMetrics()); return new TextResponse(OK, toPrometheusModel(metrics).serialize()); } catch (Exception e) { log.log(Level.WARNING, "Got exception when rendering metrics:", e); diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java index d6bde07d39a..f8e07c3f2ec 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/http/prometheus/PrometheusHandlerTest.java @@ -35,7 +35,9 @@ public class PrometheusHandlerTest extends HttpHandlerTestBase { PrometheusHandler handler = new PrometheusHandler(Executors.newSingleThreadExecutor(), getMetricsManager(), vespaServices, - getMetricsConsumers()); + getMetricsConsumers(), + getApplicationDimensions(), + getNodeDimensions()); testDriver = new RequestHandlerTestDriver(handler); valuesResponse = testDriver.sendRequest(VALUES_URI).readAll(); } diff --git a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java index 1fadc66c46a..47912e958c6 100644 --- a/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java +++ b/metrics-proxy/src/test/java/ai/vespa/metricsproxy/service/ContainerServiceTest.java @@ -7,12 +7,15 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; +import java.io.IOException; + import static ai.vespa.metricsproxy.TestUtil.getFileContents; import static ai.vespa.metricsproxy.metric.model.DimensionId.toDimensionId; import static ai.vespa.metricsproxy.metric.model.MetricId.toMetricId; import static ai.vespa.metricsproxy.service.RemoteMetricsFetcher.METRICS_PATH; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; /** * @author Unknown @@ -27,13 +30,9 @@ public class ContainerServiceTest { } @Before - public void setupHTTPServer() { - try { - String response = getFileContents("metrics-container-state-multi-chain.json"); - httpServer = new MockHttpServer(response, METRICS_PATH); - } catch (Exception e) { - e.printStackTrace(); - } + public void setupHTTPServer() throws IOException { + String response = getFileContents("metrics-container-state-multi-chain.json"); + httpServer = new MockHttpServer(response, METRICS_PATH); } @Test @@ -49,7 +48,7 @@ public class ContainerServiceTest { } else if (m.getDimensions().get(toDimensionId("chain")).equals("blendingResult")) { assertEquals(0.36666666666666664, m.getValue()); } else { - assertTrue("Unknown unknown chain", false); + fail("Unknown unknown chain"); } } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index a4a4c42d9c1..52b68708eee 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -263,8 +263,8 @@ public class NodeRepository extends AbstractComponent implements HealthCheckerPr loadBalancers.list(endpoint.applicationId()) .cluster(endpoint.clusterName()) .first() - .flatMap(LoadBalancer::instance) - .flatMap(LoadBalancerInstance::idSeed)); + .map(LoadBalancer::idSeed) + .orElseThrow(() -> new IllegalArgumentException("no load balancer for '" + endpoint + "'"))); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java index 36f2eb4bb2c..f8de6e1db16 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java @@ -17,12 +17,14 @@ import java.util.Set; public class LoadBalancer { private final LoadBalancerId id; + private final String idSeed; private final Optional<LoadBalancerInstance> instance; private final State state; private final Instant changedAt; - public LoadBalancer(LoadBalancerId id, Optional<LoadBalancerInstance> instance, State state, Instant changedAt) { + public LoadBalancer(LoadBalancerId id, String idSeed, Optional<LoadBalancerInstance> instance, State state, Instant changedAt) { this.id = Objects.requireNonNull(id, "id must be non-null"); + this.idSeed = Objects.requireNonNull(idSeed, "idSeed must be non-null"); this.instance = Objects.requireNonNull(instance, "instance must be non-null"); this.state = Objects.requireNonNull(state, "state must be non-null"); this.changedAt = Objects.requireNonNull(changedAt, "changedAt must be non-null"); @@ -40,6 +42,11 @@ public class LoadBalancer { return id; } + /** Seed to use for generating resource IDs for provisioned resources in this. */ + public String idSeed() { + return idSeed; + } + /** The instance associated with this */ public Optional<LoadBalancerInstance> instance() { return instance; @@ -64,12 +71,12 @@ public class LoadBalancer { if (this.state != State.reserved && state == State.reserved) { throw new IllegalArgumentException("Invalid state transition: " + this.state + " -> " + state); } - return new LoadBalancer(id, instance, state, changedAt); + return new LoadBalancer(id, idSeed, instance, state, changedAt); } /** Returns a copy of this with instance set to given instance */ public LoadBalancer with(LoadBalancerInstance instance) { - return new LoadBalancer(id, Optional.of(instance), state, changedAt); + return new LoadBalancer(id, idSeed, Optional.of(instance), state, changedAt); } public enum State { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java index 4dbf891b1b7..c0931ecbc70 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java @@ -20,7 +20,6 @@ import java.util.Set; */ public class LoadBalancerInstance { - private final Optional<String> idSeed; private final Optional<DomainName> hostname; private final Optional<String> ip4Address; private final Optional<String> ip6Address; @@ -32,10 +31,9 @@ public class LoadBalancerInstance { private final List<PrivateServiceId> serviceIds; private final CloudAccount cloudAccount; - public LoadBalancerInstance(Optional<String> idSeed, Optional<DomainName> hostname, Optional<String> ip4Address, Optional<String> ip6Address, + public LoadBalancerInstance(Optional<DomainName> hostname, Optional<String> ip4Address, Optional<String> ip6Address, Optional<DnsZone> dnsZone, Set<Integer> ports, Set<String> networks, Set<Real> reals, ZoneEndpoint settings, List<PrivateServiceId> serviceIds, CloudAccount cloudAccount) { - this.idSeed = Objects.requireNonNull(idSeed, "idSeed must be non-null"); this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null"); this.ip4Address = Objects.requireNonNull(ip4Address, "ip4Address must be non-null"); this.ip6Address = Objects.requireNonNull(ip6Address, "ip6Address must be non-null"); @@ -53,11 +51,6 @@ public class LoadBalancerInstance { } } - /** A unique seed to use when generating cloud-specific resource IDs for this load balancer instance. */ - public Optional<String> idSeed() { - return idSeed; - } - /** Fully-qualified domain name of this load balancer. This hostname can be used for query and feed */ public Optional<DomainName> hostname() { return hostname; @@ -128,7 +121,7 @@ public class LoadBalancerInstance { public LoadBalancerInstance with(Set<Real> reals, ZoneEndpoint settings, Optional<PrivateServiceId> serviceId) { List<PrivateServiceId> ids = new ArrayList<>(serviceIds); serviceId.filter(id -> ! ids.contains(id)).ifPresent(ids::add); - return new LoadBalancerInstance(idSeed, hostname, ip4Address, ip6Address, dnsZone, ports, networks, + return new LoadBalancerInstance(hostname, ip4Address, ip6Address, dnsZone, ports, networks, reals, settings, ids, cloudAccount); } @@ -137,7 +130,7 @@ public class LoadBalancerInstance { public LoadBalancerInstance withServiceIds(List<PrivateServiceId> serviceIds) { List<PrivateServiceId> ids = new ArrayList<>(serviceIds); for (PrivateServiceId id : this.serviceIds) if ( ! ids.contains(id)) ids.add(id); - return new LoadBalancerInstance(idSeed, hostname, ip4Address, ip6Address, dnsZone, ports, networks, reals, settings, ids, cloudAccount); + return new LoadBalancerInstance(hostname, ip4Address, ip6Address, dnsZone, ports, networks, reals, settings, ids, cloudAccount); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java index 6ddde1151dd..efd1536d108 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java @@ -7,8 +7,6 @@ import com.yahoo.config.provision.EndpointsChecker.Endpoint; import com.yahoo.config.provision.EndpointsChecker.HealthChecker; import com.yahoo.config.provision.NodeType; -import java.util.Optional; - /** * A managed load balance service. * @@ -22,16 +20,7 @@ public interface LoadBalancerService { * @param spec Load balancer specification * @return The provisioned load balancer instance */ - default LoadBalancerInstance provision(LoadBalancerSpec spec) { return provision(spec, Optional.empty()); } - - /** - * Provisions load balancers from the given specification. Implementations are expected to be idempotent - * - * @param spec Load balancer specification - * @param idSeed Seed for generating a unique ID for the load balancer instance - * @return The provisioned load balancer instance - */ - LoadBalancerInstance provision(LoadBalancerSpec spec, Optional<String> idSeed); + LoadBalancerInstance provision(LoadBalancerSpec spec); /** * Configures load balancers for the given specification. Implementations are expected to be idempotent @@ -44,7 +33,7 @@ public interface LoadBalancerService { */ LoadBalancerInstance configure(LoadBalancerInstance instance, LoadBalancerSpec spec, boolean force); - void reallocate(LoadBalancerInstance provisioned, LoadBalancerSpec spec); + void reallocate(LoadBalancerSpec spec); /** Permanently remove given load balancer */ void remove(LoadBalancer loadBalancer); @@ -56,7 +45,7 @@ public interface LoadBalancerService { boolean supports(NodeType nodeType, ClusterSpec.Type clusterType); /** See {@link HealthChecker#healthy(Endpoint)}. */ - Availability healthy(Endpoint endpoint, Optional<String> idSeed); + Availability healthy(Endpoint endpoint, String idSeed); /** Load balancer protocols */ enum Protocol { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java index 03ff17c6ebc..2c672d0eada 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java @@ -26,19 +26,16 @@ import static java.util.stream.Collectors.toMap; public class LoadBalancerServiceMock implements LoadBalancerService { private record Key(ApplicationId application, ClusterSpec.Id cluster, String idSeed) { - @Override public int hashCode() { return idSeed == null ? Objects.hash(application, cluster) : Objects.hash(idSeed); } + @Override public int hashCode() { return idSeed.hashCode(); } @Override public boolean equals(Object o) { if (o == this) return true; if ( ! (o instanceof Key key)) return false; - if (idSeed != null) return Objects.equals(idSeed, key.idSeed); - return Objects.equals(application, key.application) && - Objects.equals(cluster, key.cluster); + return Objects.equals(idSeed, key.idSeed); } } private final Map<Key, LoadBalancerInstance> instances = new HashMap<>(); private boolean throwOnCreate = false; private boolean supportsProvisioning = true; - private final AtomicBoolean uuid = new AtomicBoolean(true); public Map<LoadBalancerId, LoadBalancerInstance> instances() { return instances.entrySet().stream().collect(toMap(e -> new LoadBalancerId(e.getKey().application, e.getKey().cluster), @@ -68,10 +65,9 @@ public class LoadBalancerServiceMock implements LoadBalancerService { } @Override - public LoadBalancerInstance provision(LoadBalancerSpec spec, Optional<String> idSeed) { + public LoadBalancerInstance provision(LoadBalancerSpec spec) { if (throwOnCreate) throw new IllegalStateException("Did not expect a new load balancer to be created"); var instance = new LoadBalancerInstance( - idSeed, Optional.of(DomainName.of("lb-" + spec.application().toShortString() + "-" + spec.cluster().value())), Optional.empty(), Optional.empty(), @@ -82,13 +78,13 @@ public class LoadBalancerServiceMock implements LoadBalancerService { spec.settings(), spec.settings().isPrivateEndpoint() ? List.of(PrivateServiceId.of("service")) : List.of(), spec.cloudAccount()); - instances.put(new Key(spec.application(), spec.cluster(), idSeed.orElse(null)), instance); + instances.put(new Key(spec.application(), spec.cluster(), spec.idSeed()), instance); return instance; } @Override public LoadBalancerInstance configure(LoadBalancerInstance instance, LoadBalancerSpec spec, boolean force) { - var id = new Key(spec.application(), spec.cluster(), instance.idSeed().orElse(null)); + var id = new Key(spec.application(), spec.cluster(), spec.idSeed()); var oldInstance = requireNonNull(instances.get(id), "expected existing load balancer " + id); if (!force && !oldInstance.reals().isEmpty() && spec.reals().isEmpty()) { throw new IllegalArgumentException("Refusing to remove all reals from load balancer " + id); @@ -101,21 +97,21 @@ public class LoadBalancerServiceMock implements LoadBalancerService { } @Override - public void reallocate(LoadBalancerInstance provisioned, LoadBalancerSpec spec) { - instances.put(new Key(spec.application(), spec.cluster(), provisioned.idSeed().get()), - requireNonNull(instances.remove(new Key(null, null, provisioned.idSeed().get())))); // ᕙ༼◕_◕༽ᕤ + public void reallocate(LoadBalancerSpec spec) { + instances.put(new Key(spec.application(), spec.cluster(), spec.idSeed()), + requireNonNull(instances.remove(new Key(null, null, spec.idSeed())))); // ᕙ༼◕_◕༽ᕤ } @Override public void remove(LoadBalancer loadBalancer) { requireNonNull(instances.remove(new Key(loadBalancer.id().application(), loadBalancer.id().cluster(), - loadBalancer.instance().get().idSeed().orElse(null))), + loadBalancer.idSeed())), "expected load balancer to exist: " + loadBalancer.id()); } @Override - public Availability healthy(Endpoint endpoint, Optional<String> idSeed) { + public Availability healthy(Endpoint endpoint, String idSeed) { return Availability.ready; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java index bde736e7a28..8c0aee2b4c6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java @@ -16,20 +16,21 @@ import java.util.Set; * @author mpolden */ public record LoadBalancerSpec(ApplicationId application, ClusterSpec.Id cluster, Set<Real> reals, - ZoneEndpoint settings, CloudAccount cloudAccount) { + ZoneEndpoint settings, CloudAccount cloudAccount, String idSeed) { public static final ApplicationId preProvisionOwner = ApplicationId.from("hosted-vespa", "pre-provision", "default"); - public static LoadBalancerSpec preProvisionSpec(ClusterSpec.Id slot, CloudAccount account) { - return new LoadBalancerSpec(preProvisionOwner, slot, Set.of(), ZoneEndpoint.defaultEndpoint, account); + public static LoadBalancerSpec preProvisionSpec(ClusterSpec.Id slot, CloudAccount account, String idSeed) { + return new LoadBalancerSpec(preProvisionOwner, slot, Set.of(), ZoneEndpoint.defaultEndpoint, account, idSeed); } public LoadBalancerSpec(ApplicationId application, ClusterSpec.Id cluster, Set<Real> reals, - ZoneEndpoint settings, CloudAccount cloudAccount) { + ZoneEndpoint settings, CloudAccount cloudAccount, String idSeed) { this.application = Objects.requireNonNull(application); this.cluster = Objects.requireNonNull(cluster); this.reals = ImmutableSortedSet.copyOf(Objects.requireNonNull(reals)); this.settings = Objects.requireNonNull(settings); this.cloudAccount = Objects.requireNonNull(cloudAccount); + this.idSeed = Objects.requireNonNull(idSeed); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java index 883b8dec944..1ca7442adf5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java @@ -30,25 +30,11 @@ public class SharedLoadBalancerService implements LoadBalancerService { } @Override - public LoadBalancerInstance provision(LoadBalancerSpec spec, Optional<String> idSeed) { - return create(spec); - } - - @Override - public LoadBalancerInstance configure(LoadBalancerInstance instance, LoadBalancerSpec spec, boolean force) { - return instance.with(spec.reals(), spec.settings(), Optional.empty()); - } - - @Override - public void reallocate(LoadBalancerInstance provisioned, LoadBalancerSpec spec) { - throw new UnsupportedOperationException("reallocate is not supported with " + getClass()); - } - - private LoadBalancerInstance create(LoadBalancerSpec spec) { + public LoadBalancerInstance provision(LoadBalancerSpec spec) { if ( ! spec.settings().isPublicEndpoint()) throw new IllegalArgumentException("non-public endpoints is not supported with " + getClass()); - return new LoadBalancerInstance(Optional.empty(), - Optional.of(DomainName.of(vipHostname)), + + return new LoadBalancerInstance(Optional.of(DomainName.of(vipHostname)), Optional.empty(), Optional.empty(), Optional.empty(), @@ -61,6 +47,16 @@ public class SharedLoadBalancerService implements LoadBalancerService { } @Override + public LoadBalancerInstance configure(LoadBalancerInstance instance, LoadBalancerSpec spec, boolean force) { + return instance.with(spec.reals(), spec.settings(), Optional.empty()); + } + + @Override + public void reallocate(LoadBalancerSpec spec) { + throw new UnsupportedOperationException("reallocate is not supported with " + getClass()); + } + + @Override public void remove(LoadBalancer loadBalancer) { // Do nothing, we have no external state to modify } @@ -78,7 +74,7 @@ public class SharedLoadBalancerService implements LoadBalancerService { } @Override - public Availability healthy(Endpoint endpoint, Optional<String> idSeed) { + public Availability healthy(Endpoint endpoint, String idSeed) { return Availability.ready; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index 04ea831c45c..105928c503e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -114,7 +114,8 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { LOG.log(Level.INFO, () -> "Removing reals from inactive load balancer " + lb.id() + ": " + Sets.difference(lb.instance().get().reals(), reals)); LoadBalancerInstance instance = service.configure(lb.instance().get(), new LoadBalancerSpec(lb.id().application(), lb.id().cluster(), reals, - lb.instance().get().settings(), lb.instance().get().cloudAccount()), + lb.instance().get().settings(), + lb.instance().get().cloudAccount(), lb.idSeed()), true); db.writeLoadBalancer(lb.with(instance), lb.state()); } catch (Exception e) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java index d511570881b..31d79d34c94 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java @@ -445,7 +445,7 @@ public class CuratorDb { } public Optional<LoadBalancer> readLoadBalancer(LoadBalancerId id) { - return read(loadBalancerPath(id), LoadBalancerSerializer::fromJson); + return read(loadBalancerPath(id), bytes -> LoadBalancerSerializer.fromJson(id, bytes)); } public void writeLoadBalancer(LoadBalancer loadBalancer, LoadBalancer.State fromState) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java index 756692917e3..99cc0d1e601 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance; import com.yahoo.vespa.hosted.provision.lb.PrivateServiceId; import com.yahoo.vespa.hosted.provision.lb.Real; +import com.yahoo.vespa.hosted.provision.provisioning.LoadBalancerProvisioner; import java.io.IOException; import java.io.UncheckedIOException; @@ -71,7 +72,7 @@ public class LoadBalancerSerializer { Cursor root = slime.setObject(); root.setString(idField, loadBalancer.id().serializedForm()); - loadBalancer.instance().flatMap(LoadBalancerInstance::idSeed).ifPresent(idSeed -> root.setString(idSeedField, idSeed)); + root.setString(idSeedField, loadBalancer.idSeed()); loadBalancer.instance().flatMap(LoadBalancerInstance::hostname).ifPresent(hostname -> root.setString(hostnameField, hostname.value())); loadBalancer.instance().flatMap(LoadBalancerInstance::ip4Address).ifPresent(ip -> root.setString(lbIpAddressField, ip)); loadBalancer.instance().flatMap(LoadBalancerInstance::ip6Address).ifPresent(ip -> root.setString(lbIp6AddressField, ip)); @@ -110,7 +111,7 @@ public class LoadBalancerSerializer { } } - public static LoadBalancer fromJson(byte[] data) { + public static LoadBalancer fromJson(LoadBalancerId id, byte[] data) { Cursor object = SlimeUtils.jsonToSlime(data).get(); Set<Real> reals = new LinkedHashSet<>(); @@ -127,7 +128,8 @@ public class LoadBalancerSerializer { Set<String> networks = new LinkedHashSet<>(); object.field(networksField).traverse((ArrayTraverser) (i, network) -> networks.add(network.asString())); - Optional<String> idSeed = SlimeUtils.optionalString(object.field(idSeedField)); + // TODO jonmv: remove fallback after data is re-written. + String idSeed = SlimeUtils.optionalString(object.field(idSeedField)).orElse(id.application().tenant().value() + id.application().application().value() + id.application().instance().value() + id.cluster().value()); Optional<DomainName> hostname = SlimeUtils.optionalString(object.field(hostnameField)).map(DomainName::of); Optional<String> ip4Address = SlimeUtils.optionalString(object.field(lbIpAddressField)); Optional<String> ip6Address = SlimeUtils.optionalString(object.field(lbIp6AddressField)); @@ -140,9 +142,10 @@ public class LoadBalancerSerializer { CloudAccount cloudAccount = optionalString(object.field(cloudAccountField), CloudAccount::from).orElse(CloudAccount.empty); Optional<LoadBalancerInstance> instance = hostname.isEmpty() && ip4Address.isEmpty() && ip6Address.isEmpty() ? Optional.empty() - : Optional.of(new LoadBalancerInstance(idSeed, hostname, ip4Address, ip6Address, dnsZone, ports, networks, reals, settings, serviceIds, cloudAccount)); + : Optional.of(new LoadBalancerInstance(hostname, ip4Address, ip6Address, dnsZone, ports, networks, reals, settings, serviceIds, cloudAccount)); return new LoadBalancer(LoadBalancerId.fromSerializedForm(object.field(idField).asString()), + idSeed, instance, stateFromString(object.field(stateField).asString()), Instant.ofEpochMilli(object.field(changedAtField).asLong())); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index 24bca326f82..e6f2dc0fbfe 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.ZoneEndpoint; @@ -200,22 +201,21 @@ public class LoadBalancerProvisioner { } private void prepare(LoadBalancerId id, ZoneEndpoint zoneEndpoint, NodeSpec requested) { - Instant now = nodeRepository.clock().instant(); CloudAccount cloudAccount = requested.cloudAccount(); Optional<LoadBalancer> loadBalancer = db.readLoadBalancer(id); - LoadBalancer newLoadBalancer = loadBalancer.orElse(new LoadBalancer(id, Optional.empty(), LoadBalancer.State.reserved, now)); + LoadBalancer newLoadBalancer = null; LoadBalancer.State fromState = loadBalancer.map(LoadBalancer::state).orElse(null); try { if (loadBalancer.isPresent() && ! inAccount(cloudAccount, loadBalancer.get())) { - newLoadBalancer = newLoadBalancer.with(State.removable, now); + newLoadBalancer = loadBalancer.get().with(State.removable, nodeRepository.clock().instant()); throw new LoadBalancerServiceException("Could not (re)configure " + id + " due to change in cloud account. The operation will be retried on next deployment"); } if (loadBalancer.isPresent() && ! hasCorrectVisibility(loadBalancer.get(), zoneEndpoint)) { - newLoadBalancer = newLoadBalancer.with(State.removable, now); + newLoadBalancer = loadBalancer.get().with(State.removable, nodeRepository.clock().instant()); throw new LoadBalancerServiceException("Could not (re)configure " + id + " due to change in load balancer visibility. The operation will be retried on next deployment"); } - LoadBalancerInstance instance = provisionInstance(id, loadBalancer, zoneEndpoint, requested); - newLoadBalancer = newLoadBalancer.with(instance); + newLoadBalancer = loadBalancer.orElseGet(() -> createNewLoadBalancer(id, zoneEndpoint, requested)); // Determine id-seed. + newLoadBalancer = newLoadBalancer.with(provisionInstance(newLoadBalancer, zoneEndpoint, requested)); // Update instance. } catch (LoadBalancerServiceException e) { log.log(Level.WARNING, "Failed to provision load balancer", e); throw e; @@ -229,55 +229,43 @@ public class LoadBalancerProvisioner { newLoadBalancer.instance().get().settings().isPublicEndpoint() == zoneEndpoint.isPublicEndpoint(); } - private void activate(ApplicationTransaction transaction, ClusterSpec.Id cluster, ZoneEndpoint settings, NodeList nodes) { - Instant now = nodeRepository.clock().instant(); - LoadBalancerId id = new LoadBalancerId(transaction.application(), cluster); - Optional<LoadBalancer> loadBalancer = db.readLoadBalancer(id); - if (loadBalancer.isEmpty()) throw new IllegalArgumentException("Could not activate load balancer that was never prepared: " + id); - if (loadBalancer.get().instance().isEmpty()) throw new IllegalArgumentException("Activating " + id + ", but prepare never provisioned a load balancer instance"); - - try { - LoadBalancerInstance instance = configureInstance(id, nodes, loadBalancer.get(), settings, loadBalancer.get().instance().get().cloudAccount()); - db.writeLoadBalancers(List.of(loadBalancer.get().with(instance).with(State.active, now)), - loadBalancer.get().state(), transaction.nested()); - } catch (LoadBalancerServiceException e) { - db.writeLoadBalancers(List.of(loadBalancer.get()), loadBalancer.get().state(), transaction.nested()); - throw e; - } + /** Creates a new load balancer, with an instance if one is taken from the pool, or without otherwise. */ + private LoadBalancer createNewLoadBalancer(LoadBalancerId id, ZoneEndpoint zoneEndpoint, NodeSpec requested) { + LoadBalancerSpec spec = new LoadBalancerSpec(id.application(), id.cluster(), Set.of(), zoneEndpoint, + requested.cloudAccount(), toSeed(id, requested.type())); + return provisionFromPool(spec, requested.type()) + .orElseGet(() -> new LoadBalancer(id, spec.idSeed(), Optional.empty(), State.reserved, nodeRepository.clock().instant())); } /** Provision a load balancer instance, if necessary */ - private LoadBalancerInstance provisionInstance(LoadBalancerId id, - Optional<LoadBalancer> currentLoadBalancer, + private LoadBalancerInstance provisionInstance(LoadBalancer currentLoadBalancer, ZoneEndpoint zoneEndpoint, NodeSpec requested) { - Set<Real> reals = currentLoadBalancer.flatMap(LoadBalancer::instance) + LoadBalancerId id = currentLoadBalancer.id(); + Set<Real> reals = currentLoadBalancer.instance() .map(LoadBalancerInstance::reals) .orElse(Set.of()); // Targeted reals are changed on activation. ZoneEndpoint settings = new ZoneEndpoint(zoneEndpoint.isPublicEndpoint(), zoneEndpoint.isPrivateEndpoint(), - currentLoadBalancer.flatMap(LoadBalancer::instance) + currentLoadBalancer.instance() .map(LoadBalancerInstance::settings) .map(ZoneEndpoint::allowedUrns) .orElse(List.of())); // Allowed URNs are changed on activation. - if ( currentLoadBalancer.isPresent() - && currentLoadBalancer.get().instance().isPresent() - && currentLoadBalancer.get().instance().get().settings().equals(settings)) - return currentLoadBalancer.get().instance().get(); + if (currentLoadBalancer.instance().map(instance -> settings.equals(instance.settings())).orElse(false)) + return currentLoadBalancer.instance().get(); log.log(Level.INFO, () -> "Provisioning instance for " + id); try { - LoadBalancerSpec spec = new LoadBalancerSpec(id.application(), id.cluster(), reals, settings, requested.cloudAccount()); - return provisionFromPool(spec, requested.type()).orElseGet(() -> service.provision(spec)) - // Provisioning a private endpoint service requires hard resources to be ready, so we delay it until activation. - .withServiceIds(currentLoadBalancer.flatMap(LoadBalancer::instance).map(LoadBalancerInstance::serviceIds).orElse(List.of())); + return service.provision(new LoadBalancerSpec(id.application(), id.cluster(), reals, settings, requested.cloudAccount(), currentLoadBalancer.idSeed())) + // Provisioning a private endpoint service requires hard resources to be ready, so we delay it until activation. + .withServiceIds(currentLoadBalancer.instance().map(LoadBalancerInstance::serviceIds).orElse(List.of())); } catch (Exception e) { throw new LoadBalancerServiceException("Could not provision " + id + ". The operation will be retried on next deployment.", e); } } - private Optional<LoadBalancerInstance> provisionFromPool(LoadBalancerSpec spec, NodeType type) { + private Optional<LoadBalancer> provisionFromPool(LoadBalancerSpec spec, NodeType type) { if (type != NodeType.tenant) return Optional.empty(); if ( ! spec.settings().isDefault()) return Optional.empty(); if (preProvisionPoolSize.value() == 0) return Optional.empty(); @@ -293,9 +281,14 @@ public class LoadBalancerProvisioner { if (chosen.state() != State.active || chosen.instance().isEmpty()) throw new IllegalStateException("expected active load balancer in pre-provisioned pool, but got " + chosen); log.log(Level.INFO, "Using " + chosen + " from pre-provisioned pool"); - service.reallocate(chosen.instance().get(), spec); + service.reallocate(new LoadBalancerSpec(spec.application(), spec.cluster(), spec.reals(), spec.settings(), spec.cloudAccount(), chosen.idSeed())); db.removeLoadBalancer(chosen.id()); // Using a transaction to remove this, and write the instance, would be better, but much hassle. - return chosen.instance(); // Should be immediately written again outside of this! + // Should be immediately written again outside of this! + return Optional.of(new LoadBalancer(new LoadBalancerId(spec.application(), spec.cluster()), + chosen.idSeed(), + chosen.instance(), + State.reserved, + nodeRepository.clock().instant())); } catch (Exception e) { log.log(Level.WARNING, "Failed to provision load balancer from pool", e); @@ -327,9 +320,11 @@ public class LoadBalancerProvisioner { // No need for lock while we provision, since we'll write atomically only after we're done, and the job lock ensures single writer. while (head - tail < size) { ClusterSpec.Id slot = slotId(head); - LoadBalancerSpec spec = preProvisionSpec(slot, nodeRepository.zone().cloud().account()); - db.writeLoadBalancer(new LoadBalancer(new LoadBalancerId(preProvisionOwner, slot), - Optional.of(service.provision(spec, Optional.of(slot.value()))), + LoadBalancerId id = new LoadBalancerId(preProvisionOwner, slot); + LoadBalancerSpec spec = preProvisionSpec(slot, nodeRepository.zone().cloud().account(), toSeed(id)); + db.writeLoadBalancer(new LoadBalancer(id, + spec.idSeed(), + Optional.of(service.provision(spec)), State.active, // Keep the expirer away. nodeRepository.clock().instant()), null); @@ -337,6 +332,38 @@ public class LoadBalancerProvisioner { } } + public static String toSeed(LoadBalancerId id, NodeType type) { + return type == NodeType.tenant ? toSeed(id) : toLegacySeed(id.application(), id.cluster()); + } + + public static String toSeed(LoadBalancerId id) { + return ":" + id.serializedForm() + ":"; // ಠ_ಠ+ } + + public static String toLegacySeed(ApplicationId application, ClusterSpec.Id cluster) { + return application.tenant().value() + + application.application().value() + + application.instance().value() + + cluster.value(); // ಠ_ಠ+ } + + private void activate(ApplicationTransaction transaction, ClusterSpec.Id cluster, ZoneEndpoint settings, NodeList nodes) { + Instant now = nodeRepository.clock().instant(); + LoadBalancerId id = new LoadBalancerId(transaction.application(), cluster); + Optional<LoadBalancer> loadBalancer = db.readLoadBalancer(id); + if (loadBalancer.isEmpty()) throw new IllegalArgumentException("Could not activate load balancer that was never prepared: " + id); + if (loadBalancer.get().instance().isEmpty()) throw new IllegalArgumentException("Activating " + id + ", but prepare never provisioned a load balancer instance"); + + try { + LoadBalancerInstance instance = configureInstance(id, nodes, loadBalancer.get(), settings, loadBalancer.get().instance().get().cloudAccount()); + db.writeLoadBalancers(List.of(loadBalancer.get().with(instance).with(State.active, now)), + loadBalancer.get().state(), transaction.nested()); + } catch (LoadBalancerServiceException e) { + db.writeLoadBalancers(List.of(loadBalancer.get()), loadBalancer.get().state(), transaction.nested()); + throw e; + } + } + /** Reconfigure a load balancer instance, if necessary */ private LoadBalancerInstance configureInstance(LoadBalancerId id, NodeList nodes, LoadBalancer currentLoadBalancer, @@ -349,7 +376,7 @@ public class LoadBalancerProvisioner { log.log(Level.FINE, () -> "Configuring instance for " + id + ", targeting: " + reals); try { return service.configure(currentLoadBalancer.instance().orElseThrow(() -> new IllegalArgumentException("expected existing instance for " + id)), - new LoadBalancerSpec(id.application(), id.cluster(), reals, zoneEndpoint, cloudAccount), + new LoadBalancerSpec(id.application(), id.cluster(), reals, zoneEndpoint, cloudAccount, currentLoadBalancer.idSeed()), shouldDeactivateRouting || currentLoadBalancer.state() != LoadBalancer.State.active); } catch (Exception e) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java index 20202ca7a74..6aafea637aa 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java @@ -25,7 +25,7 @@ public interface ProvisionServiceProvider { } interface ProtoHealthChecker { - Availability healthy(Endpoint endpoint, Optional<String> idSeed); + Availability healthy(Endpoint endpoint, String idSeed); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java index f1a10134c7a..d47f5a4e4d6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java @@ -50,6 +50,7 @@ public class LoadBalancersResponse extends SlimeJsonResponse { loadBalancers.forEach(lb -> { Cursor lbObject = loadBalancerArray.addObject(); lbObject.setString("id", lb.id().serializedForm()); + lbObject.setString("idSeed", lb.idSeed()); lbObject.setString("state", lb.state().name()); lbObject.setLong("changedAt", lb.changedAt().toEpochMilli()); lbObject.setString("application", lb.id().application().application().value()); @@ -90,7 +91,6 @@ public class LoadBalancersResponse extends SlimeJsonResponse { } instance.serviceId().ifPresent(serviceId -> lbObject.setString("serviceId", serviceId.value())); lbObject.setBool("public", instance.settings().isPublicEndpoint()); - instance.idSeed().ifPresent(idSeed -> lbObject.setString("idSeed", idSeed)); }); lb.instance() .map(LoadBalancerInstance::cloudAccount) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java index 7475a3654b7..198709ba4bb 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java @@ -29,7 +29,7 @@ public class SharedLoadBalancerServiceTest { @Test public void test_create_lb() { LoadBalancerSpec spec = new LoadBalancerSpec(applicationId, clusterId, reals, - ZoneEndpoint.defaultEndpoint, CloudAccount.empty); + ZoneEndpoint.defaultEndpoint, CloudAccount.empty, "seed"); var lb = loadBalancerService.configure(loadBalancerService.provision(spec), spec, false); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java index 0d3c1994ad8..9ec63933921 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java @@ -38,8 +38,8 @@ public class LoadBalancerSerializerTest { var now = Instant.now(); { var loadBalancer = new LoadBalancer(loadBalancerId, + "1", Optional.of(new LoadBalancerInstance( - Optional.of("1"), Optional.of(DomainName.of("lb-host")), Optional.empty(), Optional.empty(), @@ -58,9 +58,9 @@ public class LoadBalancerSerializerTest { LoadBalancer.State.active, now); - var serialized = LoadBalancerSerializer.fromJson(LoadBalancerSerializer.toJson(loadBalancer)); + var serialized = LoadBalancerSerializer.fromJson(loadBalancer.id(), LoadBalancerSerializer.toJson(loadBalancer)); assertEquals(loadBalancer.id(), serialized.id()); - assertEquals(loadBalancer.instance().get().idSeed(), serialized.instance().get().idSeed()); + assertEquals(loadBalancer.idSeed(), serialized.idSeed()); assertEquals(loadBalancer.instance().get().hostname(), serialized.instance().get().hostname()); assertEquals(loadBalancer.instance().get().dnsZone(), serialized.instance().get().dnsZone()); assertEquals(loadBalancer.instance().get().ports(), serialized.instance().get().ports()); @@ -74,9 +74,9 @@ public class LoadBalancerSerializerTest { } { var loadBalancer = new LoadBalancer(loadBalancerId, + "", Optional.of(new LoadBalancerInstance( Optional.empty(), - Optional.empty(), Optional.of("1.2.3.4"), Optional.of("fd00::1"), Optional.of(new DnsZone("zone-id-1")), @@ -89,9 +89,9 @@ public class LoadBalancerSerializerTest { LoadBalancer.State.active, now); - var serialized = LoadBalancerSerializer.fromJson(LoadBalancerSerializer.toJson(loadBalancer)); + var serialized = LoadBalancerSerializer.fromJson(loadBalancer.id(), LoadBalancerSerializer.toJson(loadBalancer)); assertEquals(loadBalancer.id(), serialized.id()); - assertEquals(loadBalancer.instance().get().idSeed(), serialized.instance().get().idSeed()); + assertEquals(loadBalancer.idSeed(), serialized.idSeed()); assertEquals(loadBalancer.instance().get().hostname(), serialized.instance().get().hostname()); assertEquals(loadBalancer.instance().get().ip4Address(), serialized.instance().get().ip4Address()); assertEquals(loadBalancer.instance().get().ip6Address(), serialized.instance().get().ip6Address()); @@ -110,10 +110,11 @@ public class LoadBalancerSerializerTest { @Test public void no_instance_serialization() { var now = Instant.now(); - var loadBalancer = new LoadBalancer(loadBalancerId, Optional.empty(), LoadBalancer.State.reserved, now); + var loadBalancer = new LoadBalancer(loadBalancerId, "seed", Optional.empty(), LoadBalancer.State.reserved, now); - var serialized = LoadBalancerSerializer.fromJson(LoadBalancerSerializer.toJson(loadBalancer)); + var serialized = LoadBalancerSerializer.fromJson(loadBalancerId, LoadBalancerSerializer.toJson(loadBalancer)); assertEquals(loadBalancer.id(), serialized.id()); + assertEquals(loadBalancer.idSeed(), serialized.idSeed()); assertEquals(loadBalancer.instance(), serialized.instance()); assertEquals(loadBalancer.state(), serialized.state()); assertEquals(loadBalancer.changedAt().truncatedTo(MILLIS), serialized.changedAt()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index fb59b3077f8..7096ea3de46 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -50,6 +50,7 @@ import java.util.SortedSet; import java.util.function.Supplier; import java.util.stream.Collectors; +import static com.yahoo.vespa.hosted.provision.lb.LoadBalancerSpec.preProvisionOwner; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertSame; @@ -190,15 +191,15 @@ public class LoadBalancerProvisionerTest { provisioner.refreshPool(); expirer.run(); assertEquals(2, tester.nodeRepository().loadBalancers().list().size()); - assertEquals(2, tester.nodeRepository().loadBalancers().list(LoadBalancerSpec.preProvisionOwner).size()); + assertEquals(2, tester.nodeRepository().loadBalancers().list(preProvisionOwner).size()); // Provision a load balancer when the pool has two entries. ClusterSpec.Id containerCluster = ClusterSpec.Id.from("qrs"); prepare(app1, clusterRequest(ClusterSpec.Type.container, containerCluster)); List<LoadBalancer> loadBalancers = tester.nodeRepository().loadBalancers().list(app1).asList(); assertEquals(1, loadBalancers.size()); - assertEquals(1, tester.nodeRepository().loadBalancers().list(LoadBalancerSpec.preProvisionOwner).asList().size()); - assertEquals(Optional.of("1"), loadBalancers.get(0).instance().get().idSeed()); + assertEquals(1, tester.nodeRepository().loadBalancers().list(preProvisionOwner).asList().size()); + assertEquals(":" + preProvisionOwner.serializedForm() + ":1:", loadBalancers.get(0).idSeed()); // Shrink pool to 0 entries. flagSource.withIntFlag(PermanentFlags.PRE_PROVISIONED_LB_COUNT.id(), 0); @@ -214,9 +215,9 @@ public class LoadBalancerProvisionerTest { assertThrows(IllegalStateException.class, provisioner::refreshPool).getMessage()); tester.loadBalancerService().throwOnCreate(false); provisioner.refreshPool(); - assertEquals(List.of(Optional.of("3")), - tester.nodeRepository().loadBalancers().list(LoadBalancerSpec.preProvisionOwner) - .mapToList(lb -> lb.instance().get().idSeed())); + assertEquals(List.of(":" + preProvisionOwner.serializedForm() + ":3:"), + tester.nodeRepository().loadBalancers().list(preProvisionOwner) + .mapToList(LoadBalancer::idSeed)); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json index 620275bb033..ee8a8588c5f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers-single.json @@ -7,7 +7,7 @@ "dnsZone": "zone-id-1", "hostname": "lb-hosted-vespa.pre-provision-1", "id": "tenant4:application4:instance4:id4", - "idSeed": "1", + "idSeed": ":hosted-vespa:pre-provision:default:1:", "instance": "instance4", "networks": [ "10.2.3.0/24", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json index e8392d92522..6de3b500a00 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/load-balancers.json @@ -7,6 +7,7 @@ "dnsZone": "zone-id-1", "hostname": "lb-tenant1.application1.instance1-id1", "id": "tenant1:application1:instance1:id1", + "idSeed": ":tenant1:application1:instance1:id1:", "instance": "instance1", "networks": [ "10.2.3.0/24", @@ -47,6 +48,7 @@ "dnsZone": "zone-id-1", "hostname": "lb-hosted-vespa.zone-config-servers-zone-config-servers", "id": "hosted-vespa:zone-config-servers:default:zone-config-servers", + "idSeed": "hosted-vespazone-config-serversdefaultzone-config-servers", "instance": "default", "networks": [ "10.2.3.0/24", @@ -78,7 +80,7 @@ "dnsZone": "zone-id-1", "hostname": "lb-hosted-vespa.pre-provision-1", "id": "tenant4:application4:instance4:id4", - "idSeed": "1", + "idSeed": ":hosted-vespa:pre-provision:default:1:", "instance": "instance4", "networks": [ "10.2.3.0/24", diff --git a/parent/pom.xml b/parent/pom.xml index aeb0c9c4aa9..c6ae9cdc7e3 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -237,7 +237,7 @@ <plugin> <groupId>org.codehaus.mojo</groupId> <artifactId>exec-maven-plugin</artifactId> - <version>3.1.0</version> + <version>3.1.1</version> </plugin> <plugin> <groupId>org.codehaus.mojo</groupId> @@ -317,7 +317,7 @@ --> <groupId>org.openrewrite.maven</groupId> <artifactId>rewrite-maven-plugin</artifactId> - <version>5.10.0</version> + <version>5.13.0</version> <configuration> <activeRecipes> <recipe>org.openrewrite.java.testing.junit5.JUnit5BestPractices</recipe> @@ -1126,7 +1126,7 @@ See pluginManagement of rewrite-maven-plugin for more details --> <groupId>org.openrewrite.recipe</groupId> <artifactId>rewrite-recipe-bom</artifactId> - <version>2.4.1</version> + <version>2.5.0</version> <type>pom</type> <scope>import</scope> </dependency> diff --git a/screwdriver.yaml b/screwdriver.yaml index dd9fa5cba8b..c6707e2957f 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -6,7 +6,7 @@ cache: component-build: [/main_job_cache] shared: - image: vespaengine/vespa-build-centos-stream8:latest + image: docker.io/vespaengine/vespa-build-almalinux-8 environment: USER_SHELL_BIN: bash annotations: @@ -199,13 +199,13 @@ jobs: fi docker build --file Dockerfile.systemtest \ --progress=plain \ - --build-arg VESPA_BASE_IMAGE=vespaengine/vespa-systemtest-base-centos-stream8:latest \ + --build-arg VESPA_BASE_IMAGE=docker.io/vespaengine/vespa-systemtest-base-el8:latest \ --build-arg SYSTEMTEST_BASE_IMAGE=vespa --build-arg SKIP_M2_POPULATE=$SKIP_M2_POPULATE \ --target systemtest \ - --tag vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION \ - --tag vespaengine/vespa-systemtest-centos-stream8:latest . + --tag docker.io/vespaengine/vespa-systemtest-el8:$VESPA_VERSION \ + --tag docker.io/vespaengine/vespa-systemtest-el8:latest . - verify-test-image: | - docker run --rm -ti --entrypoint bash vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION -lc \ + docker run --rm -ti --entrypoint bash docker.io/vespaengine/vespa-systemtest-el8:$VESPA_VERSION -lc \ "/opt/vespa-systemtests/lib/node_server.rb & sleep 3 && ruby /opt/vespa-systemtests/tests/search/basicsearch/basic_search.rb --run test_basicsearch__ELASTIC" - publish-test-image: | if [[ -z "$SD_PULL_REQUEST" ]]; then @@ -213,8 +213,8 @@ jobs: set +x docker login --username aressem --password "$DOCKER_HUB_DEPLOY_KEY" eval "$OPT_STATE" - docker push docker.io/vespaengine/vespa-systemtest-centos-stream8:$VESPA_VERSION - docker push docker.io/vespaengine/vespa-systemtest-centos-stream8:latest + docker push docker.io/vespaengine/vespa-systemtest-el8:$VESPA_VERSION + docker push docker.io/vespaengine/vespa-systemtest-el8:latest fi - *save-cache - update-build-status: | @@ -233,8 +233,6 @@ jobs: fi publish-release: - image: docker.io/vespaengine/vespa-build-centos-stream8:latest - annotations: screwdriver.cd/cpu: 7 screwdriver.cd/ram: 16 @@ -278,7 +276,7 @@ jobs: meta set vespa.version $VESPA_VERSION - install-dependencies: | dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo - dnf install -y docker-ce docker-ce-cli containerd.io + dnf install -y docker-ce docker-ce-cli containerd.io python3 python3-pip docker system info - release-java-artifacts: | screwdriver/release-java-artifacts.sh $VESPA_VERSION $VESPA_REF @@ -294,8 +292,6 @@ jobs: screwdriver/factory-command.sh update-released-time $VESPA_VERSION publish-legacy-release: - image: docker.io/vespaengine/vespa-build-centos-stream8:latest - annotations: screwdriver.cd/cpu: 7 screwdriver.cd/ram: 16 @@ -307,7 +303,7 @@ jobs: screwdriver.cd/buildPeriodically: H 6 1 * * environment: - IMAGE_NAME: "vespaengine/vespa-generic-intel-x86_64" + IMAGE_NAME: "docker.io/vespaengine/vespa-generic-intel-x86_64" secrets: - DOCKER_HUB_DEPLOY_KEY @@ -374,7 +370,6 @@ jobs: fi publish-el9-preview: - image: docker.io/vespaengine/vespa-build-centos-stream8:latest annotations: screwdriver.cd/cpu: 7 screwdriver.cd/ram: 16 @@ -422,8 +417,8 @@ jobs: --build-arg VESPA_BASE_IMAGE=$BASE_IMAGE \ --build-arg VESPA_VERSION=$VESPA_VERSION \ --file Dockerfile \ - --tag vespaengine/vespa:latest \ - --tag vespaengine/$IMAGE_NAME:latest \ + --tag docker.io/vespaengine/vespa:latest \ + --tag docker.io/vespaengine/$IMAGE_NAME:latest \ . - verify-container-image: | # Run quick start guide @@ -452,7 +447,6 @@ jobs: fi publish-el8-preview: - image: docker.io/vespaengine/vespa-build-centos-stream8:latest annotations: screwdriver.cd/cpu: 7 screwdriver.cd/ram: 16 @@ -529,7 +523,7 @@ jobs: dnf install -y vespa mirror-copr-rpms-to-archive: - image: quay.io/centos/centos:stream8 + image: docker.io/almalinux:8 annotations: screwdriver.cd/cpu: LOW screwdriver.cd/ram: LOW @@ -559,7 +553,7 @@ jobs: screwdriver/delete-old-cloudsmith-artifacts.sh mirror-copr-rpms-to-artifactory: - image: quay.io/centos/centos:stream8 + image: docker.io/almalinux:8 annotations: screwdriver.cd/cpu: LOW screwdriver.cd/ram: LOW diff --git a/screwdriver/test-quick-start-guide.sh b/screwdriver/test-quick-start-guide.sh index eead39fecaa..4c3feb35a21 100755 --- a/screwdriver/test-quick-start-guide.sh +++ b/screwdriver/test-quick-start-guide.sh @@ -13,7 +13,7 @@ cd $TESTDIR # Clone and setup doc tests git clone -q --depth 1 https://github.com/vespa-engine/documentation cd documentation -python3.9 -m pip install -qqq -r test/requirements.txt --user +python3 -m pip install -qqq -r test/requirements.txt --user echo -e "urls:\n - en/vespa-quick-start.html" > test/_quick-start.yaml # Get the required vespa CLI @@ -22,5 +22,5 @@ curl -fsSL https://github.com/vespa-engine/vespa/releases/download/v${VESPA_CLI_ ln -sf /opt/vespa-cli_${VESPA_CLI_VERSION}_linux_amd64/bin/vespa /usr/local/bin/ # Run test -python3.9 test/test.py -v -c test/_quick-start.yaml +python3 test/test.py -v -c test/_quick-start.yaml diff --git a/searchcore/src/tests/grouping/grouping_test.cpp b/searchcore/src/tests/grouping/grouping_test.cpp index a2f646cba3a..fb46fb65f3b 100644 --- a/searchcore/src/tests/grouping/grouping_test.cpp +++ b/searchcore/src/tests/grouping/grouping_test.cpp @@ -45,7 +45,7 @@ struct MyWorld { bv.setInterval(0, NUM_DOCS); // attribute context { - SingleInt32ExtAttribute *attr = new SingleInt32ExtAttribute("attr0"); + auto *attr = new SingleInt32ExtAttribute("attr0"); AttributeVector::DocId docid; for (uint32_t i = 0; i < NUM_DOCS; ++i) { attr->addDoc(docid); @@ -55,7 +55,7 @@ struct MyWorld { attributeContext.add(attr); } { - SingleInt32ExtAttribute *attr = new SingleInt32ExtAttribute("attr1"); + auto *attr = new SingleInt32ExtAttribute("attr1"); AttributeVector::DocId docid; for (uint32_t i = 0; i < NUM_DOCS; ++i) { attr->addDoc(docid); @@ -65,7 +65,7 @@ struct MyWorld { attributeContext.add(attr); } { - SingleInt32ExtAttribute *attr = new SingleInt32ExtAttribute("attr2"); + auto *attr = new SingleInt32ExtAttribute("attr2"); AttributeVector::DocId docid; for (uint32_t i = 0; i < NUM_DOCS; ++i) { attr->addDoc(docid); @@ -75,7 +75,7 @@ struct MyWorld { attributeContext.add(attr); } { - SingleInt32ExtAttribute *attr = new SingleInt32ExtAttribute("attr3"); + auto *attr = new SingleInt32ExtAttribute("attr3"); AttributeVector::DocId docid; for (uint32_t i = 0; i < NUM_DOCS; ++i) { attr->addDoc(docid); @@ -94,16 +94,17 @@ using GroupingList = GroupingContext::GroupingList; SessionId createSessionId(const std::string & s) { std::vector<char> vec; - for (size_t i = 0; i < s.size(); i++) { - vec.push_back(s[i]); + for (char c : s) { + vec.push_back(c); } - return SessionId(&vec[0], vec.size()); + return {vec.data(), vec.size()}; } class CheckAttributeReferences : public vespalib::ObjectOperation, public vespalib::ObjectPredicate { public: - CheckAttributeReferences(bool log=false) : _log(log), _numrefs(0) { } + CheckAttributeReferences() : CheckAttributeReferences(false) {} + explicit CheckAttributeReferences(bool log) : _log(log), _numrefs(0) { } bool _log; uint32_t _numrefs; private: @@ -177,7 +178,7 @@ TEST_F("testGroupingContextInitialization", DoomFixture()) { baseRequest.serialize(nos); AllocatedBitVector bv(1); - GroupingContext context(bv, f1.clock.clock(), f1.timeOfDoom, os.data(), os.size(), true); + GroupingContext context(bv, f1.clock.clock(), f1.timeOfDoom, os.data(), os.size()); ASSERT_TRUE(!context.empty()); GroupingContext::GroupingList list = context.getGroupingList(); ASSERT_TRUE(list.size() == 1); @@ -303,8 +304,8 @@ TEST_F("testGroupingSession", DoomFixture()) { manager.groupInRelevanceOrder(&hit, 1); CheckAttributeReferences attrCheck_after; GroupingList &gl3(initContext.getGroupingList()); - for (unsigned int i = 0; i < gl3.size(); i++) { - gl3[i]->select(attrCheck_after, attrCheck_after); + for (auto & grouping : gl3) { + grouping->select(attrCheck_after, attrCheck_after); } EXPECT_EQUAL(attrCheck_after._numrefs, 0u); { @@ -423,9 +424,9 @@ void doGrouping(GroupingContext &ctx, { GroupingManager man(ctx); std::vector<RankedHit> hits; - hits.push_back(RankedHit(doc1, rank1)); - hits.push_back(RankedHit(doc2, rank2)); - hits.push_back(RankedHit(doc3, rank3)); + hits.emplace_back(doc1, rank1); + hits.emplace_back(doc2, rank2); + hits.emplace_back(doc3, rank3); man.groupInRelevanceOrder(&hits[0], 3); } diff --git a/searchcore/src/tests/proton/matching/match_phase_limiter/match_phase_limiter_test.cpp b/searchcore/src/tests/proton/matching/match_phase_limiter/match_phase_limiter_test.cpp index 21c572995d3..b26ed1d4765 100644 --- a/searchcore/src/tests/proton/matching/match_phase_limiter/match_phase_limiter_test.cpp +++ b/searchcore/src/tests/proton/matching/match_phase_limiter/match_phase_limiter_test.cpp @@ -352,6 +352,7 @@ TEST("require that the match phase limiter is able to pre-limit the query") { " hit_rate: 0.1," " num_docs: 100000," " max_filter_docs: 100000," + " upper_limited_corpus_size: 100000," " wanted_docs: 5000," " action: 'Will limit with prefix filter'," " max_group_size: 5000," diff --git a/searchcore/src/tests/proton/matching/unpacking_iterators_optimizer/unpacking_iterators_optimizer_test.cpp b/searchcore/src/tests/proton/matching/unpacking_iterators_optimizer/unpacking_iterators_optimizer_test.cpp index 2ef8e3dcbb9..2e1a28402a1 100644 --- a/searchcore/src/tests/proton/matching/unpacking_iterators_optimizer/unpacking_iterators_optimizer_test.cpp +++ b/searchcore/src/tests/proton/matching/unpacking_iterators_optimizer/unpacking_iterators_optimizer_test.cpp @@ -80,7 +80,7 @@ std::string dump_query(Node &root) { root.accept(dumper); } auto mem = buffer.obtain(); - return std::string(mem.data, mem.size); + return {mem.data, mem.size}; } namespace { @@ -254,26 +254,44 @@ std::string split_query_tree_dump = " Term a cheap\n" " Term b cheap\n" " Term c cheap\n"; +std::string split_query_tree_dump_always_expensive = + "And 7\n" + " Or 3\n" + " Term t2\n" + " Phrase 3 expensive\n" + " Term a\n" + " Term b\n" + " Term c\n" + " Term x1\n" + " Term x2\n" + " Phrase 3 expensive\n" + " Term a\n" + " Term b\n" + " Term c\n" + " Term t1\n" + " Term a cheap\n" + " Term b cheap\n" + " Term c cheap\n"; #endif //----------------------------------------------------------------------------- -Node::UP optimize(Node::UP root, bool white_list, bool split) { - return UnpackingIteratorsOptimizer::optimize(std::move(root), white_list, split); +Node::UP optimize(Node::UP root, bool white_list) { + return UnpackingIteratorsOptimizer::optimize(std::move(root), white_list, false); +} + +Node::UP optimize(Node::UP root, bool white_list, bool always_mark_phrase_expensive) { + return UnpackingIteratorsOptimizer::optimize(std::move(root), white_list, always_mark_phrase_expensive); } TEST(UnpackingIteratorsOptimizerTest, require_that_root_phrase_node_can_be_left_alone) { - std::string actual1 = dump_query(*optimize(make_phrase(), false, false)); - std::string actual2 = dump_query(*optimize(make_phrase(), false, true)); - std::string actual3 = dump_query(*optimize(make_phrase(), true, false)); + std::string actual1 = dump_query(*optimize(make_phrase(), false)); std::string expect = plain_phrase_dump; EXPECT_EQ(actual1, expect); - EXPECT_EQ(actual2, expect); - EXPECT_EQ(actual3, expect); } TEST(UnpackingIteratorsOptimizerTest, require_that_root_phrase_node_can_be_split) { - std::string actual1 = dump_query(*optimize(make_phrase(), true, true)); + std::string actual1 = dump_query(*optimize(make_phrase(), true)); std::string expect = split_phrase_dump; EXPECT_EQ(actual1, expect); } @@ -281,13 +299,9 @@ TEST(UnpackingIteratorsOptimizerTest, require_that_root_phrase_node_can_be_split //----------------------------------------------------------------------------- TEST(UnpackingIteratorsOptimizerTest, require_that_root_same_element_node_can_be_left_alone) { - std::string actual1 = dump_query(*optimize(make_same_element(), false, false)); - std::string actual2 = dump_query(*optimize(make_same_element(), false, true)); - std::string actual3 = dump_query(*optimize(make_same_element(), true, false)); + std::string actual1 = dump_query(*optimize(make_same_element(), false)); std::string expect = plain_same_element_dump; EXPECT_EQ(actual1, expect); - EXPECT_EQ(actual2, expect); - EXPECT_EQ(actual3, expect); } #if ENABLE_SAME_ELEMENT_SPLIT @@ -301,20 +315,22 @@ TEST(UnpackingIteratorsOptimizerTest, require_that_root_same_element_node_can_be //----------------------------------------------------------------------------- -TEST(UnpackingIteratorsOptimizerTest, require_that_query_tree_can_be_left_alone) { - std::string actual1 = dump_query(*optimize(make_query_tree(), false, false)); - std::string actual2 = dump_query(*optimize(make_query_tree(), true, false)); - std::string expect = plain_query_tree_dump; +TEST(UnpackingIteratorsOptimizerTest, require_that_query_tree_can_be_split) { + std::string actual1 = dump_query(*optimize(make_query_tree(), false)); + std::string actual2 = dump_query(*optimize(make_query_tree(), true)); + std::string expect = split_query_tree_dump; EXPECT_EQ(actual1, expect); EXPECT_EQ(actual2, expect); } -TEST(UnpackingIteratorsOptimizerTest, require_that_query_tree_can_be_split) { - std::string actual1 = dump_query(*optimize(make_query_tree(), false, true)); - std::string actual2 = dump_query(*optimize(make_query_tree(), true, true)); +TEST(UnpackingIteratorsOptimizerTest, require_that_query_tree_can_be_split_always) { + std::string actual1 = dump_query(*optimize(make_query_tree(), false, false)); + std::string actual2 = dump_query(*optimize(make_query_tree(), true, false)); + std::string actual3 = dump_query(*optimize(make_query_tree(), true, true)); std::string expect = split_query_tree_dump; EXPECT_EQ(actual1, expect); EXPECT_EQ(actual2, expect); + EXPECT_EQ(actual3, split_query_tree_dump_always_expensive); } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp b/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp index 63127d01450..882a4d1509d 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp +++ b/searchcore/src/vespa/searchcore/grouping/groupingcontext.cpp @@ -2,7 +2,6 @@ #include "groupingcontext.h" #include <vespa/searchlib/aggregation/predicates.h> -#include <vespa/searchlib/aggregation/modifiers.h> #include <vespa/searchlib/aggregation/hitsaggregationresult.h> #include <vespa/searchlib/common/bitvector.h> @@ -53,13 +52,12 @@ GroupingContext::setDistributionKey(uint32_t distributionKey) } GroupingContext::GroupingContext(const BitVector & validLids, const vespalib::Clock & clock, vespalib::steady_time timeOfDoom, - const char *groupSpec, uint32_t groupSpecLen, bool enableNested) + const char *groupSpec, uint32_t groupSpecLen) : _validLids(validLids), _clock(clock), _timeOfDoom(timeOfDoom), _os(), - _groupingList(), - _enableNestedMultivalueGrouping(enableNested) + _groupingList() { deserialize(groupSpec, groupSpecLen); } @@ -69,8 +67,7 @@ GroupingContext::GroupingContext(const BitVector & validLids, const vespalib::Cl _clock(clock), _timeOfDoom(timeOfDoom), _os(), - _groupingList(), - _enableNestedMultivalueGrouping(true) + _groupingList() { } GroupingContext::GroupingContext(const GroupingContext & rhs) @@ -78,8 +75,7 @@ GroupingContext::GroupingContext(const GroupingContext & rhs) _clock(rhs._clock), _timeOfDoom(rhs._timeOfDoom), _os(), - _groupingList(), - _enableNestedMultivalueGrouping(rhs._enableNestedMultivalueGrouping) + _groupingList() { } void @@ -100,7 +96,7 @@ GroupingContext::serialize() } bool -GroupingContext::needRanking() const +GroupingContext::needRanking() const noexcept { if (_groupingList.empty()) { return false; diff --git a/searchcore/src/vespa/searchcore/grouping/groupingcontext.h b/searchcore/src/vespa/searchcore/grouping/groupingcontext.h index c29b5122f74..e1b296df99b 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingcontext.h +++ b/searchcore/src/vespa/searchcore/grouping/groupingcontext.h @@ -34,7 +34,7 @@ public: * @param groupSpecLen The length of the grouping specification, in bytes. **/ GroupingContext(const BitVector & validLids, const vespalib::Clock & clock, vespalib::steady_time timeOfDoom, - const char *groupSpec, uint32_t groupSpecLen, bool enableNestedMultivalueGrouping); + const char *groupSpec, uint32_t groupSpecLen); /** * Create a new grouping context from a byte buffer. @@ -105,9 +105,7 @@ public: * Figure out if ranking is necessary for any of the grouping requests here. * @return true if ranking is required. */ - bool needRanking() const; - bool enableNestedMultivalueGrouping() const noexcept { return _enableNestedMultivalueGrouping; } - const search::BitVector & getValidLids() const { return _validLids; } + bool needRanking() const noexcept; void groupUnordered(const RankedHit *searchResults, uint32_t binSize, const search::BitVector * overflow); void groupInRelevanceOrder(const RankedHit *searchResults, uint32_t binSize); @@ -123,7 +121,6 @@ private: vespalib::steady_time _timeOfDoom; vespalib::nbostream _os; GroupingList _groupingList; - bool _enableNestedMultivalueGrouping; }; } diff --git a/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp b/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp index f5bcc935cb1..799d2663589 100644 --- a/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp +++ b/searchcore/src/vespa/searchcore/grouping/groupingmanager.cpp @@ -1,7 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "groupingmanager.h" -#include "groupingsession.h" #include "groupingcontext.h" #include <vespa/searchlib/aggregation/fs4hit.h> #include <vespa/searchlib/expression/attributenode.h> @@ -52,11 +51,11 @@ GroupingManager::init(const IAttributeContext &attrCtx) ExpressionNode & en = *level.getExpression().getRoot(); if (en.inherits(AttributeNode::classId)) { - AttributeNode & an = static_cast<AttributeNode &>(en); + auto & an = static_cast<AttributeNode &>(en); an.enableEnumOptimization(true); } } - ConfigureStaticParams stuff(&attrCtx, nullptr, _groupingContext.enableNestedMultivalueGrouping()); + ConfigureStaticParams stuff(&attrCtx, nullptr); grouping.configureStaticStuff(stuff); list.push_back(groupingList[i]); } catch (const std::exception & e) { @@ -96,10 +95,9 @@ void GroupingManager::prune() { GroupingContext::GroupingList &groupingList(_groupingContext.getGroupingList()); - for (size_t i = 0; i < groupingList.size(); ++i) { - Grouping &g = *groupingList[i]; - g.postMerge(); - g.sortById(); + for (const auto & g : groupingList) { + g->postMerge(); + g->sortById(); } } @@ -107,10 +105,9 @@ void GroupingManager::convertToGlobalId(const search::IDocumentMetaStore &metaStore) { GroupingContext::GroupingList & groupingList = _groupingContext.getGroupingList(); - for (size_t i = 0; i < groupingList.size(); ++i) { - Grouping & g = *groupingList[i]; - g.convertToGlobalId(metaStore); - LOG(debug, "convertToGlobalId: %s", g.asString().c_str()); + for (const auto & g : groupingList) { + g->convertToGlobalId(metaStore); + LOG(debug, "convertToGlobalId: %s", g->asString().c_str()); } } diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp index efe03c7b57e..1b891baa5c9 100644 --- a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.cpp @@ -75,7 +75,7 @@ MatchEngine::close() LOG(debug, "Closing search interface."); { std::lock_guard<std::mutex> guard(_lock); - _closed = true; + _closed.store(true, std::memory_order_relaxed); } LOG(debug, "Handshaking with task manager."); @@ -108,7 +108,7 @@ SearchReply::UP MatchEngine::search(SearchRequest::Source request, SearchClient &client) { // We continue to allow searches if the node is in Maintenance mode - if (_closed || (!_nodeUp && !_nodeMaintenance.load(std::memory_order_relaxed))) { + if (_closed.load(std::memory_order_relaxed) || (!_nodeUp && !_nodeMaintenance.load(std::memory_order_relaxed))) { auto ret = std::make_unique<SearchReply>(); ret->setDistributionKey(_distributionKey); diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h index 9d4bfc5cce5..5365cb6a290 100644 --- a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h +++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h @@ -20,7 +20,7 @@ private: std::mutex _lock; const uint32_t _distributionKey; bool _async; - bool _closed; + std::atomic<bool> _closed; std::atomic<bool> _forward_issues; HandlerMap<ISearchHandler> _handlers; vespalib::ThreadStackExecutor _executor; diff --git a/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp b/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp index 4dec253946f..d07169a0d63 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.cpp @@ -76,19 +76,19 @@ toString(AttributeLimiter::DiversityCutoffStrategy strategy) return (strategy == AttributeLimiter::DiversityCutoffStrategy::STRICT) ? STRICT_STR : LOOSE_STR; } -std::unique_ptr<SearchIterator> -AttributeLimiter::create_search(size_t want_hits, size_t max_group_size, double hit_rate, bool strictSearch) -{ +AttributeLimiter::BlueprintAndMatchData +AttributeLimiter::create_match_data(size_t want_hits, size_t max_group_size, double hit_rate, bool strictSearch) { std::lock_guard<std::mutex> guard(_lock); const uint32_t my_field_id = 0; search::fef::MatchDataLayout layout; auto my_handle = layout.allocTermField(my_field_id); - if ( ! _blueprint ) { + if (!_blueprint) { RangeLimitMetaInfo rangeInfo = _rangeQueryLocator.locate(); const uint32_t no_unique_id = 0; - string range_spec = fmt("[%s;%s;%s%zu", rangeInfo.low().c_str(), rangeInfo.high().c_str(), _descending? "-" : "", want_hits); + string range_spec = fmt("[%s;%s;%s%zu", rangeInfo.low().c_str(), rangeInfo.high().c_str(), + _descending ? "-" : "", want_hits); if (max_group_size < want_hits) { - size_t cutoffGroups = (_diversityCutoffFactor*want_hits)/max_group_size; + size_t cutoffGroups = (_diversityCutoffFactor * want_hits) / max_group_size; range_spec.append(fmt(";%s;%zu;%zu;%s]", _diversity_attribute.c_str(), max_group_size, cutoffGroups, toString(_diversityCutoffStrategy).c_str())); } else { @@ -99,12 +99,19 @@ AttributeLimiter::create_search(size_t want_hits, size_t max_group_size, double FieldSpecList field; // single field API is protected field.add(FieldSpec(_attribute_name, my_field_id, my_handle)); _blueprint = _searchable_attributes.createBlueprint(_requestContext, field, node); - _blueprint->fetchPostings(ExecuteInfo::create(strictSearch, strictSearch ? 1.0F : hit_rate, &_requestContext.getDoom())); + auto execInfo = ExecuteInfo::create(strictSearch, strictSearch ? 1.0F : hit_rate, &_requestContext.getDoom()); + _blueprint->fetchPostings(execInfo); _estimatedHits.store(_blueprint->getState().estimate().estHits, std::memory_order_relaxed); _blueprint->freeze(); } _match_datas.push_back(layout.createMatchData()); - return _blueprint->createSearch(*_match_datas.back(), strictSearch); + return {*_blueprint, *_match_datas.back()}; +} + +std::unique_ptr<SearchIterator> +AttributeLimiter::create_search(size_t want_hits, size_t max_group_size, double hit_rate, bool strictSearch) { + auto [blueprint, match_data] = create_match_data(want_hits, max_group_size, hit_rate, strictSearch); + return blueprint.createSearch(match_data, strictSearch); } } diff --git a/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.h b/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.h index df0acccbd7a..a2c73b63bee 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.h +++ b/searchcore/src/vespa/searchcore/proton/matching/attribute_limiter.h @@ -46,6 +46,8 @@ public: ssize_t getEstimatedHits() const; static DiversityCutoffStrategy toDiversityCutoffStrategy(vespalib::stringref strategy); private: + using BlueprintAndMatchData = std::pair<search::queryeval::Blueprint &, search::fef::MatchData &>; + BlueprintAndMatchData create_match_data(size_t want_hits, size_t max_group_size, double hit_rate, bool strictSearch); search::queryeval::Searchable & _searchable_attributes; const search::queryeval::IRequestContext & _requestContext; const RangeQueryLocator & _rangeQueryLocator; diff --git a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp index fbeabd89e55..cbdf5890094 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/blueprintbuilder.cpp @@ -76,7 +76,7 @@ private: } void buildWeakAnd(ProtonWeakAnd &n) { - WeakAndBlueprint *wand = new WeakAndBlueprint(n.getMinHits()); + WeakAndBlueprint *wand = new WeakAndBlueprint(n.getTargetNumHits()); Blueprint::UP result(wand); for (size_t i = 0; i < n.getChildren().size(); ++i) { search::query::Node &node = *n.getChildren()[i]; diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_phase_limiter.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_phase_limiter.cpp index b64d5ba4c05..784ce649c5f 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_phase_limiter.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_phase_limiter.cpp @@ -102,6 +102,10 @@ do_limit(AttributeLimiter &limiter_factory, SearchIterator::UP search, double ma return search; } +// When hitrate is below 1% limiting the query is often far more expensive than not. +// TODO This limit should probably be a lot higher. +constexpr double MIN_HIT_RATE_LIMIT = 0.01; + } // namespace proton::matching::<unnamed> SearchIterator::UP @@ -114,11 +118,16 @@ MatchPhaseLimiter::maybe_limit(SearchIterator::UP search, double match_freq, siz trace->setDouble("hit_rate", match_freq); trace->setLong("num_docs", num_docs); trace->setLong("max_filter_docs", max_filter_docs); + trace->setLong("upper_limited_corpus_size", upper_limited_corpus_size); trace->setLong("wanted_docs", wanted_num_docs); } - if (upper_limited_corpus_size <= wanted_num_docs) { + if ((upper_limited_corpus_size <= wanted_num_docs) || (match_freq < MIN_HIT_RATE_LIMIT)) { if (trace) { - trace->setString("action", "Will not limit !"); + if (upper_limited_corpus_size <= wanted_num_docs) { + trace->setString("action", "Will not limit due to upper_limited_corpus_size <= wanted_num_docs"); + } else if (match_freq < MIN_HIT_RATE_LIMIT) { + trace->setString("action", "Will not limit due to match_freq < MIN_HIT_RATE_LIMIT(1%)"); + } } LOG(debug, "Will not limit ! maybe_limit(hit_rate=%g, num_docs=%ld, max_filter_docs=%ld) = wanted_num_docs=%ld", match_freq, num_docs, max_filter_docs, wanted_num_docs); diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp index c6f3e215329..7beecaca613 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp @@ -193,7 +193,8 @@ MatchToolsFactory(QueryLimiter & queryLimiter, trace.addEvent(4, "Start query setup"); _query.setWhiteListBlueprint(metaStore.createWhiteListBlueprint()); trace.addEvent(5, "Deserialize and build query tree"); - _valid = _query.buildTree(queryStack, location, viewResolver, indexEnv, true); + _valid = _query.buildTree(queryStack, location, viewResolver, indexEnv, + AlwaysMarkPhraseExpensive::check(_queryEnv.getProperties(), rankSetup.always_mark_phrase_expensive())); if (_valid) { _query.extractTerms(_queryEnv.terms()); _query.extractLocations(_queryEnv.locations()); diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp index 3c8c90a229d..3e8909aa593 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp @@ -249,7 +249,7 @@ Matcher::match(const SearchRequest &request, vespalib::ThreadBundle &threadBundl { // we want to measure full set-up and tear-down time as part of // collateral time GroupingContext groupingContext(metaStore.getValidLids(), _clock, request.getTimeOfDoom(), - request.groupSpec.data(), request.groupSpec.size(), _rankSetup->enableNestedMultivalueGrouping()); + request.groupSpec.data(), request.groupSpec.size()); SessionId sessionId(request.sessionId.data(), request.sessionId.size()); bool shouldCacheSearchSession = false; bool shouldCacheGroupingSession = false; diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.cpp b/searchcore/src/vespa/searchcore/proton/matching/query.cpp index 00a3f6bab7e..071e914b405 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/query.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/query.cpp @@ -165,14 +165,8 @@ Query::~Query() = default; bool Query::buildTree(vespalib::stringref stack, const string &location, - const ViewResolver &resolver, const IIndexEnvironment &indexEnv) -{ - return buildTree(stack, location, resolver, indexEnv, true); -} -bool -Query::buildTree(vespalib::stringref stack, const string &location, const ViewResolver &resolver, const IIndexEnvironment &indexEnv, - bool split_unpacking_iterators) + bool always_mark_phrase_expensive) { SimpleQueryStackDumpIterator stack_dump_iterator(stack); _query_tree = QueryTreeCreator<ProtonNodeTypes>::create(stack_dump_iterator); @@ -180,8 +174,7 @@ Query::buildTree(vespalib::stringref stack, const string &location, SameElementModifier prefixSameElementSubIndexes; _query_tree->accept(prefixSameElementSubIndexes); exchange_location_nodes(location, _query_tree, _locations); - _query_tree = UnpackingIteratorsOptimizer::optimize(std::move(_query_tree), - bool(_whiteListBlueprint), split_unpacking_iterators); + _query_tree = UnpackingIteratorsOptimizer::optimize(std::move(_query_tree), bool(_whiteListBlueprint), always_mark_phrase_expensive); ResolveViewVisitor resolve_visitor(resolver, indexEnv); _query_tree->accept(resolve_visitor); return true; diff --git a/searchcore/src/vespa/searchcore/proton/matching/query.h b/searchcore/src/vespa/searchcore/proton/matching/query.h index 1b5d6dbca60..6ea326834a5 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/query.h +++ b/searchcore/src/vespa/searchcore/proton/matching/query.h @@ -55,12 +55,15 @@ public: bool buildTree(vespalib::stringref stack, const vespalib::string &location, const ViewResolver &resolver, - const search::fef::IIndexEnvironment &idxEnv, - bool split_unpacking_iterators); + const search::fef::IIndexEnvironment &idxEnv) + { + return buildTree(stack, location, resolver, idxEnv, false); + } bool buildTree(vespalib::stringref stack, const vespalib::string &location, const ViewResolver &resolver, - const search::fef::IIndexEnvironment &idxEnv); + const search::fef::IIndexEnvironment &idxEnv, + bool always_mark_phrase_expensive); /** * Extract query terms from the query tree; to be used to build diff --git a/searchcore/src/vespa/searchcore/proton/matching/unpacking_iterators_optimizer.cpp b/searchcore/src/vespa/searchcore/proton/matching/unpacking_iterators_optimizer.cpp index f9cdaaf7c15..c9cfbbfd40e 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/unpacking_iterators_optimizer.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/unpacking_iterators_optimizer.cpp @@ -73,21 +73,22 @@ struct TermExpander : QueryVisitor { struct NodeTraverser : TemplateTermVisitor<NodeTraverser, ProtonNodeTypes> { - bool split_unpacking_iterators; - - NodeTraverser(bool split_unpacking_iterators_in) - : split_unpacking_iterators(split_unpacking_iterators_in) {} + bool _always_mark_phrase_expensive; + NodeTraverser(bool always_mark_phrase_expensive) : _always_mark_phrase_expensive(always_mark_phrase_expensive) {} template <class TermNode> void visitTerm(TermNode &) {} void visit(ProtonNodeTypes::And &n) override { for (Node *child: n.getChildren()) { child->accept(*this); } - if (split_unpacking_iterators) { - TermExpander expander; - for (Node *child: n.getChildren()) { - child->accept(expander); - } - expander.flush(n); + TermExpander expander; + for (Node *child: n.getChildren()) { + child->accept(expander); + } + expander.flush(n); + } + void visit(Phrase &n) override { + if (_always_mark_phrase_expensive) { + n.set_expensive(true); } } }; @@ -95,15 +96,11 @@ struct NodeTraverser : TemplateTermVisitor<NodeTraverser, ProtonNodeTypes> } // namespace proton::matching::<unnamed> search::query::Node::UP -UnpackingIteratorsOptimizer::optimize(search::query::Node::UP root, - bool has_white_list, - bool split_unpacking_iterators) +UnpackingIteratorsOptimizer::optimize(search::query::Node::UP root, bool has_white_list, bool always_mark_phrase_expensive) { - if (split_unpacking_iterators) { - NodeTraverser traverser(split_unpacking_iterators); - root->accept(traverser); - } - if (has_white_list && split_unpacking_iterators) { + NodeTraverser traverser(always_mark_phrase_expensive); + root->accept(traverser); + if (has_white_list) { TermExpander expander; root->accept(expander); if (!expander.terms.empty()) { diff --git a/searchcore/src/vespa/searchcore/proton/matching/unpacking_iterators_optimizer.h b/searchcore/src/vespa/searchcore/proton/matching/unpacking_iterators_optimizer.h index d8897f4da1e..fc08ae3cfdd 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/unpacking_iterators_optimizer.h +++ b/searchcore/src/vespa/searchcore/proton/matching/unpacking_iterators_optimizer.h @@ -12,9 +12,7 @@ namespace proton::matching { * expensive. **/ struct UnpackingIteratorsOptimizer { - static search::query::Node::UP optimize(search::query::Node::UP root, - bool has_white_list, - bool split_unpacking_iterators); + static search::query::Node::UP optimize(search::query::Node::UP root, bool has_white_list, bool always_mark_phrase_expensive); }; } diff --git a/searchlib/abi-spec.json b/searchlib/abi-spec.json index 0b1cb7a103c..2d67abb0e04 100644 --- a/searchlib/abi-spec.json +++ b/searchlib/abi-spec.json @@ -1728,4 +1728,4 @@ ], "fields" : [ ] } -} +}
\ No newline at end of file diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java index a7b45feb043..c0dfc0e1d29 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/RankingExpression.java @@ -257,6 +257,9 @@ public class RankingExpression implements Serializable { * @return a list of named rank properties required to implement this expression */ public Map<String, String> getRankProperties(SerializationContext context) { + if ("".equals(name)) { + return Map.of(); + } Deque<String> path = new LinkedList<>(); String serializedRoot = root.toString(new StringBuilder(), context, path, null).toString(); Map<String, String> serializedExpressions = context.serializedFunctions(); diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java index 8af77ec1cdd..8d60f893c7c 100755 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java @@ -364,7 +364,7 @@ public class RankingExpressionTestCase { private void assertSerialization(String expectedSerialization, String expressionString) { String serializedExpression; try { - RankingExpression expression = new RankingExpression(expressionString); + RankingExpression expression = new RankingExpression("secondphase", expressionString); // No functions -> expect one rank property serializedExpression = expression.getRankProperties(new SerializationContext()).values().iterator().next(); assertEquals(expectedSerialization, serializedExpression); @@ -376,7 +376,7 @@ public class RankingExpressionTestCase { try { // No functions -> output should be parseable to a ranking expression // (but not the same one due to primitivization) - RankingExpression reparsedExpression = new RankingExpression(serializedExpression); + RankingExpression reparsedExpression = new RankingExpression("secondphase", serializedExpression); // Serializing the primitivized expression should yield the same expression again String reserializedExpression = reparsedExpression.getRankProperties(new SerializationContext()).values().iterator().next(); @@ -399,7 +399,7 @@ public class RankingExpressionTestCase { if (print) System.out.println("Parsing expression '" + expressionString + "':"); - RankingExpression expression = new RankingExpression(expressionString); + RankingExpression expression = new RankingExpression("secondphase", expressionString); Map<String, String> rankProperties = expression.getRankProperties(new SerializationContext(functions, Optional.empty())); if (print) { diff --git a/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp b/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp index 63c0b784018..dfea4901180 100644 --- a/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp +++ b/searchlib/src/tests/attribute/bitvector/bitvector_test.cpp @@ -43,9 +43,8 @@ struct BitVectorTest { using AttributePtr = AttributeVector::SP; - BitVectorTest() { } - - ~BitVectorTest() { } + BitVectorTest(); + ~BitVectorTest(); template <typename VectorType> VectorType & as(AttributePtr &v); @@ -102,6 +101,9 @@ struct BitVectorTest test(BasicType bt, CollectionType ct, const vespalib::string &pref); }; +BitVectorTest::BitVectorTest() = default; +BitVectorTest::~BitVectorTest() = default; + template <typename VectorType> VectorType & @@ -427,16 +429,14 @@ BitVectorTest::test(BasicType bt, CollectionType ct, const vespalib::string &pre SearchContextPtr sc = getSearch<VectorType>(tv, true); checkSearch(v, std::move(sc), 2, 1022, 205, !fastSearch && !filter, true); - sc = getSearch<VectorType>(tv, false); + sc = getSearch<VectorType>(tv, filter); checkSearch(v, std::move(sc), 2, 1022, 205, !filter, true); const search::IDocumentWeightAttribute *dwa = v->asDocumentWeightAttribute(); if (dwa != nullptr) { - search::IDocumentWeightAttribute::LookupResult lres = - dwa->lookup(getSearchStr<VectorType>(), dwa->get_dictionary_snapshot()); + auto lres = dwa->lookup(getSearchStr<VectorType>(), dwa->get_dictionary_snapshot()); using DWSI = search::queryeval::DocumentWeightSearchIterator; - using SI = search::queryeval::SearchIterator; TermFieldMatchData md; - SI::UP dwsi(new DWSI(md, *dwa, lres)); + auto dwsi = std::make_unique<DWSI>(md, *dwa, lres); if (!filter) { TEST_DO(checkSearch(v, std::move(dwsi), md, 2, 1022, 205, !filter, true)); } else { @@ -445,13 +445,13 @@ BitVectorTest::test(BasicType bt, CollectionType ct, const vespalib::string &pre } } populate(tv, 2, 973, false); - sc = getSearch<VectorType>(tv, true); + sc = getSearch<VectorType>(tv, filter); checkSearch(v, std::move(sc), 977, 1022, 10, !filter, true); populate(tv, 2, 973, true); sc = getSearch<VectorType>(tv, true); checkSearch(v, std::move(sc), 2, 1022, 205, !fastSearch && !filter, true); addDocs(v, 15000); - sc = getSearch<VectorType>(tv, true); + sc = getSearch<VectorType>(tv, filter); checkSearch(v, std::move(sc), 2, 1022, 205, !filter, true); populateAll(tv, 10, 15000, true); sc = getSearch<VectorType>(tv, true); diff --git a/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp b/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp index 61d94e120f5..af61301c06f 100644 --- a/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp +++ b/searchlib/src/tests/attribute/posting_store/posting_store_test.cpp @@ -96,7 +96,7 @@ protected: return sequence; } - void populate(uint32_t sequence_length); + void populate(std::vector<uint32_t> sequence_lengths, std::optional<std::function<void()>> clear_callback = std::nullopt); EntryRef get_posting_ref(int key); void test_compact_btree_nodes(uint32_t sequence_length); void test_compact_sequence(uint32_t sequence_length); @@ -120,21 +120,28 @@ PostingStoreTest::~PostingStoreTest() } void -PostingStoreTest::populate(uint32_t sequence_length) +PostingStoreTest::populate(std::vector<uint32_t> sequence_lengths, std::optional<std::function<void()>> clear_callback) { auto& store = _store; auto& dictionary = _value_store.get_dictionary(); std::vector<EntryRef> refs; - for (int i = 0; i < 9000; ++i) { - refs.emplace_back(add_sequence(i + 6, i + 6 + sequence_length)); + for (auto sequence_length : sequence_lengths) { + for (int i = 0; i < 9000; ++i) { + refs.emplace_back(add_sequence(i + 6, i + 6 + sequence_length)); + } } - dictionary.update_posting_list(_value_store.insert(1), _value_store.get_comparator(), [this, sequence_length](EntryRef) { return add_sequence(4, 4 + sequence_length); }); - dictionary.update_posting_list(_value_store.insert(2), _value_store.get_comparator(), [this, sequence_length](EntryRef) { return add_sequence(5, 5 + sequence_length); }); - for (int i = 9000; i < 11000; ++i) { - refs.emplace_back(add_sequence(i + 6, i + 6 + sequence_length)); + dictionary.update_posting_list(_value_store.insert(1), _value_store.get_comparator(), [this, sequence_length = sequence_lengths.front()](EntryRef) { return add_sequence(4, 4 + sequence_length); }); + dictionary.update_posting_list(_value_store.insert(2), _value_store.get_comparator(), [this, sequence_length = sequence_lengths.front()](EntryRef) { return add_sequence(5, 5 + sequence_length); }); + for (auto sequence_length : sequence_lengths) { + for (int i = 9000; i < 11000; ++i) { + refs.emplace_back(add_sequence(i + 6, i + 6 + sequence_length)); + } } for (auto& ref : refs) { store.clear(ref); + if (clear_callback.has_value()) { + clear_callback.value()(); + } } inc_generation(); } @@ -150,7 +157,7 @@ PostingStoreTest::get_posting_ref(int key) void PostingStoreTest::test_compact_sequence(uint32_t sequence_length) { - populate(sequence_length); + populate({sequence_length}); auto &store = _store; EntryRef old_ref1 = get_posting_ref(1); EntryRef old_ref2 = get_posting_ref(2); @@ -183,7 +190,7 @@ PostingStoreTest::test_compact_sequence(uint32_t sequence_length) void PostingStoreTest::test_compact_btree_nodes(uint32_t sequence_length) { - populate(sequence_length); + populate({sequence_length}); auto &store = _store; EntryRef old_ref1 = get_posting_ref(1); EntryRef old_ref2 = get_posting_ref(2); @@ -246,6 +253,40 @@ TEST_P(PostingStoreTest, require_that_bitvectors_are_compacted) test_compact_sequence(huge_sequence_length); } +namespace { + +/* + * Check if compaction of btree nodes or short arrays is suppressed due to + * dead ratio being too low for the sum of both data stores. + */ +bool compaction_is_suppressed(MyPostingStore& store, CompactionStrategy& compaction_strategy) +{ + store.update_stat(compaction_strategy); + auto& compaction_spec = store.get_compaction_spec(); + auto memory_usage = store.getMemoryUsage(); + return ((compaction_strategy.should_compact_memory(memory_usage.btrees) && + !compaction_spec.btree_nodes()) || + (compaction_strategy.should_compact_memory(memory_usage.short_arrays) && + !compaction_spec.store())); +} + +} + +TEST_P(PostingStoreTest, require_that_compaction_is_suppressed) +{ + CompactionStrategy compaction_strategy(0.05, 0.2); + bool suppressed_compaction = false; + auto clear_callback = [this, &compaction_strategy, &suppressed_compaction]() mutable + { + inc_generation(); + if (compaction_is_suppressed(_store, compaction_strategy)) { + suppressed_compaction = true; + } + }; + populate({ 4, 10}, clear_callback); + EXPECT_TRUE(suppressed_compaction); +} + } GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/fef/properties/properties_test.cpp b/searchlib/src/tests/fef/properties/properties_test.cpp index 5e18c41b40a..c8073739b3e 100644 --- a/searchlib/src/tests/fef/properties/properties_test.cpp +++ b/searchlib/src/tests/fef/properties/properties_test.cpp @@ -10,9 +10,8 @@ using namespace search::fef::indexproperties; struct CopyVisitor : public IPropertiesVisitor { Properties &dst; - CopyVisitor(Properties &p) : dst(p) {} - virtual void visitProperty(const Property::Value &key, - const Property &values) override + explicit CopyVisitor(Properties &p) noexcept : dst(p) {} + void visitProperty(const Property::Value &key, const Property &values) override { for (uint32_t i = 0; i < values.size(); ++i) { dst.add(key, values.getAt(i)); @@ -590,5 +589,41 @@ TEST("test query feature type properties") EXPECT_EQUAL("", type::QueryFeature::lookup(p, "bar")); } +TEST("test integer lookup") +{ + EXPECT_EQUAL(matching::NumThreadsPerSearch::NAME, vespalib::string("vespa.matching.numthreadspersearch")); + EXPECT_EQUAL(matching::NumThreadsPerSearch::DEFAULT_VALUE, std::numeric_limits<uint32_t>::max()); + { + Properties p; + p.add("vespa.matching.numthreadspersearch", "50"); + EXPECT_EQUAL(matching::NumThreadsPerSearch::lookup(p), 50u); + } + { + Properties p; + p.add("vespa.matching.numthreadspersearch", "50 "); + EXPECT_EQUAL(matching::NumThreadsPerSearch::lookup(p), 50u); + } + { + Properties p; + p.add("vespa.matching.numthreadspersearch", " 50"); + EXPECT_EQUAL(matching::NumThreadsPerSearch::lookup(p), 50u); + } + { + Properties p; + p.add("vespa.matching.numthreadspersearch", " "); + EXPECT_EQUAL(matching::NumThreadsPerSearch::lookup(p), matching::NumThreadsPerSearch::DEFAULT_VALUE); + } + { + Properties p; + p.add("vespa.matching.numthreadspersearch", "50x"); + EXPECT_EQUAL(matching::NumThreadsPerSearch::lookup(p), 50u); + } + { + Properties p; + p.add("vespa.matching.numthreadspersearch", "x"); + EXPECT_EQUAL(matching::NumThreadsPerSearch::lookup(p), matching::NumThreadsPerSearch::DEFAULT_VALUE); + } +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/query/querybuilder_test.cpp b/searchlib/src/tests/query/querybuilder_test.cpp index 606d6a2474a..40af60d4a04 100644 --- a/searchlib/src/tests/query/querybuilder_test.cpp +++ b/searchlib/src/tests/query/querybuilder_test.cpp @@ -257,7 +257,7 @@ void checkQueryTreeTypes(Node *node) { EXPECT_TRUE(checkTerm(loc_term, location, view[10], id[10], weight[10])); auto* wand = as_node<WeakAnd>(and_node->getChildren()[4]); - EXPECT_EQUAL(123u, wand->getMinHits()); + EXPECT_EQUAL(123u, wand->getTargetNumHits()); EXPECT_EQUAL(2u, wand->getChildren().size()); string_term = as_node<StringTerm>(wand->getChildren()[0]); EXPECT_TRUE(checkTerm(string_term, str[4], view[4], id[4], weight[4])); diff --git a/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp b/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp index 1d908bed568..372b76fa9af 100644 --- a/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp +++ b/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp @@ -81,7 +81,7 @@ public: } FieldSpecBaseList exposeFields() const override { - return FieldSpecBaseList(); + return {}; } bool inheritStrict(size_t i) const override { @@ -140,7 +140,7 @@ struct MyTerm : SimpleLeafBlueprint { setEstimate(HitEstimate(hitEstimate, false)); } SearchIterator::UP createLeafSearch(const search::fef::TermFieldMatchDataArray &, bool) const override { - return SearchIterator::UP(); + return {}; } SearchIteratorUP createFilterSearch(bool strict, FilterConstraint constraint) const override { return create_default_filter(strict, constraint); @@ -311,13 +311,13 @@ TEST("testHitEstimateCalculation") TEST("testHitEstimatePropagation") { - MyLeaf *leaf1 = new MyLeaf(); + auto *leaf1 = new MyLeaf(); leaf1->estimate(10); - MyLeaf *leaf2 = new MyLeaf(); + auto *leaf2 = new MyLeaf(); leaf2->estimate(20); - MyLeaf *leaf3 = new MyLeaf(); + auto *leaf3 = new MyLeaf(); leaf3->estimate(30); MyOr *parent = new MyOr(); @@ -346,7 +346,7 @@ TEST("testHitEstimatePropagation") leaf3->estimate(25); EXPECT_EQUAL(20u, root->getState().estimate().estHits); parent->addChild(std::move(tmp)); - EXPECT_TRUE(tmp.get() == 0); + EXPECT_FALSE(tmp); EXPECT_EQUAL(25u, root->getState().estimate().estHits); } @@ -572,7 +572,7 @@ TEST_F("testSearchCreation", Fixture) .addField(3, 3).create()); SearchIterator::UP leafsearch = f.create(*l); - MySearch *lw = new MySearch("leaf", true, true); + auto *lw = new MySearch("leaf", true, true); lw->addHandle(1).addHandle(2).addHandle(3); SearchIterator::UP wantleaf(lw); @@ -584,11 +584,11 @@ TEST_F("testSearchCreation", Fixture) .add(MyLeafSpec(2).addField(2, 2).create())); SearchIterator::UP andsearch = f.create(*a); - MySearch *l1 = new MySearch("leaf", true, true); - MySearch *l2 = new MySearch("leaf", true, false); + auto *l1 = new MySearch("leaf", true, true); + auto *l2 = new MySearch("leaf", true, false); l1->addHandle(1); l2->addHandle(2); - MySearch *aw = new MySearch("and", false, true); + auto *aw = new MySearch("and", false, true); aw->add(l1); aw->add(l2); SearchIterator::UP wanted(aw); @@ -600,11 +600,11 @@ TEST_F("testSearchCreation", Fixture) .add(MyLeafSpec(2).addField(2, 22).create())); SearchIterator::UP orsearch = f.create(*o); - MySearch *l1 = new MySearch("leaf", true, true); - MySearch *l2 = new MySearch("leaf", true, true); + auto *l1 = new MySearch("leaf", true, true); + auto *l2 = new MySearch("leaf", true, true); l1->addHandle(11); l2->addHandle(22); - MySearch *ow = new MySearch("or", false, true); + auto *ow = new MySearch("or", false, true); ow->add(l1); ow->add(l2); SearchIterator::UP wanted(ow); @@ -619,7 +619,7 @@ TEST("testBlueprintMakeNew") .add(MyLeafSpec(2).addField(2, 22).create())); orig->setSourceId(42); MyOr *myOr = dynamic_cast<MyOr*>(orig.get()); - ASSERT_TRUE(myOr != 0); + ASSERT_TRUE(myOr != nullptr); EXPECT_EQUAL(42u, orig->getSourceId()); EXPECT_EQUAL(2u, orig->getState().numFields()); } diff --git a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp index a663944938c..51e22dbcf2c 100644 --- a/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp +++ b/searchlib/src/tests/queryeval/blueprint/intermediate_blueprints_test.cpp @@ -8,7 +8,6 @@ #include <vespa/searchlib/queryeval/leaf_blueprints.h> #include <vespa/searchlib/queryeval/equiv_blueprint.h> #include <vespa/searchlib/queryeval/multisearch.h> -#include <vespa/searchlib/queryeval/andnotsearch.h> #include <vespa/searchlib/queryeval/wand/weak_and_search.h> #include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <vespa/searchlib/test/diskindex/testdiskindex.h> @@ -74,13 +73,13 @@ TEST("test AndNot Blueprint") { std::vector<Blueprint::HitEstimate> est; EXPECT_EQUAL(true, b.combine(est).empty); EXPECT_EQUAL(0u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(10, false)); + est.emplace_back(10, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(20, false)); + est.emplace_back(20, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(5, false)); + est.emplace_back(5, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); } @@ -120,12 +119,14 @@ TEST("test And propagates updated histestimate") { bp.addChild(ap(MyLeafSpec(20).create<RememberExecuteInfo>()->setSourceId(2))); bp.addChild(ap(MyLeafSpec(200).create<RememberExecuteInfo>()->setSourceId(2))); bp.addChild(ap(MyLeafSpec(2000).create<RememberExecuteInfo>()->setSourceId(2))); - bp.optimize_self(); + bp.optimize_self(Blueprint::OptimizePass::FIRST); + bp.optimize_self(Blueprint::OptimizePass::SECOND); + bp.optimize_self(Blueprint::OptimizePass::LAST); bp.setDocIdLimit(5000); bp.fetchPostings(ExecuteInfo::TRUE); EXPECT_EQUAL(3u, bp.childCnt()); for (uint32_t i = 0; i < bp.childCnt(); i++) { - const RememberExecuteInfo & child = dynamic_cast<const RememberExecuteInfo &>(bp.getChild(i)); + const auto & child = dynamic_cast<const RememberExecuteInfo &>(bp.getChild(i)); EXPECT_EQUAL((i == 0), child.executeInfo.isStrict()); } EXPECT_EQUAL(1.0f, dynamic_cast<const RememberExecuteInfo &>(bp.getChild(0)).executeInfo.hitRate()); @@ -133,22 +134,45 @@ TEST("test And propagates updated histestimate") { EXPECT_EQUAL(1.0f/(250*25), dynamic_cast<const RememberExecuteInfo &>(bp.getChild(2)).executeInfo.hitRate()); } +TEST("test Or propagates updated histestimate") { + OrBlueprint bp; + bp.setSourceId(2); + bp.addChild(ap(MyLeafSpec(5000).create<RememberExecuteInfo>()->setSourceId(2))); + bp.addChild(ap(MyLeafSpec(2000).create<RememberExecuteInfo>()->setSourceId(2))); + bp.addChild(ap(MyLeafSpec(800).create<RememberExecuteInfo>()->setSourceId(2))); + bp.addChild(ap(MyLeafSpec(20).create<RememberExecuteInfo>()->setSourceId(2))); + bp.optimize_self(Blueprint::OptimizePass::FIRST); + bp.optimize_self(Blueprint::OptimizePass::SECOND); + bp.optimize_self(Blueprint::OptimizePass::LAST); + bp.setDocIdLimit(5000); + bp.fetchPostings(ExecuteInfo::TRUE); + EXPECT_EQUAL(4u, bp.childCnt()); + for (uint32_t i = 0; i < bp.childCnt(); i++) { + const auto & child = dynamic_cast<const RememberExecuteInfo &>(bp.getChild(i)); + EXPECT_TRUE(child.executeInfo.isStrict()); + } + EXPECT_EQUAL(1.0f, dynamic_cast<const RememberExecuteInfo &>(bp.getChild(0)).executeInfo.hitRate()); + EXPECT_EQUAL(1.0f, dynamic_cast<const RememberExecuteInfo &>(bp.getChild(1)).executeInfo.hitRate()); + EXPECT_EQUAL(3.0f/5.0f, dynamic_cast<const RememberExecuteInfo &>(bp.getChild(2)).executeInfo.hitRate()); + EXPECT_EQUAL(3.0f*42.0f/(5.0f*50.0f), dynamic_cast<const RememberExecuteInfo &>(bp.getChild(3)).executeInfo.hitRate()); +} + TEST("test And Blueprint") { AndBlueprint b; { // combine std::vector<Blueprint::HitEstimate> est; EXPECT_EQUAL(true, b.combine(est).empty); EXPECT_EQUAL(0u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(10, false)); + est.emplace_back(10, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(20, false)); + est.emplace_back(20, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(5, false)); + est.emplace_back(5, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(5u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(0, true)); + est.emplace_back(0, true); EXPECT_EQUAL(true, b.combine(est).empty); EXPECT_EQUAL(0u, b.combine(est).estHits); } @@ -187,16 +211,16 @@ TEST("test Or Blueprint") { std::vector<Blueprint::HitEstimate> est; EXPECT_EQUAL(true, b.combine(est).empty); EXPECT_EQUAL(0u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(10, false)); + est.emplace_back(10, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(20, false)); + est.emplace_back(20, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(20u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(5, false)); + est.emplace_back(5, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(20u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(0, true)); + est.emplace_back(0, true); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(20u, b.combine(est).estHits); } @@ -259,16 +283,16 @@ TEST("test Near Blueprint") { std::vector<Blueprint::HitEstimate> est; EXPECT_EQUAL(true, b.combine(est).empty); EXPECT_EQUAL(0u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(10, false)); + est.emplace_back(10, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(20, false)); + est.emplace_back(20, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(5, false)); + est.emplace_back(5, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(5u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(0, true)); + est.emplace_back(0, true); EXPECT_EQUAL(true, b.combine(est).empty); EXPECT_EQUAL(0u, b.combine(est).estHits); } @@ -300,16 +324,16 @@ TEST("test ONear Blueprint") { std::vector<Blueprint::HitEstimate> est; EXPECT_EQUAL(true, b.combine(est).empty); EXPECT_EQUAL(0u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(10, false)); + est.emplace_back(10, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(20, false)); + est.emplace_back(20, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(5, false)); + est.emplace_back(5, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(5u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(0, true)); + est.emplace_back(0, true); EXPECT_EQUAL(true, b.combine(est).empty); EXPECT_EQUAL(0u, b.combine(est).estHits); } @@ -341,16 +365,16 @@ TEST("test Rank Blueprint") { std::vector<Blueprint::HitEstimate> est; EXPECT_EQUAL(true, b.combine(est).empty); EXPECT_EQUAL(0u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(10, false)); + est.emplace_back(10, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(20, false)); + est.emplace_back(20, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(5, false)); + est.emplace_back(5, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(0, true)); + est.emplace_back(0, true); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); } @@ -391,16 +415,16 @@ TEST("test SourceBlender Blueprint") { std::vector<Blueprint::HitEstimate> est; EXPECT_EQUAL(true, b.combine(est).empty); EXPECT_EQUAL(0u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(10, false)); + est.emplace_back(10, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(20, false)); + est.emplace_back(20, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(20u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(5, false)); + est.emplace_back(5, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(20u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(0, true)); + est.emplace_back(0, true); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(20u, b.combine(est).estHits); } @@ -453,60 +477,60 @@ TEST("test SourceBlender below AND optimization") { auto selector_1 = std::make_unique<InvalidSelector>(); // the one auto selector_2 = std::make_unique<InvalidSelector>(); // not the one //------------------------------------------------------------------------- - AndBlueprint *top = new AndBlueprint(); + auto *top = new AndBlueprint(); Blueprint::UP top_bp(top); top->addChild(ap(MyLeafSpec(2).create())); top->addChild(ap(MyLeafSpec(1).create())); top->addChild(ap(MyLeafSpec(3).create())); { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(200).create()->setSourceId(2))); blender->addChild(ap(MyLeafSpec(100).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(300).create()->setSourceId(3))); top->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(20).create()->setSourceId(2))); blender->addChild(ap(MyLeafSpec(10).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(30).create()->setSourceId(3))); top->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_2); + auto *blender = new SourceBlenderBlueprint(*selector_2); blender->addChild(ap(MyLeafSpec(10).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(20).create()->setSourceId(2))); top->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(2000).create()->setSourceId(2))); blender->addChild(ap(MyLeafSpec(1000).create()->setSourceId(1))); top->addChild(ap(blender)); } //------------------------------------------------------------------------- - AndBlueprint *expect = new AndBlueprint(); + auto *expect = new AndBlueprint(); Blueprint::UP expect_bp(expect); expect->addChild(ap(MyLeafSpec(1).create())); expect->addChild(ap(MyLeafSpec(2).create())); expect->addChild(ap(MyLeafSpec(3).create())); { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_2); + auto *blender = new SourceBlenderBlueprint(*selector_2); blender->addChild(ap(MyLeafSpec(10).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(20).create()->setSourceId(2))); expect->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender(new SourceBlenderBlueprint(*selector_1)); + auto *blender(new SourceBlenderBlueprint(*selector_1)); { - AndBlueprint *sub_and = new AndBlueprint(); + auto *sub_and = new AndBlueprint(); sub_and->setSourceId(3); sub_and->addChild(ap(MyLeafSpec(30).create()->setSourceId(3))); sub_and->addChild(ap(MyLeafSpec(300).create()->setSourceId(3))); blender->addChild(ap(sub_and)); } { - AndBlueprint *sub_and = new AndBlueprint(); + auto *sub_and = new AndBlueprint(); sub_and->setSourceId(2); sub_and->addChild(ap(MyLeafSpec(20).create()->setSourceId(2))); sub_and->addChild(ap(MyLeafSpec(200).create()->setSourceId(2))); @@ -514,7 +538,7 @@ TEST("test SourceBlender below AND optimization") { blender->addChild(ap(sub_and)); } { - AndBlueprint *sub_and = new AndBlueprint(); + auto *sub_and = new AndBlueprint(); sub_and->setSourceId(1); sub_and->addChild(ap(MyLeafSpec(10).create()->setSourceId(1))); sub_and->addChild(ap(MyLeafSpec(100).create()->setSourceId(1))); @@ -535,51 +559,51 @@ TEST("test SourceBlender below OR optimization") { auto selector_1 = std::make_unique<InvalidSelector>(); // the one auto selector_2 = std::make_unique<InvalidSelector>(); // not the one //------------------------------------------------------------------------- - OrBlueprint *top = new OrBlueprint(); + auto *top = new OrBlueprint(); Blueprint::UP top_up(top); top->addChild(ap(MyLeafSpec(2).create())); top->addChild(ap(MyLeafSpec(1).create())); top->addChild(ap(MyLeafSpec(3).create())); { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(200).create()->setSourceId(2))); blender->addChild(ap(MyLeafSpec(100).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(300).create()->setSourceId(3))); top->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(20).create()->setSourceId(2))); blender->addChild(ap(MyLeafSpec(10).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(30).create()->setSourceId(3))); top->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_2); + auto *blender = new SourceBlenderBlueprint(*selector_2); blender->addChild(ap(MyLeafSpec(10).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(20).create()->setSourceId(2))); top->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(2000).create()->setSourceId(2))); blender->addChild(ap(MyLeafSpec(1000).create()->setSourceId(1))); top->addChild(ap(blender)); } //------------------------------------------------------------------------- - OrBlueprint *expect = new OrBlueprint(); + auto *expect = new OrBlueprint(); Blueprint::UP expect_up(expect); { - SourceBlenderBlueprint *blender(new SourceBlenderBlueprint(*selector_1)); + auto *blender(new SourceBlenderBlueprint(*selector_1)); { - OrBlueprint *sub_and = new OrBlueprint(); + auto *sub_and = new OrBlueprint(); sub_and->setSourceId(3); sub_and->addChild(ap(MyLeafSpec(300).create()->setSourceId(3))); sub_and->addChild(ap(MyLeafSpec(30).create()->setSourceId(3))); blender->addChild(ap(sub_and)); } { - OrBlueprint *sub_and = new OrBlueprint(); + auto *sub_and = new OrBlueprint(); sub_and->setSourceId(2); sub_and->addChild(ap(MyLeafSpec(2000).create()->setSourceId(2))); sub_and->addChild(ap(MyLeafSpec(200).create()->setSourceId(2))); @@ -587,7 +611,7 @@ TEST("test SourceBlender below OR optimization") { blender->addChild(ap(sub_and)); } { - OrBlueprint *sub_and = new OrBlueprint(); + auto *sub_and = new OrBlueprint(); sub_and->setSourceId(1); sub_and->addChild(ap(MyLeafSpec(1000).create()->setSourceId(1))); sub_and->addChild(ap(MyLeafSpec(100).create()->setSourceId(1))); @@ -597,7 +621,7 @@ TEST("test SourceBlender below OR optimization") { expect->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_2); + auto *blender = new SourceBlenderBlueprint(*selector_2); blender->addChild(ap(MyLeafSpec(10).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(20).create()->setSourceId(2))); expect->addChild(ap(blender)); @@ -617,10 +641,10 @@ TEST("test SourceBlender below AND_NOT optimization") { auto selector_1 = std::make_unique<InvalidSelector>(); // the one auto selector_2 = std::make_unique<InvalidSelector>(); // not the one //------------------------------------------------------------------------- - AndNotBlueprint *top = new AndNotBlueprint(); + auto *top = new AndNotBlueprint(); Blueprint::UP top_up(top); { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(42).create()->setSourceId(1))); top->addChild(ap(blender)); } @@ -628,50 +652,50 @@ TEST("test SourceBlender below AND_NOT optimization") { top->addChild(ap(MyLeafSpec(1).create())); top->addChild(ap(MyLeafSpec(3).create())); { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(200).create()->setSourceId(2))); blender->addChild(ap(MyLeafSpec(100).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(300).create()->setSourceId(3))); top->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(20).create()->setSourceId(2))); blender->addChild(ap(MyLeafSpec(10).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(30).create()->setSourceId(3))); top->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_2); + auto *blender = new SourceBlenderBlueprint(*selector_2); blender->addChild(ap(MyLeafSpec(10).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(20).create()->setSourceId(2))); top->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(2000).create()->setSourceId(2))); blender->addChild(ap(MyLeafSpec(1000).create()->setSourceId(1))); top->addChild(ap(blender)); } //------------------------------------------------------------------------- - AndNotBlueprint *expect = new AndNotBlueprint(); + auto *expect = new AndNotBlueprint(); Blueprint::UP expect_up(expect); { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(42).create()->setSourceId(1))); expect->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender(new SourceBlenderBlueprint(*selector_1)); + auto *blender(new SourceBlenderBlueprint(*selector_1)); { - OrBlueprint *sub_and = new OrBlueprint(); + auto *sub_and = new OrBlueprint(); sub_and->setSourceId(3); sub_and->addChild(ap(MyLeafSpec(300).create()->setSourceId(3))); sub_and->addChild(ap(MyLeafSpec(30).create()->setSourceId(3))); blender->addChild(ap(sub_and)); } { - OrBlueprint *sub_and = new OrBlueprint(); + auto *sub_and = new OrBlueprint(); sub_and->setSourceId(2); sub_and->addChild(ap(MyLeafSpec(2000).create()->setSourceId(2))); sub_and->addChild(ap(MyLeafSpec(200).create()->setSourceId(2))); @@ -679,7 +703,7 @@ TEST("test SourceBlender below AND_NOT optimization") { blender->addChild(ap(sub_and)); } { - OrBlueprint *sub_and = new OrBlueprint(); + auto *sub_and = new OrBlueprint(); sub_and->setSourceId(1); sub_and->addChild(ap(MyLeafSpec(1000).create()->setSourceId(1))); sub_and->addChild(ap(MyLeafSpec(100).create()->setSourceId(1))); @@ -689,7 +713,7 @@ TEST("test SourceBlender below AND_NOT optimization") { expect->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_2); + auto *blender = new SourceBlenderBlueprint(*selector_2); blender->addChild(ap(MyLeafSpec(10).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(20).create()->setSourceId(2))); expect->addChild(ap(blender)); @@ -709,10 +733,10 @@ TEST("test SourceBlender below RANK optimization") { auto selector_1 = std::make_unique<InvalidSelector>(); // the one auto selector_2 = std::make_unique<InvalidSelector>(); // not the one //------------------------------------------------------------------------- - RankBlueprint *top = new RankBlueprint(); + auto *top = new RankBlueprint(); Blueprint::UP top_up(top); { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(42).create()->setSourceId(1))); top->addChild(ap(blender)); } @@ -720,36 +744,36 @@ TEST("test SourceBlender below RANK optimization") { top->addChild(ap(MyLeafSpec(1).create())); top->addChild(ap(MyLeafSpec(3).create())); { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(200).create()->setSourceId(2))); blender->addChild(ap(MyLeafSpec(100).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(300).create()->setSourceId(3))); top->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(20).create()->setSourceId(2))); blender->addChild(ap(MyLeafSpec(10).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(30).create()->setSourceId(3))); top->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_2); + auto *blender = new SourceBlenderBlueprint(*selector_2); blender->addChild(ap(MyLeafSpec(10).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(20).create()->setSourceId(2))); top->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(2000).create()->setSourceId(2))); blender->addChild(ap(MyLeafSpec(1000).create()->setSourceId(1))); top->addChild(ap(blender)); } //------------------------------------------------------------------------- - RankBlueprint *expect = new RankBlueprint(); + auto *expect = new RankBlueprint(); Blueprint::UP expect_up(expect); { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_1); + auto *blender = new SourceBlenderBlueprint(*selector_1); blender->addChild(ap(MyLeafSpec(42).create()->setSourceId(1))); expect->addChild(ap(blender)); } @@ -757,22 +781,22 @@ TEST("test SourceBlender below RANK optimization") { expect->addChild(ap(MyLeafSpec(1).create())); expect->addChild(ap(MyLeafSpec(3).create())); { - SourceBlenderBlueprint *blender = new SourceBlenderBlueprint(*selector_2); + auto *blender = new SourceBlenderBlueprint(*selector_2); blender->addChild(ap(MyLeafSpec(10).create()->setSourceId(1))); blender->addChild(ap(MyLeafSpec(20).create()->setSourceId(2))); expect->addChild(ap(blender)); } { - SourceBlenderBlueprint *blender(new SourceBlenderBlueprint(*selector_1)); + auto *blender(new SourceBlenderBlueprint(*selector_1)); { - OrBlueprint *sub_and = new OrBlueprint(); + auto *sub_and = new OrBlueprint(); sub_and->setSourceId(3); sub_and->addChild(ap(MyLeafSpec(300).create()->setSourceId(3))); sub_and->addChild(ap(MyLeafSpec(30).create()->setSourceId(3))); blender->addChild(ap(sub_and)); } { - OrBlueprint *sub_and = new OrBlueprint(); + auto *sub_and = new OrBlueprint(); sub_and->setSourceId(2); sub_and->addChild(ap(MyLeafSpec(2000).create()->setSourceId(2))); sub_and->addChild(ap(MyLeafSpec(200).create()->setSourceId(2))); @@ -780,7 +804,7 @@ TEST("test SourceBlender below RANK optimization") { blender->addChild(ap(sub_and)); } { - OrBlueprint *sub_and = new OrBlueprint(); + auto *sub_and = new OrBlueprint(); sub_and->setSourceId(1); sub_and->addChild(ap(MyLeafSpec(1000).create()->setSourceId(1))); sub_and->addChild(ap(MyLeafSpec(100).create()->setSourceId(1))); @@ -826,7 +850,7 @@ TEST("test empty root node optimization and safeness") { addChild(ap(MyLeafSpec(0, true).create())). addChild(ap(MyLeafSpec(0, true).create())))); //------------------------------------------------------------------------- - Blueprint::UP expect_up(new EmptyBlueprint()); + auto expect_up = std::make_unique<EmptyBlueprint>(); //------------------------------------------------------------------------- top1_up = Blueprint::optimize(std::move(top1_up)); top2_up = Blueprint::optimize(std::move(top2_up)); @@ -851,7 +875,7 @@ TEST("and with one empty child is optimized away") { top = Blueprint::optimize(std::move(top)); Blueprint::UP expect_up(ap((new SourceBlenderBlueprint(*selector))-> addChild(ap(MyLeafSpec(10).create())). - addChild(ap(new EmptyBlueprint())))); + addChild(std::make_unique<EmptyBlueprint>()))); EXPECT_EQUAL(expect_up->asString(), top->asString()); } @@ -964,11 +988,11 @@ TEST("require that replaced blueprints retain source id") { //------------------------------------------------------------------------- // replace empty root with empty search Blueprint::UP top1_up(ap(MyLeafSpec(0, true).create()->setSourceId(13))); - Blueprint::UP expect1_up(new EmptyBlueprint()); + auto expect1_up = std::make_unique<EmptyBlueprint>(); expect1_up->setSourceId(13); //------------------------------------------------------------------------- // replace self with single child - Blueprint::UP top2_up(ap(static_cast<AndBlueprint&>((new AndBlueprint())->setSourceId(42)). + Blueprint::UP top2_up(ap(dynamic_cast<AndBlueprint&>((new AndBlueprint())->setSourceId(42)). addChild(ap(MyLeafSpec(30).create()->setSourceId(55))))); Blueprint::UP expect2_up(ap(MyLeafSpec(30).create()->setSourceId(42))); //------------------------------------------------------------------------- @@ -1017,16 +1041,16 @@ TEST("test WeakAnd Blueprint") { std::vector<Blueprint::HitEstimate> est; EXPECT_EQUAL(true, b.combine(est).empty); EXPECT_EQUAL(0u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(10, false)); + est.emplace_back(10, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(10u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(20, false)); + est.emplace_back(20, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(20u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(5, false)); + est.emplace_back(5, false); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(20u, b.combine(est).estHits); - est.push_back(Blueprint::HitEstimate(0, true)); + est.emplace_back(0, true); EXPECT_EQUAL(false, b.combine(est).empty); EXPECT_EQUAL(20u, b.combine(est).estHits); } @@ -1057,14 +1081,14 @@ TEST("test WeakAnd Blueprint") { { WeakAndBlueprint wa(456); MatchData::UP md = MatchData::makeTestInstance(100, 10); - wa.addTerm(Blueprint::UP(new FakeBlueprint(field, x)), 120); - wa.addTerm(Blueprint::UP(new FakeBlueprint(field, z)), 140); - wa.addTerm(Blueprint::UP(new FakeBlueprint(field, y)), 130); + wa.addTerm(std::make_unique<FakeBlueprint>(field, x), 120); + wa.addTerm(std::make_unique<FakeBlueprint>(field, z), 140); + wa.addTerm(std::make_unique<FakeBlueprint>(field, y), 130); { wa.fetchPostings(ExecuteInfo::TRUE); SearchIterator::UP search = wa.createSearch(*md, true); - EXPECT_TRUE(dynamic_cast<WeakAndSearch*>(search.get()) != 0); - WeakAndSearch &s = dynamic_cast<WeakAndSearch&>(*search); + EXPECT_TRUE(dynamic_cast<WeakAndSearch*>(search.get()) != nullptr); + auto &s = dynamic_cast<WeakAndSearch&>(*search); EXPECT_EQUAL(456u, s.getN()); ASSERT_EQUAL(3u, s.getTerms().size()); EXPECT_GREATER(s.get_max_score(0), 0.0); @@ -1085,7 +1109,7 @@ TEST("test WeakAnd Blueprint") { { wa.fetchPostings(ExecuteInfo::FALSE); SearchIterator::UP search = wa.createSearch(*md, false); - EXPECT_TRUE(dynamic_cast<WeakAndSearch*>(search.get()) != 0); + EXPECT_TRUE(dynamic_cast<WeakAndSearch*>(search.get()) != nullptr); EXPECT_TRUE(search->seek(1)); EXPECT_TRUE(search->seek(2)); EXPECT_FALSE(search->seek(3)); @@ -1195,7 +1219,7 @@ namespace { SimpleStringTerm makeTerm(const std::string & term) { - return SimpleStringTerm(term, "field", 0, search::query::Weight(0)); + return {term, "field", 0, search::query::Weight(0)}; } } @@ -1231,7 +1255,7 @@ TEST("require that children does not optimize when parents refuse them to") { SearchIterator::UP search = top_up->createSearch(*md, true); EXPECT_EQUAL("search::queryeval::EquivImpl<true>", search->getClassName()); { - const MultiSearch & e = dynamic_cast<const MultiSearch &>(*search); + const auto & e = dynamic_cast<const MultiSearch &>(*search); EXPECT_EQUAL("search::BitVectorIteratorStrictT<false>", e.getChildren()[0]->getClassName()); EXPECT_EQUAL("search::diskindex::ZcRareWordPosOccIterator<true, false>", e.getChildren()[1]->getClassName()); EXPECT_EQUAL("search::diskindex::ZcRareWordPosOccIterator<true, false>", e.getChildren()[2]->getClassName()); @@ -1241,7 +1265,7 @@ TEST("require that children does not optimize when parents refuse them to") { search = top_up->createSearch(*md, true); EXPECT_EQUAL("search::queryeval::EquivImpl<true>", search->getClassName()); { - const MultiSearch & e = dynamic_cast<const MultiSearch &>(*search); + const auto & e = dynamic_cast<const MultiSearch &>(*search); EXPECT_EQUAL("search::BitVectorIteratorStrictT<false>", e.getChildren()[0]->getClassName()); EXPECT_EQUAL("search::diskindex::ZcRareWordPosOccIterator<true, false>", e.getChildren()[1]->getClassName()); EXPECT_EQUAL("search::diskindex::ZcRareWordPosOccIterator<true, false>", e.getChildren()[2]->getClassName()); @@ -1269,7 +1293,7 @@ TEST("require_that_unpack_optimization_is_not_overruled_by_equiv") { SearchIterator::UP search = top_up->createSearch(*md, true); EXPECT_EQUAL("search::queryeval::EquivImpl<true>", search->getClassName()); { - const MultiSearch & e = dynamic_cast<const MultiSearch &>(*search); + const auto & e = dynamic_cast<const MultiSearch &>(*search); EXPECT_EQUAL("search::queryeval::OrLikeSearch<true, search::queryeval::(anonymous namespace)::FullUnpack>", e.getChildren()[0]->getClassName()); } @@ -1278,7 +1302,7 @@ TEST("require_that_unpack_optimization_is_not_overruled_by_equiv") { search = top_up->createSearch(*md, true); EXPECT_EQUAL("search::queryeval::EquivImpl<true>", search->getClassName()); { - const MultiSearch & e = dynamic_cast<const MultiSearch &>(*search); + const auto & e = dynamic_cast<const MultiSearch &>(*search); EXPECT_EQUAL("search::queryeval::OrLikeSearch<true, search::queryeval::(anonymous namespace)::SelectiveUnpack>", e.getChildren()[0]->getClassName()); } @@ -1288,7 +1312,7 @@ TEST("require_that_unpack_optimization_is_not_overruled_by_equiv") { search = top_up->createSearch(*md, true); EXPECT_EQUAL("search::queryeval::EquivImpl<true>", search->getClassName()); { - const MultiSearch & e = dynamic_cast<const MultiSearch &>(*search); + const auto & e = dynamic_cast<const MultiSearch &>(*search); EXPECT_EQUAL("search::queryeval::OrLikeSearch<true, search::queryeval::NoUnpack>", e.getChildren()[0]->getClassName()); } @@ -1344,7 +1368,7 @@ TEST("require that children of onear are not optimized") { TEST("require that ANDNOT without children is optimized to empty search") { Blueprint::UP top_up(new AndNotBlueprint()); - Blueprint::UP expect_up(new EmptyBlueprint()); + auto expect_up = std::make_unique<EmptyBlueprint>(); top_up = Blueprint::optimize(std::move(top_up)); EXPECT_EQUAL(expect_up->asString(), top_up->asString()); } @@ -1395,22 +1419,57 @@ TEST("require that highest cost tier sorts last for AND") { EXPECT_EQUAL(expect_up->asString(), top_up->asString()); } -TEST("require that intermediate cost tier is minimum cost tier of children") { - Blueprint::UP bp1( - ap((new AndBlueprint())-> - addChild(ap(MyLeafSpec(10).cost_tier(1).create())). - addChild(ap(MyLeafSpec(20).cost_tier(2).create())). - addChild(ap(MyLeafSpec(30).cost_tier(3).create())))); - Blueprint::UP bp2( - ap((new AndBlueprint())-> - addChild(ap(MyLeafSpec(10).cost_tier(3).create())). - addChild(ap(MyLeafSpec(20).cost_tier(2).create())). - addChild(ap(MyLeafSpec(30).cost_tier(2).create())))); - EXPECT_EQUAL(bp1->getState().cost_tier(), 1u); - EXPECT_EQUAL(bp2->getState().cost_tier(), 2u); +template<typename BP> +void +verifyCostTierInheritance(uint8_t expected, uint8_t expected_reverse) { + auto bp1 = std::make_unique<BP>(); + bp1->addChild(ap(MyLeafSpec(10).cost_tier(1).create())). + addChild(ap(MyLeafSpec(20).cost_tier(2).create())). + addChild(ap(MyLeafSpec(30).cost_tier(3).create())); + auto bp2 = std::make_unique<BP>(); + bp2->addChild(ap(MyLeafSpec(10).cost_tier(3).create())). + addChild(ap(MyLeafSpec(20).cost_tier(2).create())). + addChild(ap(MyLeafSpec(30).cost_tier(1).create())); + EXPECT_EQUAL(bp1->getState().cost_tier(), expected); + EXPECT_EQUAL(bp2->getState().cost_tier(), expected_reverse); +} + +TEST("require that AND cost tier is minimum cost tier of children") { + verifyCostTierInheritance<AndBlueprint>(1, 1); +} + +TEST("require that OR cost tier is maximum cost tier of children") { + verifyCostTierInheritance<OrBlueprint>(3, 3); +} + +TEST("require that Rank cost tier is first childs cost tier") { + verifyCostTierInheritance<RankBlueprint>(1, 3); +} + +TEST("require that AndNot cost tier is first childs cost tier") { + verifyCostTierInheritance<AndNotBlueprint>(1, 3); +} + +struct MySourceBlender { + InvalidSelector selector; + SourceBlenderBlueprint sb; + MySourceBlender() : selector(), sb(selector) {} + IntermediateBlueprint & + addChild(Blueprint::UP child) { + return sb.addChild(std::move(child)); + } + const Blueprint::State &getState() const { + return sb.getState(); + } + +}; + +TEST("require that SourceBlender cost tier is maximum cost tier of children") { + verifyCostTierInheritance<MySourceBlender>(3, 3); } -void verify_or_est(const std::vector<Blueprint::HitEstimate> &child_estimates, Blueprint::HitEstimate expect) { +void +verify_or_est(const std::vector<Blueprint::HitEstimate> &child_estimates, Blueprint::HitEstimate expect) { OrBlueprint my_or; my_or.setDocIdLimit(32); auto my_est = my_or.combine(child_estimates); diff --git a/searchlib/src/tests/queryeval/filter_search/filter_search_test.cpp b/searchlib/src/tests/queryeval/filter_search/filter_search_test.cpp index 71033ed7d06..5933122d7a2 100644 --- a/searchlib/src/tests/queryeval/filter_search/filter_search_test.cpp +++ b/searchlib/src/tests/queryeval/filter_search/filter_search_test.cpp @@ -47,7 +47,7 @@ concept ChildCollector = requires(T a, std::unique_ptr<Blueprint> bp) { // inherit Blueprint to capture the default filter factory struct DefaultBlueprint : Blueprint { - void optimize(Blueprint* &) override { abort(); } + void optimize(Blueprint* &, OptimizePass) override { abort(); } const State &getState() const override { abort(); } void fetchPostings(const ExecuteInfo &) override { abort(); } void freeze() override { abort(); } diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp index e4047b57341..cc12b1f7825 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp @@ -204,7 +204,7 @@ createPostingIterator(fef::TermFieldMatchData *matchData, bool strict) } } // returning nullptr will trigger fallback to filter iterator - return SearchIterator::UP(); + return {}; } diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp index 8bee8b3f7f7..2e9e5470b5f 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp @@ -44,8 +44,8 @@ PostingStoreBase2::resizeBitVectors(uint32_t newSize, uint32_t newCapacity) newSize = newCapacity; if (newSize == _bvSize && newCapacity == _bvCapacity) return false; - _minBvDocFreq = std::max(newSize >> 6, 64u); - _maxBvDocFreq = std::max(newSize >> 5, 128u); + _minBvDocFreq = std::max(newSize >> 7, 64u); + _maxBvDocFreq = std::max(newSize >> 6, 128u); if (_bvs.empty()) { _bvSize = newSize; _bvCapacity = newCapacity; @@ -613,9 +613,13 @@ PostingStore<DataT>::update_stat(const CompactionStrategy& compaction_strategy) vespalib::MemoryUsage usage; auto btree_nodes_memory_usage = _allocator.getMemoryUsage(); auto store_memory_usage = _store.getMemoryUsage(); - _compaction_spec = PostingStoreCompactionSpec(compaction_strategy.should_compact_memory(btree_nodes_memory_usage), compaction_strategy.should_compact_memory(store_memory_usage)); usage.merge(btree_nodes_memory_usage); usage.merge(store_memory_usage); + if (compaction_strategy.should_compact_memory(usage)) { + _compaction_spec = PostingStoreCompactionSpec(compaction_strategy.should_compact_memory(btree_nodes_memory_usage), compaction_strategy.should_compact_memory(store_memory_usage)); + } else { + _compaction_spec = PostingStoreCompactionSpec(); + } uint64_t bvExtraBytes = _bvExtraBytes; usage.incUsedBytes(bvExtraBytes); usage.incAllocatedBytes(bvExtraBytes); diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.h b/searchlib/src/vespa/searchlib/attribute/postingstore.h index e3fb88c81f1..bd19bbd3675 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.h +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.h @@ -57,6 +57,9 @@ public: virtual ~PostingStoreBase2(); bool resizeBitVectors(uint32_t newSize, uint32_t newCapacity); virtual bool removeSparseBitVectors() = 0; + + // Only used by unit test. + const PostingStoreCompactionSpec& get_compaction_spec() const noexcept { return _compaction_spec; } }; template <typename DataT> diff --git a/searchlib/src/vespa/searchlib/expression/expressiontree.h b/searchlib/src/vespa/searchlib/expression/expressiontree.h index 52c075f3e29..127de66b6dc 100644 --- a/searchlib/src/vespa/searchlib/expression/expressiontree.h +++ b/searchlib/src/vespa/searchlib/expression/expressiontree.h @@ -50,11 +50,11 @@ public: }; ExpressionTree() noexcept; - ExpressionTree(const ExpressionNode & root); - ExpressionTree(ExpressionNode::UP root); + explicit ExpressionTree(const ExpressionNode & root); + explicit ExpressionTree(ExpressionNode::UP root); ExpressionTree(const ExpressionTree & rhs); ExpressionTree(ExpressionTree &&) noexcept = default; - ~ExpressionTree(); + ~ExpressionTree() override; ExpressionTree & operator = (ExpressionNode::UP rhs); ExpressionTree & operator = (const ExpressionTree & rhs); ExpressionTree & operator = (ExpressionTree &&) noexcept = default; diff --git a/searchlib/src/vespa/searchlib/fef/indexproperties.cpp b/searchlib/src/vespa/searchlib/fef/indexproperties.cpp index 9b111c4bd5d..9c986d0bc63 100644 --- a/searchlib/src/vespa/searchlib/fef/indexproperties.cpp +++ b/searchlib/src/vespa/searchlib/fef/indexproperties.cpp @@ -4,6 +4,7 @@ #include "properties.h" #include <vespa/vespalib/locale/c.h> #include <limits> +#include <charconv> namespace search::fef::indexproperties { @@ -49,10 +50,15 @@ uint32_t lookupUint32(const Properties &props, const vespalib::string &name, uint32_t defaultValue) { Property p = props.lookup(name); + uint32_t value(defaultValue); if (p.found()) { - return atoi(p.get().c_str()); + const auto & valS = p.get(); + const char * start = valS.c_str(); + const char * end = start + valS.size(); + while ((start != end) && isspace(start[0])) { start++; } + std::from_chars(start, end, value); } - return defaultValue; + return value; } bool @@ -173,11 +179,6 @@ namespace onsummary { namespace temporary { -const vespalib::string EnableNestedMultivalueGrouping::NAME("vespa.temporary.enable_nested_multivalue_grouping"); -bool EnableNestedMultivalueGrouping::check(const Properties &props) { - return lookupBool(props, NAME, false); -} - } namespace mutate { @@ -454,6 +455,12 @@ FuzzyAlgorithm::lookup(const Properties& props, vespalib::FuzzyMatchingAlgorithm return vespalib::fuzzy_matching_algorithm_from_string(value, default_value); } +const vespalib::string AlwaysMarkPhraseExpensive::NAME("vespa.matching.always_mark_phrase_expensive"); +const bool AlwaysMarkPhraseExpensive::DEFAULT_VALUE(false); +bool AlwaysMarkPhraseExpensive::check(const Properties &props, bool fallback) { + return lookupBool(props, NAME, fallback); +} + } // namespace matching namespace softtimeout { diff --git a/searchlib/src/vespa/searchlib/fef/indexproperties.h b/searchlib/src/vespa/searchlib/fef/indexproperties.h index c528c4366d6..1921f52276f 100644 --- a/searchlib/src/vespa/searchlib/fef/indexproperties.h +++ b/searchlib/src/vespa/searchlib/fef/indexproperties.h @@ -176,11 +176,8 @@ namespace mutate { }; } +// Add temporary flags used for safe rollout of new features here namespace temporary { - struct EnableNestedMultivalueGrouping { - static const vespalib::string NAME; - static bool check(const Properties &props); - }; } namespace mutate::on_match { @@ -339,6 +336,17 @@ namespace matching { static vespalib::FuzzyMatchingAlgorithm lookup(const Properties& props); static vespalib::FuzzyMatchingAlgorithm lookup(const Properties& props, vespalib::FuzzyMatchingAlgorithm default_value); }; + + /** + * When enabled, the unpacking part of the phrase iterator will be tagged as expensive + * under all intermediate iterators, not only AND. + **/ + struct AlwaysMarkPhraseExpensive { + static const vespalib::string NAME; + static const bool DEFAULT_VALUE; + static bool check(const Properties &props) { return check(props, DEFAULT_VALUE); } + static bool check(const Properties &props, bool fallback); + }; } namespace softtimeout { diff --git a/searchlib/src/vespa/searchlib/fef/properties.cpp b/searchlib/src/vespa/searchlib/fef/properties.cpp index bd4795fcc5a..6134807fa60 100644 --- a/searchlib/src/vespa/searchlib/fef/properties.cpp +++ b/searchlib/src/vespa/searchlib/fef/properties.cpp @@ -25,7 +25,7 @@ uint32_t Properties::rawHash(const void *buf, uint32_t len) noexcept { uint32_t res = 0; - unsigned const char *pt = (unsigned const char *) buf; + auto *pt = (unsigned const char *) buf; unsigned const char *end = pt + len; while (pt < end) { res = (res << 7) + (res >> 25) + *pt++; @@ -52,7 +52,7 @@ Properties::add(vespalib::stringref key, vespalib::stringref value) { if (!key.empty()) { Value & v = _data[key]; - v.push_back(value); + v.emplace_back(value); ++_numValues; } return *this; @@ -162,20 +162,20 @@ Property Properties::lookup(vespalib::stringref key) const noexcept { if (key.empty()) { - return Property(); + return {}; } auto node = _data.find(key); if (node == _data.end()) { - return Property(); + return {}; } - return Property(node->second); + return {node->second}; } Property Properties::lookup(vespalib::stringref namespace1, vespalib::stringref key) const noexcept { if (namespace1.empty() || key.empty()) { - return Property(); + return {}; } vespalib::string fullKey(namespace1); fullKey.append('.').append(key); @@ -187,7 +187,7 @@ Property Properties::lookup(vespalib::stringref namespace1, vespalib::stringref key) const noexcept { if (namespace1.empty() || namespace2.empty() || key.empty()) { - return Property(); + return {}; } vespalib::string fullKey(namespace1); fullKey.append('.').append(namespace2).append('.').append(key); @@ -200,7 +200,7 @@ Property Properties::lookup(vespalib::stringref namespace1, vespalib::stringref key) const noexcept { if (namespace1.empty() || namespace2.empty() || namespace3.empty() || key.empty()) { - return Property(); + return {}; } vespalib::string fullKey(namespace1); fullKey.append('.').append(namespace2).append('.').append(namespace3).append('.').append(key); diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp index 33e7dbda04b..d6b0b900516 100644 --- a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp +++ b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp @@ -60,6 +60,7 @@ RankSetup::RankSetup(const BlueprintFactory &factory, const IIndexEnvironment &i _compiled(false), _compileError(false), _degradationAscendingOrder(false), + _always_mark_phrase_expensive(false), _diversityAttribute(), _diversityMinGroups(1), _diversityCutoffFactor(10.0), @@ -74,8 +75,7 @@ RankSetup::RankSetup(const BlueprintFactory &factory, const IIndexEnvironment &i _mutateOnFirstPhase(), _mutateOnSecondPhase(), _mutateOnSummary(), - _mutateAllowQueryOverride(false), - _enableNestedMultivalueGrouping(false) + _mutateAllowQueryOverride(false) { } RankSetup::~RankSetup() = default; @@ -134,7 +134,7 @@ RankSetup::configure() _mutateOnSummary._attribute = mutate::on_summary::Attribute::lookup(_indexEnv.getProperties()); _mutateOnSummary._operation = mutate::on_summary::Operation::lookup(_indexEnv.getProperties()); _mutateAllowQueryOverride = mutate::AllowQueryOverride::check(_indexEnv.getProperties()); - _enableNestedMultivalueGrouping = temporary::EnableNestedMultivalueGrouping::check(_indexEnv.getProperties()); + _always_mark_phrase_expensive = matching::AlwaysMarkPhraseExpensive::check(_indexEnv.getProperties()); } void diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.h b/searchlib/src/vespa/searchlib/fef/ranksetup.h index 6f4651939ad..d744b38cc6e 100644 --- a/searchlib/src/vespa/searchlib/fef/ranksetup.h +++ b/searchlib/src/vespa/searchlib/fef/ranksetup.h @@ -32,7 +32,7 @@ public: : _attribute(attribute), _operation(operation) {} - bool enabled() const { return !_attribute.empty() && !_operation.empty(); } + bool enabled() const noexcept { return !_attribute.empty() && !_operation.empty(); } vespalib::string _attribute; vespalib::string _operation; }; @@ -69,6 +69,7 @@ private: bool _compiled; bool _compileError; bool _degradationAscendingOrder; + bool _always_mark_phrase_expensive; vespalib::string _diversityAttribute; uint32_t _diversityMinGroups; double _diversityCutoffFactor; @@ -84,7 +85,6 @@ private: MutateOperation _mutateOnSecondPhase; MutateOperation _mutateOnSummary; bool _mutateAllowQueryOverride; - bool _enableNestedMultivalueGrouping; void compileAndCheckForErrors(BlueprintResolver &bp); public: @@ -221,6 +221,7 @@ public: bool isDegradationOrderAscending() const { return _degradationAscendingOrder; } + bool always_mark_phrase_expensive() const noexcept { return _always_mark_phrase_expensive; } /** get number of hits to collect during graceful degradation in match phase */ uint32_t getDegradationMaxHits() const { return _degradationMaxHits; @@ -458,7 +459,6 @@ public: const MutateOperation & getMutateOnSummary() const { return _mutateOnSummary; } bool allowMutateQueryOverride() const { return _mutateAllowQueryOverride; } - bool enableNestedMultivalueGrouping() const { return _enableNestedMultivalueGrouping; } }; } diff --git a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp index d1a2c26a75d..ac54ea205be 100644 --- a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp +++ b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp @@ -1,35 +1,15 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "stackdumpiterator.h" +#include <vespa/searchlib/query/tree/predicate_query_term.h> #include <vespa/vespalib/util/compress.h> #include <vespa/vespalib/objects/nbo.h> +#include <cassert> using search::query::PredicateQueryTerm; namespace search { -namespace { - -uint64_t -readUint64(const char *&p) -{ - uint64_t value; - memcpy(&value, p, sizeof(value)); - p += sizeof(value); - return vespalib::nbo::n2h(value); -} - -double -read_double(const char *&p) -{ - double value; - memcpy(&value, p, sizeof(value)); - p += sizeof(value); - return vespalib::nbo::n2h(value); -} - -} - SimpleQueryStackDumpIterator::SimpleQueryStackDumpIterator(vespalib::stringref buf) : _buf(buf.begin()), _bufEnd(buf.end()), @@ -57,8 +37,7 @@ SimpleQueryStackDumpIterator::~SimpleQueryStackDumpIterator() = default; vespalib::stringref SimpleQueryStackDumpIterator::read_stringref(const char *&p) { - uint64_t len; - p += vespalib::compress::Integer::decompressPositive(len, p); + uint64_t len = readCompressedPositiveInt(p); if ((p + len) > _bufEnd) throw false; vespalib::stringref result(p, len); p += len; @@ -68,12 +47,40 @@ SimpleQueryStackDumpIterator::read_stringref(const char *&p) uint64_t SimpleQueryStackDumpIterator::readCompressedPositiveInt(const char *&p) { + if (p > _bufEnd || !vespalib::compress::Integer::check_decompress_space(p, _bufEnd - p)) { + throw false; + } uint64_t tmp; p += vespalib::compress::Integer::decompressPositive(tmp, p); - if (p > _bufEnd) throw false; + assert(p <= _bufEnd); return tmp; } +int64_t +SimpleQueryStackDumpIterator::readCompressedInt(const char *&p) +{ + if (p > _bufEnd || !vespalib::compress::Integer::check_decompress_positive_space(p, _bufEnd - p)) { + throw false; + } + int64_t tmp; + p += vespalib::compress::Integer::decompress(tmp, p); + assert(p <= _bufEnd); + return tmp; +} + +template <typename T> +T +SimpleQueryStackDumpIterator::read_value(const char *&p) +{ + T value; + if (p + sizeof(value) > _bufEnd) { + throw false; + } + memcpy(&value, p, sizeof(value)); + p += sizeof(value); + return vespalib::nbo::n2h(value); +} + bool SimpleQueryStackDumpIterator::next() { try { @@ -97,11 +104,8 @@ bool SimpleQueryStackDumpIterator::readNext() { _currType = ParseItem::GetType(typefield); if (ParseItem::GetFeature_Weight(typefield)) { - int64_t tmpLong; - if (p >= _bufEnd) return false; - p += vespalib::compress::Integer::decompress(tmpLong, p); + int64_t tmpLong = readCompressedInt(p); _currWeight.setPercent(tmpLong); - if (p > _bufEnd) return false; } else { _currWeight.setPercent(100); } @@ -154,13 +158,8 @@ bool SimpleQueryStackDumpIterator::readNext() { _currArity = 0; break; case ParseItem::ITEM_PURE_WEIGHTED_LONG: - { - if (p + sizeof(int64_t) > _bufEnd) { - return false; - } - _curr_integer_term = readUint64(p); - _currArity = 0; - } + _curr_integer_term = read_value<int64_t>(p); + _currArity = 0; break; case ParseItem::ITEM_WORD_ALTERNATIVES: _curr_index_name = read_stringref(p); @@ -180,20 +179,20 @@ bool SimpleQueryStackDumpIterator::readNext() { _currArity = 0; break; case ParseItem::ITEM_PREDICATE_QUERY: - if ( ! readPredicate(p)) return false; + readPredicate(p); break; case ParseItem::ITEM_WEIGHTED_SET: case ParseItem::ITEM_DOT_PRODUCT: case ParseItem::ITEM_WAND: case ParseItem::ITEM_PHRASE: - if (!readComplexTerm(p)) return false; + readComplexTerm(p); break; case ParseItem::ITEM_NEAREST_NEIGHBOR: - if ( ! readNN(p)) return false; + readNN(p); break; case ParseItem::ITEM_FUZZY: - if (!readFuzzy(p)) return false; + readFuzzy(p); break; case ParseItem::ITEM_TRUE: case ParseItem::ITEM_FALSE: @@ -210,7 +209,7 @@ bool SimpleQueryStackDumpIterator::readNext() { return (p <= _bufEnd); } -bool +void SimpleQueryStackDumpIterator::readPredicate(const char *&p) { _curr_index_name = read_stringref(p); _predicate_query_term = std::make_unique<PredicateQueryTerm>(); @@ -219,22 +218,19 @@ SimpleQueryStackDumpIterator::readPredicate(const char *&p) { for (size_t i = 0; i < count; ++i) { vespalib::stringref key = read_stringref(p); vespalib::stringref value = read_stringref(p); - if (p + sizeof(uint64_t) > _bufEnd) return false; - uint64_t sub_queries = readUint64(p); + uint64_t sub_queries = read_value<uint64_t>(p); _predicate_query_term->addFeature(key, value, sub_queries); } count = readCompressedPositiveInt(p); for (size_t i = 0; i < count; ++i) { vespalib::stringref key = read_stringref(p); - if (p + 2*sizeof(uint64_t) > _bufEnd) return false; - uint64_t value = readUint64(p); - uint64_t sub_queries = readUint64(p); + uint64_t value = read_value<uint64_t>(p); + uint64_t sub_queries = read_value<uint64_t>(p); _predicate_query_term->addRangeFeature(key, value, sub_queries); } - return true; } -bool +void SimpleQueryStackDumpIterator::readNN(const char *& p) { _curr_index_name = read_stringref(p); _curr_term = read_stringref(p); // query_tensor_name @@ -244,34 +240,35 @@ SimpleQueryStackDumpIterator::readNN(const char *& p) { // XXX: remove later when QRS doesn't send this extra flag _extraIntArg2 &= ~0x40; // QRS always sends this now: - if ((p + sizeof(double))> _bufEnd) return false; - _extraDoubleArg4 = read_double(p); // distance threshold + _extraDoubleArg4 = read_value<double>(p); // distance threshold _currArity = 0; - return true; } -bool +void SimpleQueryStackDumpIterator::readComplexTerm(const char *& p) { _currArity = readCompressedPositiveInt(p); _curr_index_name = read_stringref(p); if (_currType == ParseItem::ITEM_WAND) { _extraIntArg1 = readCompressedPositiveInt(p); // targetNumHits - if ((p + 2*sizeof(double))> _bufEnd) return false; - _extraDoubleArg4 = read_double(p); // scoreThreshold - _extraDoubleArg5 = read_double(p); // thresholdBoostFactor + _extraDoubleArg4 = read_value<double>(p); // scoreThreshold + _extraDoubleArg5 = read_value<double>(p); // thresholdBoostFactor } _curr_term = vespalib::stringref(); - return true; } -bool +void SimpleQueryStackDumpIterator::readFuzzy(const char *&p) { _curr_index_name = read_stringref(p); _curr_term = read_stringref(p); // fuzzy term _extraIntArg1 = readCompressedPositiveInt(p); // maxEditDistance _extraIntArg2 = readCompressedPositiveInt(p); // prefixLength _currArity = 0; - return true; +} + +std::unique_ptr<query::PredicateQueryTerm> +SimpleQueryStackDumpIterator::getPredicateQueryTerm() +{ + return std::move(_predicate_query_term); } } diff --git a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h index dece4ecc0b6..add26ac7938 100644 --- a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h +++ b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h @@ -3,8 +3,10 @@ #pragma once #include "parse.h" -#include <vespa/searchlib/query/tree/predicate_query_term.h> #include <vespa/vespalib/stllike/string.h> +#include <memory> + +namespace search::query { class PredicateQueryTerm; } namespace search { /** @@ -45,14 +47,17 @@ private: double _extraDoubleArg4; double _extraDoubleArg5; /** The predicate query specification */ - query::PredicateQueryTerm::UP _predicate_query_term; + std::unique_ptr<query::PredicateQueryTerm> _predicate_query_term; VESPA_DLL_LOCAL vespalib::stringref read_stringref(const char *&p); VESPA_DLL_LOCAL uint64_t readCompressedPositiveInt(const char *&p); - VESPA_DLL_LOCAL bool readPredicate(const char *&p); - VESPA_DLL_LOCAL bool readNN(const char *&p); - VESPA_DLL_LOCAL bool readComplexTerm(const char *& p); - VESPA_DLL_LOCAL bool readFuzzy(const char *&p); + VESPA_DLL_LOCAL int64_t readCompressedInt(const char *&p); + template <typename T> + VESPA_DLL_LOCAL T read_value(const char*& p); + VESPA_DLL_LOCAL void readPredicate(const char *&p); + VESPA_DLL_LOCAL void readNN(const char *&p); + VESPA_DLL_LOCAL void readComplexTerm(const char *& p); + VESPA_DLL_LOCAL void readFuzzy(const char *&p); VESPA_DLL_LOCAL bool readNext(); public: /** @@ -118,7 +123,7 @@ public: uint32_t getFuzzyMaxEditDistance() const { return _extraIntArg1; } uint32_t getFuzzyPrefixLength() const { return _extraIntArg2; } - query::PredicateQueryTerm::UP getPredicateQueryTerm() { return std::move(_predicate_query_term); } + std::unique_ptr<query::PredicateQueryTerm> getPredicateQueryTerm(); vespalib::stringref getIndexName() const { return _curr_index_name; } vespalib::stringref getTerm() const { return _curr_term; } diff --git a/searchlib/src/vespa/searchlib/query/tree/intermediatenodes.h b/searchlib/src/vespa/searchlib/query/tree/intermediatenodes.h index a644387a0df..871c258494f 100644 --- a/searchlib/src/vespa/searchlib/query/tree/intermediatenodes.h +++ b/searchlib/src/vespa/searchlib/query/tree/intermediatenodes.h @@ -30,14 +30,14 @@ public: //----------------------------------------------------------------------------- class WeakAnd : public QueryNodeMixin<WeakAnd, Intermediate> { - uint32_t _minHits; + uint32_t _targetNumHits; vespalib::string _view; public: virtual ~WeakAnd() = 0; - WeakAnd(uint32_t minHits, const vespalib::string & view) : _minHits(minHits), _view(view) {} + WeakAnd(uint32_t targetNumHits, const vespalib::string & view) : _targetNumHits(targetNumHits), _view(view) {} - uint32_t getMinHits() const { return _minHits; } + uint32_t getTargetNumHits() const { return _targetNumHits; } const vespalib::string & getView() const { return _view; } }; diff --git a/searchlib/src/vespa/searchlib/query/tree/querybuilder.h b/searchlib/src/vespa/searchlib/query/tree/querybuilder.h index 157bc13016d..4f9632e8df9 100644 --- a/searchlib/src/vespa/searchlib/query/tree/querybuilder.h +++ b/searchlib/src/vespa/searchlib/query/tree/querybuilder.h @@ -117,8 +117,8 @@ template <class NodeTypes> typename NodeTypes::Or *createOr() { return new typename NodeTypes::Or; } template <class NodeTypes> -typename NodeTypes::WeakAnd *createWeakAnd(uint32_t minHits, vespalib::stringref view) { - return new typename NodeTypes::WeakAnd(minHits, view); +typename NodeTypes::WeakAnd *createWeakAnd(uint32_t targetNumHits, vespalib::stringref view) { + return new typename NodeTypes::WeakAnd(targetNumHits, view); } template <class NodeTypes> typename NodeTypes::Equiv *createEquiv(int32_t id, Weight weight) { @@ -259,8 +259,8 @@ public: typename NodeTypes::Or &addOr(int child_count) { return addIntermediate(createOr<NodeTypes>(), child_count); } - typename NodeTypes::WeakAnd &addWeakAnd(int child_count, uint32_t minHits, stringref view) { - return addIntermediate(createWeakAnd<NodeTypes>(minHits, view), child_count); + typename NodeTypes::WeakAnd &addWeakAnd(int child_count, uint32_t targetNumHits, stringref view) { + return addIntermediate(createWeakAnd<NodeTypes>(targetNumHits, view), child_count); } typename NodeTypes::Equiv &addEquiv(int child_count, int32_t id, Weight weight) { return addIntermediate(createEquiv<NodeTypes>(id, weight), child_count); diff --git a/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h b/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h index bf77d3f77ea..b7cce0f15b1 100644 --- a/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h +++ b/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h @@ -43,7 +43,7 @@ private: } void visit(WeakAnd &node) override { - _builder.addWeakAnd(node.getChildren().size(), node.getMinHits(), node.getView()); + _builder.addWeakAnd(node.getChildren().size(), node.getTargetNumHits(), node.getView()); visitNodes(node.getChildren()); } diff --git a/searchlib/src/vespa/searchlib/query/tree/simplequery.h b/searchlib/src/vespa/searchlib/query/tree/simplequery.h index b2b84402593..51af979a230 100644 --- a/searchlib/src/vespa/searchlib/query/tree/simplequery.h +++ b/searchlib/src/vespa/searchlib/query/tree/simplequery.h @@ -38,8 +38,8 @@ struct SimpleOr : Or ~SimpleOr() override; }; struct SimpleWeakAnd : WeakAnd { - SimpleWeakAnd(uint32_t minHits, vespalib::stringref view) : - WeakAnd(minHits, view) + SimpleWeakAnd(uint32_t targetNumHits, vespalib::stringref view) : + WeakAnd(targetNumHits, view) {} }; struct SimpleEquiv : Equiv { diff --git a/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp b/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp index 3a0deff7697..465263afeb4 100644 --- a/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp @@ -9,6 +9,7 @@ #include <vespa/vespalib/util/size_literals.h> #include <vespa/searchlib/parsequery/parse.h> #include <vespa/searchlib/util/rawbuf.h> +#include <cassert> using vespalib::string; using std::vector; @@ -50,6 +51,13 @@ class QueryNodeConverter : public QueryVisitor { _buf.append(&i, sizeof(uint8_t)); } + void append_type_and_features(ParseItem::ItemType type, uint8_t item_features) { + assert(static_cast<uint32_t>(type) < 31u); + uint8_t type_and_features = (static_cast<uint8_t>(type) | item_features); + _buf.preAlloc(sizeof(uint8_t)); + _buf.append(&type_and_features, sizeof(uint8_t)); + } + void appendDouble(double i) { _buf.preAlloc(sizeof(double)); double nboVal = vespalib::nbo::n2h(i); @@ -61,7 +69,7 @@ class QueryNodeConverter : public QueryVisitor { template <typename V> void appendPredicateQueryTermVector(const V& v); - void createComplexIntermediate(const Term &node, const std::vector<Node *> & children, size_t type) { + void createComplexIntermediate(const Term &node, const std::vector<Node *> & children, ParseItem::ItemType type, uint8_t features) { uint8_t flags = 0; if (!node.isRanked()) { flags |= ParseItem::IFLAG_NORANK; @@ -70,11 +78,11 @@ class QueryNodeConverter : public QueryVisitor { flags |= ParseItem::IFLAG_NOPOSITIONDATA; } if (flags != 0) { - type |= ParseItem::IF_FLAGS; + features |= ParseItem::IF_FLAGS; } - appendByte(type); + append_type_and_features(type, features); appendCompressedNumber(node.getWeight().percent()); - if (type & ParseItem::IF_FLAGS) { + if ((features & ParseItem::IF_FLAGS) != 0) { appendByte(flags); } appendCompressedPositiveNumber(children.size()); @@ -82,30 +90,30 @@ class QueryNodeConverter : public QueryVisitor { visitNodes(children); } - void createIntermediate(const Intermediate &node, size_t type) { - appendByte(type); + void createIntermediate(const Intermediate &node, ParseItem::ItemType type) { + append_type_and_features(type, 0); appendCompressedPositiveNumber(node.getChildren().size()); visitNodes(node.getChildren()); } - void createIntermediate(const Intermediate &node, size_t type, size_t distance) { - appendByte(type); + void createIntermediate(const Intermediate &node, ParseItem::ItemType type, size_t distance) { + append_type_and_features(type, 0); appendCompressedPositiveNumber(node.getChildren().size()); appendCompressedPositiveNumber(distance); visitNodes(node.getChildren()); } - void createIntermediate(const Intermediate &node, size_t type, const vespalib::string & view) { - appendByte(type); + void createIntermediate(const Intermediate &node, ParseItem::ItemType type, const vespalib::string & view) { + append_type_and_features(type, 0); appendCompressedPositiveNumber(node.getChildren().size()); appendString(view); visitNodes(node.getChildren()); } - void createIntermediate(const Intermediate &node, size_t type, size_t distance, const vespalib::string & view) { - appendByte(type); + void createIntermediateX(const Intermediate &node, ParseItem::ItemType type, size_t target_num_hits, const vespalib::string & view) { + append_type_and_features(type, 0); appendCompressedPositiveNumber(node.getChildren().size()); - appendCompressedPositiveNumber(distance); + appendCompressedPositiveNumber(target_num_hits); appendString(view); visitNodes(node.getChildren()); } @@ -131,7 +139,7 @@ class QueryNodeConverter : public QueryVisitor { } void visit(WeakAnd &node) override { - createIntermediate(node, ParseItem::ITEM_WEAK_AND, node.getMinHits(), node.getView()); + createIntermediateX(node, ParseItem::ITEM_WEAK_AND, node.getTargetNumHits(), node.getView()); } void visit(Equiv &node) override { @@ -143,11 +151,11 @@ class QueryNodeConverter : public QueryVisitor { } void visit(Phrase &node) override { - createComplexIntermediate(node, node.getChildren(), (static_cast<uint8_t>(ParseItem::ITEM_PHRASE) | static_cast<uint8_t>(ParseItem::IF_WEIGHT))); + createComplexIntermediate(node, node.getChildren(), ParseItem::ITEM_PHRASE, static_cast<uint8_t>(ParseItem::IF_WEIGHT)); } template <typename NODE> - void createWeightedSet(NODE &node, uint8_t typefield) { + void createWeightedSet(NODE &node, ParseItem::ItemType type, uint8_t features) { uint8_t flags = 0; if (!node.isRanked()) { flags |= ParseItem::IFLAG_NORANK; @@ -158,11 +166,11 @@ class QueryNodeConverter : public QueryVisitor { flags |= ParseItem::IFLAG_NOPOSITIONDATA; } if (flags != 0) { - typefield |= ParseItem::IF_FLAGS; + features |= ParseItem::IF_FLAGS; } - appendByte(typefield); + append_type_and_features(type, features); appendCompressedNumber(node.getWeight().percent()); - if (typefield & ParseItem::IF_FLAGS) { + if ((features & ParseItem::IF_FLAGS) != 0) { appendByte(flags); } appendCompressedPositiveNumber(node.getNumTerms()); @@ -172,25 +180,24 @@ class QueryNodeConverter : public QueryVisitor { void createMultiTermNodes(const MultiTerm & mt) { for (size_t i = 0; i < mt.getNumTerms(); ++i) { auto term = mt.getAsString(i); - uint8_t typeField = static_cast<uint8_t>(ParseItem::ITEM_PURE_WEIGHTED_STRING) | static_cast<uint8_t>(ParseItem::IF_WEIGHT); - appendByte(typeField); + append_type_and_features(ParseItem::ITEM_PURE_WEIGHTED_STRING, static_cast<uint8_t>(ParseItem::IF_WEIGHT)); appendCompressedNumber(term.second.percent()); appendString(term.first); } } void visit(WeightedSetTerm &node) override { - createWeightedSet(node, static_cast<uint8_t>(ParseItem::ITEM_WEIGHTED_SET) | static_cast<uint8_t>(ParseItem::IF_WEIGHT)); + createWeightedSet(node, ParseItem::ITEM_WEIGHTED_SET, static_cast<uint8_t>(ParseItem::IF_WEIGHT)); createMultiTermNodes(node); } void visit(DotProduct &node) override { - createWeightedSet(node, static_cast<uint8_t>(ParseItem::ITEM_DOT_PRODUCT) | static_cast<uint8_t>(ParseItem::IF_WEIGHT)); + createWeightedSet(node, ParseItem::ITEM_DOT_PRODUCT, static_cast<uint8_t>(ParseItem::IF_WEIGHT)); createMultiTermNodes(node); } void visit(WandTerm &node) override { - createWeightedSet(node, static_cast<uint8_t>(ParseItem::ITEM_WAND) | static_cast<uint8_t>(ParseItem::IF_WEIGHT)); + createWeightedSet(node, ParseItem::ITEM_WAND, static_cast<uint8_t>(ParseItem::IF_WEIGHT)); appendCompressedPositiveNumber(node.getTargetNumHits()); appendDouble(node.getScoreThreshold()); appendDouble(node.getThresholdBoostFactor()); @@ -203,8 +210,8 @@ class QueryNodeConverter : public QueryVisitor { template <typename T> void appendTerm(const TermBase<T> &node); - void createTermNode(const TermNode &node, size_t type) { - uint8_t typefield = type | ParseItem::IF_WEIGHT | ParseItem::IF_UNIQUEID; + void createTermNode(const TermNode &node, ParseItem::ItemType type) { + uint8_t features = ParseItem::IF_WEIGHT | ParseItem::IF_UNIQUEID; uint8_t flags = 0; if (!node.isRanked()) { flags |= ParseItem::IFLAG_NORANK; @@ -213,19 +220,19 @@ class QueryNodeConverter : public QueryVisitor { flags |= ParseItem::IFLAG_NOPOSITIONDATA; } if (flags != 0) { - typefield |= ParseItem::IF_FLAGS; + features |= ParseItem::IF_FLAGS; } - appendByte(typefield); + append_type_and_features(type, features); appendCompressedNumber(node.getWeight().percent()); appendCompressedPositiveNumber(node.getId()); - if (typefield & ParseItem::IF_FLAGS) { + if ((features & ParseItem::IF_FLAGS) != 0) { appendByte(flags); } appendString(node.getView()); } template <class Term> - void createTerm(const Term &node, size_t type) { + void createTerm(const Term &node, ParseItem::ItemType type) { createTermNode(node, type); appendTerm(node); } @@ -239,11 +246,11 @@ class QueryNodeConverter : public QueryVisitor { } void visit(TrueQueryNode &) override { - appendByte(ParseItem::ITEM_TRUE); + append_type_and_features(ParseItem::ITEM_TRUE, 0); } void visit(FalseQueryNode &) override { - appendByte(ParseItem::ITEM_FALSE); + append_type_and_features(ParseItem::ITEM_FALSE, 0); } void visit(PrefixTerm &node) override { diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp index e006a95ae5f..5eaa2dc40ab 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp @@ -117,7 +117,7 @@ Blueprint::State::State(FieldSpecBaseList fields_in) noexcept Blueprint::State::~State() = default; Blueprint::Blueprint() noexcept - : _parent(0), + : _parent(nullptr), _sourceId(0xffffffff), _docid_limit(0), _frozen(false) @@ -129,19 +129,21 @@ Blueprint::~Blueprint() = default; Blueprint::UP Blueprint::optimize(Blueprint::UP bp) { Blueprint *root = bp.release(); - root->optimize(root); + root->optimize(root, OptimizePass::FIRST); + root->optimize(root, OptimizePass::SECOND); + root->optimize(root, OptimizePass::LAST); return Blueprint::UP(root); } void -Blueprint::optimize_self() +Blueprint::optimize_self(OptimizePass) { } Blueprint::UP Blueprint::get_replacement() { - return Blueprint::UP(); + return {}; } void @@ -163,8 +165,8 @@ std::unique_ptr<MatchingElementsSearch> Blueprint::create_matching_elements_search(const MatchingElementsFields &fields) const { (void) fields; - return std::unique_ptr<MatchingElementsSearch>(); -}; + return {}; +} namespace { @@ -196,7 +198,7 @@ template <typename Op> std::unique_ptr<SearchIterator> create_op_filter(const Blueprint::Children &children, bool strict, Blueprint::FilterConstraint constraint) { - REQUIRE(children.size() > 0); + REQUIRE( ! children.empty()); MultiSearch::Children list; std::unique_ptr<SearchIterator> spare; list.reserve(children.size()); @@ -261,7 +263,7 @@ Blueprint::create_atmost_or_filter(const Children &children, bool strict, Bluepr std::unique_ptr<SearchIterator> Blueprint::create_andnot_filter(const Children &children, bool strict, Blueprint::FilterConstraint constraint) { - REQUIRE(children.size() > 0); + REQUIRE( ! children.empty() ); MultiSearch::Children list; list.reserve(children.size()); { @@ -291,7 +293,7 @@ Blueprint::create_andnot_filter(const Children &children, bool strict, Blueprint std::unique_ptr<SearchIterator> Blueprint::create_first_child_filter(const Children &children, bool strict, Blueprint::FilterConstraint constraint) { - REQUIRE(children.size() > 0); + REQUIRE(!children.empty()); return children[0]->createFilterSearch(strict, constraint); } @@ -434,7 +436,7 @@ IntermediateBlueprint::infer_allow_termwise_eval() const } } return true; -}; +} bool IntermediateBlueprint::infer_want_global_filter() const @@ -539,19 +541,23 @@ IntermediateBlueprint::should_do_termwise_eval(const UnpackInfo &unpack, double } void -IntermediateBlueprint::optimize(Blueprint* &self) +IntermediateBlueprint::optimize(Blueprint* &self, OptimizePass pass) { assert(self == this); if (should_optimize_children()) { for (auto &child : _children) { auto *child_ptr = child.release(); - child_ptr->optimize(child_ptr); + child_ptr->optimize(child_ptr, pass); child.reset(child_ptr); } } - optimize_self(); - sort(_children); - maybe_eliminate_self(self, get_replacement()); + optimize_self(pass); + if (pass == OptimizePass::LAST) { + sort(_children); + } + if (pass == OptimizePass::FIRST) { + maybe_eliminate_self(self, get_replacement()); + } } void @@ -640,11 +646,7 @@ namespace { bool areAnyParentsEquiv(const Blueprint * node) { - return (node == nullptr) - ? false - : node->isEquiv() - ? true - : areAnyParentsEquiv(node->getParent()); + return (node != nullptr) && (node->isEquiv() || areAnyParentsEquiv(node->getParent())); } bool @@ -729,11 +731,13 @@ LeafBlueprint::getRange(vespalib::string &, vespalib::string &) const { } void -LeafBlueprint::optimize(Blueprint* &self) +LeafBlueprint::optimize(Blueprint* &self, OptimizePass pass) { assert(self == this); - optimize_self(); - maybe_eliminate_self(self, get_replacement()); + optimize_self(pass); + if (pass == OptimizePass::FIRST) { + maybe_eliminate_self(self, get_replacement()); + } } void @@ -767,7 +771,7 @@ LeafBlueprint::set_tree_size(uint32_t value) void visit(vespalib::ObjectVisitor &self, const vespalib::string &name, const search::queryeval::Blueprint *obj) { - if (obj != 0) { + if (obj != nullptr) { self.openStruct(name, obj->getClassName()); obj->visitMembers(self); self.closeStruct(); diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.h b/searchlib/src/vespa/searchlib/queryeval/blueprint.h index c9ebab7ae9d..cd0e8f2af40 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.h @@ -45,6 +45,8 @@ public: using Children = std::vector<Blueprint::UP>; using SearchIteratorUP = std::unique_ptr<SearchIterator>; + enum class OptimizePass { FIRST, SECOND, LAST }; + struct HitEstimate { uint32_t estHits; bool empty; @@ -79,8 +81,8 @@ public: static constexpr uint8_t COST_TIER_MAX = 255; State() noexcept; - State(FieldSpecBase field) noexcept; - State(FieldSpecBaseList fields_in) noexcept; + explicit State(FieldSpecBase field) noexcept; + explicit State(FieldSpecBaseList fields_in) noexcept; State(const State &rhs) = delete; State(State &&rhs) noexcept = default; State &operator=(const State &rhs) = delete; @@ -105,7 +107,7 @@ public: _estimateHits = est.estHits; _estimateEmpty = est.empty; } - HitEstimate estimate() const noexcept { return HitEstimate(_estimateHits, _estimateEmpty); } + HitEstimate estimate() const noexcept { return {_estimateHits, _estimateEmpty}; } double hit_ratio(uint32_t docid_limit) const noexcept { uint32_t total_hits = _estimateHits; uint32_t total_docs = std::max(total_hits, docid_limit); @@ -206,8 +208,8 @@ public: uint32_t get_docid_limit() const noexcept { return _docid_limit; } static Blueprint::UP optimize(Blueprint::UP bp); - virtual void optimize(Blueprint* &self) = 0; - virtual void optimize_self(); + virtual void optimize(Blueprint* &self, OptimizePass pass) = 0; + virtual void optimize_self(OptimizePass pass); virtual Blueprint::UP get_replacement(); virtual bool should_optimize_children() const { return true; } @@ -298,7 +300,7 @@ class IntermediateBlueprint : public blueprint::StateCache private: Children _children; HitEstimate calculateEstimate() const; - uint8_t calculate_cost_tier() const; + virtual uint8_t calculate_cost_tier() const; uint32_t calculate_tree_size() const; bool infer_allow_termwise_eval() const; bool infer_want_global_filter() const; @@ -326,7 +328,7 @@ public: void setDocIdLimit(uint32_t limit) noexcept final; - void optimize(Blueprint* &self) final; + void optimize(Blueprint* &self, OptimizePass pass) final; void set_global_filter(const GlobalFilter &global_filter, double estimated_hit_ratio) override; IndexList find(const IPredicate & check) const; @@ -361,7 +363,7 @@ class LeafBlueprint : public Blueprint private: State _state; protected: - void optimize(Blueprint* &self) final; + void optimize(Blueprint* &self, OptimizePass pass) final; void setEstimate(HitEstimate est) { _state.estimate(est); notifyChange(); @@ -374,7 +376,7 @@ protected: void set_want_global_filter(bool value); void set_tree_size(uint32_t value); - LeafBlueprint(bool allow_termwise_eval) noexcept + explicit LeafBlueprint(bool allow_termwise_eval) noexcept : _state() { _state.allow_termwise_eval(allow_termwise_eval); diff --git a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp index 6e0bc695fe3..c0439df1c1b 100644 --- a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp @@ -103,25 +103,27 @@ AndNotBlueprint::exposeFields() const } void -AndNotBlueprint::optimize_self() +AndNotBlueprint::optimize_self(OptimizePass pass) { if (childCnt() == 0) { return; } - if (getChild(0).isAndNot()) { - auto *child = static_cast<AndNotBlueprint *>(&getChild(0)); - while (child->childCnt() > 1) { - addChild(child->removeChild(1)); + if (pass == OptimizePass::FIRST) { + if (getChild(0).isAndNot()) { + auto *child = static_cast<AndNotBlueprint *>(&getChild(0)); + while (child->childCnt() > 1) { + addChild(child->removeChild(1)); + } + insertChild(1, child->removeChild(0)); + removeChild(0); } - insertChild(1, child->removeChild(0)); - removeChild(0); - } - for (size_t i = 1; i < childCnt(); ++i) { - if (getChild(i).getState().estimate().empty) { - removeChild(i--); + for (size_t i = 1; i < childCnt(); ++i) { + if (getChild(i).getState().estimate().empty) { + removeChild(i--); + } } } - if ( !(getParent() && getParent()->isAndNot()) ) { + if (pass == OptimizePass::LAST) { optimize_source_blenders<OrBlueprint>(*this, 1); } } @@ -191,18 +193,20 @@ AndBlueprint::exposeFields() const } void -AndBlueprint::optimize_self() -{ - for (size_t i = 0; i < childCnt(); ++i) { - if (getChild(i).isAnd()) { - auto *child = static_cast<AndBlueprint *>(&getChild(i)); - while (child->childCnt() > 0) { - addChild(child->removeChild(0)); +AndBlueprint::optimize_self(OptimizePass pass) +{ + if (pass == OptimizePass::FIRST) { + for (size_t i = 0; i < childCnt(); ++i) { + if (getChild(i).isAnd()) { + auto *child = static_cast<AndBlueprint *>(&getChild(i)); + while (child->childCnt() > 0) { + addChild(child->removeChild(0)); + } + removeChild(i--); } - removeChild(i--); } } - if ( !(getParent() && getParent()->isAnd()) ) { + if (pass == OptimizePass::LAST) { optimize_source_blenders<AndBlueprint>(*this, 0); } } @@ -263,6 +267,17 @@ AndBlueprint::computeNextHitRate(const Blueprint & child, double hitRate) const return hitRate * child.hit_ratio(); } +double +OrBlueprint::computeNextHitRate(const Blueprint & child, double hitRate) const { + // Avoid dropping hitRate to zero when meeting a conservatively high hitrate in a child. + // Happens at least when using non fast-search attributes, and with AND nodes. + constexpr double MIN_INVERSE_HIT_RATIO = 0.10; + double inverse_child_hit_ratio = 1.0 - child.hit_ratio(); + return (inverse_child_hit_ratio > MIN_INVERSE_HIT_RATIO) + ? hitRate * inverse_child_hit_ratio + : hitRate; +} + //----------------------------------------------------------------------------- OrBlueprint::~OrBlueprint() = default; @@ -280,20 +295,22 @@ OrBlueprint::exposeFields() const } void -OrBlueprint::optimize_self() -{ - for (size_t i = 0; (childCnt() > 1) && (i < childCnt()); ++i) { - if (getChild(i).isOr()) { - auto *child = static_cast<OrBlueprint *>(&getChild(i)); - while (child->childCnt() > 0) { - addChild(child->removeChild(0)); +OrBlueprint::optimize_self(OptimizePass pass) +{ + if (pass == OptimizePass::FIRST) { + for (size_t i = 0; (childCnt() > 1) && (i < childCnt()); ++i) { + if (getChild(i).isOr()) { + auto *child = static_cast<OrBlueprint *>(&getChild(i)); + while (child->childCnt() > 0) { + addChild(child->removeChild(0)); + } + removeChild(i--); + } else if (getChild(i).getState().estimate().empty) { + removeChild(i--); } - removeChild(i--); - } else if (getChild(i).getState().estimate().empty) { - removeChild(i--); } } - if ( !(getParent() && getParent()->isOr()) ) { + if (pass == OptimizePass::LAST) { optimize_source_blenders<OrBlueprint>(*this, 0); } } @@ -344,6 +361,16 @@ OrBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) const return create_or_filter(get_children(), strict, constraint); } +uint8_t +OrBlueprint::calculate_cost_tier() const +{ + uint8_t cost_tier = State::COST_TIER_NORMAL; + for (const Blueprint::UP &child : get_children()) { + cost_tier = std::max(cost_tier, child->getState().cost_tier()); + } + return cost_tier; +} + //----------------------------------------------------------------------------- WeakAndBlueprint::~WeakAndBlueprint() = default; @@ -531,14 +558,18 @@ RankBlueprint::exposeFields() const } void -RankBlueprint::optimize_self() +RankBlueprint::optimize_self(OptimizePass pass) { - for (size_t i = 1; i < childCnt(); ++i) { - if (getChild(i).getState().estimate().empty) { - removeChild(i--); + if (pass == OptimizePass::FIRST) { + for (size_t i = 1; i < childCnt(); ++i) { + if (getChild(i).getState().estimate().empty) { + removeChild(i--); + } } } - optimize_source_blenders<OrBlueprint>(*this, 1); + if (pass == OptimizePass::LAST) { + optimize_source_blenders<OrBlueprint>(*this, 1); + } } Blueprint::UP @@ -596,7 +627,7 @@ RankBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) cons //----------------------------------------------------------------------------- -SourceBlenderBlueprint::SourceBlenderBlueprint(const ISourceSelector &selector) +SourceBlenderBlueprint::SourceBlenderBlueprint(const ISourceSelector &selector) noexcept : _selector(selector) { } @@ -626,27 +657,6 @@ SourceBlenderBlueprint::inheritStrict(size_t) const return true; } -class FindSource : public Blueprint::IPredicate -{ -public: - explicit FindSource(uint32_t sourceId) noexcept : _sourceId(sourceId) { } - bool check(const Blueprint & bp) const override { return bp.getSourceId() == _sourceId; } -private: - uint32_t _sourceId; -}; - -ssize_t -SourceBlenderBlueprint::findSource(uint32_t sourceId) const -{ - ssize_t index(-1); - FindSource fs(sourceId); - IndexList list = find(fs); - if ( ! list.empty()) { - index = list.front(); - } - return index; -} - SearchIterator::UP SourceBlenderBlueprint::createIntermediateSearch(MultiSearch::Children sub_searches, bool strict, search::fef::MatchData &) const @@ -673,6 +683,16 @@ SourceBlenderBlueprint::isCompatibleWith(const SourceBlenderBlueprint &other) co return (&_selector == &other._selector); } +uint8_t +SourceBlenderBlueprint::calculate_cost_tier() const +{ + uint8_t cost_tier = State::COST_TIER_NORMAL; + for (const Blueprint::UP &child : get_children()) { + cost_tier = std::max(cost_tier, child->getState().cost_tier()); + } + return cost_tier; +} + //----------------------------------------------------------------------------- } diff --git a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h index b8ca46f7549..75dc47272af 100644 --- a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h +++ b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h @@ -17,7 +17,7 @@ public: bool supports_termwise_children() const override { return true; } HitEstimate combine(const std::vector<HitEstimate> &data) const override; FieldSpecBaseList exposeFields() const override; - void optimize_self() override; + void optimize_self(OptimizePass pass) override; bool isAndNot() const override { return true; } Blueprint::UP get_replacement() override; void sort(Children &children) const override; @@ -28,6 +28,9 @@ public: SearchIterator::UP createFilterSearch(bool strict, FilterConstraint constraint) const override; private: + uint8_t calculate_cost_tier() const override { + return (childCnt() > 0) ? get_children()[0]->getState().cost_tier() : State::COST_TIER_NORMAL; + } bool isPositive(size_t index) const override { return index == 0; } }; @@ -40,7 +43,7 @@ public: bool supports_termwise_children() const override { return true; } HitEstimate combine(const std::vector<HitEstimate> &data) const override; FieldSpecBaseList exposeFields() const override; - void optimize_self() override; + void optimize_self(OptimizePass pass) override; bool isAnd() const override { return true; } Blueprint::UP get_replacement() override; void sort(Children &children) const override; @@ -64,7 +67,7 @@ public: bool supports_termwise_children() const override { return true; } HitEstimate combine(const std::vector<HitEstimate> &data) const override; FieldSpecBaseList exposeFields() const override; - void optimize_self() override; + void optimize_self(OptimizePass pass) override; bool isOr() const override { return true; } Blueprint::UP get_replacement() override; void sort(Children &children) const override; @@ -74,6 +77,9 @@ public: bool strict, fef::MatchData &md) const override; SearchIterator::UP createFilterSearch(bool strict, FilterConstraint constraint) const override; +private: + double computeNextHitRate(const Blueprint & child, double hitRate) const override; + uint8_t calculate_cost_tier() const override; }; //----------------------------------------------------------------------------- @@ -95,8 +101,8 @@ public: bool strict, fef::MatchData &md) const override; SearchIterator::UP createFilterSearch(bool strict, FilterConstraint constraint) const override; - WeakAndBlueprint(uint32_t n) : _n(n) {} - ~WeakAndBlueprint(); + explicit WeakAndBlueprint(uint32_t n) noexcept : _n(n) {} + ~WeakAndBlueprint() override; void addTerm(Blueprint::UP bp, uint32_t weight) { addChild(std::move(bp)); _weights.push_back(weight); @@ -124,7 +130,7 @@ public: bool strict, fef::MatchData &md) const override; SearchIterator::UP createFilterSearch(bool strict, FilterConstraint constraint) const override; - NearBlueprint(uint32_t window) : _window(window) {} + explicit NearBlueprint(uint32_t window) noexcept : _window(window) {} }; //----------------------------------------------------------------------------- @@ -146,7 +152,7 @@ public: bool strict, fef::MatchData &md) const override; SearchIterator::UP createFilterSearch(bool strict, FilterConstraint constraint) const override; - ONearBlueprint(uint32_t window) : _window(window) {} + explicit ONearBlueprint(uint32_t window) noexcept : _window(window) {} }; //----------------------------------------------------------------------------- @@ -156,7 +162,7 @@ class RankBlueprint final : public IntermediateBlueprint public: HitEstimate combine(const std::vector<HitEstimate> &data) const override; FieldSpecBaseList exposeFields() const override; - void optimize_self() override; + void optimize_self(OptimizePass pass) override; Blueprint::UP get_replacement() override; void sort(Children &children) const override; bool inheritStrict(size_t i) const override; @@ -166,6 +172,9 @@ public: bool strict, fef::MatchData &md) const override; SearchIterator::UP createFilterSearch(bool strict, FilterConstraint constraint) const override; + uint8_t calculate_cost_tier() const override { + return (childCnt() > 0) ? get_children()[0]->getState().cost_tier() : State::COST_TIER_NORMAL; + } }; //----------------------------------------------------------------------------- @@ -176,18 +185,12 @@ private: const ISourceSelector &_selector; public: - SourceBlenderBlueprint(const ISourceSelector &selector); + explicit SourceBlenderBlueprint(const ISourceSelector &selector) noexcept; ~SourceBlenderBlueprint() override; HitEstimate combine(const std::vector<HitEstimate> &data) const override; FieldSpecBaseList exposeFields() const override; void sort(Children &children) const override; bool inheritStrict(size_t i) const override; - /** - * Will return the index matching the given sourceId. - * @param sourceId The sourceid to find. - * @return The index to the child representing the sourceId. -1 if not found. - */ - ssize_t findSource(uint32_t sourceId) const; SearchIterator::UP createIntermediateSearch(MultiSearch::Children subSearches, bool strict, fef::MatchData &md) const override; @@ -197,6 +200,7 @@ public: /** check if this blueprint has the same source selector as the other */ bool isCompatibleWith(const SourceBlenderBlueprint &other) const; bool isSourceBlender() const override { return true; } + uint8_t calculate_cost_tier() const override; }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp index 6b2faac847e..5c795679d48 100644 --- a/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.cpp @@ -45,12 +45,14 @@ SameElementBlueprint::addTerm(Blueprint::UP term) } void -SameElementBlueprint::optimize_self() +SameElementBlueprint::optimize_self(OptimizePass pass) { - std::sort(_terms.begin(), _terms.end(), - [](const auto &a, const auto &b) { - return (a->getState().estimate() < b->getState().estimate()); - }); + if (pass == OptimizePass::LAST) { + std::sort(_terms.begin(), _terms.end(), + [](const auto &a, const auto &b) { + return (a->getState().estimate() < b->getState().estimate()); + }); + } } void diff --git a/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.h b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.h index 240d064c5cc..cc6331375cf 100644 --- a/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/same_element_blueprint.h @@ -34,7 +34,7 @@ public: // used by create visitor void addTerm(Blueprint::UP term); - void optimize_self() override; + void optimize_self(OptimizePass pass) override; void fetchPostings(const ExecuteInfo &execInfo) override; std::unique_ptr<SameElementSearch> create_same_element_search(search::fef::TermFieldMatchData& tfmd, bool strict) const; diff --git a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.cpp index 8de16189c86..3cb69f00240 100644 --- a/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/simple_phrase_blueprint.cpp @@ -2,11 +2,8 @@ #include "simple_phrase_blueprint.h" #include "simple_phrase_search.h" -#include "emptysearch.h" #include "field_spec.hpp" -#include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/vespalib/objects/visit.hpp> -#include <algorithm> #include <map> namespace search::queryeval { @@ -28,7 +25,7 @@ SimplePhraseBlueprint::~SimplePhraseBlueprint() = default; FieldSpec SimplePhraseBlueprint::getNextChildField(const FieldSpec &outer) { - return FieldSpec(outer.getName(), outer.getFieldId(), _layout.allocTermField(outer.getFieldId()), false); + return {outer.getName(), outer.getFieldId(), _layout.allocTermField(outer.getFieldId()), false}; } void diff --git a/security-utils/src/main/java/com/yahoo/security/Pkcs10CsrBuilder.java b/security-utils/src/main/java/com/yahoo/security/Pkcs10CsrBuilder.java index c34e3acb09f..629b610564a 100644 --- a/security-utils/src/main/java/com/yahoo/security/Pkcs10CsrBuilder.java +++ b/security-utils/src/main/java/com/yahoo/security/Pkcs10CsrBuilder.java @@ -90,7 +90,8 @@ public class Pkcs10CsrBuilder { .toArray(GeneralName[]::new)); extGen.addExtension(Extension.subjectAlternativeName, false, generalNames); } - requestBuilder.addAttribute(PKCSObjectIdentifiers.pkcs_9_at_extensionRequest, extGen.generate()); + if (!extGen.isEmpty()) + requestBuilder.addAttribute(PKCSObjectIdentifiers.pkcs_9_at_extensionRequest, extGen.generate()); ContentSigner contentSigner = new JcaContentSignerBuilder(signatureAlgorithm.getAlgorithmName()) .setProvider(BouncyCastleProviderHolder.getInstance()) .build(keyPair.getPrivate()); diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java index 860d14a9333..4e7ae5418a1 100644 --- a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java +++ b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java @@ -49,7 +49,6 @@ import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; import static com.yahoo.collections.CollectionUtil.first; @@ -296,21 +295,21 @@ public class StandaloneContainerApplication implements Application { container.initService(deployState); } - private static Element getJDiscInServices(Element element) { - List<Element> jDiscElements = new ArrayList<>(); + private static Element getContainerElementInServices(Element element) { + List<Element> containerElements = new ArrayList<>(); for (ConfigModelId cid : ContainerModelBuilder.configModelIds) { List<Element> children = XML.getChildren(element, cid.getName()); - jDiscElements.addAll(children); + containerElements.addAll(children); } - if (jDiscElements.size() == 1) { - return jDiscElements.get(0); - } else if (jDiscElements.isEmpty()) { - throw new RuntimeException("No jdisc element found under services."); + if (containerElements.size() == 1) { + return containerElements.get(0); + } else if (containerElements.isEmpty()) { + throw new RuntimeException("No container element found under services."); } else { - List<String> nameAndId = jDiscElements.stream().map(e -> e.getNodeName() + " id='" + e.getAttribute("id") + "'") + List<String> nameAndId = containerElements.stream().map(e -> e.getNodeName() + " id='" + e.getAttribute("id") + "'") .toList(); - throw new RuntimeException("Found multiple JDisc elements: " + String.join(", ", nameAndId)); + throw new RuntimeException("Found multiple container elements: " + String.join(", ", nameAndId)); } } @@ -321,7 +320,7 @@ public class StandaloneContainerApplication implements Application { if (ContainerModelBuilder.configModelIds.stream().anyMatch(id -> id.getName().equals(nodeName))) { return element; } else { - return getJDiscInServices(element); + return getContainerElementInServices(element); } } diff --git a/storage/src/tests/common/metricstest.cpp b/storage/src/tests/common/metricstest.cpp index 61332bb9ad6..899c1979e86 100644 --- a/storage/src/tests/common/metricstest.cpp +++ b/storage/src/tests/common/metricstest.cpp @@ -122,9 +122,6 @@ void MetricsTest::createFakeLoad() metrics.docs.inc(10 * n); metrics.bytes.inc(10240 * n); } - _filestorMetrics->directoryEvents.inc(5); - _filestorMetrics->partitionEvents.inc(4); - _filestorMetrics->diskEvents.inc(3); { FileStorMetrics& disk(*_filestorMetrics); disk.queueSize.addValue(4 * n); @@ -147,18 +144,10 @@ void MetricsTest::createFakeLoad() thread.update.notFound.inc(1 * n); thread.update.latencyRead.addValue(2 * n); thread.update.latency.addValue(7 * n); - thread.revert.count.inc(2 * n); - thread.revert.notFound.inc(n / 2); - thread.revert.latency.addValue(2 * n); thread.visit.count.inc(6 * n); thread.deleteBuckets.count.inc(1 * n); - thread.repairs.count.inc(3 * n); - thread.repairFixed.inc(1 * n); thread.splitBuckets.count.inc(20 * n); - thread.movedBuckets.count.inc(1 * n); - thread.readBucketInfo.count.inc(2 * n); - thread.internalJoin.count.inc(3 * n); thread.mergeBuckets.count.inc(2 * n); thread.getBucketDiff.count.inc(4 * n); diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index 96618eb9206..4911ad88692 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -1421,6 +1421,14 @@ void FileStorManagerTest::do_test_delete_bucket(bool use_throttled_delete) { StorBucketDatabase::WrappedEntry entry(_node->getStorageBucketDatabase().get(bid, "foo")); EXPECT_FALSE(entry.exists()); } + if (use_throttled_delete) { + auto& metrics = thread_metrics_of(*c.manager)->remove_by_gid; + EXPECT_EQ(metrics.failed.getValue(), 0); + EXPECT_EQ(metrics.count.getValue(), 1); + // We can't reliably test the actual latency here without wiring mock clock bumping into + // the async remove by GID execution, but we can at least test that we updated the metric. + EXPECT_EQ(metrics.latency.getCount(), 1); + } } // TODO remove once throttled behavior is the default diff --git a/storage/src/tests/storageserver/mergethrottlertest.cpp b/storage/src/tests/storageserver/mergethrottlertest.cpp index 6f80ffe0727..a480ba2740f 100644 --- a/storage/src/tests/storageserver/mergethrottlertest.cpp +++ b/storage/src/tests/storageserver/mergethrottlertest.cpp @@ -1603,6 +1603,39 @@ TEST_F(MergeThrottlerTest, queued_merges_are_not_counted_towards_memory_usage) { EXPECT_EQ(throttler(0).getMetrics().estimated_merge_memory_usage.getLast(), 0_Mi); } +TEST_F(MergeThrottlerTest, enqueued_merge_not_started_if_insufficient_memory_available) { + // See `queued_merges_are_not_counted_towards_memory_usage` test for magic number rationale + const auto max_pending = throttler_max_merges_pending(0); + ASSERT_LT(max_pending, 1000); + ASSERT_GT(max_pending, 1); + throttler(0).set_max_merge_memory_usage_bytes_locking(10_Mi); + + // Fill up entire active window and enqueue a single merge + fill_throttler_queue_with_n_commands(0, 0); + _topLinks[0]->sendDown(MergeBuilder(document::BucketId(16, 1000)).nodes(0, 1, 2).unordered(true).memory_usage(11_Mi).create()); + waitUntilMergeQueueIs(throttler(0), 1, _messageWaitTime); // Should end up in queue + + // Drain all active merges. As long as we have other active merges, the enqueued merge should not + // be allowed through since it's too large. Eventually it will hit the "at least one merge must + // be allowed at any time regardless of size" exception and is dequeued. + for (uint32_t i = 0; i < max_pending; ++i) { + auto fwd_cmd = _topLinks[0]->getAndRemoveMessage(MessageType::MERGEBUCKET); + auto fwd_reply = dynamic_cast<api::MergeBucketCommand&>(*fwd_cmd).makeReply(); + + ASSERT_NO_FATAL_FAILURE(send_and_expect_reply( + std::shared_ptr<api::StorageReply>(std::move(fwd_reply)), + MessageType::MERGEBUCKET_REPLY, ReturnCode::OK)); // Unwind reply for completed merge + + if (i < max_pending - 1) { + // Merge should still be in the queue, as it requires 11 MiB, and we only have 10 MiB. + // It will eventually be executed when the window is empty (see below). + waitUntilMergeQueueIs(throttler(0), 1, _messageWaitTime); + } + } + // We've freed up the entire send window, so the over-sized merge can finally squeeze through. + waitUntilMergeQueueIs(throttler(0), 0, _messageWaitTime); +} + namespace { vespalib::HwInfo make_mem_info(uint64_t mem_size) { diff --git a/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp b/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp index d5d4accd70a..e6de5449b60 100644 --- a/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp +++ b/storage/src/vespa/storage/distributor/distributor_bucket_space_repo.cpp @@ -5,7 +5,9 @@ #include <vespa/vdslib/state/cluster_state_bundle.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/document/bucket/fixed_bucket_spaces.h> +#include <vespa/vespalib/util/backtrace.h> #include <cassert> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".distributor.distributor_bucket_space_repo"); @@ -34,7 +36,10 @@ DistributorBucketSpace & DistributorBucketSpaceRepo::get(BucketSpace bucketSpace) { auto itr = _map.find(bucketSpace); - assert(itr != _map.end()); + if (itr == _map.end()) [[unlikely]] { + LOG(error, "Bucket space %" PRIu64 " does not have a valid mapping. %s", bucketSpace.getId(), vespalib::getStackTrace(0).c_str()); + abort(); + } return *itr->second; } @@ -42,7 +47,10 @@ const DistributorBucketSpace & DistributorBucketSpaceRepo::get(BucketSpace bucketSpace) const { auto itr = _map.find(bucketSpace); - assert(itr != _map.end()); + if (itr == _map.end()) [[unlikely]] { + LOG(error, "Bucket space %" PRIu64 " does not have a valid mapping. %s", bucketSpace.getId(), vespalib::getStackTrace(0).c_str()); + abort(); + } return *itr->second; } diff --git a/storage/src/vespa/storage/persistence/asynchandler.cpp b/storage/src/vespa/storage/persistence/asynchandler.cpp index bb3952c8c33..a69f9e55afb 100644 --- a/storage/src/vespa/storage/persistence/asynchandler.cpp +++ b/storage/src/vespa/storage/persistence/asynchandler.cpp @@ -300,15 +300,26 @@ AsyncHandler::handle_delete_bucket_throttling(api::DeleteBucketCommand& cmd, Mes }); _spi.deleteBucketAsync(spi_bucket, std::make_unique<ResultTaskOperationDone>(_sequencedExecutor, bucket.getBucketId(), std::move(task))); }); + auto& throttler = _env._fileStorHandler.operation_throttler(); + auto* remove_by_gid_metric = &_env._metrics.remove_by_gid; + for (auto& meta : meta_entries) { auto token = throttler.blocking_acquire_one(); + remove_by_gid_metric->count.inc(); std::vector<spi::DocTypeGidAndTimestamp> to_remove = {{meta->getDocumentType(), meta->getGid(), meta->getTimestamp()}}; - auto task = makeResultTask([bucket = cmd.getBucket(), token = std::move(token), invoke_delete_on_zero_refs]([[maybe_unused]] spi::Result::UP ignored) { - LOG(spam, "%s: completed removeByGidAsync operation", bucket.toString().c_str()); - // Nothing else clever to do here. Throttle token and deleteBucket dispatch refs dropped implicitly. - }); - LOG(spam, "%s: about to invoke removeByGidAsync(%s, %s, %zu)", cmd.getBucket().toString().c_str(), + auto task = makeResultTask([bucket = cmd.getBucket(), token = std::move(token), + invoke_delete_on_zero_refs, remove_by_gid_metric, + op_timer = framework::MilliSecTimer(_env._component.getClock())] + (spi::Result::UP result) { + if (result->hasError()) { + remove_by_gid_metric->failed.inc(); + } + remove_by_gid_metric->latency.addValue(op_timer.getElapsedTimeAsDouble()); + LOG(spam, "%s: completed removeByGidAsync operation", bucket.toString().c_str()); + // Nothing else clever to do here. Throttle token and deleteBucket dispatch refs dropped implicitly. + }); + LOG(spam, "%s: about to invoke removeByGidAsync(%s, %s, %" PRIu64 ")", cmd.getBucket().toString().c_str(), vespalib::string(meta->getDocumentType()).c_str(), meta->getGid().toString().c_str(), meta->getTimestamp().getValue()); _spi.removeByGidAsync(spi_bucket, std::move(to_remove), std::make_unique<ResultTaskOperationDone>(_sequencedExecutor, cmd.getBucketId(), std::move(task))); } diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp index 6072a1dbc3e..9ed15605fb2 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.cpp @@ -154,13 +154,11 @@ FileStorThreadMetrics::FileStorThreadMetrics(const std::string& name, const std: removeLocation("remove_location", "Remove location", this), statBucket("stat_bucket", "Stat bucket", this), update(this), - revert("revert", "Revert", this), createIterator("createiterator", {}, this), visit(this), createBuckets("createbuckets", "Number of buckets that has been created.", this), deleteBuckets("deletebuckets", "Number of buckets that has been deleted.", this), - repairs("bucketverified", "Number of times buckets have been checked.", this), - repairFixed("bucketfixed", {}, "Number of times bucket has been fixed because of corruption", this), + remove_by_gid("remove_by_gid", "Internal single-document remove operations used by DeleteBucket", this), recheckBucketInfo("recheckbucketinfo", "Number of times bucket info has been explicitly " "rechecked due to buckets being marked modified by " @@ -169,11 +167,6 @@ FileStorThreadMetrics::FileStorThreadMetrics(const std::string& name, const std: splitBuckets("splitbuckets", "Number of times buckets have been split.", this), joinBuckets("joinbuckets", "Number of times buckets have been joined.", this), setBucketStates("setbucketstates", "Number of times buckets have been activated or deactivated.", this), - movedBuckets("movedbuckets", "Number of buckets moved between disks", this), - readBucketList("readbucketlist", "Number of read bucket list requests", this), - readBucketInfo("readbucketinfo", "Number of read bucket info requests", this), - internalJoin("internaljoin", "Number of joins to join buckets on multiple disks during " - "storage initialization.", this), mergeBuckets("mergebuckets", "Number of times buckets have been merged.", this), getBucketDiff("getbucketdiff", "Number of getbucketdiff commands that have been processed.", this), applyBucketDiff("applybucketdiff", "Number of applybucketdiff commands that have been processed.", this), @@ -211,18 +204,11 @@ FileStorMetrics::FileStorMetrics() throttle_window_size("throttle_window_size", {}, "Current size of async operation throttler window size", this), throttle_waiting_threads("throttle_waiting_threads", {}, "Number of threads waiting to acquire a throttle token", this), throttle_active_tokens("throttle_active_tokens", {}, "Current number of active throttle tokens", this), - waitingForLockHitRate("waitingforlockrate", {}, - "Amount of times a filestor thread has needed to wait for " - "lock to take next message in queue.", this), active_operations(this), - directoryEvents("directoryevents", {}, "Number of directory events received.", this), - partitionEvents("partitionevents", {}, "Number of partition events received.", this), - diskEvents("diskevents", {}, "Number of disk events received.", this), bucket_db_init_latency("bucket_db_init_latency", {}, "Time taken (in ms) to initialize bucket databases with " "information from the persistence provider", this) { pendingMerges.unsetOnZeroValue(); - waitingForLockHitRate.unsetOnZeroValue(); } FileStorMetrics::~FileStorMetrics() = default; diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h index 85a3813c9eb..75182705323 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormetrics.h @@ -1,11 +1,6 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. /** - * @class storage::FileStorMetrics - * @ingroup filestorage - * - * @brief Metrics for the file store threads. - * - * @version $Id$ + * Metrics for the peristence threads. */ #pragma once @@ -98,21 +93,15 @@ struct FileStorThreadMetrics : public metrics::MetricSet Op removeLocation; Op statBucket; Update update; - OpWithNotFound revert; Op createIterator; Visitor visit; Op createBuckets; Op deleteBuckets; - Op repairs; - metrics::LongCountMetric repairFixed; + Op remove_by_gid; Op recheckBucketInfo; Op splitBuckets; Op joinBuckets; Op setBucketStates; - Op movedBuckets; - Op readBucketList; - Op readBucketInfo; - Op internalJoin; Op mergeBuckets; Op getBucketDiff; Op applyBucketDiff; @@ -148,11 +137,7 @@ struct FileStorMetrics : public metrics::MetricSet metrics::LongAverageMetric throttle_window_size; metrics::LongAverageMetric throttle_waiting_threads; metrics::LongAverageMetric throttle_active_tokens; - metrics::DoubleAverageMetric waitingForLockHitRate; ActiveOperationsMetrics active_operations; - metrics::LongCountMetric directoryEvents; - metrics::LongCountMetric partitionEvents; - metrics::LongCountMetric diskEvents; metrics::LongAverageMetric bucket_db_init_latency; FileStorMetrics(); diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.cpp b/storage/src/vespa/storage/storageserver/mergethrottler.cpp index 54a4ddbc780..b99c238f9ab 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.cpp +++ b/storage/src/vespa/storage/storageserver/mergethrottler.cpp @@ -419,6 +419,13 @@ MergeThrottler::getNextQueuedMerge() return entry._msg; } +const api::MergeBucketCommand& +MergeThrottler::peek_merge_queue() const noexcept +{ + assert(!_queue.empty()); + return dynamic_cast<const api::MergeBucketCommand&>(*_queue.begin()->_msg); +} + void MergeThrottler::enqueue_merge_for_later_processing( const api::StorageMessage::SP& msg, @@ -549,45 +556,40 @@ MergeThrottler::rejectOutdatedQueuedMerges( // If there's a merge queued and the throttling policy allows for // the merge to be processed, do so. bool -MergeThrottler::attemptProcessNextQueuedMerge( - MessageGuard& msgGuard) +MergeThrottler::attemptProcessNextQueuedMerge(MessageGuard& msgGuard) { - if (!canProcessNewMerge()) { + if (_queue.empty()) { + return false; + } + if ( ! (canProcessNewMerge() && accepting_merge_is_within_memory_limits(peek_merge_queue()))) { // Should never reach a non-sending state when there are // no to-be-replied merges that can trigger a new processing assert(!_merges.empty()); return false; } + // If we get here, there must be something to dequeue. api::StorageMessage::SP msg = getNextQueuedMerge(); - if (msg) { - // In case of resends and whatnot, it's possible for a merge - // command to be in the queue while another higher priority - // command for the same bucket sneaks in front of it and gets - // a slot. Send BUSY in this case to make the distributor retry - // later, at which point the existing merge has hopefully gone - // through and the new one will be effectively a no-op to perform - if (!isMergeAlreadyKnown(msg)) { - LOG(spam, "Processing queued merge %s", msg->toString().c_str()); - processNewMergeCommand(msg, msgGuard); - } else { - vespalib::asciistream oss; - oss << "Queued merge " << msg->toString() << " is out of date; it has already " - "been started by someone else since it was queued"; - LOG(debug, "%s", oss.c_str()); - sendReply(dynamic_cast<const api::MergeBucketCommand&>(*msg), - api::ReturnCode(api::ReturnCode::BUSY, oss.str()), - msgGuard, _metrics->chaining); - } - return true; + assert(msg); + // In case of resends and whatnot, it's possible for a merge + // command to be in the queue while another higher priority + // command for the same bucket sneaks in front of it and gets + // a slot. Send BUSY in this case to make the distributor retry + // later, at which point the existing merge has hopefully gone + // through and the new one will be effectively a no-op to perform + if (!isMergeAlreadyKnown(msg)) { + LOG(spam, "Processing queued merge %s", msg->toString().c_str()); + processNewMergeCommand(msg, msgGuard); } else { - if (_queue.empty()) { - LOG(spam, "Queue empty - no merges to process"); - } else { - LOG(spam, "Merges queued, but throttle policy disallows further merges at this time"); - } + vespalib::asciistream oss; + oss << "Queued merge " << msg->toString() << " is out of date; it has already " + "been started by someone else since it was queued"; + LOG(debug, "%s", oss.c_str()); + sendReply(dynamic_cast<const api::MergeBucketCommand&>(*msg), + api::ReturnCode(api::ReturnCode::BUSY, oss.str()), + msgGuard, _metrics->chaining); } - return false; + return true; } bool diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.h b/storage/src/vespa/storage/storageserver/mergethrottler.h index a5559c159bf..e210a8bfb8b 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.h +++ b/storage/src/vespa/storage/storageserver/mergethrottler.h @@ -372,6 +372,7 @@ private: [[nodiscard]] bool allow_merge_despite_full_window(const api::MergeBucketCommand& cmd) const noexcept; [[nodiscard]] bool accepting_merge_is_within_memory_limits(const api::MergeBucketCommand& cmd) const noexcept; [[nodiscard]] bool may_allow_into_queue(const api::MergeBucketCommand& cmd) const noexcept; + [[nodiscard]] const api::MergeBucketCommand& peek_merge_queue() const noexcept; void sendReply(const api::MergeBucketCommand& cmd, const api::ReturnCode& result, diff --git a/vespalib/src/tests/btree/btree_test.cpp b/vespalib/src/tests/btree/btree_test.cpp index f6fa962fa9d..21838676906 100644 --- a/vespalib/src/tests/btree/btree_test.cpp +++ b/vespalib/src/tests/btree/btree_test.cpp @@ -231,6 +231,7 @@ protected: template <typename TreeType> void requireThatUpperBoundWorksT(); void requireThatIteratorDistanceWorks(int numEntries); + void test_step_forward(int num_entries); }; template <typename LeafNodeType> @@ -1519,6 +1520,33 @@ BTreeTest::requireThatIteratorDistanceWorks(int numEntries) } } +void +BTreeTest::test_step_forward(int num_entries) +{ + GenerationHandler g; + MyTree tree; + for (int i = 0; i < num_entries; ++i) { + tree.insert(i, toStr(i)); + } + auto it = tree.begin(); + auto ite = it; + ite.end(); + for (int i = 0; i <= num_entries; ++i) { + auto iit = tree.lowerBound(i); + auto iit2 = iit; + iit2 += (num_entries - i); + EXPECT_TRUE(iit2.identical(ite)); + iit2 = iit; + iit2 += (1000000 + num_entries); + EXPECT_TRUE(iit2.identical(ite)); + for (int j = i; j <= num_entries; ++j) { + auto jit = tree.lowerBound(j); + auto iit3 = iit; + iit3 += (j - i); + EXPECT_TRUE(iit3.identical(jit)); + } + } +} TEST_F(BTreeTest, require_that_iterator_distance_works) { @@ -1530,6 +1558,16 @@ TEST_F(BTreeTest, require_that_iterator_distance_works) requireThatIteratorDistanceWorks(400); } +TEST_F(BTreeTest, require_that_step_forward_works) +{ + test_step_forward(1); + test_step_forward(3); + test_step_forward(8); + test_step_forward(20); + test_step_forward(100); + test_step_forward(400); +} + TEST_F(BTreeTest, require_that_foreach_key_works) { using Tree = BTree<int, int, btree::NoAggregated, MyComp, MyTraits>; diff --git a/vespalib/src/tests/compress/compress_test.cpp b/vespalib/src/tests/compress/compress_test.cpp index 04f71d5cc18..c4383a1c193 100644 --- a/vespalib/src/tests/compress/compress_test.cpp +++ b/vespalib/src/tests/compress/compress_test.cpp @@ -26,6 +26,9 @@ CompressTest::verifyPositiveNumber(uint64_t n, const uint8_t * expected, size_t for (size_t i(0); i < sz; i++) { EXPECT_EQUAL(expected[i], buf[i]); } + EXPECT_FALSE(Integer::check_decompress_positive_space(expected, 0u)); + EXPECT_FALSE(Integer::check_decompress_positive_space(expected, sz - 1)); + EXPECT_TRUE(Integer::check_decompress_positive_space(expected, sz)); uint64_t v(0); EXPECT_EQUAL(sz, Integer::decompressPositive(v, expected)); EXPECT_EQUAL(n, v); @@ -39,6 +42,9 @@ CompressTest::verifyNumber(int64_t n, const uint8_t * expected, size_t sz) { for (size_t i(0); i < sz; i++) { EXPECT_EQUAL(expected[i], buf[i]); } + EXPECT_FALSE(Integer::check_decompress_space(expected, 0u)); + EXPECT_FALSE(Integer::check_decompress_space(expected, sz - 1)); + EXPECT_TRUE(Integer::check_decompress_space(expected, sz)); int64_t v(0); EXPECT_EQUAL(sz, Integer::decompress(v, expected)); EXPECT_EQUAL(n, v); diff --git a/vespalib/src/vespa/vespalib/btree/btreeiterator.h b/vespalib/src/vespa/vespalib/btree/btreeiterator.h index 153fb005f00..9519951f9e2 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeiterator.h +++ b/vespalib/src/vespa/vespalib/btree/btreeiterator.h @@ -166,7 +166,7 @@ private: /* * Find the previous leaf node, called by operator--() as needed. */ - VESPA_DLL_LOCAL void findPrevLeafNode(); + void findPrevLeafNode(); protected: /* @@ -219,7 +219,23 @@ protected: * Step iterator backwards. If at end then place it at last valid * position in tree (cf. rbegin()) */ - BTreeIteratorBase & operator--(); + BTreeIteratorBase & operator--() { + if (_leaf.getNode() == nullptr) { + rbegin(); + return *this; + } + if (_leaf.getIdx() > 0u) { + _leaf.decIdx(); + return *this; + } + findPrevLeafNode(); + return *this; + } + + /* + * Step iterator forwards the given number of steps. + */ + void step_forward(size_t steps); ~BTreeIteratorBase(); BTreeIteratorBase(const BTreeIteratorBase &other); @@ -491,6 +507,7 @@ protected: using ParentType::_compatLeafNode; using ParentType::clearPath; using ParentType::setupEmpty; + using ParentType::step_forward; public: using ParentType::end; @@ -546,6 +563,13 @@ public: return *this; } + /* + * Step iterator forwards the given number of steps. + */ + BTreeConstIterator & operator+=(size_t steps) { + step_forward(steps); + return *this; + } /** * Position iterator at first position with a key that is greater * than or equal to the key argument. The iterator must be set up @@ -688,6 +712,7 @@ public: using ParentType::_leafRoot; using ParentType::_compatLeafNode; using ParentType::end; + using ParentType::step_forward; using EntryRef = datastore::EntryRef; BTreeIterator(BTreeNode::Ref root, const NodeAllocatorType &allocator) @@ -716,6 +741,11 @@ public: return *this; } + BTreeIterator & operator+=(size_t steps) { + step_forward(steps); + return *this; + } + NodeAllocatorType & getAllocator() const { return const_cast<NodeAllocatorType &>(*_allocator); } diff --git a/vespalib/src/vespa/vespalib/btree/btreeiterator.hpp b/vespalib/src/vespa/vespalib/btree/btreeiterator.hpp index 5f15ebc8b61..3119d05cfd9 100644 --- a/vespalib/src/vespa/vespalib/btree/btreeiterator.hpp +++ b/vespalib/src/vespa/vespalib/btree/btreeiterator.hpp @@ -461,25 +461,6 @@ BTreeIteratorBase() noexcept template <typename KeyT, typename DataT, typename AggrT, uint32_t INTERNAL_SLOTS, uint32_t LEAF_SLOTS, uint32_t PATH_SIZE> -BTreeIteratorBase<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS, PATH_SIZE> & -BTreeIteratorBase<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS, PATH_SIZE>:: -operator--() -{ - if (_leaf.getNode() == nullptr) { - rbegin(); - return *this; - } - if (_leaf.getIdx() > 0u) { - _leaf.decIdx(); - return *this; - } - findPrevLeafNode(); - return *this; -} - - -template <typename KeyT, typename DataT, typename AggrT, - uint32_t INTERNAL_SLOTS, uint32_t LEAF_SLOTS, uint32_t PATH_SIZE> size_t BTreeIteratorBase<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS, PATH_SIZE>:: size() const noexcept @@ -545,6 +526,85 @@ identical(const BTreeIteratorBase &rhs) const } +template <typename KeyT, typename DataT, typename AggrT, + uint32_t INTERNAL_SLOTS, uint32_t LEAF_SLOTS, uint32_t PATH_SIZE> +void +BTreeIteratorBase<KeyT, DataT, AggrT, INTERNAL_SLOTS, LEAF_SLOTS, PATH_SIZE>:: +step_forward(size_t steps) +{ + auto lnode = _leaf.getNode(); + if (lnode == nullptr) { + return; + } + auto idx = _leaf.getIdx(); + if (idx + steps < lnode->validSlots()) { + _leaf.setIdx(idx + steps); + return; + } + if (_pathSize == 0) { + _leaf.invalidate(); + return; + } + size_t remaining_steps = steps - (lnode->validSlots() - idx); + uint32_t level = 0; + uint32_t levels = _pathSize; + const InternalNodeType* node; + /* + * Find intermediate node representing subtree containing old and new + * position. + */ + for (;;) { + node = _path[level].getNode(); + idx = _path[level].getIdx() + 1; + while (idx < node->validSlots()) { + auto ref = node->getChild(idx); + auto valid_leaves = (level != 0) ? + _allocator->mapInternalRef(ref)->validLeaves() : + _allocator->mapLeafRef(ref)->validLeaves(); + if (remaining_steps < valid_leaves) { + break; + } + remaining_steps -= valid_leaves; + ++idx; + } + if (idx < node->validSlots()) { + break; + } else { + ++level; + if (level == levels) { + end(); + return; + } + } + } + /* + * Walk down subtree adjusting iterator for new position. + */ + _path[level].setIdx(idx); + while (level > 0) { + --level; + node = _allocator->mapInternalRef(node->getChild(idx)); + assert(remaining_steps < node->validLeaves()); + idx = 0; + while (idx < node->validSlots()) { + auto ref = node->getChild(idx); + auto valid_leaves = (level != 0) ? + _allocator->mapInternalRef(ref)->validLeaves() : + _allocator->mapLeafRef(ref)->validLeaves(); + if (remaining_steps < valid_leaves) { + break; + } + remaining_steps -= valid_leaves; + ++idx; + } + assert(idx < node->validSlots()); + _path[level].setNodeAndIdx(node, idx); + } + lnode = _allocator->mapLeafRef(node->getChild(idx)); + assert(remaining_steps < lnode->validSlots()); + _leaf.setNodeAndIdx(lnode, remaining_steps); +} + template <typename KeyT, typename DataT, typename AggrT, typename CompareT, typename TraitsT> void diff --git a/vespalib/src/vespa/vespalib/util/compress.h b/vespalib/src/vespa/vespalib/util/compress.h index 5cf2814f3ea..66234da0e32 100644 --- a/vespalib/src/vespa/vespalib/util/compress.h +++ b/vespalib/src/vespa/vespalib/util/compress.h @@ -105,6 +105,32 @@ public: } return numbytes; } + + static bool check_decompress_space(const void* srcv, size_t len) { + if (len == 0u) { + return false; + } + const uint8_t * src = static_cast<const uint8_t *>(srcv); + const uint8_t c = src[0]; + if ((c & 0x40) != 0) { + return (((c & 0x20) != 0) ? (len >= 4u) : (len >= 2u)); + } else { + return true; + } + } + + static bool check_decompress_positive_space(const void* srcv, size_t len) { + if (len == 0u) { + return false; + } + const uint8_t * src = static_cast<const uint8_t *>(srcv); + const uint8_t c = src[0]; + if ((c & 0x80) != 0) { + return (((c & 0x40) != 0) ? (len >= 4u) : (len >= 2u)); + } else { + return true; + } + } private: [[ noreturn ]] static void throw_too_big(int64_t n); [[ noreturn ]] static void throw_too_big(uint64_t n); |