diff options
164 files changed, 2419 insertions, 1594 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 4fbc756236d..8972fada5bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,15 +51,21 @@ add_subdirectory(ann_benchmark) add_subdirectory(application-model) add_subdirectory(application-preprocessor) add_subdirectory(athenz-identity-provider-service) +add_subdirectory(client) +add_subdirectory(cloud-tenant-cd) +add_subdirectory(clustercontroller-apps) +add_subdirectory(clustercontroller-core) +add_subdirectory(clustercontroller-reindexer) +add_subdirectory(clustercontroller-utils) +add_subdirectory(config) add_subdirectory(config-bundle) +add_subdirectory(configd) +add_subdirectory(configdefinitions) add_subdirectory(config-model) add_subdirectory(config-model-api) +add_subdirectory(config-model-fat) add_subdirectory(config-provisioning) add_subdirectory(config-proxy) -add_subdirectory(config) -add_subdirectory(config-model-fat) -add_subdirectory(configd) -add_subdirectory(configdefinitions) add_subdirectory(configserver) add_subdirectory(configserver-flags) add_subdirectory(configutil) @@ -68,14 +74,9 @@ add_subdirectory(container-core) add_subdirectory(container-disc) add_subdirectory(container-messagebus) add_subdirectory(container-search) -add_subdirectory(container-search-gui) add_subdirectory(container-search-and-docproc) +add_subdirectory(container-search-gui) add_subdirectory(container-spifly) -add_subdirectory(cloud-tenant-cd) -add_subdirectory(clustercontroller-apps) -add_subdirectory(clustercontroller-core) -add_subdirectory(clustercontroller-reindexer) -add_subdirectory(clustercontroller-utils) add_subdirectory(defaults) add_subdirectory(docproc) add_subdirectory(docprocs) @@ -98,8 +99,8 @@ add_subdirectory(jrt_test) add_subdirectory(linguistics) add_subdirectory(linguistics-components) add_subdirectory(logd) -add_subdirectory(logserver) add_subdirectory(logforwarder) +add_subdirectory(logserver) add_subdirectory(lowercasing_test) add_subdirectory(messagebus) add_subdirectory(messagebus_test) @@ -127,22 +128,22 @@ add_subdirectory(tenant-cd-api) add_subdirectory(vbench) add_subdirectory(vdslib) add_subdirectory(vdstestlib) -add_subdirectory(vespa-athenz) -add_subdirectory(vespa-feed-client) -add_subdirectory(vespa-feed-client-cli) -add_subdirectory(vespa-osgi-testrunner) -add_subdirectory(vespa-testrunner-components) -add_subdirectory(vespa_feed_perf) add_subdirectory(vespa-3party-bundles) +add_subdirectory(vespa-athenz) add_subdirectory(vespabase) add_subdirectory(vespaclient) -add_subdirectory(vespaclient-core) add_subdirectory(vespaclient-container-plugin) +add_subdirectory(vespaclient-core) add_subdirectory(vespaclient-java) +add_subdirectory(vespa-feed-client) +add_subdirectory(vespa-feed-client-cli) +add_subdirectory(vespa_feed_perf) add_subdirectory(vespajlib) add_subdirectory(vespalib) add_subdirectory(vespalog) add_subdirectory(vespamalloc) +add_subdirectory(vespa-osgi-testrunner) +add_subdirectory(vespa-testrunner-components) add_subdirectory(zkfacade) add_subdirectory(zookeeper-command-line-client) add_subdirectory(zookeeper-server) diff --git a/client/CMakeLists.txt b/client/CMakeLists.txt new file mode 100644 index 00000000000..1ca6d2b4e60 --- /dev/null +++ b/client/CMakeLists.txt @@ -0,0 +1,14 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +set(GODIR ${CMAKE_CURRENT_SOURCE_DIR}/go) + +file(GLOB_RECURSE GOSRCFILES ${GODIR}/*.go) + +add_custom_command(OUTPUT ${GODIR}/bin/vespa-logfmt + COMMAND make + DEPENDS ${GODIR}/Makefile ${GOSRCFILES} + WORKING_DIRECTORY ${GODIR}) + +add_custom_target(vespalog_logfmt ALL DEPENDS ${GODIR}/bin/vespa-logfmt) + +install(PROGRAMS ${GODIR}/bin/vespa-logfmt DESTINATION bin) diff --git a/client/go/.gitignore b/client/go/.gitignore index 8933bc220cb..baab7c638c6 100644 --- a/client/go/.gitignore +++ b/client/go/.gitignore @@ -6,3 +6,4 @@ share/ !target/ mytestapp/ mytestapp-cache/ +mytmp/ diff --git a/client/go/Makefile b/client/go/Makefile index 78adf299f0e..8a07f880c24 100644 --- a/client/go/Makefile +++ b/client/go/Makefile @@ -131,7 +131,8 @@ clean: rmdir -p $(BIN) $(SHARE)/man/man1 > /dev/null 2>&1 || true test: ci - go test ./... + mkdir -p mytmp + TMPDIR=`pwd`/mytmp go test ./... checkfmt: @bash -c "diff --line-format='%L' <(echo -n) <(gofmt -l .)" || { echo "one or more files need to be formatted: try make fmt to fix this automatically"; exit 1; } diff --git a/client/go/cmd/logfmt/runlogfmt.go b/client/go/cmd/logfmt/runlogfmt.go index 9d782692f50..5abc4cc1cb8 100644 --- a/client/go/cmd/logfmt/runlogfmt.go +++ b/client/go/cmd/logfmt/runlogfmt.go @@ -8,12 +8,18 @@ import ( "bufio" "fmt" "os" - - "github.com/mattn/go-isatty" ) -func inputIsTty() bool { - return isatty.IsTerminal(os.Stdin.Fd()) +func inputIsPipe() bool { + fi, err := os.Stdin.Stat() + if err != nil { + return false + } + if fi.Mode()&os.ModeNamedPipe == 0 { + return false + } else { + return true + } } func vespaHome() string { @@ -28,7 +34,7 @@ func vespaHome() string { func RunLogfmt(opts *Options, args []string) { if len(args) == 0 { - if inputIsTty() { + if !inputIsPipe() { args = append(args, vespaHome()+"/logs/vespa/vespa.log") } else { formatFile(opts, os.Stdin) @@ -39,7 +45,10 @@ func RunLogfmt(opts *Options, args []string) { fmt.Fprintf(os.Stderr, "Must have exact 1 file for 'follow' option, got %d\n", len(args)) return } - tailFile(opts, args[0]) + if err := tailFile(opts, args[0]); err != nil { + fmt.Fprintln(os.Stderr, err) + return + } return } for _, arg := range args { @@ -62,11 +71,15 @@ func formatLine(opts *Options, line string) { } } -func tailFile(opts *Options, fn string) { - tailed := FollowFile(fn) - for line := range tailed.Lines { +func tailFile(opts *Options, fn string) error { + tailed, err := FollowFile(fn) + if err != nil { + return err + } + for line := range tailed.Lines() { formatLine(opts, line.Text) } + return nil } func formatFile(opts *Options, arg *os.File) { diff --git a/client/go/cmd/logfmt/tail.go b/client/go/cmd/logfmt/tail.go index 44674012548..75e7cbb0693 100644 --- a/client/go/cmd/logfmt/tail.go +++ b/client/go/cmd/logfmt/tail.go @@ -1,169 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. // vespa logfmt command -// Author: arnej +// Author: mpolden package logfmt -import ( - "bufio" - "fmt" - "io" - "os" - "time" - - "golang.org/x/sys/unix" -) - -const lastLinesSize = 4 * 1024 - type Line struct { Text string } -// an active "tail -f" like object - -type Tail struct { - Lines chan Line - lineBuf []byte - curFile *os.File - fn string - reader *bufio.Reader - curStat unix.Stat_t -} - -// API for starting to follow a log file - -func FollowFile(fn string) (res Tail) { - res.fn = fn - res.lineBuf = make([]byte, lastLinesSize) - res.openTail() - res.Lines = make(chan Line, 20) - res.lineBuf = res.lineBuf[:0] - go runTailWith(&res) - return res -} - -func (t *Tail) setFile(f *os.File) { - if t.curFile != nil { - t.curFile.Close() - } - t.curFile = f - if f != nil { - err := unix.Fstat(int(f.Fd()), &t.curStat) - if err != nil { - f.Close() - fmt.Fprintf(os.Stderr, "unexpected failure: %v\n", err) - return - } - t.reader = bufio.NewReaderSize(f, 1024*1024) - } else { - t.reader = nil - } -} - -// open log file and seek to the start of a line near the end, if possible. -func (t *Tail) openTail() { - file, err := os.Open(t.fn) - if err != nil { - return - } - sz, err := file.Seek(0, os.SEEK_END) - if err != nil { - return - } - if sz < lastLinesSize { - sz, err = file.Seek(0, os.SEEK_SET) - if err == nil { - // just read from start of file, all OK - t.setFile(file) - } - return - } - sz, _ = file.Seek(-lastLinesSize, os.SEEK_END) - n, err := file.Read(t.lineBuf) - if err != nil { - return - } - for i := 0; i < n; i++ { - if t.lineBuf[i] == '\n' { - sz, err = file.Seek(sz+int64(i+1), os.SEEK_SET) - if err == nil { - t.setFile(file) - } - return - } - } -} - -func (t *Tail) reopen(cur *unix.Stat_t) { - for cnt := 0; cnt < 100; cnt++ { - file, err := os.Open(t.fn) - if err != nil { - t.setFile(nil) - if cnt == 0 { - fmt.Fprintf(os.Stderr, "%v (waiting for log file to appear)\n", err) - } - time.Sleep(1000 * time.Millisecond) - continue - } - var stat unix.Stat_t - err = unix.Fstat(int(file.Fd()), &stat) - if err != nil { - file.Close() - fmt.Fprintf(os.Stderr, "unexpected failure: %v\n", err) - time.Sleep(5000 * time.Millisecond) - continue - } - if cur != nil && cur.Dev == stat.Dev && cur.Ino == stat.Ino { - // same file, continue following it - file.Close() - return - } - // new file, start following it - t.setFile(file) - return - } -} - -// runs as a goroutine -func runTailWith(t *Tail) { - defer t.setFile(nil) -loop: - for { - for t.curFile == nil { - t.reopen(nil) - } - bytes, err := t.reader.ReadSlice('\n') - t.lineBuf = append(t.lineBuf, bytes...) - if err == bufio.ErrBufferFull { - continue - } - if err == nil { - ll := len(t.lineBuf) - 1 - t.Lines <- Line{Text: string(t.lineBuf[:ll])} - t.lineBuf = t.lineBuf[:0] - continue - } - if err == io.EOF { - pos, _ := t.curFile.Seek(0, os.SEEK_CUR) - for cnt := 0; cnt < 100; cnt++ { - time.Sleep(10 * time.Millisecond) - sz, _ := t.curFile.Seek(0, os.SEEK_END) - if sz != pos { - if sz < pos { - // truncation case - pos = 0 - } - t.curFile.Seek(pos, os.SEEK_SET) - continue loop - } - } - // no change in file size, try reopening - t.reopen(&t.curStat) - } else { - fmt.Fprintf(os.Stderr, "error tailing '%s': %v\n", t.fn, err) - close(t.Lines) - return - } - } +type Tail interface { + Lines() chan Line } diff --git a/client/go/cmd/logfmt/tail_not_unix.go b/client/go/cmd/logfmt/tail_not_unix.go new file mode 100644 index 00000000000..7030572575d --- /dev/null +++ b/client/go/cmd/logfmt/tail_not_unix.go @@ -0,0 +1,15 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: mpolden + +//go:build windows + +package logfmt + +import ( + "fmt" +) + +func FollowFile(fn string) (Tail, error) { + return nil, fmt.Errorf("tail is not supported on this platform") +} diff --git a/client/go/cmd/logfmt/tail_unix.go b/client/go/cmd/logfmt/tail_unix.go new file mode 100644 index 00000000000..7703844da48 --- /dev/null +++ b/client/go/cmd/logfmt/tail_unix.go @@ -0,0 +1,170 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa logfmt command +// Author: arnej + +//go:build !windows + +package logfmt + +import ( + "bufio" + "fmt" + "io" + "os" + "time" + + "golang.org/x/sys/unix" +) + +const lastLinesSize = 4 * 1024 + +// an active "tail -f" like object + +type unixTail struct { + lines chan Line + lineBuf []byte + curFile *os.File + fn string + reader *bufio.Reader + curStat unix.Stat_t +} + +func (t *unixTail) Lines() chan Line { return t.lines } + +// API for starting to follow a log file + +func FollowFile(fn string) (Tail, error) { + res := unixTail{} + res.fn = fn + res.lineBuf = make([]byte, lastLinesSize) + res.openTail() + res.lines = make(chan Line, 20) + res.lineBuf = res.lineBuf[:0] + go runTailWith(&res) + return &res, nil +} + +func (t *unixTail) setFile(f *os.File) { + if t.curFile != nil { + t.curFile.Close() + } + t.curFile = f + if f != nil { + err := unix.Fstat(int(f.Fd()), &t.curStat) + if err != nil { + f.Close() + fmt.Fprintf(os.Stderr, "unexpected failure: %v\n", err) + return + } + t.reader = bufio.NewReaderSize(f, 1024*1024) + } else { + t.reader = nil + } +} + +// open log file and seek to the start of a line near the end, if possible. +func (t *unixTail) openTail() { + file, err := os.Open(t.fn) + if err != nil { + return + } + sz, err := file.Seek(0, os.SEEK_END) + if err != nil { + return + } + if sz < lastLinesSize { + sz, err = file.Seek(0, os.SEEK_SET) + if err == nil { + // just read from start of file, all OK + t.setFile(file) + } + return + } + sz, _ = file.Seek(-lastLinesSize, os.SEEK_END) + n, err := file.Read(t.lineBuf) + if err != nil { + return + } + for i := 0; i < n; i++ { + if t.lineBuf[i] == '\n' { + sz, err = file.Seek(sz+int64(i+1), os.SEEK_SET) + if err == nil { + t.setFile(file) + } + return + } + } +} + +func (t *unixTail) reopen(cur *unix.Stat_t) { + for cnt := 0; cnt < 100; cnt++ { + file, err := os.Open(t.fn) + if err != nil { + t.setFile(nil) + if cnt == 0 { + fmt.Fprintf(os.Stderr, "%v (waiting for log file to appear)\n", err) + } + time.Sleep(1000 * time.Millisecond) + continue + } + var stat unix.Stat_t + err = unix.Fstat(int(file.Fd()), &stat) + if err != nil { + file.Close() + fmt.Fprintf(os.Stderr, "unexpected failure: %v\n", err) + time.Sleep(5000 * time.Millisecond) + continue + } + if cur != nil && cur.Dev == stat.Dev && cur.Ino == stat.Ino { + // same file, continue following it + file.Close() + return + } + // new file, start following it + t.setFile(file) + return + } +} + +// runs as a goroutine +func runTailWith(t *unixTail) { + defer t.setFile(nil) +loop: + for { + for t.curFile == nil { + t.reopen(nil) + } + bytes, err := t.reader.ReadSlice('\n') + t.lineBuf = append(t.lineBuf, bytes...) + if err == bufio.ErrBufferFull { + continue + } + if err == nil { + ll := len(t.lineBuf) - 1 + t.lines <- Line{Text: string(t.lineBuf[:ll])} + t.lineBuf = t.lineBuf[:0] + continue + } + if err == io.EOF { + pos, _ := t.curFile.Seek(0, os.SEEK_CUR) + for cnt := 0; cnt < 100; cnt++ { + time.Sleep(10 * time.Millisecond) + sz, _ := t.curFile.Seek(0, os.SEEK_END) + if sz != pos { + if sz < pos { + // truncation case + pos = 0 + } + t.curFile.Seek(pos, os.SEEK_SET) + continue loop + } + } + // no change in file size, try reopening + t.reopen(&t.curStat) + } else { + fmt.Fprintf(os.Stderr, "error tailing '%s': %v\n", t.fn, err) + close(t.lines) + return + } + } +} diff --git a/client/go/cmd/testdata/applications/withSource/src/main/application/services.xml b/client/go/cmd/testdata/applications/withSource/src/main/application/services.xml index ceaaa0f5e3c..60267517c33 100644 --- a/client/go/cmd/testdata/applications/withSource/src/main/application/services.xml +++ b/client/go/cmd/testdata/applications/withSource/src/main/application/services.xml @@ -32,8 +32,9 @@ </config> </handler> - <nodes jvmargs="-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=*:8998"> + <nodes> <node hostalias="node1" /> + <jvm options="-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=*:8998"/> </nodes> </container> diff --git a/client/go/go.mod b/client/go/go.mod index 15b45a291a6..4181d4ee2f2 100644 --- a/client/go/go.mod +++ b/client/go/go.mod @@ -17,7 +17,7 @@ require ( github.com/stretchr/objx v0.1.1 // indirect github.com/stretchr/testify v1.7.0 github.com/zalando/go-keyring v0.1.1 - golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect + golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) diff --git a/client/go/vespa-logfmt/cmd.go b/client/go/vespa-logfmt/cmd.go index 749cac5fa56..c1b8261c0c1 100644 --- a/client/go/vespa-logfmt/cmd.go +++ b/client/go/vespa-logfmt/cmd.go @@ -34,6 +34,7 @@ and converts it to something human-readable`, cmd.Flags().BoolVar(&curOptions.TruncateService, "ts", false, "") cmd.Flags().BoolVarP(&curOptions.FollowTail, "follow", "f", false, "follow logfile with tail -f") cmd.Flags().BoolVarP(&curOptions.DequoteNewlines, "nldequote", "N", false, "dequote newlines embedded in message") + cmd.Flags().BoolVarP(&curOptions.DequoteNewlines, "dequotenewlines", "n", false, "dequote newlines embedded in message") cmd.Flags().BoolVarP(&curOptions.TruncateComponent, "truncatecomponent", "t", false, "truncate component name") cmd.Flags().BoolVar(&curOptions.TruncateComponent, "tc", false, "") cmd.Flags().StringVarP(&curOptions.OnlyHostname, "host", "H", "", "select only one host") @@ -41,5 +42,6 @@ and converts it to something human-readable`, cmd.Flags().StringVarP(&curOptions.OnlyService, "service", "S", "", "select only one service") cmd.Flags().MarkHidden("tc") cmd.Flags().MarkHidden("ts") + cmd.Flags().MarkHidden("dequotenewlines") return cmd } diff --git a/client/js/app/README.md b/client/js/app/README.md index ae6a8d1cc25..9dbce216224 100644 --- a/client/js/app/README.md +++ b/client/js/app/README.md @@ -3,17 +3,71 @@  # Vespa client +This app contains the **Query Builder** and the **Trace Visualizer**. -This app is work in progress. -It currently contains the **Query Builder** and the **Trace Visualizer**. +Install and start: + + $ nvm install --lts node # in case the installed node.js is too old + $ yarn install + $ yarn dev # then open link, like http://127.0.0.1:3000/ + +Alternatively, use Docker to start it without installing node: + + $ docker run -v `pwd`:/w -w /w --publish 3000:3000 node:16 sh -c 'yarn install && yarn dev --host' + +When started, open [http://127.0.0.1:3000/](http://127.0.0.1:3000/). + +*Troubleshooting:* Remove the generated `node_modules` directory and try again. +This is also relevant when switching between running local and in the container. ## Query Builder -The Query Builder is a tool for creating Vespa queries to send to a local backend. +The Query Builder is a tool for creating Vespa queries to send to a local Vespa application. The tool provides all of the options for query parameters from dropdowns. The input fields provide hints to what is the expected type of value. + + +To access a Vespa endpoint from the Query Builder, +deploy with [CORS filters](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS). +To the query-serving container, add an `http` element in _services.xml_ like: +``` + <container id="default" version="1.0"> + + <http> + <filtering strict-mode="false"> + <request-chain id="request-chain"> + <filter id="com.yahoo.jdisc.http.filter.security.cors.CorsPreflightRequestFilter" + bundle="jdisc-security-filters"> + <config name="jdisc.http.filter.security.cors.cors-filter"> + <allowedUrls> + <item>*</item> + </allowedUrls> + </config> + </filter> + <binding>http://*/search/</binding> + </request-chain> + + <response-chain id="response-chain"> + <filter id="com.yahoo.jdisc.http.filter.security.cors.CorsResponseFilter" + bundle="jdisc-security-filters"> + <config name="jdisc.http.filter.security.cors.cors-filter"> + <allowedUrls> + <item>*</item> + </allowedUrls> + </config> + </filter> + <binding>http://*/search/</binding> + </response-chain> + </filtering> + + <server port="8080" id="default"/> + </http> +``` + +Deploy again, and (possibly) restart Vespa (internal port changes can be triggered by this). + ## Trace Visualizer @@ -45,13 +99,3 @@ Press the _JSON File_ button as shown in the image, and drag and drop the trace Then click on the newly added trace and see it displayed as a flame graph:  - - - -## Client install and start - - nvm install --lts node # in case your current node.js is too old - yarn install - yarn dev # then open link, like http://127.0.0.1:3000/ - -<!-- ToDo: publish a Docker image with all the clients ... --> diff --git a/client/js/app/img/querybuilder.png b/client/js/app/img/querybuilder.png Binary files differnew file mode 100644 index 00000000000..fa68f5bb82f --- /dev/null +++ b/client/js/app/img/querybuilder.png diff --git a/client/js/app/src/app/pages/querybuilder/TransformVespaTrace.jsx b/client/js/app/src/app/pages/querybuilder/TransformVespaTrace.jsx index 7299f1cb777..7a630a017cb 100644 --- a/client/js/app/src/app/pages/querybuilder/TransformVespaTrace.jsx +++ b/client/js/app/src/app/pages/querybuilder/TransformVespaTrace.jsx @@ -128,6 +128,10 @@ function createProtonSpans(children, parentID) { [{ refType: 'CHILD_OF', traceID: traceID, spanID: parentID }] ); data.push(newSpan); + // eslint-disable-next-line no-prototype-builtins + if (!child.hasOwnProperty('traces')) { + return; + } let traces = child['traces']; for (let k = 0; k < traces.length; k++) { let trace = traces[k]; diff --git a/client/js/app/src/app/pages/querybuilder/query-response/download-jeager.jsx b/client/js/app/src/app/pages/querybuilder/query-response/download-jaeger.jsx index 4130172a329..795c53f8c21 100644 --- a/client/js/app/src/app/pages/querybuilder/query-response/download-jeager.jsx +++ b/client/js/app/src/app/pages/querybuilder/query-response/download-jaeger.jsx @@ -20,7 +20,7 @@ function downloadFile(filename, blob) { URL.revokeObjectURL(href); } -export function DownloadJeager({ response, ...props }) { +export function DownloadJaeger({ response, ...props }) { const handleClick = () => { try { const json = JSON.parse(response); diff --git a/client/js/app/src/app/pages/querybuilder/query-response/query-response.jsx b/client/js/app/src/app/pages/querybuilder/query-response/query-response.jsx index 56562ae1717..794e163ef04 100644 --- a/client/js/app/src/app/pages/querybuilder/query-response/query-response.jsx +++ b/client/js/app/src/app/pages/querybuilder/query-response/query-response.jsx @@ -7,7 +7,7 @@ import { CopyButton, Textarea, } from '@mantine/core'; -import { DownloadJeager } from 'app/pages/querybuilder/query-response/download-jeager'; +import { DownloadJaeger } from 'app/pages/querybuilder/query-response/download-jaeger'; import { useQueryBuilderContext } from 'app/pages/querybuilder/context/query-builder-provider'; import { Icon } from 'app/components'; @@ -33,7 +33,7 @@ export function QueryResponse() { </Button> )} </CopyButton> - <DownloadJeager + <DownloadJaeger variant="outline" size="xs" compact diff --git a/client/js/app/src/app/pages/querytracer/query-tracer.jsx b/client/js/app/src/app/pages/querytracer/query-tracer.jsx index 758182d4f3f..ca1a4e99c8f 100644 --- a/client/js/app/src/app/pages/querytracer/query-tracer.jsx +++ b/client/js/app/src/app/pages/querytracer/query-tracer.jsx @@ -1,6 +1,6 @@ import React, { useState } from 'react'; import { Stack, Textarea } from '@mantine/core'; -import { DownloadJeager } from 'app/pages/querybuilder/query-response/download-jeager'; +import { DownloadJaeger } from 'app/pages/querybuilder/query-response/download-jaeger'; export function QueryTracer() { const [response, setResponse] = useState(''); @@ -19,7 +19,7 @@ export function QueryTracer() { value={response} onChange={({ target }) => setResponse(target.value)} /> - <DownloadJeager fullWidth response={response} /> + <DownloadJaeger fullWidth response={response} /> </Stack> ); } diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index a30cbb86299..5862731ced7 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -27,7 +27,7 @@ <httpclient.version>4.5.13</httpclient.version> <httpcore.version>4.4.13</httpcore.version> <junit5.version>5.8.1</junit5.version> <!-- TODO: in parent this is named 'junit.version' --> - <onnxruntime.version>1.11.0</onnxruntime.version> + <onnxruntime.version>1.12.1</onnxruntime.version> <!-- END parent/pom.xml --> diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java index 6a7c75c629e..789c86bd6cf 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/RpcServerTest.java @@ -22,7 +22,6 @@ import com.yahoo.vespa.clustercontroller.core.rpc.RpcServer; import com.yahoo.vespa.clustercontroller.core.testutils.LogFormatter; import com.yahoo.vespa.clustercontroller.core.testutils.WaitCondition; import org.junit.jupiter.api.Test; - import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -31,7 +30,10 @@ import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author humbe @@ -55,7 +57,7 @@ public class RpcServerTest extends FleetControllerTest { Slobrok slobrok = new Slobrok(); String[] slobrokConnectionSpecs = getSlobrokConnectionSpecs(slobrok); RpcServer server = new RpcServer(timer, new Object(), "mycluster", 0, new BackOff()); - server.setSlobrokConnectionSpecs(slobrokConnectionSpecs, 18347); + server.setSlobrokConnectionSpecs(slobrokConnectionSpecs, 0); int portUsed = server.getPort(); server.setSlobrokConnectionSpecs(slobrokConnectionSpecs, portUsed); server.disconnect(); diff --git a/component/abi-spec.json b/component/abi-spec.json index d990a9077b4..cfbbdf4b306 100644 --- a/component/abi-spec.json +++ b/component/abi-spec.json @@ -138,6 +138,7 @@ "public void <init>(java.lang.String)", "public void <init>(com.yahoo.text.Utf8Array)", "public static com.yahoo.component.Version fromString(java.lang.String)", + "public com.yahoo.component.Version withQualifier(java.lang.String)", "public java.lang.String toFullString()", "public int getMajor()", "public int getMinor()", diff --git a/component/src/main/java/com/yahoo/component/Version.java b/component/src/main/java/com/yahoo/component/Version.java index 1d4546c0c58..db8606d31fa 100644 --- a/component/src/main/java/com/yahoo/component/Version.java +++ b/component/src/main/java/com/yahoo/component/Version.java @@ -202,6 +202,12 @@ public final class Version implements Comparable<Version> { return (versionString == null) ? emptyVersion :new Version(versionString); } + public Version withQualifier(String qualifier) { + if (qualifier.indexOf('.') != -1) + throw new IllegalArgumentException("Qualifier cannot contain '.'"); + return new Version(major, minor, micro, qualifier); + } + /** * Must be called on construction after the component values are set * diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 4536257f8a3..e45ab5de253 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -76,6 +76,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"vekterli"}) default boolean useThreePhaseUpdates() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Select sequencer type use while feeding") default String feedSequencerType() { return "THROUGHPUT"; } @ModelFeatureFlag(owners = {"baldersheim"}) default String responseSequencerType() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"baldersheim"}) default String queryDispatchPolicy() { return "adaptive"; } @ModelFeatureFlag(owners = {"baldersheim"}) default int defaultNumResponseThreads() { return 2; } @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter="7.last") default boolean skipCommunicationManagerThread() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}, removeAfter="7.last") default boolean skipMbusRequestThread() { return true; } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 400aea86834..8e2f3feb010 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -41,6 +41,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean useThreePhaseUpdates = false; private double defaultTermwiseLimit = 1.0; private String jvmGCOptions = null; + private String queryDispatchPolicy = "adaptive"; private String sequencerType = "THROUGHPUT"; private boolean firstTimeDeployment = false; private String responseSequencerType = "ADAPTIVE"; @@ -147,6 +148,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public int mbusCppEventsBeforeWakeup() { return mbus_cpp_events_before_wakeup; } @Override public int rpcNumTargets() { return rpc_num_targets; } @Override public int rpcEventsBeforeWakeup() { return rpc_events_before_wakeup; } + @Override public String queryDispatchPolicy() { return queryDispatchPolicy; } public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) { @@ -192,6 +194,10 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea jvmGCOptions = gcOptions; return this; } + public TestProperties setQueryDispatchPolicy(String policy) { + queryDispatchPolicy = policy; + return this; + } public TestProperties setFeedSequencerType(String type) { sequencerType = type; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index ff39aa0903c..6dad3c5f06f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -158,7 +158,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> String clusterName, ContentSearchCluster search) { List<ModelElement> indexedDefs = getIndexedSchemas(clusterElem); if (!indexedDefs.isEmpty()) { - IndexedSearchCluster isc = new IndexedSearchCluster(search, clusterName, 0); + IndexedSearchCluster isc = new IndexedSearchCluster(search, clusterName, 0, deployState.featureFlags()); isc.setRoutingSelector(clusterElem.childAsString("documents.selection")); Double visibilityDelay = clusterElem.childAsDouble("engine.proton.visibility-delay"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java index 6cad778dccf..1d1e1a8e3dc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java @@ -11,10 +11,16 @@ public class DispatchTuning { public static final DispatchTuning empty = new DispatchTuning.Builder().build(); - public enum DispatchPolicy { ROUNDROBIN, ADAPTIVE } + public enum DispatchPolicy { + ROUNDROBIN, + LATENCY_AMORTIZED_OVER_REQUESTS, + LATENCY_AMORTIZED_OVER_TIME, + BEST_OF_RANDOM_2, + ADAPTIVE + } private final Integer maxHitsPerPartition; - private DispatchPolicy dispatchPolicy; + private final DispatchPolicy dispatchPolicy; private final Double minActiveDocsCoverage; public Double getTopkProbability() { @@ -36,9 +42,6 @@ public class DispatchTuning { /** Returns the policy used to select which group to dispatch a query to */ public DispatchPolicy getDispatchPolicy() { return dispatchPolicy; } - @SuppressWarnings("unused") - public void setDispatchPolicy(DispatchPolicy dispatchPolicy) { this.dispatchPolicy = dispatchPolicy; } - /** Returns the percentage of documents which must be available in a group for that group to receive queries */ public Double getMinActiveDocsCoverage() { return minActiveDocsCoverage; } @@ -67,10 +70,13 @@ public class DispatchTuning { return this; } - private DispatchPolicy toDispatchPolicy(String policy) { + public static DispatchPolicy toDispatchPolicy(String policy) { switch (policy.toLowerCase()) { case "adaptive": case "random": return DispatchPolicy.ADAPTIVE; // TODO: Deprecate 'random' on Vespa 9 case "round-robin": return DispatchPolicy.ROUNDROBIN; + case "latency-amortized-over-requests" : return DispatchPolicy.LATENCY_AMORTIZED_OVER_REQUESTS; + case "latency-amortized-over-time" : return DispatchPolicy.LATENCY_AMORTIZED_OVER_TIME; + case "best-of-random-2" : return DispatchPolicy.BEST_OF_RANDOM_2; default: throw new IllegalArgumentException("Unknown dispatch policy '" + policy + "'"); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 2c83e87df97..56fb915797b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.ConfigInstance; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; @@ -18,6 +19,7 @@ import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.model.container.docproc.DocprocChain; import com.yahoo.vespa.model.content.DispatchSpec; +import com.yahoo.vespa.model.content.DispatchTuning; import com.yahoo.vespa.model.content.SearchCoverage; import java.util.ArrayList; @@ -56,6 +58,7 @@ public class IndexedSearchCluster extends SearchCluster private final DispatchGroup rootDispatch; private DispatchSpec dispatchSpec; private final List<SearchNode> searchNodes = new ArrayList<>(); + private final DispatchTuning.DispatchPolicy defaultDispatchPolicy; /** * Returns the document selector that is able to resolve what documents are to be routed to this search cluster. @@ -67,10 +70,11 @@ public class IndexedSearchCluster extends SearchCluster return routingSelector; } - public IndexedSearchCluster(AbstractConfigProducer<SearchCluster> parent, String clusterName, int index) { + public IndexedSearchCluster(AbstractConfigProducer<SearchCluster> parent, String clusterName, int index, ModelContext.FeatureFlags featureFlags) { super(parent, clusterName, index); documentDbsConfigProducer = new MultipleDocumentDatabasesConfigProducer(this, documentDbs); rootDispatch = new DispatchGroup(this); + defaultDispatchPolicy = DispatchTuning.Builder.toDispatchPolicy(featureFlags.queryDispatchPolicy()); } @Override @@ -273,6 +277,15 @@ public class IndexedSearchCluster extends SearchCluster return dispatchSpec; } + private static DistributionPolicy.Enum toDistributionPolicy(DispatchTuning.DispatchPolicy tuning) { + return switch (tuning) { + case ADAPTIVE: yield DistributionPolicy.ADAPTIVE; + case ROUNDROBIN: yield DistributionPolicy.ROUNDROBIN; + case BEST_OF_RANDOM_2: yield DistributionPolicy.BEST_OF_RANDOM_2; + case LATENCY_AMORTIZED_OVER_REQUESTS: yield DistributionPolicy.LATENCY_AMORTIZED_OVER_REQUESTS; + case LATENCY_AMORTIZED_OVER_TIME: yield DistributionPolicy.LATENCY_AMORTIZED_OVER_TIME; + }; + } @Override public void getConfig(DispatchConfig.Builder builder) { for (SearchNode node : getSearchNodes()) { @@ -289,14 +302,9 @@ public class IndexedSearchCluster extends SearchCluster if (tuning.dispatch.getMinActiveDocsCoverage() != null) builder.minActivedocsPercentage(tuning.dispatch.getMinActiveDocsCoverage()); if (tuning.dispatch.getDispatchPolicy() != null) { - switch (tuning.dispatch.getDispatchPolicy()) { - case ADAPTIVE: - builder.distributionPolicy(DistributionPolicy.ADAPTIVE); - break; - case ROUNDROBIN: - builder.distributionPolicy(DistributionPolicy.ROUNDROBIN); - break; - } + builder.distributionPolicy(toDistributionPolicy(tuning.dispatch.getDispatchPolicy())); + } else { + builder.distributionPolicy(toDistributionPolicy(defaultDispatchPolicy)); } if (tuning.dispatch.getMaxHitsPerPartition() != null) builder.maxHitsPerNode(tuning.dispatch.getMaxHitsPerPartition()); diff --git a/config-model/src/main/resources/schema/content.rnc b/config-model/src/main/resources/schema/content.rnc index ff45b127b8b..5ca9a0792a2 100644 --- a/config-model/src/main/resources/schema/content.rnc +++ b/config-model/src/main/resources/schema/content.rnc @@ -82,7 +82,7 @@ ClusterControllerTuning = element cluster-controller { DispatchTuning = element dispatch { element max-hits-per-partition { xsd:nonNegativeInteger }? & - element dispatch-policy { string "round-robin" | string "adaptive" | string "random" }? & + element dispatch-policy { string "round-robin" | string "adaptive" | string "random" | "best-of-random-2" | "latency-amortized-over-requests" }? & element min-active-docs-coverage { xsd:double }? & element top-k-probability { xsd:double }? } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index cfb2fceb60b..17ce43bba2c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1005,6 +1005,37 @@ public class ContentClusterTest extends ContentBaseTest { verifyTopKProbabilityPropertiesControl(); } + private void verifyQueryDispatchPolicy(String policy, DispatchConfig.DistributionPolicy.Enum expected) { + TestProperties properties = new TestProperties(); + if (policy != null) { + properties.setQueryDispatchPolicy(policy); + } + VespaModel model = createEnd2EndOneNode(properties); + + ContentCluster cc = model.getContentClusters().get("storage"); + DispatchConfig.Builder builder = new DispatchConfig.Builder(); + cc.getSearch().getConfig(builder); + + DispatchConfig cfg = new DispatchConfig(builder); + assertEquals(expected, cfg.distributionPolicy()); + } + + @Test + public void default_dispatch_controlled_by_properties() { + verifyQueryDispatchPolicy(null, DispatchConfig.DistributionPolicy.ADAPTIVE); + verifyQueryDispatchPolicy("adaptive", DispatchConfig.DistributionPolicy.ADAPTIVE); + verifyQueryDispatchPolicy("round-robin", DispatchConfig.DistributionPolicy.ROUNDROBIN); + verifyQueryDispatchPolicy("best-of-random-2", DispatchConfig.DistributionPolicy.BEST_OF_RANDOM_2); + verifyQueryDispatchPolicy("latency-amortized-over-requests", DispatchConfig.DistributionPolicy.LATENCY_AMORTIZED_OVER_REQUESTS); + verifyQueryDispatchPolicy("latency-amortized-over-time", DispatchConfig.DistributionPolicy.LATENCY_AMORTIZED_OVER_TIME); + try { + verifyQueryDispatchPolicy("unknown", DispatchConfig.DistributionPolicy.ADAPTIVE); + fail(); + } catch (IllegalArgumentException e) { + assertEquals("Unknown dispatch policy 'unknown'", e.getMessage()); + } + } + private boolean resolveThreePhaseUpdateConfigWithFeatureFlag(boolean flagEnableThreePhase) { VespaModel model = createEnd2EndOneNode(new TestProperties().setUseThreePhaseUpdates(flagEnableThreePhase)); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/DispatchTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/DispatchTuningTest.java index cddbe267628..af547965749 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/DispatchTuningTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/DispatchTuningTest.java @@ -20,7 +20,7 @@ public class DispatchTuningTest { .setTopKProbability(18.3) .build(); assertEquals(69, dispatch.getMaxHitsPerPartition().intValue()); - assertEquals(12.5, dispatch.getMinActiveDocsCoverage().doubleValue(), 0.0); + assertEquals(12.5, dispatch.getMinActiveDocsCoverage(), 0.0); assertEquals(DispatchTuning.DispatchPolicy.ROUNDROBIN, dispatch.getDispatchPolicy()); assertEquals(18.3, dispatch.getTopkProbability(), 0.0); } @@ -44,6 +44,33 @@ public class DispatchTuningTest { } @Test + void requireThatLatencyAmortizedOverRequestsDispatchWork() { + DispatchTuning dispatch = new DispatchTuning.Builder() + .setDispatchPolicy("latency-amortized-over-requests") + .build(); + assertEquals(DispatchTuning.DispatchPolicy.LATENCY_AMORTIZED_OVER_REQUESTS, dispatch.getDispatchPolicy()); + assertNull(dispatch.getMinActiveDocsCoverage()); + } + + @Test + void requireThatLatencyAmortizedOverTimeDispatchWork() { + DispatchTuning dispatch = new DispatchTuning.Builder() + .setDispatchPolicy("latency-amortized-over-time") + .build(); + assertEquals(DispatchTuning.DispatchPolicy.LATENCY_AMORTIZED_OVER_TIME, dispatch.getDispatchPolicy()); + assertNull(dispatch.getMinActiveDocsCoverage()); + } + + @Test + void requireThatBestOfRandom2DispatchWork() { + DispatchTuning dispatch = new DispatchTuning.Builder() + .setDispatchPolicy("best-of-random-2") + .build(); + assertEquals(DispatchTuning.DispatchPolicy.BEST_OF_RANDOM_2, dispatch.getDispatchPolicy()); + assertNull(dispatch.getMinActiveDocsCoverage()); + } + + @Test void requireThatDefaultsAreNull() { DispatchTuning dispatch = new DispatchTuning.Builder().build(); assertNull(dispatch.getMaxHitsPerPartition()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomDispatchTuningBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomDispatchTuningBuilderTest.java index 564d6024acf..9eaa4ea6ed3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomDispatchTuningBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/DomDispatchTuningBuilderTest.java @@ -61,36 +61,37 @@ public class DomDispatchTuningBuilderTest { " </tuning>" + "</content>"); assertEquals(69, dispatch.getMaxHitsPerPartition().intValue()); - assertEquals(12.5, dispatch.getMinActiveDocsCoverage().doubleValue(), 0.0); - assertEquals(0.999, dispatch.getTopkProbability().doubleValue(), 0.0); + assertEquals(12.5, dispatch.getMinActiveDocsCoverage(), 0.0); + assertEquals(0.999, dispatch.getTopkProbability(), 0.0); } - @Test - void requireThatTuningDispatchPolicyRoundRobin() throws Exception { - DispatchTuning dispatch = newTuningDispatch( - "<content>" + - " <tuning>" + - " <dispatch>" + - " <dispatch-policy>round-robin</dispatch-policy>" + - " </dispatch>" + - " </tuning>" + - "</content>"); - assertEquals(DispatchTuning.DispatchPolicy.ROUNDROBIN, dispatch.getDispatchPolicy()); + private static String dispatchPolicy(String policy) { + return "<content>" + + " <tuning>" + + " <dispatch>" + + " <dispatch-policy>" + policy +"</dispatch-policy>" + + " </dispatch>" + + " </tuning>" + + "</content>"; } @Test - void requireThatTuningDispatchPolicyRandom() throws Exception { - DispatchTuning dispatch = newTuningDispatch( - "<content>" + - " <tuning>" + - " <dispatch>" + - " <dispatch-policy>random</dispatch-policy>" + - " </dispatch>" + - " </tuning>" + - "</content>"); - assertEquals(DispatchTuning.DispatchPolicy.ADAPTIVE, dispatch.getDispatchPolicy()); + void requireThatTuningDispatchPolicies() throws Exception { + assertEquals(DispatchTuning.DispatchPolicy.ROUNDROBIN, + newTuningDispatch(dispatchPolicy("round-robin")).getDispatchPolicy()); + assertEquals(DispatchTuning.DispatchPolicy.ADAPTIVE, + newTuningDispatch(dispatchPolicy("random")).getDispatchPolicy()); + assertEquals(DispatchTuning.DispatchPolicy.ADAPTIVE, + newTuningDispatch(dispatchPolicy("adaptive")).getDispatchPolicy()); + assertEquals(DispatchTuning.DispatchPolicy.BEST_OF_RANDOM_2, + newTuningDispatch(dispatchPolicy("best-of-random-2")).getDispatchPolicy()); + assertEquals(DispatchTuning.DispatchPolicy.LATENCY_AMORTIZED_OVER_REQUESTS, + newTuningDispatch(dispatchPolicy("latency-amortized-over-requests")).getDispatchPolicy()); + assertEquals(DispatchTuning.DispatchPolicy.LATENCY_AMORTIZED_OVER_TIME, + newTuningDispatch(dispatchPolicy("latency-amortized-over-time")).getDispatchPolicy()); } + private static DispatchTuning newTuningDispatch(String xml) throws Exception { return DomTuningDispatchBuilder.build( new ModelElement(DocumentBuilderFactory.newInstance() diff --git a/configdefinitions/src/vespa/dispatch.def b/configdefinitions/src/vespa/dispatch.def index 37b49ecaeb9..e26a136d245 100644 --- a/configdefinitions/src/vespa/dispatch.def +++ b/configdefinitions/src/vespa/dispatch.def @@ -8,7 +8,7 @@ namespace=vespa.config.search minActivedocsPercentage double default=97.0 # Distribution policy for group selection -distributionPolicy enum { ROUNDROBIN, ADAPTIVE } default=ADAPTIVE +distributionPolicy enum { ROUNDROBIN, BEST_OF_RANDOM_2, LATENCY_AMORTIZED_OVER_REQUESTS, LATENCY_AMORTIZED_OVER_TIME, ADAPTIVE } default=ADAPTIVE ## Maximum number of hits that will be requested from a single node ## in this dataset. If not set, there is no limit. Using this option diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 1aa70ff4b5b..59a48ad3c7e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -25,7 +25,6 @@ import com.yahoo.vespa.config.server.application.ConfigNotConvergedException; import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; import com.yahoo.vespa.config.server.configchange.ReindexActions; import com.yahoo.vespa.config.server.configchange.RestartActions; -import com.yahoo.vespa.config.server.session.LocalSession; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.session.Session; import com.yahoo.vespa.config.server.session.SessionRepository; @@ -161,9 +160,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { } private void deleteSession() { - SessionRepository sessionRepository = sessionRepository(); - LocalSession localSession = sessionRepository.getLocalSession(session.getSessionId()); - sessionRepository.deleteLocalSession(localSession); + sessionRepository().deleteLocalSession(session.getSessionId()); } private SessionRepository sessionRepository() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 068323f7784..7c7a12bbf36 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -166,6 +166,7 @@ public class ModelContextImpl implements ModelContext { public static class FeatureFlags implements ModelContext.FeatureFlags { + private final String queryDispatchPolicy; private final double defaultTermwiseLimit; private final boolean useThreePhaseUpdates; private final String feedSequencer; @@ -276,8 +277,10 @@ public class ModelContextImpl implements ModelContext { this.mbus_cpp_events_before_wakeup = flagValue(source, appId, version, Flags.MBUS_CPP_EVENTS_BEFORE_WAKEUP); this.rpc_num_targets = flagValue(source, appId, version, Flags.RPC_NUM_TARGETS); this.rpc_events_before_wakeup = flagValue(source, appId, version, Flags.RPC_EVENTS_BEFORE_WAKEUP); + this.queryDispatchPolicy = flagValue(source, appId, version, Flags.QUERY_DISPATCH_POLICY); } + @Override public String queryDispatchPolicy() { return queryDispatchPolicy;} @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @Override public boolean useThreePhaseUpdates() { return useThreePhaseUpdates; } @Override public String feedSequencerType() { return feedSequencer; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index f2fbff84907..fc33830e707 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -39,7 +39,6 @@ import com.yahoo.vespa.config.server.http.v2.response.ReindexingResponse; import com.yahoo.vespa.config.server.tenant.Tenant; import java.io.IOException; -import java.io.OutputStream; import java.net.URI; import java.time.Duration; import java.time.Instant; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java index a6bbd6c20a2..2e9c28bdc3b 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionRepository.java @@ -221,15 +221,31 @@ public class SessionRepository { Set<LocalSession> sessionIds = new HashSet<>(); for (File session : sessions) { long sessionId = Long.parseLong(session.getName()); - SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); - File sessionDir = getAndValidateExistingSessionAppDir(sessionId); - ApplicationPackage applicationPackage = FilesApplicationPackage.fromFile(sessionDir); - LocalSession localSession = new LocalSession(tenantName, sessionId, applicationPackage, sessionZKClient); + LocalSession localSession = getSessionFromFile(sessionId); sessionIds.add(localSession); } return sessionIds; } + private LocalSession getSessionFromFile(long sessionId) { + SessionZooKeeperClient sessionZKClient = createSessionZooKeeperClient(sessionId); + File sessionDir = getAndValidateExistingSessionAppDir(sessionId); + ApplicationPackage applicationPackage = FilesApplicationPackage.fromFile(sessionDir); + return new LocalSession(tenantName, sessionId, applicationPackage, sessionZKClient); + } + + public Set<Long> getLocalSessionsIdsFromFileSystem() { + File[] sessions = tenantFileSystemDirs.sessionsPath().listFiles(sessionApplicationsFilter); + if (sessions == null) return Set.of(); + + Set<Long> sessionIds = new HashSet<>(); + for (File session : sessions) { + long sessionId = Long.parseLong(session.getName()); + sessionIds.add(sessionId); + } + return sessionIds; + } + public ConfigChangeActions prepareLocalSession(Session session, DeployLogger logger, PrepareParams params, Instant now) { params.vespaVersion().ifPresent(version -> { if ( ! params.isBootstrap() && ! modelFactoryRegistry.allVersions().contains(version)) @@ -310,8 +326,7 @@ public class SessionRepository { } // Will delete session data in ZooKeeper and file system - public void deleteLocalSession(LocalSession session) { - long sessionId = session.getSessionId(); + public void deleteLocalSession(long sessionId) { log.log(Level.FINE, () -> "Deleting local session " + sessionId); SessionStateWatcher watcher = sessionStateWatchers.remove(sessionId); if (watcher != null) watcher.close(); @@ -323,7 +338,7 @@ public class SessionRepository { private void deleteAllSessions() { for (LocalSession session : getLocalSessions()) { - deleteLocalSession(session); + deleteLocalSession(session.getSessionId()); } } @@ -586,35 +601,48 @@ public class SessionRepository { public void deleteExpiredSessions(Map<ApplicationId, Long> activeSessions) { log.log(Level.FINE, () -> "Deleting expired local sessions for tenant '" + tenantName + "'"); - Set<LocalSession> toDelete = new HashSet<>(); + Set<Long> sessionIdsToDelete = new HashSet<>(); Set<Long> newSessions = findNewSessionsInFileSystem(); try { - for (LocalSession candidate : getLocalSessionsFromFileSystem()) { + for (long sessionId : getLocalSessionsIdsFromFileSystem()) { // Skip sessions newly added (we might have a session in the file system, but not in ZooKeeper, // we don't want to touch any of them) - if (newSessions.contains(candidate.getSessionId())) + if (newSessions.contains(sessionId)) continue; - Instant createTime = candidate.getCreateTime(); - log.log(Level.FINE, () -> "Candidate local session for deletion: " + candidate.getSessionId() + - ", created: " + createTime + ", state " + candidate.getStatus() + ", can be deleted: " + canBeDeleted(candidate)); + var sessionZooKeeperClient = createSessionZooKeeperClient(sessionId); + Instant createTime = sessionZooKeeperClient.readCreateTime(); + Session.Status status = sessionZooKeeperClient.readStatus(); - if (hasExpired(createTime) && canBeDeleted(candidate)) { - toDelete.add(candidate); + log.log(Level.FINE, () -> "Candidate local session for deletion: " + sessionId + + ", created: " + createTime + ", status " + status + ", can be deleted: " + canBeDeleted(sessionId, status) + + ", hasExpired: " + hasExpired(createTime)); + + if (hasExpired(createTime) && canBeDeleted(sessionId, status)) { + log.log(Level.FINE, () -> "expired: " + hasExpired(createTime) + ", can be deleted: " + canBeDeleted(sessionId, status)); + sessionIdsToDelete.add(sessionId); } else if (createTime.plus(Duration.ofDays(1)).isBefore(clock.instant())) { - Optional<ApplicationId> applicationId = candidate.getOptionalApplicationId(); + LocalSession session; + log.log(Level.FINE, () -> "not expired, but more than 1 day old: " + sessionId); + try { + session = getSessionFromFile(sessionId); + } catch (Exception e) { + log.log(Level.FINE, () -> "could not get session from file: " + sessionId + ": " + e.getMessage()); + continue; + } + Optional<ApplicationId> applicationId = session.getOptionalApplicationId(); if (applicationId.isEmpty()) continue; Long activeSession = activeSessions.get(applicationId.get()); - if (activeSession == null || activeSession != candidate.getSessionId()) { - toDelete.add(candidate); - log.log(Level.FINE, () -> "Will delete inactive session " + candidate.getSessionId() + " created " + + if (activeSession == null || activeSession != sessionId) { + sessionIdsToDelete.add(sessionId); + log.log(Level.FINE, () -> "Will delete inactive session " + sessionId + " created " + createTime + " for '" + applicationId + "'"); } } } - toDelete.forEach(this::deleteLocalSession); + sessionIdsToDelete.forEach(this::deleteLocalSession); // Make sure to catch here, to avoid executor just dying in case of issues ... } catch (Throwable e) { @@ -628,16 +656,16 @@ public class SessionRepository { } // Sessions with state other than UNKNOWN or ACTIVATE or old sessions in UNKNOWN state - private boolean canBeDeleted(LocalSession candidate) { - return ( ! List.of(Session.Status.UNKNOWN, Session.Status.ACTIVATE).contains(candidate.getStatus())) - || oldSessionDirWithUnknownStatus(candidate); + private boolean canBeDeleted(long sessionId, Session.Status status) { + return ( ! List.of(Session.Status.UNKNOWN, Session.Status.ACTIVATE).contains(status)) + || oldSessionDirWithUnknownStatus(sessionId, status); } - private boolean oldSessionDirWithUnknownStatus(LocalSession session) { + private boolean oldSessionDirWithUnknownStatus(long sessionId, Session.Status status) { Duration expiryTime = Duration.ofHours(configserverConfig.keepSessionsWithUnknownStatusHours()); - File sessionDir = tenantFileSystemDirs.getUserApplicationDir(session.getSessionId()); + File sessionDir = tenantFileSystemDirs.getUserApplicationDir(sessionId); return sessionDir.exists() - && session.getStatus() == Session.Status.UNKNOWN + && status == Session.Status.UNKNOWN && created(sessionDir).plus(expiryTime).isBefore(clock.instant()); } diff --git a/configserver/src/main/sh/start-configserver b/configserver/src/main/sh/start-configserver index 8127b0bfafc..f223c0a8fb9 100755 --- a/configserver/src/main/sh/start-configserver +++ b/configserver/src/main/sh/start-configserver @@ -151,9 +151,7 @@ export standalone_jdisc_container__deployment_profile=configserver # class path CP="${VESPA_HOME}/lib/jars/jdisc_core-jar-with-dependencies.jar" -baseuserargs="$VESPA_CONFIGSERVER_JVMARGS" -serveruserargs="$cloudconfig_server__jvmargs" -jvmargs="$baseuserargs $serveruserargs" +jvmoptions="$VESPA_CONFIGSERVER_JVMARGS" export LD_PRELOAD=${VESPA_HOME}/lib64/vespa/malloc/libvespamalloc.so @@ -161,8 +159,8 @@ rm -f $cfpfile vespa-run-as-vespa-user sh -c "printenv > $cfpfile" fixddir $bundlecachedir -heap_min=$(get_min_heap_mb "${jvmargs}" 128) -heap_max=$(get_max_heap_mb "${jvmargs}" 2048) +heap_min=$(get_min_heap_mb "${jvmoptions}" 128) +heap_max=$(get_max_heap_mb "${jvmoptions}" 2048) vespa-run-as-vespa-user vespa-runserver -s ${VESPA_SERVICE_NAME} -r 30 -p $pidfile -- \ java \ -Xms${heap_min}m -Xmx${heap_max}m \ @@ -174,7 +172,7 @@ vespa-run-as-vespa-user vespa-runserver -s ${VESPA_SERVICE_NAME} -r 30 -p $pidfi -XX:+ExitOnOutOfMemoryError \ -XX:-OmitStackTraceInFastThrow \ -XX:MaxJavaStackTraceDepth=1000000 \ - $jvmargs \ + $jvmoptions \ --add-opens=java.base/java.io=ALL-UNNAMED \ --add-opens=java.base/java.lang=ALL-UNNAMED \ --add-opens=java.base/java.net=ALL-UNNAMED \ diff --git a/configserver/src/test/apps/illegalApp2/hosts.xml b/configserver/src/test/apps/illegalApp2/hosts.xml new file mode 100644 index 00000000000..a515a4e97da --- /dev/null +++ b/configserver/src/test/apps/illegalApp2/hosts.xml @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<hosts> + <host name="mytesthost"> + <alias>node1</alias> + </host> +</hosts> diff --git a/configserver/src/test/apps/illegalApp2/schemas/music.sd b/configserver/src/test/apps/illegalApp2/schemas/music.sd new file mode 100644 index 00000000000..f4b11d1e8e4 --- /dev/null +++ b/configserver/src/test/apps/illegalApp2/schemas/music.sd @@ -0,0 +1,50 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +# A basic search definition - called music, should be saved to music.sd +search music { + + # It contains one document type only - called music as well + document music { + + field title type string { + indexing: summary | index # How this field should be indexed + # index-to: title, default # Create two indexes + weight: 75 # Ranking importancy of this field, used by the built in nativeRank feature + } + + field artist type string { + indexing: summary | attribute | index + # index-to: artist, default + + weight: 25 + } + + field year type int { + indexing: summary | attribute + } + + # Increase query + field popularity type int { + indexing: summary | attribute + } + + field url type uri { + indexing: summary | index + } + + } + + rank-profile default inherits default { + first-phase { + expression: nativeRank(title,artist) + attribute(popularity) + } + + } + + rank-profile textmatch inherits default { + first-phase { + expression: nativeRank(title,artist) + } + + } + +} diff --git a/configserver/src/test/apps/illegalApp2/services.xml b/configserver/src/test/apps/illegalApp2/services.xml new file mode 100644 index 00000000000..3e957a0e228 --- /dev/null +++ b/configserver/src/test/apps/illegalApp2/services.xml @@ -0,0 +1,39 @@ +<?xml version="1.0" encoding="utf-8" ?> +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<services version="1.0"> + + <admin version="2.0"> + <adminserver hostalias="node1"/> + <logserver hostalias="node1" /> + </admin> + + <content version="1.0"> + <redundancy>2</redundancy> + <documents> + <document type="music" mode="index"/> + </documents> + <nodes> + <node hostalias="node1" distribution-key="0"/> + </nodes> + + </content> + + <container version="1.0"> + <include dir='file:///etc/passwd'/> + <document-processing compressdocuments="true"> + <chain id="ContainerWrapperTest"> + <documentprocessor id="com.yahoo.vespa.config.AppleDocProc"/> + </chain> + </document-processing> + + <config name="project.specific"> + <value>someval</value> + </config> + + <nodes> + <node hostalias="node1" /> + </nodes> + + </container> + +</services> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index d1d8c165124..a8439a9061c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -59,9 +59,7 @@ import org.junit.rules.TemporaryFolder; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; -import java.io.UncheckedIOException; import java.nio.file.Files; -import java.nio.file.attribute.BasicFileAttributeView; import java.nio.file.attribute.FileTime; import java.time.Duration; import java.time.Instant; @@ -94,6 +92,7 @@ public class ApplicationRepositoryTest { private final static File testAppLogServerWithContainer = new File("src/test/apps/app-logserver-with-container"); private final static File app1 = new File("src/test/apps/cs1"); private final static File app2 = new File("src/test/apps/cs2"); + private final static File illegalApp2 = new File("src/test/apps/illegalapp2"); private final static TenantName tenant1 = TenantName.from("test1"); private final static TenantName tenant2 = TenantName.from("test2"); @@ -456,7 +455,7 @@ public class ApplicationRepositoryTest { deployment4.get().prepare(); // session 5 (not activated) assertEquals(2, sessionRepository.getLocalSessions().size()); - sessionRepository.deleteLocalSession(localSession); + sessionRepository.deleteLocalSession(localSession.getSessionId()); assertEquals(1, sessionRepository.getLocalSessions().size()); // Create a local session without any data in zookeeper (corner case seen in production occasionally) @@ -464,28 +463,29 @@ public class ApplicationRepositoryTest { int sessionId = 6; TenantName tenantName = tester.tenant().getName(); Instant session6CreateTime = clock.instant(); - Files.createDirectory(new TenantFileSystemDirs(serverdb, tenantName).getUserApplicationDir(sessionId).toPath()); - LocalSession localSession2 = new LocalSession(tenant1, + TenantFileSystemDirs tenantFileSystemDirs = new TenantFileSystemDirs(serverdb, tenantName); + Files.createDirectory(tenantFileSystemDirs.getUserApplicationDir(sessionId).toPath()); + String hostName = ConfigUtils.getCanonicalHostName(); + LocalSession localSession2 = new LocalSession(tenantName, sessionId, FilesApplicationPackage.fromFile(testApp), - new SessionZooKeeperClient(curator, - tenantName, - sessionId, - ConfigUtils.getCanonicalHostName())); + new SessionZooKeeperClient(curator, tenantName, sessionId, hostName)); sessionRepository.addLocalSession(localSession2); assertEquals(2, sessionRepository.getLocalSessions().size()); // Create a session, set status to UNKNOWN, we don't want to expire those (creation time is then EPOCH, // so will be candidate for expiry) - Session session = sessionRepository.createRemoteSession(7); - sessionRepository.createSetStatusTransaction(session, Session.Status.UNKNOWN); + sessionId = 7; + Session session = sessionRepository.createRemoteSession(sessionId); + sessionRepository.createSessionZooKeeperClient(sessionId).createNewSession(clock.instant()); + sessionRepository.createSetStatusTransaction(session, Session.Status.UNKNOWN).commit(); assertEquals(2, sessionRepository.getLocalSessions().size()); // Still 2, no new local session // Check that trying to expire local session when there exists a local session without any data in zookeeper // should not delete session if this is a new file ... deleteExpiredLocalSessionsAndAssertNumberOfSessions(2, tester, sessionRepository); - // ... but it should be deleted if some time has passed + // ... but it should be deleted when some time has passed clock.advance(Duration.ofSeconds(60)); deleteExpiredLocalSessionsAndAssertNumberOfSessions(1, tester, sessionRepository); @@ -495,6 +495,21 @@ public class ApplicationRepositoryTest { // Advance time, session SHOULD be deleted clock.advance(Duration.ofHours(configserverConfig.keepSessionsWithUnknownStatusHours()).plus(Duration.ofMinutes(1))); deleteExpiredLocalSessionsAndAssertNumberOfSessions(0, tester, sessionRepository); + + // Create a local session with invalid application package and check that expiring local sessions still works + sessionId = 8; + java.nio.file.Path applicationPath = tenantFileSystemDirs.getUserApplicationDir(sessionId).toPath(); + session = sessionRepository.createRemoteSession(sessionId); + sessionRepository.createSessionZooKeeperClient(sessionId).createNewSession(clock.instant()); + sessionRepository.createSetStatusTransaction(session, Session.Status.PREPARE).commit(); + Files.createDirectory(applicationPath); + Files.writeString(Files.createFile(applicationPath.resolve("services.xml")), "non-legal xml"); + assertEquals(0, sessionRepository.getLocalSessions().size()); // Will not show up in local sessions + + // Advance time, session SHOULD be deleted + clock.advance(Duration.ofHours(configserverConfig.keepSessionsWithUnknownStatusHours()).plus(Duration.ofMinutes(1))); + deleteExpiredLocalSessionsAndAssertNumberOfSessions(0, tester, sessionRepository); + assertFalse(applicationPath.toFile().exists()); // App has been deleted } @Test @@ -737,16 +752,6 @@ public class ApplicationRepositoryTest { return applicationRepository.getMetadataFromLocalSession(tenant, sessionId); } - private void setCreatedTime(java.nio.file.Path file, Instant createdTime) { - try { - BasicFileAttributeView attributes = Files.getFileAttributeView(file, BasicFileAttributeView.class); - FileTime time = FileTime.fromMillis(createdTime.toEpochMilli()); - attributes.setTimes(time, time, time); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - /** Stores all added or set values for each metric and context. */ static class MockMetric implements Metric { diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java index 7782e54dc90..7ea91726673 100644 --- a/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/ConfiguredApplication.java @@ -56,7 +56,6 @@ import java.util.Collections; import java.util.IdentityHashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.concurrent.Phaser; import java.util.concurrent.atomic.AtomicBoolean; @@ -85,7 +84,7 @@ public final class ConfiguredApplication implements Application { // Subscriber that is used when this is not a standalone-container. Subscribes // to config to make sure that container will be registered in slobrok (by {@link com.yahoo.jrt.slobrok.api.Register}) // if slobrok config changes (typically slobroks moving to other nodes) - private final Optional<SlobrokConfigSubscriber> slobrokConfigSubscriber; + private final SlobrokConfigSubscriber slobrokConfigSubscriber; private final ShutdownDeadline shutdownDeadline; //TODO: FilterChainRepository should instead always be set up in the model. @@ -151,8 +150,8 @@ public final class ConfiguredApplication implements Application { this.metric = metric; this.configId = System.getProperty("config.id"); this.slobrokConfigSubscriber = (subscriberFactory instanceof CloudSubscriberFactory) - ? Optional.of(new SlobrokConfigSubscriber(configId)) - : Optional.empty(); + ? new SlobrokConfigSubscriber(configId) + : null; this.restrictedOsgiFramework = new DisableOsgiFramework(new RestrictedBundleContext(osgiFramework.bundleContext())); this.shutdownDeadline = new ShutdownDeadline(configId); this.reconfigurerThread = new Thread(this::doReconfigurationLoop, "configured-application-reconfigurer"); @@ -161,8 +160,8 @@ public final class ConfiguredApplication implements Application { @Override public void start() { - qrConfig = getConfig(QrConfig.class, true); - reconfigure(qrConfig); + qrConfig = getConfig(QrConfig.class); + reconfigure(qrConfig.shutdown()); hackToInitializeServer(qrConfig); ContainerBuilder builder = createBuilderWithGuiceBindings(); @@ -173,27 +172,37 @@ public final class ConfiguredApplication implements Application { portWatcher.setDaemon(true); portWatcher.start(); - if (setupRpc()) { - slobrokRegistrator = registerInSlobrok(qrConfig); // marks this as up - } + setupRpc(qrConfig); } - private boolean setupRpc() { - if ( ! qrConfig.rpc().enabled()) return false; + private synchronized void setupRpc(QrConfig cfg) { + if (!cfg.rpc().enabled()) return; supervisor = new Supervisor(new Transport("configured-application")).setDropEmptyBuffers(true); supervisor.addMethod(new Method("prepareStop", "d", "", this::prepareStop)); - Spec listenSpec = new Spec(qrConfig.rpc().port()); + listenRpc(cfg); + } + + private synchronized void listenRpc(QrConfig cfg) { + Spec listenSpec = new Spec(cfg.rpc().port()); try { acceptor = supervisor.listen(listenSpec); - return true; + slobrokRegistrator = registerInSlobrok(cfg, acceptor.port()); } catch (ListenFailedException e) { throw new RuntimeException("Could not create rpc server listening on " + listenSpec, e); } } - private Register registerInSlobrok(QrConfig qrConfig) { + private void reListenRpc(QrConfig cfg) { + unregisterInSlobrok(); + if (supervisor == null) { + setupRpc(cfg); + } else if (cfg.rpc().enabled()) { + listenRpc(cfg); + } + } + private Register registerInSlobrok(QrConfig qrConfig, int port) { SlobrokList slobrokList = getSlobrokList(); - Spec mySpec = new Spec(HostName.getLocalhost(), acceptor.port()); + Spec mySpec = new Spec(HostName.getLocalhost(), port); Register slobrokRegistrator = new Register(supervisor, slobrokList, mySpec); slobrokRegistrator.registerName(qrConfig.rpc().slobrokId()); log.log(Level.INFO, "Registered name '" + qrConfig.rpc().slobrokId() + @@ -205,23 +214,25 @@ public final class ConfiguredApplication implements Application { // or need to get the config directly (standalone container) private SlobrokList getSlobrokList() { SlobrokList slobrokList; - if (slobrokConfigSubscriber.isPresent()) { - slobrokList = slobrokConfigSubscriber.get().getSlobroks(); + if (slobrokConfigSubscriber != null) { + slobrokList = slobrokConfigSubscriber.getSlobroks(); } else { slobrokList = new SlobrokList(); - SlobroksConfig slobrokConfig = getConfig(SlobroksConfig.class, true); + SlobroksConfig slobrokConfig = getConfig(SlobroksConfig.class); slobrokList.setup(slobrokConfig.slobrok().stream().map(SlobroksConfig.Slobrok::connectionspec).toArray(String[]::new)); } return slobrokList; } - private void unregisterInSlobrok() { - if (slobrokRegistrator != null) + private synchronized void unregisterInSlobrok() { + if (slobrokRegistrator != null) { slobrokRegistrator.shutdown(); - if (acceptor != null) + slobrokRegistrator = null; + } + if (acceptor != null) { acceptor.shutdown().join(); - if (supervisor != null) - supervisor.transport().shutdown().join(); + acceptor = null; + } } private static void hackToInitializeServer(QrConfig config) { @@ -234,11 +245,11 @@ public final class ConfiguredApplication implements Application { } } - private <T extends ConfigInstance> T getConfig(Class<T> configClass, boolean isInitializing) { + private <T extends ConfigInstance> T getConfig(Class<T> configClass) { Subscriber subscriber = subscriberFactory.getSubscriber(Collections.singleton(new ConfigKey<>(configClass, configId)), configClass.getName()); try { - subscriber.waitNextGeneration(isInitializing); + subscriber.waitNextGeneration(true); return configClass.cast(first(subscriber.config().values())); } finally { subscriber.close(); @@ -251,24 +262,32 @@ public final class ConfiguredApplication implements Application { try { while (true) { subscriber.waitNextGeneration(false); - QrConfig newConfig = QrConfig.class.cast(first(subscriber.config().values())); - reconfigure(qrConfig); - if (qrConfig.rpc().port() != newConfig.rpc().port()) { - com.yahoo.protect.Process.logAndDie( - "Rpc port config has changed from " + - qrConfig.rpc().port() + " to " + newConfig.rpc().port() + - ". This we can not handle without a restart so we will just bail out."); + if (first(subscriber.config().values()) instanceof QrConfig newConfig) { + reconfigure(newConfig.shutdown()); + synchronized (this) { + if (qrConfig.rpc().port() != newConfig.rpc().port()) { + log.log(Level.INFO, "Rpc port changed from " + qrConfig.rpc().port() + " to " + newConfig.rpc().port()); + try { + reListenRpc(newConfig); + } catch (Throwable e) { + com.yahoo.protect.Process.logAndDie("Rpc port config has changed from " + + qrConfig.rpc().port() + " to " + newConfig.rpc().port() + + ", and we were not able to reconfigure so we will just bail out and restart.", e); + } + } + qrConfig = newConfig; + } + log.fine("Received new QrConfig :" + newConfig); } - log.fine("Received new QrConfig :" + newConfig); } } finally { subscriber.close(); } } - void reconfigure(QrConfig qrConfig) { - dumpHeapOnShutdownTimeout.set(qrConfig.shutdown().dumpHeapOnTimeout()); - shutdownTimeoutS.set(qrConfig.shutdown().timeout()); + private void reconfigure(QrConfig.Shutdown shutdown) { + dumpHeapOnShutdownTimeout.set(shutdown.dumpHeapOnTimeout()); + shutdownTimeoutS.set(shutdown.timeout()); } private void initializeAndActivateContainer(ContainerBuilder builder, Runnable cleanupTask) { @@ -454,9 +473,13 @@ public final class ConfiguredApplication implements Application { if (configurer != null) { configurer.shutdown(); } - slobrokConfigSubscriber.ifPresent(SlobrokConfigSubscriber::shutdown); + if (slobrokConfigSubscriber != null) { + slobrokConfigSubscriber.shutdown(); + } Container.get().shutdown(); unregisterInSlobrok(); + if (supervisor != null) + supervisor.transport().shutdown().join(); shutdownDeadline.cancel(); log.info("Destroy: Finished"); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java index 9f57512f657..5da1f1a07be 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AbstractParser.java @@ -5,12 +5,21 @@ import com.yahoo.language.Language; import com.yahoo.language.process.Segmenter; import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.query.*; +import com.yahoo.prelude.query.AndSegmentItem; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.IndexedItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.NullItem; +import com.yahoo.prelude.query.PhraseItem; +import com.yahoo.prelude.query.PhraseSegmentItem; +import com.yahoo.prelude.query.WordItem; import com.yahoo.search.query.QueryTree; import com.yahoo.search.query.parser.Parsable; import com.yahoo.search.query.parser.ParserEnvironment; -import java.util.*; +import java.util.Iterator; +import java.util.List; +import java.util.ListIterator; /** * The Vespa query parser. @@ -20,6 +29,7 @@ import java.util.*; */ public abstract class AbstractParser implements CustomParser { + /** The current submodes of this parser */ protected Submodes submodes = new Submodes(); @@ -32,6 +42,8 @@ public abstract class AbstractParser implements CustomParser { /** The IndexFacts.Session of this query */ protected IndexFacts.Session indexFacts; + protected String defaultIndex; + /** * The counter for braces in URLs, braces in URLs are accepted so long as * they are balanced. @@ -125,28 +137,29 @@ public abstract class AbstractParser implements CustomParser { @Override public final Item parse(String queryToParse, String filterToParse, Language parsingLanguage, - IndexFacts.Session indexFacts, String defaultIndexName) { - return parse(queryToParse, filterToParse, parsingLanguage, indexFacts, defaultIndexName, null); + IndexFacts.Session indexFacts, String defaultIndex) { + return parse(queryToParse, filterToParse, parsingLanguage, indexFacts, defaultIndex, null); } private Item parse(String queryToParse, String filterToParse, Language parsingLanguage, - IndexFacts.Session indexFacts, String defaultIndexName, Parsable parsable) { + IndexFacts.Session indexFacts, String defaultIndex, Parsable parsable) { if (queryToParse == null) return null; - if (defaultIndexName != null) - defaultIndexName = indexFacts.getCanonicName(defaultIndexName); + if (defaultIndex != null) + defaultIndex = indexFacts.getCanonicName(defaultIndex); - tokenize(queryToParse, defaultIndexName, indexFacts, parsingLanguage); + tokenize(queryToParse, defaultIndex, indexFacts, parsingLanguage); if (parsingLanguage == null && parsable != null) { - String detectionText = generateLanguageDetectionTextFrom(tokens, indexFacts, defaultIndexName); + String detectionText = generateLanguageDetectionTextFrom(tokens, indexFacts, defaultIndex); if (detectionText.isEmpty()) // heuristic detection text extraction is fallible detectionText = queryToParse; parsingLanguage = parsable.getOrDetectLanguage(detectionText); } - setState(parsingLanguage, indexFacts); - Item root = parseItems(defaultIndexName); + setState(parsingLanguage, indexFacts, defaultIndex); + Item root = parseItems(); + if (filterToParse != null) { AnyParser filterParser = new AnyParser(environment); if (root == null) { @@ -155,11 +168,9 @@ public abstract class AbstractParser implements CustomParser { root = filterParser.applyFilter(root, filterToParse, parsingLanguage, indexFacts); } } - root = simplifyPhrases(root); - if (defaultIndexName != null) { - assignDefaultIndex(indexFacts.getCanonicName(defaultIndexName), root); - } - return root; + if (defaultIndex != null) + assignDefaultIndex(indexFacts.getCanonicName(defaultIndex), root); + return simplifyPhrases(root); } /** @@ -221,16 +232,11 @@ public abstract class AbstractParser implements CustomParser { return detectionText.toString(); } - private boolean is(Token.Kind kind, Token tokenOrNull) { - if (tokenOrNull == null) return false; - return kind.equals(tokenOrNull.kind); - } - - protected abstract Item parseItems(String defaultIndexName); - /** - * Assigns the default index to query terms having no default index. The - * parser _should_ have done this, for some reason it doesn't. + * Assigns the default index to query terms having no default index. + * + * This will apply the default index to terms without it added through the filter parameter, + * where setting defaultIndex into state causes incorrect parsing. * * @param defaultIndex the default index to assign * @param item the item to check @@ -243,7 +249,7 @@ public abstract class AbstractParser implements CustomParser { if ("".equals(indexName.getIndexName())) indexName.setIndexName(defaultIndex); - } + } else if (item instanceof CompositeItem) { Iterator<Item> items = ((CompositeItem)item).getItemIterator(); while (items.hasNext()) @@ -251,6 +257,13 @@ public abstract class AbstractParser implements CustomParser { } } + private boolean is(Token.Kind kind, Token tokenOrNull) { + if (tokenOrNull == null) return false; + return kind.equals(tokenOrNull.kind); + } + + protected abstract Item parseItems(); + /** * Unicode normalizes some piece of natural language text. The chosen form * is compatibility decomposition, canonical composition (NFKC). @@ -261,10 +274,11 @@ public abstract class AbstractParser implements CustomParser { return environment.getLinguistics().getNormalizer().normalize(input); } - protected void setState(Language queryLanguage, IndexFacts.Session indexFacts) { + protected void setState(Language queryLanguage, IndexFacts.Session indexFacts, String defaultIndex) { this.indexFacts = indexFacts; - language = queryLanguage; - submodes.reset(); + this.defaultIndex = defaultIndex; + this.language = queryLanguage; + this.submodes.reset(); } /** @@ -293,8 +307,7 @@ public abstract class AbstractParser implements CustomParser { return unwashed; } else if (unwashed instanceof PhraseItem) { return collapsePhrase((PhraseItem) unwashed); - } else if (unwashed instanceof CompositeItem) { - CompositeItem composite = (CompositeItem) unwashed; + } else if (unwashed instanceof CompositeItem composite) { ListIterator<Item> i = composite.getItemIterator(); while (i.hasNext()) { @@ -312,9 +325,8 @@ public abstract class AbstractParser implements CustomParser { } private static Item collapsePhrase(PhraseItem phrase) { - if (phrase.getItemCount() == 1 && phrase.getItem(0) instanceof WordItem) { + if (phrase.getItemCount() == 1 && phrase.getItem(0) instanceof WordItem word) { // TODO: Other stuff which needs propagation? - WordItem word = (WordItem) phrase.getItem(0); word.setWeight(phrase.getWeight()); return word; } else { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java index 3358075d670..8f98763a838 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AdvancedParser.java @@ -1,14 +1,25 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.query.parser; -import com.yahoo.prelude.query.*; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.EquivItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.NearItem; +import com.yahoo.prelude.query.NotItem; +import com.yahoo.prelude.query.ONearItem; +import com.yahoo.prelude.query.OrItem; +import com.yahoo.prelude.query.RankItem; +import com.yahoo.prelude.query.SegmentItem; +import com.yahoo.prelude.query.WeakAndItem; +import com.yahoo.prelude.query.WordItem; import com.yahoo.search.query.parser.ParserEnvironment; import static com.yahoo.prelude.query.parser.Token.Kind.LBRACE; import static com.yahoo.prelude.query.parser.Token.Kind.NUMBER; /** - * Parser for queries of type advanced. + * Parser for queries of type 'advanced'. * * @author Steinar Knutsen * @deprecated YQL should be used for formal queries @@ -20,7 +31,8 @@ public class AdvancedParser extends StructuredParser { super(environment); } - protected Item parseItems(String defaultIndexName) { + @Override + protected Item parseItems() { return advancedItems(true); } @@ -53,46 +65,40 @@ public class AdvancedParser extends StructuredParser { boolean expectingOperator = false; do { - item = null; - + item = indexableItem().getFirst(); if (item == null) { - item = indexableItem(); - if (item == null) { - item = compositeItem(); - itemIsComposite = true; - } else { - itemIsComposite = false; - } - if (item != null) { - Item newTop = null; + item = compositeItem(); + itemIsComposite = true; + } else { + itemIsComposite = false; + } + if (item != null) { + Item newTop = null; - if (expectingOperator) { - newTop = handleAdvancedOperator(topLevelItem, item, - topLevelIsClosed); - } - if (newTop != null) { // Operator found - topLevelIsClosed = false; - expectingOperator = false; - topLevelItem = newTop; - } else if (topLevelItem == null) { - topLevelItem = item; - if (itemIsComposite) { - topLevelIsClosed = true; - } - expectingOperator = true; - } else if (topLevelItem instanceof CompositeItem - && !(topLevelItem instanceof SegmentItem)) { - ((CompositeItem) topLevelItem).addItem(item); - expectingOperator = true; - } else { - AndItem and = new AndItem(); - - and.addItem(topLevelItem); - and.addItem(item); - topLevelItem = and; - topLevelIsClosed = false; - expectingOperator = true; + if (expectingOperator) { + newTop = handleAdvancedOperator(topLevelItem, item, topLevelIsClosed); + } + if (newTop != null) { // Operator found + topLevelIsClosed = false; + expectingOperator = false; + topLevelItem = newTop; + } else if (topLevelItem == null) { + topLevelItem = item; + if (itemIsComposite) { + topLevelIsClosed = true; } + expectingOperator = true; + } else if (topLevelItem instanceof CompositeItem && !(topLevelItem instanceof SegmentItem)) { + ((CompositeItem) topLevelItem).addItem(item); + expectingOperator = true; + } else { + AndItem and = new AndItem(); + + and.addItem(topLevelItem); + and.addItem(item); + topLevelItem = and; + topLevelIsClosed = false; + expectingOperator = true; } } @@ -178,7 +184,7 @@ public class AdvancedParser extends StructuredParser { int distance = consumeNumericArgument(); if (distance==0) distance=NearItem.defaultDistance; - if (topLevelIsClosed || !(topLevelItem instanceof NearItem) || distance!=((NearItem)topLevelItem).getDistance()) { + if (topLevelIsClosed || !(topLevelItem instanceof NearItem) || distance != ((NearItem)topLevelItem).getDistance()) { NearItem near = new NearItem(distance); near.addItem(topLevelItem); @@ -188,7 +194,7 @@ public class AdvancedParser extends StructuredParser { } else if (isTheWord("onear", item)) { int distance = consumeNumericArgument(); if (distance==0) - distance=ONearItem.defaultDistance; + distance= ONearItem.defaultDistance; if (topLevelIsClosed || !(topLevelItem instanceof ONearItem) || distance!=((ONearItem)topLevelItem).getDistance()) { ONearItem oNear = new ONearItem(distance); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java index 09caa72ca59..9a60eaef76b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java @@ -41,16 +41,16 @@ public class AllParser extends SimpleParser { } @Override - protected Item parseItems(String defaultIndexName) { + protected Item parseItems() { int position = tokens.getPosition(); try { - return parseItemsBody(defaultIndexName); + return parseItemsBody(); } finally { tokens.setPosition(position); } } - protected Item parseItemsBody(String defaultIndexName) { + protected Item parseItemsBody() { // Algorithm: Collect positive, negative, and and'ed items, then combine. CompositeItem and = null; NotItem not = null; // Store negatives here as we go @@ -65,7 +65,7 @@ public class AllParser extends SimpleParser { current = positiveItem(); if (current == null) - current = indexableItem(defaultIndexName); + current = indexableItem().getFirst(); if (current == null) current = compositeItem(); @@ -129,8 +129,9 @@ public class AllParser extends SimpleParser { try { if ( ! tokens.skip(MINUS)) return null; if (tokens.currentIsNoIgnore(SPACE)) return null; - - item = indexableItem(); + var itemAndExplicitIndex = indexableItem(); + item = itemAndExplicitIndex.getFirst(); + boolean explicitIndex = itemAndExplicitIndex.getSecond(); if (item == null) { item = compositeItem(); @@ -155,11 +156,11 @@ public class AllParser extends SimpleParser { // but interpret -(N) as a negative item matching a positive number // but interpret --N as a negative item matching a negative number if (item instanceof IntItem && - ((IntItem)item).getIndexName().isEmpty() && + ! explicitIndex && ! isComposited && - ! ((IntItem)item).getNumber().startsWith(("-"))) + ! ((IntItem)item).getNumber().startsWith(("-"))) { item = null; - + } return item; } finally { if (item == null) { @@ -204,8 +205,7 @@ public class AllParser extends SimpleParser { rank.addItem(topLevelItem); } return rank; - } else if ((item instanceof RankItem) && (((RankItem)item).getItem(0) instanceof OrItem)) { - RankItem itemAsRank = (RankItem) item; + } else if ((item instanceof RankItem itemAsRank) && (((RankItem)item).getItem(0) instanceof OrItem)) { OrItem or = (OrItem) itemAsRank.getItem(0); ((RankItem) topLevelItem).addItem(0, or); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java index f4ff769ad05..7fc4c823018 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AnyParser.java @@ -14,9 +14,7 @@ import com.yahoo.prelude.query.RankItem; import com.yahoo.prelude.query.TermItem; import com.yahoo.search.query.parser.ParserEnvironment; -import java.util.Collections; import java.util.Iterator; -import java.util.Set; import static com.yahoo.prelude.query.parser.Token.Kind.*; @@ -31,12 +29,13 @@ public class AnyParser extends SimpleParser { super(environment); } - protected Item parseItems(String defaultIndexName) { - return anyItems(true, defaultIndexName); + @Override + protected Item parseItems() { + return anyItems(true); } Item parseFilter(String filter, Language queryLanguage, IndexFacts.Session indexFacts) { - setState(queryLanguage, indexFacts); + setState(queryLanguage, indexFacts, null); tokenize(filter, null, indexFacts, queryLanguage); Item filterRoot = anyItems(true); @@ -55,7 +54,7 @@ public class AnyParser extends SimpleParser { if ( ! tokens.skipMultiple(MINUS)) return null; if (tokens.currentIsNoIgnore(SPACE)) return null; - item = indexableItem(); + item = indexableItem().getFirst(); if (item == null) { item = compositeItem(); @@ -124,7 +123,7 @@ public class AnyParser extends SimpleParser { } Item applyFilter(Item root, String filter, Language queryLanguage, IndexFacts.Session indexFacts) { - setState(queryLanguage, indexFacts); + setState(queryLanguage, indexFacts, null); tokenize(filter, null, indexFacts, queryLanguage); return filterItems(root); } @@ -148,16 +147,14 @@ public class AnyParser extends SimpleParser { private Item filterItems(Item root) { while (tokens.hasNext()) { - Item item = null; - - item = positiveItem(); + Item item = positiveItem(); root = addAndFilter(root, item); if (item == null) { item = negativeItem(); root = addNotFilter(root, item); } if (item == null) { - item = indexableItem(); + item = indexableItem().getFirst(); root = addRankFilter(root, item); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/CustomParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/CustomParser.java index e867def5903..e3b2278475b 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/CustomParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/CustomParser.java @@ -7,7 +7,6 @@ import com.yahoo.prelude.query.Item; import com.yahoo.search.query.parser.Parser; import java.util.Collections; -import java.util.Objects; import java.util.Set; /** diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/ParseException.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/ParseException.java index bef2ca9ffe9..82515c51c05 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/ParseException.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/ParseException.java @@ -1,13 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.query.parser; - /** * Parser exceptions. JavaCC legacy, never thrown. * - * @author bratseth + * @author bratseth */ -@SuppressWarnings("serial") public class ParseException extends RuntimeException { public ParseException(String message) { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java index 72eb56dd0fb..01b5b943829 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/PhraseParser.java @@ -16,7 +16,8 @@ public class PhraseParser extends AbstractParser { super(environment); } - protected Item parseItems(String defaultIndex) { + @Override + protected Item parseItems() { return forcedPhrase(); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/ProgrammaticParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/ProgrammaticParser.java index 6a005bc0ec9..209753a596c 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/ProgrammaticParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/ProgrammaticParser.java @@ -9,8 +9,6 @@ import com.yahoo.search.query.QueryTree; import com.yahoo.search.query.parser.Parsable; import com.yahoo.search.query.textserialize.TextSerialize; -import java.util.Set; - /** * @author Simon Thoresen Hult */ @@ -32,4 +30,5 @@ public final class ProgrammaticParser implements CustomParser { if (queryToParse == null) return null; return TextSerialize.parse(queryToParse); } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java index fafbf55a522..b7355c43f81 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/SimpleParser.java @@ -1,7 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.query.parser; -import com.yahoo.prelude.query.*; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.BlockItem; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.NotItem; +import com.yahoo.prelude.query.OrItem; +import com.yahoo.prelude.query.PhraseItem; +import com.yahoo.prelude.query.RankItem; +import com.yahoo.prelude.query.TermItem; +import com.yahoo.prelude.query.TrueItem; import com.yahoo.search.query.parser.ParserEnvironment; import java.util.Iterator; @@ -33,12 +42,12 @@ abstract class SimpleParser extends StructuredParser { * If there's a explicit composite and some other terms, * a rank terms combines them */ - protected Item anyItems(boolean topLevel, String defaultIndexName) { + protected Item anyItems(boolean topLevel) { int position = tokens.getPosition(); Item item = null; try { - item = anyItemsBody(topLevel, defaultIndexName); + item = anyItemsBody(topLevel); return item; } finally { if (item == null) { @@ -47,14 +56,10 @@ abstract class SimpleParser extends StructuredParser { } } - protected Item anyItems(boolean topLevel) { - return anyItems(topLevel, null); - } - - private Item anyItemsBody(boolean topLevel, String defaultIndexName) { + private Item anyItemsBody(boolean topLevel) { Item topLevelItem = null; NotItem not = null; - Item item = null; + Item item; do { item = positiveItem(); if (item != null) { @@ -92,7 +97,7 @@ abstract class SimpleParser extends StructuredParser { } if (item == null) { - item = indexableItem(defaultIndexName); + item = indexableItem().getFirst(); if (item != null) { if (topLevelItem == null) { topLevelItem = item; @@ -177,9 +182,7 @@ abstract class SimpleParser extends StructuredParser { return null; } - if (item == null) { - item = indexableItem(); - } + item = indexableItem().getFirst(); if (item == null) { item = compositeItem(); @@ -200,12 +203,10 @@ abstract class SimpleParser extends StructuredParser { * (+ items) are not found, but negatives are. */ private Item getItemAsPositiveItem(Item item, NotItem not) { - if (!(item instanceof RankItem)) { + if (!(item instanceof RankItem rank)) { return item; } - RankItem rank = (RankItem) item; - // Remove the not from the rank item, the rank should generally // be the first, but this is not always the case int limit = rank.getItemCount(); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java index c668cf66447..9d9aee54df0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java @@ -1,13 +1,30 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.query.parser; +import com.yahoo.collections.Pair; import com.yahoo.prelude.IndexFacts; -import com.yahoo.prelude.query.*; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.AndSegmentItem; +import com.yahoo.prelude.query.BlockItem; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.IntItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.MarkerWordItem; +import com.yahoo.prelude.query.PhraseItem; +import com.yahoo.prelude.query.PhraseSegmentItem; +import com.yahoo.prelude.query.PrefixItem; +import com.yahoo.prelude.query.SegmentItem; +import com.yahoo.prelude.query.Substring; +import com.yahoo.prelude.query.SubstringItem; +import com.yahoo.prelude.query.SuffixItem; +import com.yahoo.prelude.query.TaggableItem; +import com.yahoo.prelude.query.TermItem; +import com.yahoo.prelude.query.UriItem; +import com.yahoo.prelude.query.WordItem; import com.yahoo.search.query.parser.ParserEnvironment; import java.util.ArrayList; import java.util.List; -import java.util.Objects; import static com.yahoo.prelude.query.parser.Token.Kind.*; @@ -52,19 +69,22 @@ abstract class StructuredParser extends AbstractParser { submodes.setFromIndex(indexName, indexFacts); } - protected Item indexableItem() { - return indexableItem(null); - } - - protected Item indexableItem(String defaultIndexName) { + /** + * Returns an item and whether it had an explicit index ('indexname:' prefix). + * + * @return an item and whether it has an explicit index, or a Pair with the first element null if none + */ + protected Pair<Item, Boolean> indexableItem() { int position = tokens.getPosition(); Item item = null; try { + boolean explicitIndex = false; String indexName = indexPrefix(); - if (Objects.isNull(indexName)) { - indexName = defaultIndexName; - } + if (indexName != null) + explicitIndex = true; + else + indexName = this.defaultIndex; setSubmodeFromIndex(indexName, indexFacts); item = number(); @@ -86,7 +106,6 @@ abstract class StructuredParser extends AbstractParser { if (item != null) { weight = weightSuffix(); } - if (indexName != null && item != null) { item.setIndexName(indexName); } @@ -95,7 +114,7 @@ abstract class StructuredParser extends AbstractParser { item.setWeight(weight); } - return item; + return new Pair<>(item, explicitIndex); } finally { if (item == null) { tokens.setPosition(position); @@ -109,8 +128,7 @@ abstract class StructuredParser extends AbstractParser { if (tokens.currentIsNoIgnore(SPACE)) { return false; } - if (tokens.currentIsNoIgnore(NUMBER) - || tokens.currentIsNoIgnore(WORD)) { + if (tokens.currentIsNoIgnore(NUMBER) || tokens.currentIsNoIgnore(WORD)) { return true; } tokens.skipNoIgnore(); @@ -201,7 +219,6 @@ abstract class StructuredParser extends AbstractParser { item = indexPrefix(); } } - return item; } finally { if (item == null) { @@ -286,7 +303,6 @@ abstract class StructuredParser extends AbstractParser { tokens.skip(LSQUAREBRACKET); if (item == null) tokens.skipNoIgnore(SPACE); - // TODO: Better definition of start and end of numeric items if (item == null && tokens.currentIsNoIgnore(MINUS) && (tokens.currentNoIgnore(1).kind == NUMBER)) { tokens.skipNoIgnore(); @@ -592,7 +608,7 @@ abstract class StructuredParser extends AbstractParser { if (firstWord instanceof IntItem) { IntItem asInt = (IntItem) firstWord; firstWord = new WordItem(asInt.stringValue(), asInt.getIndexName(), - true, asInt.getOrigin()); + true, asInt.getOrigin()); } composite.addItem(firstWord); composite.addItem(word); diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/Token.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/Token.java index b668df9208c..3bf4d9dcf01 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/Token.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/Token.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.prelude.query.parser; - import com.yahoo.prelude.query.Substring; /** @@ -11,7 +10,7 @@ import com.yahoo.prelude.query.Substring; */ public class Token { - public static enum Kind { + public enum Kind { EOF("<EOF>"), NUMBER("<NUMBER>"), WORD("<WORD>"), @@ -77,31 +76,6 @@ public class Token { /** Returns whether this is a <i>special token</i> */ public boolean isSpecial() { return special; } - public String toString() { return image; } - - public boolean equals(Object object) { - if (this == object) { - return true; - } - if (object == null) { - return false; - } - if (object.getClass() != this.getClass()) { - return false; - } - - Token other = (Token) object; - - if (this.kind != other.kind) { - return false; - } - if (!(this.image.equals(other.image))) { - return false; - } - - return true; - } - /** * Returns the substring containing the image ins original form (including casing), * as well as all the text surrounding the token @@ -110,6 +84,22 @@ public class Token { */ public Substring getSubstring() { return substring; } + @Override + public String toString() { return image; } + + @Override + public boolean equals(Object object) { + if (this == object) return true; + if (object == null) return false; + if (object.getClass() != this.getClass()) return false; + + Token other = (Token) object; + if (this.kind != other.kind) return false; + if (!(this.image.equals(other.image))) return false; + return true; + } + + @Override public int hashCode() { return image.hashCode() ^ kind.hashCode(); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenPosition.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenPosition.java index 9c60abab637..5ead962e430 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenPosition.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenPosition.java @@ -37,9 +37,7 @@ final class TokenPosition { * Returns null (no exception) if there are no more tokens. */ public Token current() { - Token token = current(0); - - return token; + return current(0); } /** diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenizeParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenizeParser.java index eefbe5fa0d0..dbbc321d057 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenizeParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/TokenizeParser.java @@ -22,7 +22,7 @@ public final class TokenizeParser extends AbstractParser { } @Override - protected Item parseItems(String defaultIndex) { + protected Item parseItems() { WeakAndItem weakAnd = new WeakAndItem(); Token token; while (null != (token = tokens.next())) { diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java index 93b8cf1ed83..c1d415b8e27 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/Tokenizer.java @@ -84,7 +84,6 @@ public final class Tokenizer { * @param indexFacts information about the indexes we will search * @return a read-only list of tokens. This list can only be used by this thread */ - @SuppressWarnings({"deprecation"}) // To avoid this we need to pass an IndexFacts.session down instead - easily done but not without breaking API's public List<Token> tokenize(String string, String defaultIndexName, IndexFacts.Session indexFacts) { this.source = string; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/UnicodePropertyDump.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/UnicodePropertyDump.java index b01b1295f45..8d2adfe0d78 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/UnicodePropertyDump.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/UnicodePropertyDump.java @@ -26,13 +26,13 @@ class UnicodePropertyDump { boolean debug = false; if (arg.length > 0) { - start = Integer.valueOf(arg[0]).intValue(); + start = Integer.parseInt(arg[0]); } if (arg.length > 1) { - end = Integer.valueOf(arg[1]).intValue(); + end = Integer.parseInt(arg[1]); } if (arg.length > 2) { - debug = Boolean.valueOf(arg[2]).booleanValue(); + debug = Boolean.parseBoolean(arg[2]); } dumpProperties(start, end, debug, System.out); } @@ -109,4 +109,5 @@ class UnicodePropertyDump { out.println(); } } + } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/WebParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/WebParser.java index 40497d94a6d..aff28179050 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/parser/WebParser.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/WebParser.java @@ -28,7 +28,7 @@ public class WebParser extends AllParser { } @Override - protected Item parseItemsBody(String defaultIndexName) { + protected Item parseItemsBody() { // Algorithm: Collect positive, negative, and'ed and or'ed elements, then combine. CompositeItem and = null; OrItem or = null; @@ -45,7 +45,7 @@ public class WebParser extends AllParser { current = positiveItem(); if (current == null) - current = indexableItem(defaultIndexName); + current = indexableItem().getFirst(); if (current != null) { if (and != null && (current instanceof WordItem) && "OR".equals(((WordItem)current).getRawWord())) { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/CloseableInvoker.java b/container-search/src/main/java/com/yahoo/search/dispatch/CloseableInvoker.java index e66c48ddb74..9da89c6bfd8 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/CloseableInvoker.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/CloseableInvoker.java @@ -2,6 +2,7 @@ package com.yahoo.search.dispatch; import java.io.Closeable; +import java.time.Duration; import java.util.function.BiConsumer; /** @@ -15,13 +16,13 @@ public abstract class CloseableInvoker implements Closeable { protected abstract void release(); - private BiConsumer<Boolean, Long> teardown = null; + private BiConsumer<Boolean, Duration> teardown = null; private boolean success = false; private long startTime = 0; - public void teardown(BiConsumer<Boolean, Long> teardown) { + public void teardown(BiConsumer<Boolean, Duration> teardown) { this.teardown = teardown; - this.startTime = System.currentTimeMillis(); + this.startTime = System.nanoTime(); } protected void setFinalStatus(boolean success) { @@ -31,7 +32,7 @@ public abstract class CloseableInvoker implements Closeable { @Override public final void close() { if (teardown != null) { - teardown.accept(success, System.currentTimeMillis() - startTime); + teardown.accept(success, Duration.ofNanos(System.nanoTime() - startTime)); teardown = null; } release(); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index 68a8e351b34..9ae97dabccd 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -25,11 +25,11 @@ import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.result.ErrorMessage; import com.yahoo.vespa.config.search.DispatchConfig; +import java.time.Duration; import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; /** * A dispatcher communicates with search nodes to perform queries and fill hits. @@ -96,6 +96,15 @@ public class Dispatcher extends AbstractComponent { this(new ClusterMonitor<>(searchCluster, true), searchCluster, dispatchConfig, new RpcInvokerFactory(resourcePool, searchCluster), metric); } + private static LoadBalancer.Policy toLoadBalancerPolicy(DispatchConfig.DistributionPolicy.Enum policy) { + return switch (policy) { + case ROUNDROBIN: yield LoadBalancer.Policy.ROUNDROBIN; + case BEST_OF_RANDOM_2: yield LoadBalancer.Policy.BEST_OF_RANDOM_2; + case ADAPTIVE,LATENCY_AMORTIZED_OVER_REQUESTS: yield LoadBalancer.Policy.LATENCY_AMORTIZED_OVER_REQUESTS; + case LATENCY_AMORTIZED_OVER_TIME: yield LoadBalancer.Policy.LATENCY_AMORTIZED_OVER_TIME; + }; + } + /* Protected for simple mocking in tests. Beware that searchCluster is shutdown on in deconstruct() */ protected Dispatcher(ClusterMonitor<Node> clusterMonitor, SearchCluster searchCluster, @@ -107,8 +116,7 @@ public class Dispatcher extends AbstractComponent { this.searchCluster = searchCluster; this.clusterMonitor = clusterMonitor; - this.loadBalancer = new LoadBalancer(searchCluster, - dispatchConfig.distributionPolicy() == DispatchConfig.DistributionPolicy.ROUNDROBIN); + this.loadBalancer = new LoadBalancer(searchCluster, toLoadBalancerPolicy(dispatchConfig.distributionPolicy())); this.invokerFactory = invokerFactory; this.metric = metric; this.metricContext = metric.createContext(null); @@ -219,7 +227,7 @@ public class Dispatcher extends AbstractComponent { invoker.get().teardown((success, time) -> loadBalancer.releaseGroup(group, success, time)); return invoker.get(); } else { - loadBalancer.releaseGroup(group, false, 0); + loadBalancer.releaseGroup(group, false, Duration.ZERO); if (rejected == null) { rejected = new HashSet<>(); } @@ -239,7 +247,7 @@ public class Dispatcher extends AbstractComponent { */ private Set<Integer> rejectGroupBlockingFeed(List<Group> groups) { if (groups.size() == 1) return null; - List<Group> groupsRejectingFeed = groups.stream().filter(Group::isBlockingWrites).collect(Collectors.toList()); + List<Group> groupsRejectingFeed = groups.stream().filter(Group::isBlockingWrites).toList(); if (groupsRejectingFeed.size() != 1) return null; Set<Integer> rejected = new HashSet<>(); rejected.add(groupsRejectingFeed.get(0).id()); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java index 4c0bcb38d15..6d8bb1dce3d 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/LoadBalancer.java @@ -4,6 +4,7 @@ package com.yahoo.search.dispatch; import com.yahoo.search.dispatch.searchcluster.Group; import com.yahoo.search.dispatch.searchcluster.SearchCluster; +import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -31,16 +32,22 @@ public class LoadBalancer { private final List<GroupStatus> scoreboard; private final GroupScheduler scheduler; - public LoadBalancer(SearchCluster searchCluster, boolean roundRobin) { + public enum Policy { ROUNDROBIN, LATENCY_AMORTIZED_OVER_REQUESTS, LATENCY_AMORTIZED_OVER_TIME, BEST_OF_RANDOM_2} + + public LoadBalancer(SearchCluster searchCluster, Policy policy) { this.scoreboard = new ArrayList<>(searchCluster.groups().size()); for (Group group : searchCluster.orderedGroups()) { scoreboard.add(new GroupStatus(group)); } - if (roundRobin || scoreboard.size() == 1) { - this.scheduler = new RoundRobinScheduler(scoreboard); - } else { - this.scheduler = new AdaptiveScheduler(new Random(), scoreboard); - } + if (scoreboard.size() == 1) + policy = Policy.ROUNDROBIN; + + this.scheduler = switch (policy) { + case ROUNDROBIN: yield new RoundRobinScheduler(scoreboard); + case BEST_OF_RANDOM_2: yield new BestOfRandom2(new Random(), scoreboard); + case LATENCY_AMORTIZED_OVER_REQUESTS: yield new AdaptiveScheduler(new Random(), scoreboard); + case LATENCY_AMORTIZED_OVER_TIME: yield new AdaptiveScheduler(new Random(), scoreboard); // TODO Intentionally the same for now + }; } /** @@ -71,13 +78,13 @@ public class LoadBalancer { * * @param group previously allocated group * @param success was the query successful - * @param searchTimeMs query execution time in milliseconds, used for adaptive load balancing + * @param searchTime query execution time, used for adaptive load balancing */ - public void releaseGroup(Group group, boolean success, double searchTimeMs) { + public void releaseGroup(Group group, boolean success, Duration searchTime) { synchronized (this) { for (GroupStatus sched : scoreboard) { if (sched.group.id() == group.id()) { - sched.release(success, searchTimeMs / 1000.0); + sched.release(success, searchTime); break; } } @@ -99,22 +106,23 @@ public class LoadBalancer { allocations++; } - void release(boolean success, double searchTime) { + void release(boolean success, Duration searchTime) { + double searchSeconds = searchTime.toMillis()/1000.0; allocations--; if (allocations < 0) { log.warning("Double free of query target group detected"); allocations = 0; } if (success) { - searchTime = Math.max(searchTime, MIN_QUERY_TIME); + searchSeconds = Math.max(searchSeconds, MIN_QUERY_TIME); double decayRate = Math.min(queries + MIN_LATENCY_DECAY_RATE, DEFAULT_LATENCY_DECAY_RATE); - averageSearchTime = (searchTime + (decayRate - 1) * averageSearchTime) / decayRate; + averageSearchTime = (searchSeconds + (decayRate - 1) * averageSearchTime) / decayRate; queries++; } } - double averageSearchTime() { - return averageSearchTime; + Duration averageSearchTime() { + return Duration.ofNanos((long)(averageSearchTime*1000000000)); } double averageSearchTimeInverse() { @@ -125,9 +133,9 @@ public class LoadBalancer { return group.id(); } - void setQueryStatistics(long queries, double averageSearchTime) { + void setQueryStatistics(long queries, Duration averageSearchTime) { this.queries = queries; - this.averageSearchTime = averageSearchTime; + this.averageSearchTime = averageSearchTime.toMillis()/1000.0; } } @@ -239,4 +247,47 @@ public class LoadBalancer { } } + static class BestOfRandom2 implements GroupScheduler { + private final Random random; + private final List<GroupStatus> scoreboard; + public BestOfRandom2(Random random, List<GroupStatus> scoreboard) { + this.random = random; + this.scoreboard = scoreboard; + } + @Override + public Optional<GroupStatus> takeNextGroup(Set<Integer> rejectedGroups) { + GroupStatus gs = selectBestOf2(rejectedGroups, true); + return (gs != null) + ? Optional.of(gs) + : Optional.ofNullable(selectBestOf2(rejectedGroups, false)); + } + + private GroupStatus selectBestOf2(Set<Integer> rejectedGroups, boolean requireCoverage) { + List<Integer> candidates = new ArrayList<>(scoreboard.size()); + for (int i=0; i < scoreboard.size(); i++) { + GroupStatus gs = scoreboard.get(i); + if (rejectedGroups == null || !rejectedGroups.contains(gs.group.id())) { + if (!requireCoverage || gs.group.hasSufficientCoverage()) { + candidates.add(i); + } + } + } + GroupStatus candA = selectRandom(candidates); + GroupStatus candB = selectRandom(candidates); + if (candA == null) return candB; + if (candB == null) return candA; + if (candB.allocations < candA.allocations) return candB; + return candA; + } + private GroupStatus selectRandom(List<Integer> candidates) { + if ( ! candidates.isEmpty()) { + int index = random.nextInt(candidates.size()); + Integer groupIndex = candidates.remove(index); + return scoreboard.get(groupIndex); + } + return null; + } + + } + } diff --git a/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java b/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java index 95ecf3c2dba..06b6eca5f84 100644 --- a/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/test/QueryTestCase.java @@ -195,7 +195,7 @@ public class QueryTestCase { assertTrue(p.hashCode() != q.hashCode()); } - /** Test using the defauultindex feature */ + /** Test using the defaultindex feature */ @Test void testDefaultIndex() { Query q = newQuery("?query=hi hello keyword:kanoo " + diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java index e9ed1c48302..c9981b3598b 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java @@ -2,6 +2,7 @@ package com.yahoo.search.dispatch; import com.yahoo.search.dispatch.LoadBalancer.AdaptiveScheduler; +import com.yahoo.search.dispatch.LoadBalancer.BestOfRandom2; import com.yahoo.search.dispatch.LoadBalancer.GroupStatus; import com.yahoo.search.dispatch.searchcluster.Group; import com.yahoo.search.dispatch.searchcluster.Node; @@ -9,6 +10,7 @@ import com.yahoo.search.dispatch.searchcluster.SearchCluster; import org.junit.jupiter.api.Test; import org.opentest4j.AssertionFailedError; +import java.time.Duration; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -29,7 +31,7 @@ public class LoadBalancerTest { void requireThatLoadBalancerServesSingleNodeSetups() { Node n1 = new Node(0, "test-node1", 0); SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1), null, null); - LoadBalancer lb = new LoadBalancer(cluster, true); + LoadBalancer lb = new LoadBalancer(cluster, LoadBalancer.Policy.ROUNDROBIN); Optional<Group> grp = lb.takeGroup(null); Group group = grp.orElseGet(() -> { @@ -43,7 +45,7 @@ public class LoadBalancerTest { Node n1 = new Node(0, "test-node1", 0); Node n2 = new Node(1, "test-node2", 1); SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), null, null); - LoadBalancer lb = new LoadBalancer(cluster, true); + LoadBalancer lb = new LoadBalancer(cluster, LoadBalancer.Policy.ROUNDROBIN); Optional<Group> grp = lb.takeGroup(null); Group group = grp.orElseGet(() -> { @@ -59,7 +61,7 @@ public class LoadBalancerTest { Node n3 = new Node(0, "test-node3", 1); Node n4 = new Node(1, "test-node4", 1); SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2, n3, n4), null, null); - LoadBalancer lb = new LoadBalancer(cluster, true); + LoadBalancer lb = new LoadBalancer(cluster, LoadBalancer.Policy.ROUNDROBIN); Optional<Group> grp = lb.takeGroup(null); assertTrue(grp.isPresent()); @@ -70,14 +72,14 @@ public class LoadBalancerTest { Node n1 = new Node(0, "test-node1", 0); Node n2 = new Node(1, "test-node2", 1); SearchCluster cluster = new SearchCluster("a", createDispatchConfig(n1, n2), null, null); - LoadBalancer lb = new LoadBalancer(cluster, true); + LoadBalancer lb = new LoadBalancer(cluster, LoadBalancer.Policy.ROUNDROBIN); // get first group Optional<Group> grp = lb.takeGroup(null); Group group = grp.get(); int id1 = group.id(); // release allocation - lb.releaseGroup(group, true, 1.0); + lb.releaseGroup(group, true, Duration.ofMillis(1)); // get second group grp = lb.takeGroup(null); @@ -90,28 +92,28 @@ public class LoadBalancerTest { final double delta = 0.00001; GroupStatus gs = newGroupStatus(1); - gs.setQueryStatistics(0, 1.0); - updateSearchTime(gs, 1.0); - assertEquals(1.0, gs.averageSearchTime(), delta); - updateSearchTime(gs, 2.0); - assertEquals(1.02326, gs.averageSearchTime(), delta); - updateSearchTime(gs, 2.0); - assertEquals(1.04545, gs.averageSearchTime(), delta); - updateSearchTime(gs, 0.1); - updateSearchTime(gs, 0.1); - updateSearchTime(gs, 0.1); - updateSearchTime(gs, 0.1); - assertEquals(0.966667, gs.averageSearchTime(), delta); + gs.setQueryStatistics(0, Duration.ofSeconds(1)); + updateSearchTime(gs, Duration.ofSeconds(1)); + assertEquals(Duration.ofSeconds(1), gs.averageSearchTime()); + updateSearchTime(gs, Duration.ofSeconds(2)); + assertEquals(Duration.ofNanos(1023255813), gs.averageSearchTime()); + updateSearchTime(gs, Duration.ofSeconds(2)); + assertEquals(Duration.ofNanos(1045454545), gs.averageSearchTime()); + updateSearchTime(gs, Duration.ofMillis(100)); + updateSearchTime(gs, Duration.ofMillis(100)); + updateSearchTime(gs, Duration.ofMillis(100)); + updateSearchTime(gs, Duration.ofMillis(100)); + assertEquals(Duration.ofNanos(966666666), gs.averageSearchTime()); for (int i = 0; i < 10000; i++) { - updateSearchTime(gs, 1.0); + updateSearchTime(gs, Duration.ofSeconds(1)); } - assertEquals(1.0, gs.averageSearchTime(), delta); - updateSearchTime(gs, 0.1); - assertEquals(0.9991, gs.averageSearchTime(), delta); + assertEquals(Duration.ofNanos(999999812), gs.averageSearchTime()); + updateSearchTime(gs, Duration.ofMillis(100)); + assertEquals(Duration.ofNanos(999099812), gs.averageSearchTime()); for (int i = 0; i < 10000; i++) { - updateSearchTime(gs, 0.0); + updateSearchTime(gs, Duration.ZERO); } - assertEquals(0.001045, gs.averageSearchTime(), delta); + assertEquals(Duration.ofNanos(1045087), gs.averageSearchTime()); } @Test @@ -138,7 +140,7 @@ public class LoadBalancerTest { List<GroupStatus> scoreboard = new ArrayList<>(); for (int i = 0; i < 5; i++) { GroupStatus gs = newGroupStatus(i); - gs.setQueryStatistics(1, 0.1 * (i + 1)); + gs.setQueryStatistics(1, Duration.ofMillis((long)(0.1 * (i + 1)*1000.0))); scoreboard.add(gs); } Random seq = sequence(0.0, 0.4379, 0.4380, 0.6569, 0.6570, 0.8029, 0.8030, 0.9124, 0.9125); @@ -155,7 +157,40 @@ public class LoadBalancerTest { assertEquals(4, sched.takeNextGroup(null).get().groupId()); } - private static void updateSearchTime(GroupStatus gs, double time) { + private static GroupStatus allocate(GroupStatus gs) { + gs.allocate(); + return gs; + } + @Test + void requireBestOfRandom2Scheduler() { + List<GroupStatus> scoreboard = new ArrayList<>(); + for (int i = 0; i < 5; i++) { + scoreboard.add(newGroupStatus(i)); + } + Random seq = sequence( + 0.1, 0.125, + 0.1, 0.125, + 0.1, 0.125, + 0.1, 0.125, + 0.1, 0.375, + 0.9, 0.125, + 0.9, 0.125, + 0.9, 0.125 + ); + BestOfRandom2 sched = new BestOfRandom2(seq, scoreboard); + + assertEquals(0, allocate(sched.takeNextGroup(null).get()).groupId()); + assertEquals(1, allocate(sched.takeNextGroup(null).get()).groupId()); + assertEquals(0, allocate(sched.takeNextGroup(null).get()).groupId()); + assertEquals(1, allocate(sched.takeNextGroup(null).get()).groupId()); + assertEquals(2, allocate(sched.takeNextGroup(null).get()).groupId()); + assertEquals(4, allocate(sched.takeNextGroup(null).get()).groupId()); + assertEquals(4, allocate(sched.takeNextGroup(null).get()).groupId()); + assertEquals(4, allocate(sched.takeNextGroup(null).get()).groupId()); + assertEquals(0, allocate(sched.takeNextGroup(null).get()).groupId()); + } + + private static void updateSearchTime(GroupStatus gs, Duration time) { gs.allocate(); gs.release(true, time); } @@ -183,6 +218,10 @@ public class LoadBalancerTest { } return retv; } + @Override + public int nextInt(int bound) { + return (int)(nextDouble() * bound); + } }; } diff --git a/container-search/src/test/java/com/yahoo/search/test/QueryWithFilterTestCase.java b/container-search/src/test/java/com/yahoo/search/test/QueryWithFilterTestCase.java new file mode 100644 index 00000000000..3920a95bd98 --- /dev/null +++ b/container-search/src/test/java/com/yahoo/search/test/QueryWithFilterTestCase.java @@ -0,0 +1,80 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.test; + +import com.yahoo.language.Language; +import com.yahoo.language.Linguistics; +import com.yahoo.language.simple.SimpleLinguistics; +import com.yahoo.prelude.Index; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.IndexModel; +import com.yahoo.prelude.SearchDefinition; +import com.yahoo.search.Query; +import com.yahoo.search.searchchain.Execution; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author bratseth + */ +public class QueryWithFilterTestCase { + + /** Tests that default-index is not applied to ALL filters */ + @Test + void testRankFilter() { + Query q = newQueryFromEncoded("?query=trump" + + "&model.type=all" + + "&model.defaultIndex=text" + + "&filter=filterattribute%3Afrontpage_US_en-US"); + assertEquals("RANK text:trump |filterattribute:frontpage_US_en-US", + q.getModel().getQueryTree().toString()); + } + + /** Tests that default-index is not applied to NOT filters */ + @Test + void testAndFilter() { + Query q = newQueryFromEncoded("?query=trump" + + "&model.type=all" + + "&model.defaultIndex=text" + + "&filter=%2B%28filterattribute%3Afrontpage_US_en-US%29"); + assertEquals("AND text:trump |filterattribute:frontpage_US_en-US", + q.getModel().getQueryTree().toString()); + } + + /** Tests that default-index is not applied to NOT filters */ + @Test + void testAndFilterWithoutExplicitIndex() { + Query q = newQueryFromEncoded("?query=trump" + + "&model.type=all" + + "&model.defaultIndex=text" + + "&filter=%2B%28filterTerm%29"); + assertEquals("AND text:trump |text:filterTerm", + q.getModel().getQueryTree().toString()); + } + + private Query newQueryFromEncoded(String queryString) { + return newQueryFromEncoded(queryString, null, new SimpleLinguistics()); + } + + private Query newQueryFromEncoded(String encodedQueryString, Language language, Linguistics linguistics) { + Query query = new Query(encodedQueryString); + query.getModel().setExecution(new Execution(Execution.Context.createContextStub(createIndexFacts(), + linguistics))); + query.getModel().setLanguage(language); + return query; + } + + private IndexFacts createIndexFacts() { + SearchDefinition sd = new SearchDefinition("test"); + sd.addIndex(new Index("test")); + sd.addIndex(attribute("filterattribute")); + return new IndexFacts(new IndexModel(sd)); + } + + private Index attribute(String name) { + Index index = new Index(name); + index.setExact(true, null); + return index; + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java index 8b4c00e9b9d..e0524091c39 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.zone; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeType; @@ -34,6 +35,9 @@ public interface ZoneRegistry { /** Returns whether the system of this registry contains the given zone */ boolean hasZone(ZoneId zoneId); + /** Returns whether cloudAccount in this system supports given zone */ + boolean hasZone(ZoneId zoneId, CloudAccount cloudAccount); + /** Returns a list containing the id of all zones in this registry */ ZoneFilter zones(); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/SimplePrincipal.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/SimplePrincipal.java index 780171d0ccb..363d0726a1f 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/SimplePrincipal.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/SimplePrincipal.java @@ -18,6 +18,10 @@ public class SimplePrincipal implements Principal { this.name = name; } + public static SimplePrincipal of(Principal principal) { + return new SimplePrincipal(principal.getName()); + } + @Override public String getName() { return name; diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java index 44f9c0ea3b8..ae0467fcc86 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java @@ -5,6 +5,7 @@ import com.google.common.collect.BiMap; import com.google.common.collect.ImmutableBiMap; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; +import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; import java.security.Principal; import java.security.PublicKey; @@ -20,16 +21,16 @@ import java.util.Optional; */ public class CloudTenant extends Tenant { - private final Optional<Principal> creator; - private final BiMap<PublicKey, Principal> developerKeys; + private final Optional<SimplePrincipal> creator; + private final BiMap<PublicKey, SimplePrincipal> developerKeys; private final TenantInfo info; private final List<TenantSecretStore> tenantSecretStores; private final ArchiveAccess archiveAccess; private final Optional<Instant> invalidateUserSessionsBefore; /** Public for the serialization layer — do not use! */ - public CloudTenant(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Optional<Principal> creator, - BiMap<PublicKey, Principal> developerKeys, TenantInfo info, + public CloudTenant(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Optional<SimplePrincipal> creator, + BiMap<PublicKey, SimplePrincipal> developerKeys, TenantInfo info, List<TenantSecretStore> tenantSecretStores, ArchiveAccess archiveAccess, Optional<Instant> invalidateUserSessionsBefore) { super(name, createdAt, lastLoginInfo, Optional.empty()); this.creator = creator; @@ -45,12 +46,12 @@ public class CloudTenant extends Tenant { return new CloudTenant(requireName(tenantName), createdAt, LastLoginInfo.EMPTY, - Optional.ofNullable(creator), + Optional.ofNullable(creator).map(SimplePrincipal::of), ImmutableBiMap.of(), TenantInfo.empty(), List.of(), new ArchiveAccess(), Optional.empty()); } /** The user that created the tenant */ - public Optional<Principal> creator() { + public Optional<SimplePrincipal> creator() { return creator; } @@ -60,7 +61,7 @@ public class CloudTenant extends Tenant { } /** Returns the set of developer keys and their corresponding developers for this tenant. */ - public BiMap<PublicKey, Principal> developerKeys() { return developerKeys; } + public BiMap<PublicKey, SimplePrincipal> developerKeys() { return developerKeys; } /** List of configured secret stores */ public List<TenantSecretStore> tenantSecretStores() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java index ac7c6319c1b..da40f63d543 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; +import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; import com.yahoo.vespa.hosted.controller.tenant.ArchiveAccess; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; @@ -124,15 +125,15 @@ public abstract class LockedTenant { /** A locked CloudTenant. */ public static class Cloud extends LockedTenant { - private final Optional<Principal> creator; - private final BiMap<PublicKey, Principal> developerKeys; + private final Optional<SimplePrincipal> creator; + private final BiMap<PublicKey, SimplePrincipal> developerKeys; private final TenantInfo info; private final List<TenantSecretStore> tenantSecretStores; private final ArchiveAccess archiveAccess; private final Optional<Instant> invalidateUserSessionsBefore; - private Cloud(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Optional<Principal> creator, - BiMap<PublicKey, Principal> developerKeys, TenantInfo info, + private Cloud(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Optional<SimplePrincipal> creator, + BiMap<PublicKey, SimplePrincipal> developerKeys, TenantInfo info, List<TenantSecretStore> tenantSecretStores, ArchiveAccess archiveAccess, Optional<Instant> invalidateUserSessionsBefore) { super(name, createdAt, lastLoginInfo); this.developerKeys = ImmutableBiMap.copyOf(developerKeys); @@ -153,15 +154,18 @@ public abstract class LockedTenant { } public Cloud withDeveloperKey(PublicKey key, Principal principal) { - BiMap<PublicKey, Principal> keys = HashBiMap.create(developerKeys); + BiMap<PublicKey, SimplePrincipal> keys = HashBiMap.create(developerKeys); + SimplePrincipal simplePrincipal = new SimplePrincipal(principal.getName()); if (keys.containsKey(key)) throw new IllegalArgumentException("Key " + KeyUtils.toPem(key) + " is already owned by " + keys.get(key)); - keys.put(key, principal); + if (keys.inverse().containsKey(simplePrincipal)) + throw new IllegalArgumentException(principal + " is already associated with key " + KeyUtils.toPem(keys.inverse().get(simplePrincipal))); + keys.put(key, simplePrincipal); return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore); } public Cloud withoutDeveloperKey(PublicKey key) { - BiMap<PublicKey, Principal> keys = HashBiMap.create(developerKeys); + BiMap<PublicKey, SimplePrincipal> keys = HashBiMap.create(developerKeys); keys.remove(key); return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores, archiveAccess, invalidateUserSessionsBefore); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java index 6178bfbb89e..153ce87ff6b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java @@ -36,14 +36,19 @@ public class InstanceList extends AbstractFilteringList<ApplicationId, InstanceL } /** - * Returns the subset of instances where all production deployments are compatible with the given version. + * Returns the subset of instances where all production deployments are compatible with the given version, + * and at least one known build is compatible with the given version. * * @param platform the version which applications returned are compatible with */ public InstanceList compatibleWithPlatform(Version platform, Function<ApplicationId, VersionCompatibility> compatibility) { - return matching(id -> instance(id).productionDeployments().values().stream() - .flatMap(deployment -> application(id).revisions().get(deployment.revision()).compileVersion().stream()) - .noneMatch(version -> compatibility.apply(id).refuse(platform, version))); + return matching(id -> instance(id).productionDeployments().values().stream() + .flatMap(deployment -> application(id).revisions().get(deployment.revision()).compileVersion().stream()) + .noneMatch(version -> compatibility.apply(id).refuse(platform, version)) + && application(id).revisions().production().stream() + .anyMatch(revision -> revision.compileVersion() + .map(compiled -> compatibility.apply(id).accept(platform, compiled)) + .orElse(true))); } /** diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java index 281ac50e63a..b3df417bd80 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java @@ -11,6 +11,7 @@ import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; @@ -54,8 +55,8 @@ public class ApplicationPackageValidator { * @throws IllegalArgumentException if any validations fail */ public void validate(Application application, ApplicationPackage applicationPackage, Instant instant) { - validateSteps(applicationPackage.deploymentSpec()); validateCloudAccounts(application, applicationPackage.deploymentSpec()); + validateSteps(applicationPackage.deploymentSpec()); validateEndpointRegions(applicationPackage.deploymentSpec()); validateEndpointChange(application, applicationPackage, instant); validateCompactedEndpoint(applicationPackage); @@ -88,12 +89,22 @@ public class ApplicationPackageValidator { private void validateSteps(DeploymentSpec deploymentSpec) { for (var spec : deploymentSpec.instances()) { for (var zone : spec.zones()) { - if (zone.environment().isManuallyDeployed()) - throw new IllegalArgumentException("region must be one with automated deployments, but got: " + zone.environment()); + Environment environment = zone.environment(); + if (environment.isManuallyDeployed()) + throw new IllegalArgumentException("region must be one with automated deployments, but got: " + environment); - if ( zone.environment() == Environment.prod - && ! controller.zoneRegistry().hasZone(ZoneId.from(zone.environment(), zone.region().orElseThrow()))) - throw new IllegalArgumentException("Zone " + zone + " in deployment spec was not found in this system!"); + if (environment == Environment.prod) { + RegionName region = zone.region().orElseThrow(); + if (!controller.zoneRegistry().hasZone(ZoneId.from(environment, region))) { + throw new IllegalArgumentException("Zone " + zone + " in deployment spec was not found in this system!"); + } + Optional<CloudAccount> cloudAccount = spec.cloudAccount(environment, region); + if (cloudAccount.isPresent() && !controller.zoneRegistry().hasZone(ZoneId.from(environment, region), cloudAccount.get())) { + throw new IllegalArgumentException("Zone " + zone + " in deployment spec is not configured for " + + "use in cloud account '" + cloudAccount.get().value() + + "', in this system"); + } + } } } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index d83f552ab25..1f2a016f630 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -323,14 +323,6 @@ public class DeploymentTrigger { instance -> instance.withJobPause(jobType, OptionalLong.empty())))); } - /** Triggers a change of this application, unless it already has a change. */ - public void triggerChange(ApplicationId instanceId, Change change) { - applications().lockApplicationOrThrow(TenantAndApplicationId.from(instanceId), application -> { - if ( ! application.get().require(instanceId.instance()).change().hasTargets()) - forceChange(instanceId, change); - }); - } - /** Overrides the given instance's platform and application changes with any contained in the given change. */ public void forceChange(ApplicationId instanceId, Change change) { applications().lockApplicationOrThrow(TenantAndApplicationId.from(instanceId), application -> { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 7114402f824..11a784ce899 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -233,41 +233,45 @@ public class JobController { Run run = run(id); return run.stepStatus(copyVespaLogs).map(succeeded::equals).orElse(false) ? controller.serviceRegistry().runDataStore().getLogs(id, tester) - : getVespaLogsFromLogserver(run, fromMillis, tester); + : getVespaLogsFromLogserver(run, fromMillis, tester).orElse(InputStream.nullInputStream()); } public static Optional<Instant> deploymentCompletedAt(Run run, boolean tester) { return (tester ? run.stepInfo(installTester) : run.stepInfo(installInitialReal).or(() -> run.stepInfo(installReal))) - .flatMap(StepInfo::startTime); + .flatMap(StepInfo::startTime).map(start -> start.minusSeconds(10)); } public void storeVespaLogs(RunId id) { Run run = run(id); if ( ! id.type().isProduction()) { - try (InputStream logs = getVespaLogsFromLogserver(run, 0, false)) { - controller.serviceRegistry().runDataStore().putLogs(id, false, logs); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } + getVespaLogsFromLogserver(run, 0, false).ifPresent(logs -> { + try (logs) { + controller.serviceRegistry().runDataStore().putLogs(id, false, logs); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + }); } if (id.type().isTest()) { - try (InputStream logs = getVespaLogsFromLogserver(run, 0, true)) { - controller.serviceRegistry().runDataStore().putLogs(id, true, logs); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } + getVespaLogsFromLogserver(run, 0, true).ifPresent(logs -> { + try (logs) { + controller.serviceRegistry().runDataStore().putLogs(id, true, logs); + } + catch(IOException e){ + throw new UncheckedIOException(e); + } + }); } } - private InputStream getVespaLogsFromLogserver(Run run, long fromMillis, boolean tester) { - long deploymentCompletedAtMillis = deploymentCompletedAt(run, tester).orElse(Instant.EPOCH).toEpochMilli(); - return controller.serviceRegistry().configServer().getLogs(new DeploymentId(tester ? run.id().tester().id() : run.id().application(), - run.id().type().zone()), - Map.of("from", Long.toString(Math.max(fromMillis, deploymentCompletedAtMillis)), - "to", Long.toString(run.end().orElse(controller.clock().instant()).toEpochMilli()))); + private Optional<InputStream> getVespaLogsFromLogserver(Run run, long fromMillis, boolean tester) { + return deploymentCompletedAt(run, tester).map(at -> + controller.serviceRegistry().configServer().getLogs(new DeploymentId(tester ? run.id().tester().id() : run.id().application(), + run.id().type().zone()), + Map.of("from", Long.toString(Math.max(fromMillis, at.toEpochMilli())), + "to", Long.toString(run.end().orElse(controller.clock().instant()).toEpochMilli())))); } /** Fetches any new test log entries, and records the id of the last of these, for continuation. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index 1932dc65657..269623de3f0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -119,12 +119,18 @@ public class Upgrader extends ControllerMaintainer { } int numberToUpgrade = policy == UpgradePolicy.canary ? instances.size() : numberOfApplicationsToUpgrade(); - for (ApplicationId id : instances.matching(targets.keySet()::contains).first(numberToUpgrade)) { - log.log(Level.INFO, "Triggering upgrade to " + targets.get(id) + " for " + id); - if (failingRevision.contains(id)) + for (ApplicationId id : instances.matching(targets.keySet()::contains)) { + if (failingRevision.contains(id)) { + log.log(Level.INFO, "Cancelling failing revision for " + id); controller().applications().deploymentTrigger().cancelChange(id, ChangesToCancel.APPLICATION); - - controller().applications().deploymentTrigger().triggerChange(id, Change.of(targets.get(id))); + } + + if (controller().applications().requireInstance(id).change().isEmpty()) { + log.log(Level.INFO, "Triggering upgrade to " + targets.get(id) + " for " + id); + controller().applications().deploymentTrigger().forceChange(id, Change.of(targets.get(id))); + --numberToUpgrade; + } + if (numberToUpgrade <= 0) break; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java index 8a363405c41..b6e18881dab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java @@ -82,7 +82,7 @@ public class Notification { feedBlock, /** Application cluster is reindexing document(s) */ - reindex; + reindex } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java index ecb9db8195f..32f5e6714f2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/BufferedLogStore.java @@ -69,8 +69,8 @@ public class BufferedLogStore { List<LogEntry> stepEntries = log.computeIfAbsent(step, __ -> new ArrayList<>()); for (LogEntry entry : entries) { if (sizeLowerBound > chunkSize) { - buffer.writeLastLogEntryId(id, type, lastEntryId); buffer.writeLog(id, type, lastChunkId, logSerializer.toJson(log)); + buffer.writeLastLogEntryId(id, type, lastEntryId); lastChunkId = lastEntryId + 1; if (++numberOfChunks > maxLogSize / chunkSize && ! forceLog) { log = Map.of(step, List.of(new LogEntry(++lastEntryId, @@ -87,8 +87,8 @@ public class BufferedLogStore { stepEntries.add(new LogEntry(++lastEntryId, entry.at(), entry.type(), entry.message())); sizeLowerBound += entry.message().length(); } - buffer.writeLastLogEntryId(id, type, lastEntryId); buffer.writeLog(id, type, lastChunkId, logSerializer.toJson(log)); + buffer.writeLastLogEntryId(id, type, lastEntryId); } /** Reads all log entries after the given threshold, from the buffered log, i.e., for an active run. */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java index e91fbe8b1b7..fc7cafe4c89 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java @@ -137,7 +137,7 @@ public class TenantSerializer { root.setLong(deletedAtField, tenant.deletedAt().toEpochMilli()); } - private void developerKeysToSlime(BiMap<PublicKey, Principal> keys, Cursor array) { + private void developerKeysToSlime(BiMap<PublicKey, ? extends Principal> keys, Cursor array) { keys.forEach((key, user) -> { Cursor object = array.addObject(); object.setString("key", KeyUtils.toPem(key)); @@ -184,8 +184,8 @@ public class TenantSerializer { TenantName name = TenantName.from(tenantObject.field(nameField).asString()); Instant createdAt = SlimeUtils.instant(tenantObject.field(createdAtField)); LastLoginInfo lastLoginInfo = lastLoginInfoFromSlime(tenantObject.field(lastLoginInfoField)); - Optional<Principal> creator = SlimeUtils.optionalString(tenantObject.field(creatorField)).map(SimplePrincipal::new); - BiMap<PublicKey, Principal> developerKeys = developerKeysFromSlime(tenantObject.field(pemDeveloperKeysField)); + Optional<SimplePrincipal> creator = SlimeUtils.optionalString(tenantObject.field(creatorField)).map(SimplePrincipal::new); + BiMap<PublicKey, SimplePrincipal> developerKeys = developerKeysFromSlime(tenantObject.field(pemDeveloperKeysField)); TenantInfo info = tenantInfoFromSlime(tenantObject.field(tenantInfoField)); List<TenantSecretStore> tenantSecretStores = secretStoresFromSlime(tenantObject.field(secretStoresField)); ArchiveAccess archiveAccess = archiveAccessFromSlime(tenantObject); @@ -200,8 +200,8 @@ public class TenantSerializer { return new DeletedTenant(name, createdAt, deletedAt); } - private BiMap<PublicKey, Principal> developerKeysFromSlime(Inspector array) { - ImmutableBiMap.Builder<PublicKey, Principal> keys = ImmutableBiMap.builder(); + private BiMap<PublicKey, SimplePrincipal> developerKeysFromSlime(Inspector array) { + ImmutableBiMap.Builder<PublicKey, SimplePrincipal> keys = ImmutableBiMap.builder(); array.traverse((ArrayTraverser) (__, keyObject) -> keys.put(KeyUtils.fromPemEncodedPublicKey(keyObject.field("key").asString()), new SimplePrincipal(keyObject.field("user").asString()))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index a4bb9034a85..7e186cbe34c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -59,7 +59,6 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbi import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.integration.aws.TenantRoles; -import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Cluster; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; @@ -165,7 +164,6 @@ import static java.util.Map.Entry.comparingByKey; import static java.util.stream.Collectors.collectingAndThen; import static java.util.stream.Collectors.joining; import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toUnmodifiableList; /** * This implements the application/v4 API which is used to deploy and manage applications @@ -202,15 +200,15 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { public HttpResponse auditAndHandle(HttpRequest request) { try { Path path = new Path(request.getUri()); - switch (request.getMethod()) { - case GET: return handleGET(path, request); - case PUT: return handlePUT(path, request); - case POST: return handlePOST(path, request); - case PATCH: return handlePATCH(path, request); - case DELETE: return handleDELETE(path, request); - case OPTIONS: return handleOPTIONS(); - default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); - } + return switch (request.getMethod()) { + case GET: yield handleGET(path, request); + case PUT: yield handlePUT(path, request); + case POST: yield handlePOST(path, request); + case PATCH: yield handlePATCH(path, request); + case DELETE: yield handleDELETE(path, request); + case OPTIONS: yield handleOPTIONS(); + default: yield ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported"); + }; } catch (RestApiException.Forbidden e) { return ErrorResponse.forbidden(Exceptions.toMessageString(e)); @@ -225,16 +223,12 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return ErrorResponse.badRequest(Exceptions.toMessageString(e)); } catch (ConfigServerException e) { - switch (e.code()) { - case NOT_FOUND: - return ErrorResponse.notFoundError(Exceptions.toMessageString(e)); - case ACTIVATION_CONFLICT: - return new ErrorResponse(CONFLICT, e.code().name(), Exceptions.toMessageString(e)); - case INTERNAL_SERVER_ERROR: - return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); - default: - return new ErrorResponse(BAD_REQUEST, e.code().name(), Exceptions.toMessageString(e)); - } + return switch (e.code()) { + case NOT_FOUND: yield ErrorResponse.notFoundError(Exceptions.toMessageString(e)); + case ACTIVATION_CONFLICT: yield new ErrorResponse(CONFLICT, e.code().name(), Exceptions.toMessageString(e)); + case INTERNAL_SERVER_ERROR: yield ErrorResponse.internalServerError(Exceptions.toMessageString(e)); + default: yield new ErrorResponse(BAD_REQUEST, e.code().name(), Exceptions.toMessageString(e)); + }; } catch (RuntimeException e) { log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); @@ -714,19 +708,18 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { } private static TenantContacts.Audience fromAudience(String value) { - switch (value) { - case "tenant": return TenantContacts.Audience.TENANT; - case "notifications": return TenantContacts.Audience.NOTIFICATIONS; + return switch (value) { + case "tenant": yield TenantContacts.Audience.TENANT; + case "notifications": yield TenantContacts.Audience.NOTIFICATIONS; default: throw new IllegalArgumentException("Unknown contact audience '" + value + "'."); - } + }; } private static String toAudience(TenantContacts.Audience audience) { - switch (audience) { - case TENANT: return "tenant"; - case NOTIFICATIONS: return "notifications"; - default: throw new IllegalArgumentException("Unexpected contact audience '" + audience + "'."); - } + return switch (audience) { + case TENANT: yield "tenant"; + case NOTIFICATIONS: yield "notifications"; + }; } @@ -820,18 +813,18 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private TenantContacts updateTenantInfoContacts(Inspector insp, TenantContacts oldContacts) { if (!insp.valid()) return oldContacts; - List<TenantContacts.Contact> contacts = SlimeUtils.entriesStream(insp).map(inspector -> { + List<TenantContacts.EmailContact> contacts = SlimeUtils.entriesStream(insp).map(inspector -> { String email = inspector.field("email").asString().trim(); List<TenantContacts.Audience> audiences = SlimeUtils.entriesStream(inspector.field("audiences")) .map(audience -> fromAudience(audience.asString())) - .collect(Collectors.toUnmodifiableList()); + .toList(); if (!email.contains("@")) { throw new IllegalArgumentException("'email' needs to be an email address"); } return new TenantContacts.EmailContact(audiences, email); - }).collect(toUnmodifiableList()); + }).toList(); return new TenantContacts(contacts); } @@ -884,24 +877,21 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { } private static String notificationTypeAsString(Notification.Type type) { - switch (type) { - case submission: - case applicationPackage: return "applicationPackage"; - case testPackage: return "testPackage"; - case deployment: return "deployment"; - case feedBlock: return "feedBlock"; - case reindex: return "reindex"; - default: throw new IllegalArgumentException("No serialization defined for notification type " + type); - } + return switch (type) { + case submission, applicationPackage: yield "applicationPackage"; + case testPackage: yield "testPackage"; + case deployment: yield "deployment"; + case feedBlock: yield "feedBlock"; + case reindex: yield "reindex"; + }; } private static String notificationLevelAsString(Notification.Level level) { - switch (level) { - case info: return "info"; - case warning: return "warning"; - case error: return "error"; - default: throw new IllegalArgumentException("No serialization defined for notification level " + level); - } + return switch (level) { + case info: yield "info"; + case warning: yield "warning"; + case error: yield "error"; + }; } private HttpResponse applications(String tenantName, Optional<String> applicationName, HttpRequest request) { @@ -1081,7 +1071,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { return new SlimeJsonResponse(root); } - private void toSlime(Cursor keysArray, Map<PublicKey, Principal> keys) { + private void toSlime(Cursor keysArray, Map<PublicKey, ? extends Principal> keys) { keys.forEach((key, principal) -> { Cursor keyObject = keysArray.addObject(); keyObject.setString("key", KeyUtils.toPem(key)); @@ -1334,17 +1324,19 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { } private static String valueOf(Node.State state) { - switch (state) { - case failed: return "failed"; - case parked: return "parked"; - case dirty: return "dirty"; - case ready: return "ready"; - case active: return "active"; - case inactive: return "inactive"; - case reserved: return "reserved"; - case provisioned: return "provisioned"; + return switch (state) { + case failed: yield "failed"; + case parked: yield "parked"; + case dirty: yield "dirty"; + case ready: yield "ready"; + case active: yield "active"; + case inactive: yield "inactive"; + case reserved: yield "reserved"; + case provisioned: yield "provisioned"; + case breakfixed: yield "breakfixed"; + case deprovisioned: yield "deprovisioned"; default: throw new IllegalArgumentException("Unexpected node state '" + state + "'."); - } + }; } static String valueOf(Node.ServiceState state) { @@ -1360,31 +1352,29 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { } private static String valueOf(Node.ClusterType type) { - switch (type) { - case admin: return "admin"; - case content: return "content"; - case container: return "container"; - case combined: return "combined"; - default: throw new IllegalArgumentException("Unexpected node cluster type '" + type + "'."); - } + return switch (type) { + case admin: yield "admin"; + case content: yield "content"; + case container: yield "container"; + case combined: yield "combined"; + case unknown: throw new IllegalArgumentException("Unexpected node cluster type '" + type + "'."); + }; } private static String valueOf(NodeResources.DiskSpeed diskSpeed) { - switch (diskSpeed) { - case fast : return "fast"; - case slow : return "slow"; - case any : return "any"; - default: throw new IllegalArgumentException("Unknown disk speed '" + diskSpeed.name() + "'"); - } + return switch (diskSpeed) { + case fast : yield "fast"; + case slow : yield "slow"; + case any : yield "any"; + }; } private static String valueOf(NodeResources.StorageType storageType) { - switch (storageType) { - case remote : return "remote"; - case local : return "local"; - case any : return "any"; - default: throw new IllegalArgumentException("Unknown storage type '" + storageType.name() + "'"); - } + return switch (storageType) { + case remote : yield "remote"; + case local : yield "local"; + case any : yield "any"; + }; } private HttpResponse logs(String tenantName, String applicationName, String instanceName, String environment, String region, Map<String, String> queryParameters) { @@ -2146,11 +2136,11 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { List<String> clusterNames = Optional.ofNullable(request.getProperty("clusterId")).stream() .flatMap(clusters -> Stream.of(clusters.split(","))) .filter(cluster -> ! cluster.isBlank()) - .collect(toUnmodifiableList()); + .toList(); List<String> documentTypes = Optional.ofNullable(request.getProperty("documentType")).stream() .flatMap(types -> Stream.of(types.split(","))) .filter(type -> ! type.isBlank()) - .collect(toUnmodifiableList()); + .toList(); Double speed = request.hasProperty("speed") ? Double.parseDouble(request.getProperty("speed")) : null; boolean indexedOnly = request.getBooleanProperty("indexedOnly"); @@ -2209,13 +2199,12 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { } private static String toString(ApplicationReindexing.State state) { - switch (state) { - case PENDING: return "pending"; - case RUNNING: return "running"; - case FAILED: return "failed"; - case SUCCESSFUL: return "successful"; - default: return null; - } + return switch (state) { + case PENDING: yield "pending"; + case RUNNING: yield "running"; + case FAILED: yield "failed"; + case SUCCESSFUL: yield "successful"; + }; } /** Enables reindexing of an application in a zone. */ @@ -2893,12 +2882,11 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { } private static String tenantType(Tenant tenant) { - switch (tenant.type()) { - case athenz: return "ATHENS"; - case cloud: return "CLOUD"; - case deleted: return "DELETED"; - default: throw new IllegalArgumentException("Unknown tenant type: " + tenant.getClass().getSimpleName()); - } + return switch (tenant.type()) { + case athenz: yield "ATHENS"; + case cloud: yield "CLOUD"; + case deleted: yield "DELETED"; + }; } private static ApplicationId appIdFromPath(Path path) { @@ -3003,29 +2991,27 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { } private static String rotationStateString(RotationState state) { - switch (state) { - case in: return "IN"; - case out: return "OUT"; - } - return "UNKNOWN"; + return switch (state) { + case in: yield "IN"; + case out: yield "OUT"; + case unknown: yield "UNKNOWN"; + }; } private static String endpointScopeString(Endpoint.Scope scope) { - switch (scope) { - case weighted: return "weighted"; - case application: return "application"; - case global: return "global"; - case zone: return "zone"; - } - throw new IllegalArgumentException("Unknown endpoint scope " + scope); + return switch (scope) { + case weighted: yield "weighted"; + case application: yield "application"; + case global: yield "global"; + case zone: yield "zone"; + }; } private static String routingMethodString(RoutingMethod method) { - switch (method) { - case exclusive: return "exclusive"; - case sharedLayer4: return "sharedLayer4"; - } - throw new IllegalArgumentException("Unknown routing method " + method); + return switch (method) { + case exclusive: yield "exclusive"; + case sharedLayer4: yield "sharedLayer4"; + }; } private static <T> T getAttribute(HttpRequest request, String attributeName, Class<T> cls) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index b5177cd1d3e..6c193e9b539 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -9,6 +9,7 @@ import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; @@ -1194,13 +1195,25 @@ public class ControllerTest { .cloudAccount(cloudAccount) .region(zone.region()) .build(); + // Deployment fails because cloud account is not declared for this tenant try { context.submit(applicationPackage).deploy(); - fail("Expected exception"); // Account invalid for tenant - } catch (IllegalArgumentException ignored) { + fail("Expected exception"); + } catch (IllegalArgumentException e) { + assertEquals("Cloud account '012345678912' is not valid for tenant 'tenant'", e.getMessage()); } + // Deployment fails because requested region is not configured in cloud account tester.controllerTester().flagSource().withListFlag(PermanentFlags.CLOUD_ACCOUNTS.id(), List.of(cloudAccount), String.class); + try { + context.submit(applicationPackage).deploy(); + fail("Expected exception"); + } catch (IllegalArgumentException e) { + assertEquals("Zone prod.us-west-1 in deployment spec is not configured for use in cloud account '012345678912', in this system", e.getMessage()); + } + + // Deployment succeeds + tester.controllerTester().zoneRegistry().setCloudAccountZones(new CloudAccount(cloudAccount), zone); context.submit(applicationPackage).deploy(); assertEquals(cloudAccount, tester.controllerTester().configServer().cloudAccount(context.deploymentIdIn(zone)).get().value()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index 62fafa12993..6dfeaf55ea4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -1221,7 +1221,7 @@ public class DeploymentTriggerTest { assertEquals(Change.empty(), app.instance().change()); // Application is pinned to previous version, and downgrades to that. Tests are re-run. - tester.deploymentTrigger().triggerChange(app.instanceId(), Change.of(version0).withPin()); + tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(version0).withPin()); app.runJob(stagingTest).runJob(productionUsEast3); tester.clock().advance(Duration.ofMinutes(1)); app.failDeployment(testUsEast3); @@ -2106,12 +2106,12 @@ public class DeploymentTriggerTest { Version version2 = new Version("7.8.9"); Version version3 = new Version("8.9.10"); tester.controllerTester().upgradeSystem(version2); - tester.deploymentTrigger().triggerChange(appToAvoidVersionGC.instanceId(), Change.of(version2)); + tester.deploymentTrigger().forceChange(appToAvoidVersionGC.instanceId(), Change.of(version2)); appToAvoidVersionGC.deployPlatform(version2); // app upgrades first zone to version3, and then the other two to version2. tester.controllerTester().upgradeSystem(version3); - tester.deploymentTrigger().triggerChange(app.instanceId(), Change.of(version3)); + tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(version3)); app.runJob(systemTest).runJob(stagingTest); tester.triggerJobs(); tester.upgrader().overrideConfidence(version3, VespaVersion.Confidence.broken); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index 4131bfd8b9f..680e69998b6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -5,6 +5,7 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeType; @@ -43,6 +44,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry private final Map<Environment, RegionName> defaultRegionForEnvironment = new HashMap<>(); private final Map<CloudName, UpgradePolicy> osUpgradePolicies = new HashMap<>(); private final Map<ZoneApi, List<RoutingMethod>> zoneRoutingMethods = new HashMap<>(); + private final Map<CloudAccount, Set<ZoneId>> cloudAccountZones = new HashMap<>(); private final Set<ZoneApi> reprovisionToUpgradeOs = new HashSet<>(); private final SystemName system; // Don't even think about making it non-final! ƪ(`▿▿▿▿´ƪ) @@ -146,6 +148,11 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry return this; } + public ZoneRegistryMock setCloudAccountZones(CloudAccount cloudAccount, ZoneId... zones) { + this.cloudAccountZones.put(cloudAccount, Set.of(zones)); + return this; + } + @Override public SystemName system() { return system; @@ -254,6 +261,11 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry } @Override + public boolean hasZone(ZoneId zoneId, CloudAccount cloudAccount) { + return hasZone(zoneId) && cloudAccountZones.getOrDefault(cloudAccount, Set.of()).contains(zoneId); + } + + @Override public URI getConfigServerVipUri(ZoneId zoneId) { return URI.create(Text.format("https://cfg.%s.test.vip:4443/", zoneId.value())); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java index 9b2a1607e76..15855770c0b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OutstandingChangeDeployerTest.java @@ -32,7 +32,7 @@ public class OutstandingChangeDeployerTest { var app = tester.newDeploymentContext().submit(applicationPackage).deploy(); Version version = new Version(6, 2); - tester.deploymentTrigger().triggerChange(app.instanceId(), Change.of(version)); + tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(version)); assertEquals(Change.of(version), app.instance().change()); assertFalse(app.deploymentStatus().outstandingChange(app.instance().name()).hasTargets()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java index 324c9706df9..a927439de1c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java @@ -80,7 +80,7 @@ public class ControllerContainerCloudTest extends ControllerContainerTest { } public RequestBuilder data(byte[] data) { this.data = data; return this; } public RequestBuilder data(String data) { this.data = data.getBytes(StandardCharsets.UTF_8); return this; } - public RequestBuilder principal(String principal) { this.principal = new SimplePrincipal(principal); return this; } + public RequestBuilder principal(String principal) { this.principal = new SimplePrincipal(principal){ }; return this; } public RequestBuilder user(User user) { this.user = user; return this; } public RequestBuilder roles(Set<Role> roles) { this.roles = roles; return this; } public RequestBuilder roles(Role... roles) { return roles(Set.of(roles)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java index ec9be1f04c3..fcbecfa2e68 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java @@ -121,7 +121,7 @@ public class SignatureFilterTest { Instant.EPOCH, LastLoginInfo.EMPTY, Optional.empty(), - ImmutableBiMap.of(publicKey, () -> "user"), + ImmutableBiMap.of(publicKey, new SimplePrincipal("user")), TenantInfo.empty(), List.of(), new ArchiveAccess(), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java index 1344b106bbe..f34dd3fe629 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java @@ -136,6 +136,14 @@ public class UserApiTest extends ControllerContainerCloudTest { "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Key " + quotedPemPublicKey + " is already owned by joe@dev\"}", 400); + // POST a different developer key for an existing user is forbidden + tester.assertResponse(request("/application/v4/tenant/my-tenant/key", POST) + .principal("joe@dev") + .roles(Set.of(Role.developer(id.tenant()))) + .data("{\"key\":\"" + otherPemPublicKey + "\"}"), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"joe@dev is already associated with key " + quotedPemPublicKey + "\"}", + 400); + // POST in a different pem developer key tester.assertResponse(request("/application/v4/tenant/my-tenant/key", POST) .principal("developer@tenant") diff --git a/default_build_settings.cmake b/default_build_settings.cmake index 009cd6d615e..e973cfcf7f7 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -64,7 +64,7 @@ endfunction() function(setup_vespa_default_build_settings_darwin) message("-- Setting up default build settings for darwin") - set(DEFAULT_VESPA_LLVM_VERSION "13" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "14" PARENT_SCOPE) set(DEFAULT_CMAKE_PREFIX_PATH "${VESPA_DEPS}" "/usr/local/opt/bison" "/usr/local/opt/flex" "/usr/local/opt/openssl@1.1" "/usr/local/opt/openblas" "/usr/local/opt/icu4c" PARENT_SCOPE) set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib" "/usr/local/opt/bison/lib" "/usr/local/opt/flex/lib" "/usr/local/opt/icu4c/lib" "/usr/local/opt/openssl@1.1/lib" "/usr/local/opt/openblas/lib") list(APPEND DEFAULT_EXTRA_LINK_DIRECTORY "/usr/local/lib") @@ -74,12 +74,6 @@ function(setup_vespa_default_build_settings_darwin) set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${DEFAULT_EXTRA_INCLUDE_DIRECTORY}" PARENT_SCOPE) endfunction() -function(setup_vespa_default_build_settings_fedora_34) - message("-- Setting up default build settings for fedora 34") - set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) - set(DEFAULT_VESPA_LLVM_VERSION "12" PARENT_SCOPE) -endfunction() - function(setup_vespa_default_build_settings_fedora_35) message("-- Setting up default build settings for fedora 35") set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) @@ -98,6 +92,12 @@ function(setup_vespa_default_build_settings_fedora_37) set(DEFAULT_VESPA_LLVM_VERSION "14" PARENT_SCOPE) endfunction() +function(setup_vespa_default_build_settings_fedora_38) + message("-- Setting up default build settings for fedora 38") + set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" "/usr/include/openblas" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "14" PARENT_SCOPE) +endfunction() + function(setup_vespa_default_build_settings_amzn_2) message("-- Setting up default build settings for amzn 2") set(DEFAULT_EXTRA_LINK_DIRECTORY "${VESPA_DEPS}/lib64" "/usr/lib64/llvm7.0/lib" PARENT_SCOPE) @@ -236,14 +236,14 @@ function(vespa_use_default_build_settings) setup_vespa_default_build_settings_almalinux_9_0() elseif(VESPA_OS_DISTRO STREQUAL "darwin") setup_vespa_default_build_settings_darwin() - elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 34") - setup_vespa_default_build_settings_fedora_34() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 35") setup_vespa_default_build_settings_fedora_35() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 36") setup_vespa_default_build_settings_fedora_36() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 37") setup_vespa_default_build_settings_fedora_37() + elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 38") + setup_vespa_default_build_settings_fedora_38() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "amzn 2") setup_vespa_default_build_settings_amzn_2() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "amzn 2022") @@ -351,8 +351,8 @@ function(vespa_use_default_cxx_compiler) unset(DEFAULT_CMAKE_CXX_COMPILER) if(NOT DEFINED VESPA_COMPILER_VARIANT OR VESPA_COMPILER_VARIANT STREQUAL "gcc") if(APPLE) - set(DEFAULT_CMAKE_C_COMPILER "/usr/local/bin/gcc-11") - set(DEFAULT_CMAKE_CXX_COMPILER "/usr/local/bin/g++-11") + set(DEFAULT_CMAKE_C_COMPILER "/usr/local/bin/gcc-12") + set(DEFAULT_CMAKE_CXX_COMPILER "/usr/local/bin/g++-12") elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "amzn 2") set(DEFAULT_CMAKE_C_COMPILER "/usr/bin/gcc10-gcc") set(DEFAULT_CMAKE_CXX_COMPILER "/usr/bin/gcc10-g++") @@ -403,12 +403,12 @@ function(vespa_use_default_java_home) if (DEFINED JAVA_HOME) return() endif() - set(DEFAULT_JAVA_HOME "/usr/lib/jvm/java-11-openjdk") + set(DEFAULT_JAVA_HOME "/usr/lib/jvm/java-17-openjdk") if(APPLE) execute_process(COMMAND "/usr/libexec/java_home" OUTPUT_VARIABLE DEFAULT_JAVA_HOME) string(STRIP "${DEFAULT_JAVA_HOME}" DEFAULT_JAVA_HOME) - elseif(VESPA_OS_DISTRO STREQUAL "ubuntu") - set(DEFAULT_JAVA_HOME "/usr/lib/jvm/java-11-openjdk-amd64" PARENT_SCOPE) + elseif(VESPA_OS_DISTRO STREQUAL "ubuntu" OR VESPA_OS_DISTRO STREQUAL "debian") + set(DEFAULT_JAVA_HOME "/usr/lib/jvm/java-17-openjdk-amd64" PARENT_SCOPE) endif() if(COMMAND vespa_use_specific_java_home) vespa_use_specific_java_home() diff --git a/dist/vespa.spec b/dist/vespa.spec index b94a77491b8..4b4c82c436b 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -111,7 +111,7 @@ BuildRequires: vespa-gtest = 1.11.0 %define _use_vespa_gtest 1 BuildRequires: vespa-icu-devel >= 65.1.0-1 BuildRequires: vespa-lz4-devel >= 1.9.2-2 -BuildRequires: vespa-onnxruntime-devel = 1.11.0 +BuildRequires: vespa-onnxruntime-devel = 1.12.1 BuildRequires: vespa-openssl-devel >= 1.1.1o-1 %define _use_vespa_openssl 1 BuildRequires: vespa-protobuf-devel = 3.19.1 @@ -139,7 +139,7 @@ BuildRequires: vespa-openssl-devel >= 1.1.1o-1 BuildRequires: vespa-gtest = 1.11.0 %define _use_vespa_gtest 1 BuildRequires: vespa-lz4-devel >= 1.9.2-2 -BuildRequires: vespa-onnxruntime-devel = 1.11.0 +BuildRequires: vespa-onnxruntime-devel = 1.12.1 BuildRequires: vespa-protobuf-devel = 3.19.1 BuildRequires: vespa-libzstd-devel >= 1.4.5-2 %endif @@ -149,7 +149,7 @@ BuildRequires: maven BuildRequires: maven-openjdk17 BuildRequires: openssl-devel BuildRequires: vespa-lz4-devel >= 1.9.2-2 -BuildRequires: vespa-onnxruntime-devel = 1.11.0 +BuildRequires: vespa-onnxruntime-devel = 1.12.1 BuildRequires: vespa-libzstd-devel >= 1.4.5-2 BuildRequires: protobuf-devel %if 0%{?_centos_stream} @@ -169,15 +169,11 @@ BuildRequires: maven-openjdk17 %endif BuildRequires: openssl-devel BuildRequires: vespa-lz4-devel >= 1.9.2-2 -BuildRequires: vespa-onnxruntime-devel = 1.11.0 +BuildRequires: vespa-onnxruntime-devel = 1.12.1 BuildRequires: vespa-libzstd-devel >= 1.4.5-2 -%if 0%{?fc34} -BuildRequires: protobuf-devel %if 0%{?amzn2022} +BuildRequires: protobuf-devel BuildRequires: llvm-devel >= 13.0.0 -%else -BuildRequires: llvm-devel >= 12.0.0 -%endif BuildRequires: boost-devel >= 1.75 BuildRequires: gtest-devel BuildRequires: gmock-devel @@ -198,8 +194,15 @@ BuildRequires: gmock-devel %endif %if 0%{?fc37} BuildRequires: protobuf-devel -BuildRequires: llvm-devel >= 14.0.0 -BuildRequires: boost-devel >= 1.76 +BuildRequires: llvm-devel >= 14.0.5 +BuildRequires: boost-devel >= 1.78 +BuildRequires: gtest-devel +BuildRequires: gmock-devel +%endif +%if 0%{?fc38} +BuildRequires: protobuf-devel +BuildRequires: llvm-devel >= 14.0.5 +BuildRequires: boost-devel >= 1.78 BuildRequires: gtest-devel BuildRequires: gmock-devel %endif @@ -328,12 +331,8 @@ Requires: gtest %endif %if 0%{?fedora} Requires: gtest -%if 0%{?fc34} %if 0%{?amzn2022} %define _vespa_llvm_version 13 -%else -%define _vespa_llvm_version 12 -%endif %endif %if 0%{?fc35} %define _vespa_llvm_version 13 @@ -344,6 +343,9 @@ Requires: gtest %if 0%{?fc37} %define _vespa_llvm_version 14 %endif +%if 0%{?fc38} +%define _vespa_llvm_version 14 +%endif %define _extra_link_directory %{_vespa_deps_prefix}/lib64 %define _extra_include_directory %{_vespa_deps_prefix}/include;/usr/include/openblas %endif @@ -459,12 +461,8 @@ Requires: protobuf %endif %if 0%{?fedora} Requires: protobuf -%if 0%{?fc34} %if 0%{?amzn2022} Requires: llvm-libs >= 13.0.0 -%else -Requires: llvm-libs >= 12.0.0 -%endif %endif %if 0%{?fc35} Requires: llvm-libs >= 13.0.0 @@ -473,10 +471,13 @@ Requires: llvm-libs >= 13.0.0 Requires: llvm-libs >= 14.0.0 %endif %if 0%{?fc37} -Requires: llvm-libs >= 14.0.0 +Requires: llvm-libs >= 14.0.5 +%endif +%if 0%{?fc38} +Requires: llvm-libs >= 14.0.5 %endif %endif -Requires: vespa-onnxruntime = 1.11.0 +Requires: vespa-onnxruntime = 1.12.1 %description libs @@ -571,7 +572,7 @@ nearest neighbor search used for low-level benchmarking. %endif %else %setup -q -%if ( 0%{?el8} || 0%{?fc34} ) && %{_vespa_llvm_version} < 13 +%if 0%{?el8} && %{_vespa_llvm_version} < 13 if grep -qs 'result_pair<R>(' /usr/include/llvm/ADT/STLExtras.h then patch /usr/include/llvm/ADT/STLExtras.h < dist/STLExtras.h.diff diff --git a/document/src/vespa/document/bucket/bucketid.cpp b/document/src/vespa/document/bucket/bucketid.cpp index f01bfb67674..9ca4db1e51c 100644 --- a/document/src/vespa/document/bucket/bucketid.cpp +++ b/document/src/vespa/document/bucket/bucketid.cpp @@ -8,6 +8,7 @@ #include <vespa/vespalib/stllike/hash_set.hpp> #include <vespa/vespalib/util/stringfmt.h> #include <limits> +#include <xxh3.h> using vespalib::nbostream; using vespalib::asciistream; @@ -76,6 +77,12 @@ void BucketId::initialize() noexcept { fillStripMasks(BucketId::_stripMasks, BucketId::maxNumBits); } +uint64_t +BucketId::hash::operator () (const BucketId& bucketId) const noexcept { + const uint64_t raw_id = bucketId.getId(); + return XXH3_64bits(&raw_id, sizeof(uint64_t)); +} + vespalib::string BucketId::toString() const { diff --git a/document/src/vespa/document/bucket/bucketid.h b/document/src/vespa/document/bucket/bucketid.h index d54f86e4ae7..370948c1acc 100644 --- a/document/src/vespa/document/bucket/bucketid.h +++ b/document/src/vespa/document/bucket/bucketid.h @@ -37,9 +37,7 @@ class BucketId { public: struct hash { - size_t operator () (const BucketId& g) const noexcept { - return g.getId(); - } + uint64_t operator () (const BucketId& g) const noexcept; }; /** diff --git a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java index 9f40013f627..c1877373ce2 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/FetchVector.java @@ -20,7 +20,7 @@ public class FetchVector { * Note: If this enum is changed, you must also change {@link DimensionHelper}. */ public enum Dimension { - /** A legal value for TenantName, e.g. vespa-team */ + /** Value from TenantName::value, e.g. vespa-team */ TENANT_ID, /** Value from ApplicationId::serializedForm of the form tenant:applicationName:instance. */ @@ -29,7 +29,7 @@ public class FetchVector { /** Node type from com.yahoo.config.provision.NodeType::name, e.g. tenant, host, confighost, controller, etc. */ NODE_TYPE, - /** Cluster type from com.yahoo.config.provision.ClusterSpec.Type::value, e.g. content, container, admin */ + /** Cluster type from com.yahoo.config.provision.ClusterSpec.Type::name, e.g. content, container, admin */ CLUSTER_TYPE, /** Cluster ID from com.yahoo.config.provision.ClusterSpec.Id::value, e.g. cluster-controllers, logserver. */ diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index e349165b855..21f95c7ac6a 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -11,6 +11,7 @@ import java.time.format.DateTimeFormatter; import java.util.List; import java.util.Optional; import java.util.TreeMap; +import java.util.function.Predicate; import static com.yahoo.vespa.flags.FetchVector.Dimension.APPLICATION_ID; import static com.yahoo.vespa.flags.FetchVector.Dimension.CONSOLE_USER_EMAIL; @@ -61,6 +62,14 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundStringFlag QUERY_DISPATCH_POLICY = defineStringFlag( + "query-dispatch-policy", "adaptive", + List.of("baldersheim"), "2022-08-20", "2023-01-01", + "Select query dispatch policy, valid values are adaptive, round-robin, best-of-random-2," + + " latency-amortized-over-requests, latency-amortized-over-time", + "Takes effect at redeployment (requires restart)", + ZONE_ID, APPLICATION_ID); + public static final UnboundStringFlag FEED_SEQUENCER_TYPE = defineStringFlag( "feed-sequencer-type", "THROUGHPUT", List.of("baldersheim"), "2020-12-02", "2023-01-01", @@ -332,7 +341,7 @@ public class Flags { public static final UnboundBooleanFlag ENABLE_DATA_HIGHWAY_IN_AWS = defineFeatureFlag( "enable-data-highway-in-aws", false, - List.of("hmusum"), "2022-01-06", "2022-09-01", + List.of("hmusum"), "2022-01-06", "2022-10-01", "Enable Data Highway in AWS", "Takes effect on restart of Docker container", ZONE_ID, APPLICATION_ID); @@ -410,8 +419,8 @@ public class Flags { public static final UnboundStringFlag APPLICATION_FILES_WITH_UNKNOWN_EXTENSION = defineStringFlag( "fail-deployment-for-files-with-unknown-extension", "FAIL", - List.of("hmusum"), "2022-04-27", "2022-09-01", - "Whether to log, fail or do nothing for deployments when app has a file with unknown extension (valid values: LOG, FAIL)", + List.of("hmusum"), "2022-04-27", "2022-10-01", + "Whether to log or fail for deployments when app has a file with unknown extension (valid values: LOG, FAIL)", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); @@ -431,14 +440,14 @@ public class Flags { public static final UnboundListFlag<String> FILE_DISTRIBUTION_ACCEPTED_COMPRESSION_TYPES = defineListFlag( "file-distribution-accepted-compression-types", List.of("gzip", "lz4"), String.class, - List.of("hmusum"), "2022-07-05", "2022-09-05", + List.of("hmusum"), "2022-07-05", "2022-10-01", "´List of accepted compression types used when asking for a file reference. Valid values: gzip, lz4", "Takes effect on restart of service", APPLICATION_ID); public static final UnboundListFlag<String> FILE_DISTRIBUTION_COMPRESSION_TYPES_TO_SERVE = defineListFlag( "file-distribution-compression-types-to-use", List.of("lz4", "gzip"), String.class, - List.of("hmusum"), "2022-07-05", "2022-09-05", + List.of("hmusum"), "2022-07-05", "2022-10-01", "List of compression types to use (in preferred order), matched with accepted compression types when serving file references. Valid values: gzip, lz4", "Takes effect on restart of service", APPLICATION_ID); @@ -490,7 +499,19 @@ public class Flags { public static UnboundStringFlag defineStringFlag(String flagId, String defaultValue, List<String> owners, String createdAt, String expiresAt, String description, String modificationEffect, FetchVector.Dimension... dimensions) { - return define(UnboundStringFlag::new, flagId, defaultValue, owners, createdAt, expiresAt, description, modificationEffect, dimensions); + return defineStringFlag(flagId, defaultValue, owners, + createdAt, expiresAt, description, + modificationEffect, value -> true, + dimensions); + } + + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ + public static UnboundStringFlag defineStringFlag(String flagId, String defaultValue, List<String> owners, + String createdAt, String expiresAt, String description, + String modificationEffect, Predicate<String> validator, + FetchVector.Dimension... dimensions) { + return define((i, d, v) -> new UnboundStringFlag(i, d, v, validator), + flagId, defaultValue, owners, createdAt, expiresAt, description, modificationEffect, dimensions); } /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ @@ -532,7 +553,7 @@ public class Flags { @FunctionalInterface private interface TypedUnboundFlagFactory<T, U extends UnboundFlag<?, ?, ?>> { - U create(FlagId id, T defaultVale, FetchVector defaultFetchVector); + U create(FlagId id, T defaultValue, FetchVector defaultFetchVector); } /** diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index b5a292a554d..e193fca59f5 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -9,6 +9,8 @@ import java.time.ZoneOffset; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.List; +import java.util.function.Predicate; +import java.util.regex.Pattern; import static com.yahoo.vespa.flags.FetchVector.Dimension.APPLICATION_ID; import static com.yahoo.vespa.flags.FetchVector.Dimension.CLUSTER_ID; @@ -17,6 +19,7 @@ import static com.yahoo.vespa.flags.FetchVector.Dimension.CONSOLE_USER_EMAIL; import static com.yahoo.vespa.flags.FetchVector.Dimension.HOSTNAME; import static com.yahoo.vespa.flags.FetchVector.Dimension.NODE_TYPE; import static com.yahoo.vespa.flags.FetchVector.Dimension.TENANT_ID; +import static com.yahoo.vespa.flags.FetchVector.Dimension.VESPA_VERSION; import static com.yahoo.vespa.flags.FetchVector.Dimension.ZONE_ID; /** @@ -113,6 +116,19 @@ public class PermanentFlags { "Takes effect on next deployment from controller", ZONE_ID, APPLICATION_ID); + private static final String VERSION_QUALIFIER_REGEX = "[a-zA-Z0-9_-]+"; + private static final Pattern QUALIFIER_PATTERN = Pattern.compile("^" + VERSION_QUALIFIER_REGEX + "$"); + private static final Pattern VERSION_PATTERN = Pattern.compile("^\\d\\.\\d\\.\\d(\\." + VERSION_QUALIFIER_REGEX + ")?$"); + + public static final UnboundStringFlag WANTED_DOCKER_TAG = defineStringFlag( + "wanted-docker-tag", "", + "If non-empty the flag value overrides the docker image tag of the wantedDockerImage of the node object. " + + "If the flag value contains '.', it must specify a valid Vespa version like '8.83.42'. " + + "Otherwise a '.' + the flag value will be appended.", + "Takes effect on the next host admin tick. The upgrade to the new wanted docker image is orchestrated.", + value -> value.isEmpty() || QUALIFIER_PATTERN.matcher(value).find() || VERSION_PATTERN.matcher(value).find(), + HOSTNAME, NODE_TYPE, TENANT_ID, APPLICATION_ID, CLUSTER_TYPE, CLUSTER_ID, VESPA_VERSION); + public static final UnboundStringFlag ZOOKEEPER_SERVER_VERSION = defineStringFlag( "zookeeper-server-version", "3.7.1", // Note: Nodes running Vespa 7 have 3.7.1 as the only available version "ZooKeeper server version, a jar file zookeeper-server-<ZOOKEEPER_SERVER_VERSION>-jar-with-dependencies.jar must exist", @@ -280,6 +296,11 @@ public class PermanentFlags { return Flags.defineStringFlag(flagId, defaultValue, OWNERS, toString(CREATED_AT), toString(EXPIRES_AT), description, modificationEffect, dimensions); } + private static UnboundStringFlag defineStringFlag( + String flagId, String defaultValue, String description, String modificationEffect, Predicate<String> validator, FetchVector.Dimension... dimensions) { + return Flags.defineStringFlag(flagId, defaultValue, OWNERS, toString(CREATED_AT), toString(EXPIRES_AT), description, modificationEffect, validator, dimensions); + } + private static UnboundIntFlag defineIntFlag( String flagId, int defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { return Flags.defineIntFlag(flagId, defaultValue, OWNERS, toString(CREATED_AT), toString(EXPIRES_AT), description, modificationEffect, dimensions); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/UnboundStringFlag.java b/flags/src/main/java/com/yahoo/vespa/flags/UnboundStringFlag.java index f96be55e2eb..9c69e917fa6 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/UnboundStringFlag.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/UnboundStringFlag.java @@ -4,6 +4,10 @@ package com.yahoo.vespa.flags; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.TextNode; +import java.util.function.Consumer; +import java.util.function.Function; +import java.util.function.Predicate; + /** * @author hakonhall */ @@ -13,8 +17,26 @@ public class UnboundStringFlag extends UnboundFlagImpl<String, StringFlag, Unbou } public UnboundStringFlag(FlagId id, String defaultValue, FetchVector defaultFetchVector) { - super(id, defaultValue, defaultFetchVector, - new SimpleFlagSerializer<>(TextNode::new, JsonNode::isTextual, JsonNode::asText), - UnboundStringFlag::new, StringFlag::new); + this(id, defaultValue, defaultFetchVector, + new SimpleFlagSerializer<>(TextNode::new, JsonNode::isTextual, JsonNode::asText)); + } + + public UnboundStringFlag(FlagId id, String defaultValue, Predicate<String> validator) { + this(id, defaultValue, new FetchVector(), validator); + } + + public UnboundStringFlag(FlagId id, String defaultValue, FetchVector fetchVector, Predicate<String> validator) { + this(id, defaultValue, fetchVector, + new SimpleFlagSerializer<>(stringValue -> { + if (!validator.test(stringValue)) + throw new IllegalArgumentException("Invalid value: '" + stringValue + "'"); + return new TextNode(stringValue); + }, + JsonNode::isTextual, JsonNode::asText)); + } + + public UnboundStringFlag(FlagId id, String defaultValue, FetchVector defaultFetchVector, + FlagSerializer<String> serializer) { + super(id, defaultValue, defaultFetchVector, serializer, UnboundStringFlag::new, StringFlag::new); } } diff --git a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java index c8e45f0f61a..29942998083 100644 --- a/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java +++ b/indexinglanguage/src/main/java/com/yahoo/vespa/indexinglanguage/expressions/ScriptExpression.java @@ -91,7 +91,6 @@ public final class ScriptExpression extends ExpressionList<StatementExpression> } /** Creates an expression with simple lingustics for testing */ - @SuppressWarnings("deprecation") public static ScriptExpression fromString(String expression) throws ParseException { return fromString(expression, new SimpleLinguistics(), Embedder.throwsOnUse.asMap()); } diff --git a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cors/CorsLogic.java b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cors/CorsLogic.java index 3f6801eebe7..1ff76fd45ac 100644 --- a/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cors/CorsLogic.java +++ b/jdisc-security-filters/src/main/java/com/yahoo/jdisc/http/filter/security/cors/CorsLogic.java @@ -41,6 +41,6 @@ class CorsLogic { } private static boolean requestOriginMatchesAnyAllowed(String requestOrigin, Set<String> allowedUrls) { - return allowedUrls.stream().anyMatch(requestOrigin::startsWith) || allowedUrls.contains("*"); + return allowedUrls.stream().anyMatch(requestOrigin::equals) || allowedUrls.contains("*"); } } diff --git a/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/cors/CorsPreflightRequestFilterTest.java b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/cors/CorsPreflightRequestFilterTest.java index b5b94d5a2c2..7ba050b7cc0 100644 --- a/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/cors/CorsPreflightRequestFilterTest.java +++ b/jdisc-security-filters/src/test/java/com/yahoo/jdisc/http/filter/security/cors/CorsPreflightRequestFilterTest.java @@ -43,6 +43,14 @@ public class CorsPreflightRequestFilterTest { } @Test + void extended_request_origin_does_not_yield_allow_origin_header_in_response() { + final String ALLOWED_ORIGIN = "https://allowed.origin"; + final String EXTENDED_ORIGIN = "https://allowed.origin.as.subdomain.com"; + HeaderFields headers = doFilterRequest(newRequestFilter(ALLOWED_ORIGIN), EXTENDED_ORIGIN); + assertNull(headers.getFirst(ALLOW_ORIGIN_HEADER)); + } + + @Test void allowed_wildcard_origin_yields_origin_header_in_response() { final String ALLOWED_ORIGIN = "http://allowed.origin"; HeaderFields headers = doFilterRequest(newRequestFilter("*"), ALLOWED_ORIGIN); diff --git a/jrt/src/com/yahoo/jrt/Acceptor.java b/jrt/src/com/yahoo/jrt/Acceptor.java index 8368941007b..d4bdb7007f1 100644 --- a/jrt/src/com/yahoo/jrt/Acceptor.java +++ b/jrt/src/com/yahoo/jrt/Acceptor.java @@ -1,14 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jrt; - import java.nio.channels.ClosedChannelException; import java.nio.channels.ServerSocketChannel; import java.util.concurrent.CountDownLatch; import java.util.logging.Level; import java.util.logging.Logger; - /** * A class used to listen on a network socket. A separate thread is * used to accept connections and register them with the underlying diff --git a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java index dba19b47821..8080dc92729 100644 --- a/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java +++ b/linguistics/src/main/java/com/yahoo/language/opennlp/OpenNlpTokenizer.java @@ -7,24 +7,21 @@ import com.yahoo.language.process.Normalizer; import com.yahoo.language.process.SpecialTokenRegistry; import com.yahoo.language.process.StemMode; import com.yahoo.language.process.Token; -import com.yahoo.language.process.TokenType; import com.yahoo.language.process.Tokenizer; import com.yahoo.language.process.Transformer; import com.yahoo.language.simple.SimpleNormalizer; -import com.yahoo.language.simple.SimpleToken; -import com.yahoo.language.simple.SimpleTokenType; import com.yahoo.language.simple.SimpleTokenizer; import com.yahoo.language.simple.SimpleTransformer; import opennlp.tools.stemmer.Stemmer; import opennlp.tools.stemmer.snowball.SnowballStemmer; -import java.util.ArrayList; import java.util.List; /** * Tokenizer using OpenNlp * * @author matskin + * @author bratseth */ public class OpenNlpTokenizer implements Tokenizer { @@ -51,26 +48,11 @@ public class OpenNlpTokenizer implements Tokenizer { @Override public Iterable<Token> tokenize(String input, Language language, StemMode stemMode, boolean removeAccents) { - if (input.isEmpty()) return List.of(); Stemmer stemmer = stemmerFor(language, stemMode); - if (stemmer == null) return simpleTokenizer.tokenize(input, language, stemMode, removeAccents); - - List<Token> tokens = new ArrayList<>(); - int nextCode = input.codePointAt(0); - TokenType prevType = SimpleTokenType.valueOf(nextCode); - for (int prev = 0, next = Character.charCount(nextCode); next <= input.length(); ) { - nextCode = next < input.length() ? input.codePointAt(next) : SPACE_CODE; - TokenType nextType = SimpleTokenType.valueOf(nextCode); - if (!prevType.isIndexable() || !nextType.isIndexable()) { - String original = input.substring(prev, next); - String token = processToken(original, language, stemMode, removeAccents, stemmer); - tokens.add(new SimpleToken(original).setOffset(prev).setType(prevType).setTokenString(token)); - prev = next; - prevType = nextType; - } - next += Character.charCount(nextCode); - } - return tokens; + if (stemmer == null) + return simpleTokenizer.tokenize(input, language, stemMode, removeAccents); + else + return simpleTokenizer.tokenize(input, token -> processToken(token, language, stemMode, removeAccents, stemmer)); } private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents, diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java index 7479e326b45..b6ca219afc8 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleToken.java @@ -111,45 +111,28 @@ public class SimpleToken implements Token { } @Override - public int hashCode() { - return orig.hashCode(); - } - - @Override - public boolean equals(Object obj) { - if (!(obj instanceof Token)) { - return false; - } - Token rhs = (Token)obj; - if (!getType().equals(rhs.getType())) { - return false; - } - if (!equalsOpt(getOrig(), rhs.getOrig())) { - return false; - } - if (getOffset() != rhs.getOffset()) { - return false; - } - if (!equalsOpt(getScript(), rhs.getScript())) { - return false; - } - if (!equalsOpt(getTokenString(), rhs.getTokenString())) { - return false; - } - if (isSpecialToken() != rhs.isSpecialToken()) { - return false; - } - if (getNumComponents() != rhs.getNumComponents()) { - return false; - } + public boolean equals(Object o) { + if (!(o instanceof Token other)) return false; + + if (getType() != other.getType()) return false; + if (!equalsOpt(getOrig(), other.getOrig())) return false; + if (getOffset() != other.getOffset()) return false; + if (!equalsOpt(getScript(), other.getScript())) return false; + if (!equalsOpt(getTokenString(), other.getTokenString())) return false; + if (isSpecialToken() != other.isSpecialToken()) return false; + if (getNumComponents() != other.getNumComponents()) return false; for (int i = 0, len = getNumComponents(); i < len; ++i) { - if (!equalsOpt(getComponent(i), rhs.getComponent(i))) { + if (!equalsOpt(getComponent(i), other.getComponent(i))) return false; - } } return true; } + @Override + public int hashCode() { + return orig.hashCode(); + } + private static boolean equalsOpt(Object lhs, Object rhs) { if (lhs == null || rhs == null) { return lhs == rhs; diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenType.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenType.java index ace2fd3246e..5c321e4da9b 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenType.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenType.java @@ -10,58 +10,58 @@ public class SimpleTokenType { public static TokenType valueOf(int codePoint) { switch (Character.getType(codePoint)) { - case Character.NON_SPACING_MARK: - // "combining grave accent" - // and "DEVANAGARI VOWEL SIGN SHORT E" etc - // (letter-like) - case Character.COMBINING_SPACING_MARK: - // "DEVANAGARI VOWEL SIGN SHORT O" - // and similar (letter-like) - case Character.LETTER_NUMBER: - // "SMALL ROMAN NUMERAL SIX" etc (letter-like) - case Character.UPPERCASE_LETTER: - case Character.LOWERCASE_LETTER: - case Character.TITLECASE_LETTER: - case Character.MODIFIER_LETTER: - case Character.OTHER_LETTER: - return TokenType.ALPHABETIC; + case Character.NON_SPACING_MARK: + // "combining grave accent" + // and "DEVANAGARI VOWEL SIGN SHORT E" etc + // (letter-like) + case Character.COMBINING_SPACING_MARK: + // "DEVANAGARI VOWEL SIGN SHORT O" + // and similar (letter-like) + case Character.LETTER_NUMBER: + // "SMALL ROMAN NUMERAL SIX" etc (letter-like) + case Character.UPPERCASE_LETTER: + case Character.LOWERCASE_LETTER: + case Character.TITLECASE_LETTER: + case Character.MODIFIER_LETTER: + case Character.OTHER_LETTER: + return TokenType.ALPHABETIC; - case Character.ENCLOSING_MARK: - // "enclosing circle" etc is symbol-like - case Character.MATH_SYMBOL: - case Character.CURRENCY_SYMBOL: - case Character.MODIFIER_SYMBOL: - case Character.OTHER_SYMBOL: - return TokenType.SYMBOL; + case Character.ENCLOSING_MARK: + // "enclosing circle" etc is symbol-like + case Character.MATH_SYMBOL: + case Character.CURRENCY_SYMBOL: + case Character.MODIFIER_SYMBOL: + case Character.OTHER_SYMBOL: + return TokenType.SYMBOL; - case Character.OTHER_NUMBER: - // "SUPERSCRIPT TWO", - // "DINGBAT CIRCLED SANS-SERIF DIGIT THREE" - // and more numbers that should mostly normalize - // to digits - case Character.DECIMAL_DIGIT_NUMBER: - return TokenType.NUMERIC; + case Character.OTHER_NUMBER: + // "SUPERSCRIPT TWO", + // "DINGBAT CIRCLED SANS-SERIF DIGIT THREE" + // and more numbers that should mostly normalize + // to digits + case Character.DECIMAL_DIGIT_NUMBER: + return TokenType.NUMERIC; - case Character.SPACE_SEPARATOR: - case Character.LINE_SEPARATOR: - case Character.PARAGRAPH_SEPARATOR: - return TokenType.SPACE; + case Character.SPACE_SEPARATOR: + case Character.LINE_SEPARATOR: + case Character.PARAGRAPH_SEPARATOR: + return TokenType.SPACE; - case Character.DASH_PUNCTUATION: - case Character.START_PUNCTUATION: - case Character.END_PUNCTUATION: - case Character.CONNECTOR_PUNCTUATION: - case Character.OTHER_PUNCTUATION: - case Character.INITIAL_QUOTE_PUNCTUATION: - case Character.FINAL_QUOTE_PUNCTUATION: - return TokenType.PUNCTUATION; + case Character.DASH_PUNCTUATION: + case Character.START_PUNCTUATION: + case Character.END_PUNCTUATION: + case Character.CONNECTOR_PUNCTUATION: + case Character.OTHER_PUNCTUATION: + case Character.INITIAL_QUOTE_PUNCTUATION: + case Character.FINAL_QUOTE_PUNCTUATION: + return TokenType.PUNCTUATION; - case Character.CONTROL: - case Character.FORMAT: - case Character.SURROGATE: - case Character.PRIVATE_USE: - case Character.UNASSIGNED: - return TokenType.UNKNOWN; + case Character.CONTROL: + case Character.FORMAT: + case Character.SURROGATE: + case Character.PRIVATE_USE: + case Character.UNASSIGNED: + return TokenType.UNKNOWN; } throw new UnsupportedOperationException(String.valueOf(Character.getType(codePoint))); } diff --git a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java index 3dc28b99144..b791c843357 100644 --- a/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java +++ b/linguistics/src/main/java/com/yahoo/language/simple/SimpleTokenizer.java @@ -7,8 +7,8 @@ import com.yahoo.language.process.*; import com.yahoo.language.simple.kstem.KStemmer; import java.util.ArrayList; -import java.util.Collections; import java.util.List; +import java.util.function.Function; import java.util.logging.Logger; import java.util.logging.Level; @@ -49,30 +49,46 @@ public class SimpleTokenizer implements Tokenizer { this.specialTokenRegistry = specialTokenRegistry; } + /** Tokenize the input, applying the transform of this to each token string. */ @Override public Iterable<Token> tokenize(String input, Language language, StemMode stemMode, boolean removeAccents) { - if (input.isEmpty()) return Collections.emptyList(); + return tokenize(input, + token -> processToken(token, language, stemMode, removeAccents)); + } + + /** Tokenize the input, and apply the given transform to each token string. */ + public Iterable<Token> tokenize(String input, Function<String, String> tokenProocessor) { + if (input.isEmpty()) return List.of(); List<Token> tokens = new ArrayList<>(); int nextCode = input.codePointAt(0); TokenType prevType = SimpleTokenType.valueOf(nextCode); + TokenType tokenType = prevType; for (int prev = 0, next = Character.charCount(nextCode); next <= input.length(); ) { nextCode = next < input.length() ? input.codePointAt(next) : SPACE_CODE; TokenType nextType = SimpleTokenType.valueOf(nextCode); if (!prevType.isIndexable() || !nextType.isIndexable()) { String original = input.substring(prev, next); - String token = processToken(original, language, stemMode, removeAccents); tokens.add(new SimpleToken(original).setOffset(prev) - .setType(prevType) - .setTokenString(token)); + .setType(tokenType) + .setTokenString(tokenProocessor.apply(original))); prev = next; prevType = nextType; + tokenType = prevType; + } + else { + tokenType = determineType(tokenType, nextType); } next += Character.charCount(nextCode); } return tokens; } + private TokenType determineType(TokenType tokenType, TokenType characterType) { + if (characterType == TokenType.ALPHABETIC) return TokenType.ALPHABETIC; + return tokenType; + } + private String processToken(String token, Language language, StemMode stemMode, boolean removeAccents) { String original = token; log.log(Level.FINEST, () -> "processToken '" + original + "'"); diff --git a/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpTokenizationTestCase.java b/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpTokenizationTestCase.java index 77489f2eb44..cd2a0f73895 100644 --- a/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpTokenizationTestCase.java +++ b/linguistics/src/test/java/com/yahoo/language/opennlp/OpenNlpTokenizationTestCase.java @@ -4,6 +4,7 @@ package com.yahoo.language.opennlp; import com.yahoo.language.Language; import com.yahoo.language.process.StemMode; import com.yahoo.language.process.Token; +import com.yahoo.language.process.TokenType; import com.yahoo.language.process.Tokenizer; import org.junit.Test; @@ -151,11 +152,9 @@ public class OpenNlpTokenizationTestCase { String input = "tafsirnya\u0648\u0643\u064F\u0646\u0652"; for (StemMode stemMode : new StemMode[] { StemMode.NONE, StemMode.SHORTEST }) { - for (Language l : new Language[] { Language.INDONESIAN, - Language.ENGLISH, Language.ARABIC }) { + for (Language l : List.of(Language.INDONESIAN, Language.ENGLISH, Language.ARABIC)) { for (boolean accentDrop : new boolean[] { true, false }) { - for (Token token : tokenizer.tokenize(input, - l, stemMode, accentDrop)) { + for (Token token : tokenizer.tokenize(input, l, stemMode, accentDrop)) { if (token.getTokenString().length() == 0) { assertFalse(token.isIndexable()); } @@ -165,6 +164,31 @@ public class OpenNlpTokenizationTestCase { } } + @Test + public void testTokenTypes() { + testTokenTypes(Language.ENGLISH); + testTokenTypes(Language.SPANISH); + } + + public void testTokenTypes(Language language) { + assertEquals(TokenType.ALPHABETIC, tokenize("word", language).iterator().next().getType()); + assertEquals(TokenType.NUMERIC, tokenize("123", language).iterator().next().getType()); + assertEquals(TokenType.SPACE, tokenize(" ", language).iterator().next().getType()); + assertEquals(TokenType.PUNCTUATION, tokenize(".", language).iterator().next().getType()); + assertEquals(TokenType.ALPHABETIC, tokenize("123word", language).iterator().next().getType()); + + var tokens = tokenize("123 123word word123", language).iterator(); + assertEquals(TokenType.NUMERIC, tokens.next().getType()); + assertEquals(TokenType.SPACE, tokens.next().getType()); + assertEquals(TokenType.ALPHABETIC, tokens.next().getType()); + assertEquals(TokenType.SPACE, tokens.next().getType()); + assertEquals(TokenType.ALPHABETIC, tokens.next().getType()); + } + + private Iterable<Token> tokenize(String input, Language language) { + return tokenizer.tokenize(input, language, StemMode.SHORTEST, true); + } + private void recurseDecompose(Token t) { assertTrue(t.getOffset() >= 0); assertTrue(t.getOrig().length() >= 0); diff --git a/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java b/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java index 5054f5a9bff..fa8419e200f 100644 --- a/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java +++ b/linguistics/src/test/java/com/yahoo/language/process/GramSplitterTestCase.java @@ -171,21 +171,30 @@ public class GramSplitterTestCase { public void testChineseComma() { String text = "我喜欢红色、蓝色和紫色"; Iterator<GramSplitter.Gram> grams = gramSplitter.split(text, 2); - for (; grams.hasNext(); ) { - System.out.println(grams.next().extractFrom(text)); - } + assertEquals("我喜", grams.next().extractFrom(text)); + assertEquals("喜欢", grams.next().extractFrom(text)); + assertEquals("欢红", grams.next().extractFrom(text)); + assertEquals("红色", grams.next().extractFrom(text)); + assertEquals("蓝色", grams.next().extractFrom(text)); + assertEquals("色和", grams.next().extractFrom(text)); + assertEquals("和紫", grams.next().extractFrom(text)); + assertEquals("紫色", grams.next().extractFrom(text)); } @Test public void testEnglishComma() { String text = "我喜欢红色,蓝色和紫色"; Iterator<GramSplitter.Gram> grams = gramSplitter.split(text, 2); - for (; grams.hasNext(); ) { - System.out.println(grams.next().extractFrom(text)); - } + assertEquals("我喜", grams.next().extractFrom(text)); + assertEquals("喜欢", grams.next().extractFrom(text)); + assertEquals("欢红", grams.next().extractFrom(text)); + assertEquals("红色", grams.next().extractFrom(text)); + assertEquals("蓝色", grams.next().extractFrom(text)); + assertEquals("色和", grams.next().extractFrom(text)); + assertEquals("和紫", grams.next().extractFrom(text)); + assertEquals("紫色", grams.next().extractFrom(text)); } - private void assertGramSplits(String input, int gramSize, String ... expected) { assertEquals(Arrays.asList(expected), gramSplitter.split(input, gramSize).toExtractedList()); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java index a371cdcde25..a112c0d2697 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java @@ -49,9 +49,9 @@ public class NodeAdminImpl implements NodeAdmin { private final Gauge jvmHeapUsed; private final Gauge jvmHeapFree; private final Gauge jvmHeapTotal; - private final Gauge memoryOverhead; private final Gauge containerCount; private final Counter numberOfUnhandledExceptions; + private final Metrics metrics; public NodeAdminImpl(NodeAgentFactory nodeAgentFactory, Metrics metrics, Clock clock, FileSystem fileSystem) { this(nodeAgentContext -> create(clock, nodeAgentFactory, nodeAgentContext), @@ -82,8 +82,8 @@ public class NodeAdminImpl implements NodeAdmin { this.jvmHeapUsed = metrics.declareGauge("mem.heap.used"); this.jvmHeapFree = metrics.declareGauge("mem.heap.free"); this.jvmHeapTotal = metrics.declareGauge("mem.heap.total"); - this.memoryOverhead = metrics.declareGauge("mem.system.overhead"); this.containerCount = metrics.declareGauge("container.count"); + this.metrics = metrics; } @Override @@ -139,7 +139,8 @@ public class NodeAdminImpl implements NodeAdmin { if (!isSuspended) { containerCount.sample(numContainers); ProcMeminfo meminfo = procMeminfoReader.read(); - memoryOverhead.sample(meminfo.memTotalBytes() - meminfo.memAvailableBytes() - totalContainerMemoryBytes); + metrics.declareGauge("mem.system.overhead", new Dimensions(Map.of("containers", Long.toString(numContainers)))) + .sample(meminfo.memTotalBytes() - meminfo.memAvailableBytes() - totalContainerMemoryBytes); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 6408132fe0b..025723fe71c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.nodeagent; +import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; @@ -212,15 +213,15 @@ public class NodeAgentImpl implements NodeAgent { changed = true; } - Optional<DockerImage> actualDockerImage = context.node().wantedDockerImage().filter(n -> containerState == UNKNOWN); - if (!Objects.equals(context.node().currentDockerImage(), actualDockerImage)) { + Optional<DockerImage> wantedDockerImage = context.node().wantedDockerImage().filter(n -> containerState == UNKNOWN); + if (!Objects.equals(context.node().currentDockerImage(), wantedDockerImage)) { DockerImage currentImage = context.node().currentDockerImage().orElse(DockerImage.EMPTY); - DockerImage newImage = actualDockerImage.orElse(DockerImage.EMPTY); + DockerImage newImage = wantedDockerImage.orElse(DockerImage.EMPTY); currentNodeAttributes.withDockerImage(currentImage); - currentNodeAttributes.withVespaVersion(currentImage.tagAsVersion()); + currentNodeAttributes.withVespaVersion(context.node().currentVespaVersion().orElse(Version.emptyVersion)); newNodeAttributes.withDockerImage(newImage); - newNodeAttributes.withVespaVersion(newImage.tagAsVersion()); + newNodeAttributes.withVespaVersion(context.node().wantedVespaVersion().orElse(Version.emptyVersion)); changed = true; } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java index 62bde5ff1a8..4cbfac4842e 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java @@ -25,7 +25,10 @@ public class RestartTest { DockerImage dockerImage = DockerImage.fromString("registry.example.com/repo/image:1.2.3"); try (ContainerTester tester = new ContainerTester(List.of(dockerImage))) { String hostname = "host1.test.yahoo.com"; - NodeSpec nodeSpec = NodeSpec.Builder.testSpec(hostname).wantedDockerImage(dockerImage).build(); + NodeSpec nodeSpec = NodeSpec.Builder.testSpec(hostname) + .wantedDockerImage(dockerImage) + .wantedVespaVersion(dockerImage.tagAsVersion()) + .build(); tester.addChildNodeRepositoryNode(nodeSpec); ContainerName host1 = new ContainerName("host1"); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index e4c2f595164..a7697e5cb5f 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -187,7 +187,7 @@ public class NodeAgentImplTest { inOrder.verify(containerOperations, times(1)).resumeNode(eq(context)); inOrder.verify(healthChecker, times(1)).verifyHealth(eq(context)); inOrder.verify(nodeRepository).updateNodeAttributes( - hostName, new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(dockerImage.tagAsVersion()).withRebootGeneration(0)); + hostName, new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(vespaVersion).withRebootGeneration(0)); inOrder.verify(orchestrator, never()).resume(hostName); } @@ -614,7 +614,7 @@ public class NodeAgentImplTest { inOrder.verify(aclMaintainer, times(1)).converge(eq(context)); inOrder.verify(containerOperations, times(1)).resumeNode(eq(context)); inOrder.verify(nodeRepository).updateNodeAttributes( - hostName, new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(dockerImage.tagAsVersion()).withRebootGeneration(0)); + hostName, new NodeAttributes().withDockerImage(dockerImage).withVespaVersion(vespaVersion).withRebootGeneration(0)); inOrder.verify(orchestrator).resume(hostName); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index 87a9735f91e..309280c8f15 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.restapi; +import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.DockerImage; @@ -10,10 +11,14 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.vespa.applicationmodel.HostName; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.PermanentFlags; +import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Address; +import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.node.TrustStoreItem; import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter; @@ -47,6 +52,7 @@ class NodesResponse extends SlimeJsonResponse { private final boolean recursive; private final Function<HostName, Optional<HostInfo>> orchestrator; private final NodeRepository nodeRepository; + private final StringFlag wantedDockerTagFlag; public NodesResponse(ResponseType responseType, HttpRequest request, Orchestrator orchestrator, NodeRepository nodeRepository) { @@ -56,6 +62,7 @@ class NodesResponse extends SlimeJsonResponse { this.recursive = request.getBooleanProperty("recursive"); this.orchestrator = orchestrator.getHostResolver(); this.nodeRepository = nodeRepository; + this.wantedDockerTagFlag = PermanentFlags.WANTED_DOCKER_TAG.bindTo(nodeRepository.flagSource()); Cursor root = slime.setObject(); switch (responseType) { @@ -146,7 +153,7 @@ class NodesResponse extends SlimeJsonResponse { toSlime(allocation.membership(), object.setObject("membership")); object.setLong("restartGeneration", allocation.restartGeneration().wanted()); object.setLong("currentRestartGeneration", allocation.restartGeneration().current()); - object.setString("wantedDockerImage", nodeRepository.containerImages().get(node).withTag(allocation.membership().cluster().vespaVersion()).asString()); + object.setString("wantedDockerImage", nodeRepository.containerImages().get(node).withTag(resolveVersionFlag(wantedDockerTagFlag, node, allocation)).asString()); object.setString("wantedVespaVersion", allocation.membership().cluster().vespaVersion().toFullString()); NodeResourcesSerializer.toSlime(allocation.requestedResources(), object.setObject("requestedResources")); allocation.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, object.setArray("networkPorts"))); @@ -189,6 +196,24 @@ class NodesResponse extends SlimeJsonResponse { node.cloudAccount().ifPresent(cloudAccount -> object.setString("cloudAccount", cloudAccount.value())); } + private Version resolveVersionFlag(StringFlag flag, Node node, Allocation allocation) { + String value = flag + .with(FetchVector.Dimension.HOSTNAME, node.hostname()) + .with(FetchVector.Dimension.NODE_TYPE, node.type().name()) + .with(FetchVector.Dimension.TENANT_ID, allocation.owner().tenant().value()) + .with(FetchVector.Dimension.APPLICATION_ID, allocation.owner().serializedForm()) + .with(FetchVector.Dimension.CLUSTER_TYPE, allocation.membership().cluster().type().name()) + .with(FetchVector.Dimension.CLUSTER_ID, allocation.membership().cluster().id().value()) + .with(FetchVector.Dimension.VESPA_VERSION, allocation.membership().cluster().vespaVersion().toFullString()) + .value(); + + return value.isEmpty() ? + allocation.membership().cluster().vespaVersion() : + value.indexOf('.') == -1 ? + allocation.membership().cluster().vespaVersion().withQualifier(value) : + new Version(value); + } + private void toSlime(ApplicationId id, Cursor object) { object.setString("tenant", id.tenant().value()); object.setString("application", id.application().value()); diff --git a/parent/pom.xml b/parent/pom.xml index a83c7f50960..cab654eb4a5 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -1066,7 +1066,7 @@ <maven-site-plugin.version>3.9.1</maven-site-plugin.version> <maven-source-plugin.version>3.2.1</maven-source-plugin.version> <mockito.version>4.0.0</mockito.version> - <onnxruntime.version>1.11.0</onnxruntime.version> <!-- WARNING: sync cloud-tenant-base-dependencies-enforcer/pom.xml --> + <onnxruntime.version>1.12.1</onnxruntime.version> <!-- WARNING: sync cloud-tenant-base-dependencies-enforcer/pom.xml --> <org.json.version>20220320</org.json.version> <org.lz4.version>1.8.0</org.lz4.version> <prometheus.client.version>0.6.0</prometheus.client.version> diff --git a/screwdriver.yaml b/screwdriver.yaml index 0ffc3316f95..21ab6625eb4 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -283,6 +283,17 @@ jobs: exit 1 fi + verify-opensource-rpm-installable: + image: quay.io/centos/centos:stream8 + annotations: + screwdriver.cd/buildPeriodically: H 0 * * * + steps: + - install: | + dnf config-manager --add-repo https://raw.githubusercontent.com/vespa-engine/vespa/master/dist/vespa-engine.repo + dnf config-manager --enable powertools + dnf install -y epel-release + dnf install -y vespa + mirror-copr-rpms-to-artifactory: annotations: screwdriver.cd/cpu: LOW diff --git a/searchcore/src/apps/verify_ranksetup/CMakeLists.txt b/searchcore/src/apps/verify_ranksetup/CMakeLists.txt index cc8434ef46f..536392739d9 100644 --- a/searchcore/src/apps/verify_ranksetup/CMakeLists.txt +++ b/searchcore/src/apps/verify_ranksetup/CMakeLists.txt @@ -1,12 +1,20 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_verify_ranksetup_app +vespa_add_library(searchcore_verify_ranksetup SOURCES verify_ranksetup.cpp - OUTPUT_NAME vespa-verify-ranksetup-bin - INSTALL bin + INSTALL lib64 DEPENDS searchcore_fconfig searchcore_matching searchcore_documentmetastore ) -vespa_generate_config(searchcore_verify_ranksetup_app verify-ranksetup.def) +vespa_generate_config(searchcore_verify_ranksetup verify-ranksetup.def) + +vespa_add_executable(searchcore_verify_ranksetup_app + SOURCES + verify_ranksetup_app.cpp + OUTPUT_NAME vespa-verify-ranksetup-bin + INSTALL bin + DEPENDS + searchcore_verify_ranksetup +) diff --git a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp index e72771f5aa6..dcfbc34653d 100644 --- a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp +++ b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp @@ -1,5 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "verify_ranksetup.h" #include "config-verify-ranksetup.h" #include <vespa/config-attributes.h> #include <vespa/config-indexschema.h> @@ -21,12 +22,10 @@ #include <vespa/searchlib/fef/fef.h> #include <vespa/searchlib/fef/test/plugin/setup.h> #include <vespa/config/subscription/configsubscriber.hpp> -#include <vespa/vespalib/util/signalhandler.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/stllike/asciistream.h> #include <optional> -#include <vespa/log/log.h> -LOG_SETUP("vespa-verify-ranksetup"); - using config::ConfigContext; using config::ConfigHandle; using config::ConfigRuntimeException; @@ -50,8 +49,10 @@ using vespalib::eval::SimpleConstantValue; using vespalib::eval::TensorSpec; using vespalib::eval::Value; using vespalib::eval::ValueType; +using vespalib::make_string_short::fmt; -std::optional<vespalib::string> get_file(const vespalib::string &ref, const VerifyRanksetupConfig &myCfg) { +std::optional<vespalib::string> +get_file(const vespalib::string &ref, const VerifyRanksetupConfig &myCfg) { for (const auto &entry: myCfg.file) { if (ref == entry.ref) { return entry.path; @@ -60,37 +61,43 @@ std::optional<vespalib::string> get_file(const vespalib::string &ref, const Veri return std::nullopt; } -RankingExpressions make_expressions(const RankingExpressionsConfig &expressionsCfg, const VerifyRanksetupConfig &myCfg) { +RankingExpressions +make_expressions(const RankingExpressionsConfig &expressionsCfg, const VerifyRanksetupConfig &myCfg, + std::vector<search::fef::Message> & messages) { RankingExpressions expressions; for (const auto &entry: expressionsCfg.expression) { if (auto file = get_file(entry.fileref, myCfg)) { - LOG(debug, "Add expression %s with ref=%s and path=%s", entry.name.c_str(), entry.fileref.c_str(), file.value().c_str()); expressions.add(entry.name, file.value()); } else { - LOG(warning, "could not find file name for ranking expression '%s' (ref:'%s')", - entry.name.c_str(), entry.fileref.c_str()); + messages.emplace_back(search::fef::Level::WARNING, + fmt("could not find file name for ranking expression '%s' (ref:'%s')", + entry.name.c_str(), entry.fileref.c_str())); } } return expressions; } -OnnxModels make_models(const OnnxModelsConfig &modelsCfg, const VerifyRanksetupConfig &myCfg) { +OnnxModels +make_models(const OnnxModelsConfig &modelsCfg, const VerifyRanksetupConfig &myCfg, + std::vector<search::fef::Message> & messages) { OnnxModels::Vector model_list; for (const auto &entry: modelsCfg.model) { if (auto file = get_file(entry.fileref, myCfg)) { model_list.emplace_back(entry.name, file.value()); OnnxModels::configure(entry, model_list.back()); } else { - LOG(warning, "could not find file name for onnx model '%s' (ref:'%s')", - entry.name.c_str(), entry.fileref.c_str()); + messages.emplace_back(search::fef::Level::WARNING, + fmt("could not find file name for onnx model '%s' (ref:'%s')", + entry.name.c_str(), entry.fileref.c_str())); } } return OnnxModels(model_list); } -class App +class VerifyRankSetup { -public: +private: + std::vector<search::fef::Message> _messages; bool verify(const search::index::Schema &schema, const search::fef::Properties &props, const IConstantValueRepo &repo, @@ -105,30 +112,42 @@ public: const RankingExpressionsConfig &expressionsCfg, const OnnxModelsConfig &modelsCfg); - int usage(); - int main(int argc, char **argv); +public: + VerifyRankSetup(); + ~VerifyRankSetup(); + const std::vector<search::fef::Message> & getMessages() const { return _messages; } + bool verify(const std::string & configId); }; struct DummyConstantValueRepo : IConstantValueRepo { const RankingConstantsConfig &cfg; DummyConstantValueRepo(const RankingConstantsConfig &cfg_in) : cfg(cfg_in) {} - virtual vespalib::eval::ConstantValue::UP getConstant(const vespalib::string &name) const override { - for (const auto &entry: cfg.constant) { - if (entry.name == name) { - try { - auto tensor = vespalib::eval::value_from_spec(TensorSpec(entry.type), FastValueBuilderFactory::get()); - return std::make_unique<SimpleConstantValue>(std::move(tensor)); - } catch (std::exception &) { - return std::make_unique<BadConstantValue>(); - } + vespalib::eval::ConstantValue::UP getConstant(const vespalib::string &name) const override; +}; + +vespalib::eval::ConstantValue::UP +DummyConstantValueRepo::getConstant(const vespalib::string &name) const { + for (const auto &entry: cfg.constant) { + if (entry.name == name) { + try { + auto tensor = vespalib::eval::value_from_spec(TensorSpec(entry.type), FastValueBuilderFactory::get()); + return std::make_unique<SimpleConstantValue>(std::move(tensor)); + } catch (std::exception &) { + return std::make_unique<BadConstantValue>(); } } - return vespalib::eval::ConstantValue::UP(nullptr); } -}; + return vespalib::eval::ConstantValue::UP(nullptr); +} + +VerifyRankSetup::VerifyRankSetup() + : _messages() +{ } + +VerifyRankSetup::~VerifyRankSetup() = default; bool -App::verify(const search::index::Schema &schema, +VerifyRankSetup::verify(const search::index::Schema &schema, const search::fef::Properties &props, const IConstantValueRepo &repo, const RankingExpressions &expressions, @@ -143,25 +162,25 @@ App::verify(const search::index::Schema &schema, rankSetup.configure(); // reads config values from the property map bool ok = true; if (!rankSetup.getFirstPhaseRank().empty()) { - ok = verifyFeature(factory, indexEnv, rankSetup.getFirstPhaseRank(), "first phase ranking") && ok; + ok = verifyFeature(factory, indexEnv, rankSetup.getFirstPhaseRank(), "first phase ranking", _messages) && ok; } if (!rankSetup.getSecondPhaseRank().empty()) { - ok = verifyFeature(factory, indexEnv, rankSetup.getSecondPhaseRank(), "second phase ranking") && ok; + ok = verifyFeature(factory, indexEnv, rankSetup.getSecondPhaseRank(), "second phase ranking", _messages) && ok; } for (size_t i = 0; i < rankSetup.getSummaryFeatures().size(); ++i) { - ok = verifyFeature(factory, indexEnv, rankSetup.getSummaryFeatures()[i], "summary features") && ok; + ok = verifyFeature(factory, indexEnv, rankSetup.getSummaryFeatures()[i], "summary features", _messages) && ok; } for (const auto & feature : rankSetup.get_match_features()) { - ok = verifyFeature(factory, indexEnv, feature, "match features") && ok; + ok = verifyFeature(factory, indexEnv, feature, "match features", _messages) && ok; } for (size_t i = 0; i < rankSetup.getDumpFeatures().size(); ++i) { - ok = verifyFeature(factory, indexEnv, rankSetup.getDumpFeatures()[i], "dump features") && ok; + ok = verifyFeature(factory, indexEnv, rankSetup.getDumpFeatures()[i], "dump features", _messages) && ok; } return ok; } bool -App::verifyConfig(const VerifyRanksetupConfig &myCfg, +VerifyRankSetup::verifyConfig(const VerifyRanksetupConfig &myCfg, const RankProfilesConfig &rankCfg, const IndexschemaConfig &schemaCfg, const AttributesConfig &attributeCfg, @@ -174,8 +193,8 @@ App::verifyConfig(const VerifyRanksetupConfig &myCfg, search::index::SchemaBuilder::build(schemaCfg, schema); search::index::SchemaBuilder::build(attributeCfg, schema); DummyConstantValueRepo repo(constantsCfg); - auto expressions = make_expressions(expressionsCfg, myCfg); - auto models = make_models(modelsCfg, myCfg); + auto expressions = make_expressions(expressionsCfg, myCfg, _messages); + auto models = make_models(modelsCfg, myCfg, _messages); for(size_t i = 0; i < rankCfg.rankprofile.size(); i++) { search::fef::Properties properties; const RankProfilesConfig::Rankprofile &profile = rankCfg.rankprofile[i]; @@ -184,33 +203,20 @@ App::verifyConfig(const VerifyRanksetupConfig &myCfg, profile.fef.property[j].value); } if (verify(schema, properties, repo, expressions, models)) { - LOG(info, "rank profile '%s': pass", profile.name.c_str()); + _messages.emplace_back(search::fef::Level::INFO, + fmt("rank profile '%s': pass", profile.name.c_str())); } else { - LOG(error, "rank profile '%s': FAIL", profile.name.c_str()); + _messages.emplace_back(search::fef::Level::ERROR, + fmt("rank profile '%s': FAIL", profile.name.c_str())); ok = false; } } return ok; } -int -App::usage() -{ - fprintf(stderr, "Usage: vespa-verify-ranksetup <config-id>\n"); - return 1; -} - -int -App::main(int argc, char **argv) +bool +VerifyRankSetup::verify(const std::string & configid) { - if (argc != 2) { - return usage(); - } - - const std::string configid = argv[1]; - LOG(debug, "verifying rank setup for config id '%s' ...", - configid.c_str()); - bool ok = false; try { auto ctx = std::make_shared<ConfigContext>(*config::legacyConfigId2Spec(configid)); @@ -233,18 +239,19 @@ App::main(int argc, char **argv) *expressionsHandle->getConfig(), *modelsHandle->getConfig()); } catch (ConfigRuntimeException & e) { - LOG(error, "Unable to subscribe to config: %s", e.getMessage().c_str()); + _messages.emplace_back(search::fef::Level::ERROR, + fmt("Unable to subscribe to config: %s", e.getMessage().c_str())); } catch (InvalidConfigException & e) { - LOG(error, "Error getting config: %s", e.getMessage().c_str()); - } - if (!ok) { - return 1; + _messages.emplace_back(search::fef::Level::ERROR, + fmt("Error getting config: %s", e.getMessage().c_str())); } - return 0; + return ok; } -int main(int argc, char **argv) { - vespalib::SignalHandler::PIPE.ignore(); - App app; - return app.main(argc, argv); +std::pair<bool, std::vector<search::fef::Message>> +verifyRankSetup(const char * configId) { + VerifyRankSetup verifier; + bool ok = verifier.verify(configId); + + return {ok, verifier.getMessages()}; } diff --git a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.h b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.h new file mode 100644 index 00000000000..d65dede8696 --- /dev/null +++ b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.h @@ -0,0 +1,7 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchlib/fef/verify_feature.h> + +std::pair<bool, std::vector<search::fef::Message>> verifyRankSetup(const char * configId); diff --git a/searchcore/src/apps/verify_ranksetup/verify_ranksetup_app.cpp b/searchcore/src/apps/verify_ranksetup/verify_ranksetup_app.cpp new file mode 100644 index 00000000000..4f1bf1acbed --- /dev/null +++ b/searchcore/src/apps/verify_ranksetup/verify_ranksetup_app.cpp @@ -0,0 +1,56 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "verify_ranksetup.h" +#include <vespa/vespalib/util/signalhandler.h> + +#include <vespa/log/log.h> +LOG_SETUP("vespa-verify-ranksetup"); + +class App +{ +public: + int usage(); + int main(int argc, char **argv); +}; + +int +App::usage() +{ + fprintf(stderr, "Usage: vespa-verify-ranksetup <config-id>\n"); + return 1; +} + +namespace { +ns_log::Logger::LogLevel +toLogLevel(search::fef::Level level) { + switch (level) { + case search::fef::Level::INFO: + return ns_log::Logger::LogLevel::info; + case search::fef::Level::WARNING: + return ns_log::Logger::LogLevel::warning; + case search::fef::Level::ERROR: + return ns_log::Logger::LogLevel::error; + } + abort(); +} +} +int +App::main(int argc, char **argv) +{ + if (argc != 2) { + return usage(); + } + + auto [ok, messages] = verifyRankSetup(argv[1]); + + for (const auto & msg : messages) { + VLOG(toLogLevel(msg.first), "%s", msg.second.c_str()); + } + return ok ? 0 : 1; +} + +int main(int argc, char **argv) { + vespalib::SignalHandler::PIPE.ignore(); + App app; + return app.main(argc, argv); +} diff --git a/searchcore/src/vespa/searchcore/bmcluster/bucket_db_snapshot.cpp b/searchcore/src/vespa/searchcore/bmcluster/bucket_db_snapshot.cpp index 3ddb1afdb35..486472aec0f 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/bucket_db_snapshot.cpp +++ b/searchcore/src/vespa/searchcore/bmcluster/bucket_db_snapshot.cpp @@ -2,7 +2,6 @@ #include "bucket_db_snapshot.h" #include <vespa/persistence/spi/persistenceprovider.h> -#include <vespa/vespalib/stllike/hash_set.hpp> #include <vespa/vespalib/stllike/hash_map.hpp> #include <cassert> @@ -66,9 +65,4 @@ BucketDbSnapshot::try_get_bucket_info(BucketId bucket_id) const } -namespace vespalib { - -template class hash_map<BucketId, BucketInfo, BucketId::hash>; -template class hash_set<BucketId, BucketId::hash>; - -} +VESPALIB_HASH_MAP_INSTANTIATE_H(document::BucketId, storage::spi::BucketInfo, document::BucketId::hash); diff --git a/searchcore/src/vespa/searchcore/bmcluster/bucket_db_snapshot.h b/searchcore/src/vespa/searchcore/bmcluster/bucket_db_snapshot.h index d750a73f207..189250194b4 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/bucket_db_snapshot.h +++ b/searchcore/src/vespa/searchcore/bmcluster/bucket_db_snapshot.h @@ -22,6 +22,10 @@ class BucketDbSnapshot public: using BucketIdSet = vespalib::hash_set<document::BucketId, document::BucketId::hash>; BucketDbSnapshot(); + BucketDbSnapshot(const BucketDbSnapshot &) = delete; + BucketDbSnapshot & operator=(const BucketDbSnapshot &) = delete; + BucketDbSnapshot(BucketDbSnapshot &&) noexcept = default; + BucketDbSnapshot & operator=(BucketDbSnapshot &&) noexcept = default; ~BucketDbSnapshot(); void populate(document::BucketSpace bucket_space, storage::spi::PersistenceProvider& provider); uint32_t count_new_documents(const BucketDbSnapshot &old) const; diff --git a/searchcore/src/vespa/searchcore/bmcluster/bucket_db_snapshot_vector.h b/searchcore/src/vespa/searchcore/bmcluster/bucket_db_snapshot_vector.h index 7799ed8fc58..df6183fd6f9 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/bucket_db_snapshot_vector.h +++ b/searchcore/src/vespa/searchcore/bmcluster/bucket_db_snapshot_vector.h @@ -16,8 +16,9 @@ class BucketDbSnapshotVector vespalib::hash_map<document::BucketSpace, std::vector<BucketDbSnapshot>, document::BucketSpace::hash> _snapshots; using BucketIdSet = BucketDbSnapshot::BucketIdSet; public: - BucketDbSnapshotVector(const std::vector<storage::spi::PersistenceProvider *>& providers, const storage::lib::ClusterStateBundle &cluster_state_bundle); + BucketDbSnapshotVector(const BucketDbSnapshotVector &) = delete; + BucketDbSnapshotVector & operator = (const BucketDbSnapshotVector &) = delete; ~BucketDbSnapshotVector(); uint32_t count_moved_documents(const BucketDbSnapshotVector &old) const; uint32_t count_lost_unique_documents(const BucketDbSnapshotVector &old) const; diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp index a99d5cbd573..0fee6e494fe 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.cpp @@ -14,6 +14,7 @@ namespace proton { using bucketdb::RemoveBatchEntry; + BucketDB::BucketDB() : _map(), _cachedBucketId(), @@ -95,7 +96,6 @@ BucketDB::modify(const GlobalId &gid, } } - bucketdb::BucketState BucketDB::get(BucketId bucketId) const { diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h index bd6aa16504b..7c1c336ab48 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketdb.h @@ -13,15 +13,14 @@ namespace proton { class BucketDB { -public: - typedef document::GlobalId GlobalId; - typedef document::BucketId BucketId; - typedef storage::spi::Timestamp Timestamp; - typedef storage::spi::BucketChecksum BucketChecksum; - typedef bucketdb::BucketState BucketState; - typedef vespalib::hash_map<BucketId, BucketState, BucketId::hash> Map; - private: + using GlobalId = document::GlobalId; + using BucketId = document::BucketId; + using Timestamp = storage::spi::Timestamp; + using BucketChecksum = storage::spi::BucketChecksum; + using BucketState = bucketdb::BucketState; + using Map = vespalib::hash_map<BucketId, BucketState, document::BucketId::hash>; + Map _map; BucketId _cachedBucketId; BucketState _cachedBucketState; diff --git a/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp b/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp index 4e78bf04df6..697028e42cc 100644 --- a/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp +++ b/searchcore/src/vespa/searchcore/proton/common/handlermap.hpp @@ -31,7 +31,7 @@ public: * using event barriers to resolve visibility constraints in the * future without changing the external API. **/ - class Snapshot : public vespalib::Sequence<T*> + class Snapshot final : public vespalib::Sequence<T*> { private: std::vector<HandlerSP> _handlers; @@ -41,7 +41,7 @@ public: Snapshot() : _handlers(), _offset(0) { } Snapshot(const StdMap &map) : _handlers(), _offset(0) { _handlers.reserve(map.size()); - for (auto itr : map) { + for (const auto & itr : map) { _handlers.push_back(itr.second); } } @@ -51,11 +51,39 @@ public: Snapshot(const Snapshot &) = delete; Snapshot & operator = (const Snapshot &) = delete; + size_t size() const { return _handlers.size(); } + bool valid() const override { return (_offset < _handlers.size()); } T *get() const override { return _handlers[_offset].get(); } void next() override { ++_offset; } }; + class UnsafeSnapshot final : public vespalib::Sequence<T*> + { + private: + std::vector<T *> _handlers; + size_t _offset; + + public: + UnsafeSnapshot() : _handlers(), _offset(0) { } + UnsafeSnapshot(const StdMap &map) : _handlers(), _offset(0) { + _handlers.reserve(map.size()); + for (const auto & itr : map) { + _handlers.push_back(itr.second.get()); + } + } + UnsafeSnapshot(UnsafeSnapshot &&) noexcept = default; + UnsafeSnapshot & operator = (UnsafeSnapshot &&) noexcept = default; + UnsafeSnapshot(const UnsafeSnapshot &) = delete; + UnsafeSnapshot & operator = (const UnsafeSnapshot &) = delete; + + size_t size() const { return _handlers.size(); } + + bool valid() const override { return (_offset < _handlers.size()); } + T *get() const override { return _handlers[_offset]; } + void next() override { ++_offset; } + }; + /** * Convenience typedefs. */ @@ -170,12 +198,19 @@ public: **/ Snapshot snapshot() const { return Snapshot(_handlers); } + /** + * Create a snapshot of the handlers currently contained in this + * map and return it as a sequence. Note that any lifetime guarantees + * must be be given at a higher level + * + * @return handler sequence + **/ + UnsafeSnapshot unsafeSnapshot() const { return UnsafeSnapshot(_handlers); } + // we want to use snapshots rather than direct iteration to reduce locking; // the below functions should be deprecated when possible. - iterator begin() { return _handlers.begin(); } const_iterator begin() const { return _handlers.begin(); } - iterator end() { return _handlers.end(); } const_iterator end() const { return _handlers.end(); } size_t size() const { return _handlers.size(); } }; diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp index 011d97d4609..fd3deb2abd7 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp @@ -367,7 +367,7 @@ FlushEngine::initFlush(const FlushContext &ctx) void FlushEngine::flushDone(const FlushContext &ctx, uint32_t taskId) { - vespalib::duration duration = vespalib::duration::zero(); + vespalib::duration duration; { std::lock_guard<std::mutex> guard(_lock); duration = _flushing[taskId].elapsed(); @@ -422,7 +422,7 @@ FlushEngine::getCurrentlyFlushingSet() const uint32_t FlushEngine::initFlush(const IFlushHandler::SP &handler, const IFlushTarget::SP &target) { - uint32_t taskId(0); + uint32_t taskId; { std::lock_guard<std::mutex> guard(_lock); taskId = _taskId++; diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp index 826860b743e..108397e6e5a 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp @@ -32,10 +32,12 @@ using search::MatchingElements; using search::attribute::IAttributeContext; using search::fef::MatchDataLayout; using search::fef::MatchData; +using search::fef::RankSetup; using search::fef::indexproperties::hitcollector::HeapSize; using search::queryeval::Blueprint; using search::queryeval::SearchIterator; using vespalib::Doom; +using vespalib::make_string_short::fmt; namespace proton::matching { @@ -117,7 +119,8 @@ Matcher::Matcher(const search::index::Schema &schema, const Properties &props, c _rankSetup = std::make_shared<search::fef::RankSetup>(_blueprintFactory, _indexEnv); _rankSetup->configure(); // reads config values from the property map if (!_rankSetup->compile()) { - throw vespalib::IllegalArgumentException("failed to compile rank setup", VESPA_STRLOC); + throw vespalib::IllegalArgumentException(fmt("failed to compile rank setup :\n%s", + _rankSetup->getJoinedWarnings().c_str()), VESPA_STRLOC); } } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp index cc619c3a09a..1bcb96702a4 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.cpp @@ -6,6 +6,7 @@ namespace proton { using HandlerSnapshot = PersistenceHandlerMap::HandlerSnapshot; +using UnsafeHandlerSnapshot = PersistenceHandlerMap::UnsafeHandlerSnapshot; PersistenceHandlerMap::PersistenceHandlerMap() : _map() @@ -51,8 +52,7 @@ PersistenceHandlerMap::getHandlerSnapshot() const handlers.push_back(handlerItr.second); } } - size_t handlersSize = handlers.size(); - return HandlerSnapshot(DocTypeToHandlerMap::Snapshot(std::move(handlers)), handlersSize); + return HandlerSnapshot(DocTypeToHandlerMap::Snapshot(std::move(handlers))); } HandlerSnapshot @@ -60,9 +60,22 @@ PersistenceHandlerMap::getHandlerSnapshot(document::BucketSpace bucketSpace) con { auto itr = _map.find(bucketSpace); if (itr != _map.end()) { - return HandlerSnapshot(itr->second.snapshot(), itr->second.size()); + return HandlerSnapshot(itr->second.snapshot()); } return HandlerSnapshot(); } +UnsafeHandlerSnapshot +PersistenceHandlerMap::getUnsafeHandlerSnapshot(document::BucketSpace bucketSpace) const +{ + auto itr = _map.find(bucketSpace); + if (itr != _map.end()) { + return UnsafeHandlerSnapshot(itr->second.unsafeSnapshot()); + } + return UnsafeHandlerSnapshot(); +} + +PersistenceHandlerMap::HandlerSnapshot::~HandlerSnapshot() = default; +PersistenceHandlerMap::UnsafeHandlerSnapshot::~UnsafeHandlerSnapshot() = default; + } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.h index 749f044ff0e..807b0f0b5d7 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistence_handler_map.h @@ -21,30 +21,42 @@ class IPersistenceHandler; class PersistenceHandlerMap { public: using DocTypeToHandlerMap = HandlerMap<IPersistenceHandler>; - using PersistenceHandlerSequence = DocTypeToHandlerMap::Snapshot; using PersistenceHandlerSP = std::shared_ptr<IPersistenceHandler>; class HandlerSnapshot { private: - PersistenceHandlerSequence _handlers; - size_t _size; + DocTypeToHandlerMap::Snapshot _handlers; public: - HandlerSnapshot() : _handlers(), _size(0) {} - HandlerSnapshot(DocTypeToHandlerMap::Snapshot handlers_, size_t size_) - : _handlers(std::move(handlers_)), - _size(size_) + HandlerSnapshot() : _handlers() {} + HandlerSnapshot(DocTypeToHandlerMap::Snapshot handlers_) + : _handlers(std::move(handlers_)) {} HandlerSnapshot(const HandlerSnapshot &) = delete; HandlerSnapshot & operator = (const HandlerSnapshot &) = delete; + ~HandlerSnapshot(); - size_t size() const { return _size; } - PersistenceHandlerSequence &handlers() { return _handlers; } - static PersistenceHandlerSequence release(HandlerSnapshot &&rhs) { return std::move(rhs._handlers); } + size_t size() const { return _handlers.size(); } + vespalib::Sequence<IPersistenceHandler*> &handlers() { return _handlers; } + static DocTypeToHandlerMap::Snapshot release(HandlerSnapshot &&rhs) { return std::move(rhs._handlers); } }; -private: + class UnsafeHandlerSnapshot { + private: + DocTypeToHandlerMap::UnsafeSnapshot _handlers; + public: + UnsafeHandlerSnapshot() : _handlers() {} + UnsafeHandlerSnapshot(DocTypeToHandlerMap::UnsafeSnapshot handlers_) + : _handlers(std::move(handlers_)) + {} + UnsafeHandlerSnapshot(const UnsafeHandlerSnapshot &) = delete; + UnsafeHandlerSnapshot & operator = (const UnsafeHandlerSnapshot &) = delete; + ~UnsafeHandlerSnapshot(); + size_t size() const { return _handlers.size(); } + vespalib::Sequence<IPersistenceHandler*> &handlers() { return _handlers; } + }; +private: struct BucketSpaceHash { std::size_t operator() (const document::BucketSpace &bucketSpace) const { return bucketSpace.getId(); } }; @@ -59,6 +71,7 @@ public: IPersistenceHandler * getHandler(document::BucketSpace bucketSpace, const DocTypeName &docType) const; HandlerSnapshot getHandlerSnapshot() const; HandlerSnapshot getHandlerSnapshot(document::BucketSpace bucketSpace) const; + UnsafeHandlerSnapshot getUnsafeHandlerSnapshot(document::BucketSpace bucketSpace) const; }; } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index 81a7244ba1d..df3dcdcf66a 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -5,7 +5,6 @@ #include "transport_latch.h" #include <vespa/persistence/spi/bucketexecutor.h> #include <vespa/persistence/spi/catchresult.h> -#include <vespa/vespalib/stllike/hash_set.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/document/update/documentupdate.h> @@ -200,11 +199,16 @@ PersistenceEngine::getHandlerSnapshot(const WriteGuard &) const } PersistenceEngine::HandlerSnapshot -PersistenceEngine::getHandlerSnapshot(const ReadGuard &, document::BucketSpace bucketSpace) const +PersistenceEngine::getSafeHandlerSnapshot(const ReadGuard &, document::BucketSpace bucketSpace) const { return _handlers.getHandlerSnapshot(bucketSpace); } +PersistenceEngine::UnsafeHandlerSnapshot +PersistenceEngine::getHandlerSnapshot(const ReadGuard &, document::BucketSpace bucketSpace) const { + return _handlers.getUnsafeHandlerSnapshot(bucketSpace); +} + PersistenceEngine::HandlerSnapshot PersistenceEngine::getHandlerSnapshot(const WriteGuard &, document::BucketSpace bucketSpace) const { @@ -279,7 +283,7 @@ PersistenceEngine::listBuckets(BucketSpace bucketSpace) const // Runs in SPI thread. // No handover to write threads in persistence handlers. ReadGuard rguard(_rwMutex); - HandlerSnapshot snap = getHandlerSnapshot(rguard, bucketSpace); + auto snap = getHandlerSnapshot(rguard, bucketSpace); BucketIdListResultHandler resultHandler; for (; snap.handlers().valid(); snap.handlers().next()) { IPersistenceHandler *handler = snap.handlers().get(); @@ -294,7 +298,7 @@ PersistenceEngine::setClusterState(BucketSpace bucketSpace, const ClusterState & { ReadGuard rguard(_rwMutex); saveClusterState(bucketSpace, calc); - HandlerSnapshot snap = getHandlerSnapshot(rguard, bucketSpace); + auto snap = getHandlerSnapshot(rguard, bucketSpace); auto catchResult = std::make_unique<storage::spi::CatchResult>(); auto futureResult = catchResult->future_result(); GenericResultHandler resultHandler(snap.size(), std::move(catchResult)); @@ -312,7 +316,7 @@ void PersistenceEngine::setActiveStateAsync(const Bucket & bucket, BucketInfo::ActiveState newState, OperationComplete::UP onComplete) { ReadGuard rguard(_rwMutex); - HandlerSnapshot snap = getHandlerSnapshot(rguard, bucket.getBucketSpace()); + auto snap = getHandlerSnapshot(rguard, bucket.getBucketSpace()); auto resultHandler = std::make_shared<GenericResultHandler>(snap.size(), std::move(onComplete)); while (snap.handlers().valid()) { IPersistenceHandler *handler = snap.handlers().get(); @@ -332,7 +336,7 @@ PersistenceEngine::getBucketInfo(const Bucket& b) const // Runs in SPI thread. // No handover to write threads in persistence handlers. ReadGuard rguard(_rwMutex); - HandlerSnapshot snap = getHandlerSnapshot(rguard, b.getBucketSpace()); + auto snap = getHandlerSnapshot(rguard, b.getBucketSpace()); BucketInfoResultHandler resultHandler; for (; snap.handlers().valid(); snap.handlers().next()) { IPersistenceHandler *handler = snap.handlers().get(); @@ -482,10 +486,10 @@ PersistenceEngine::GetResult PersistenceEngine::get(const Bucket& b, const document::FieldSet& fields, const DocumentId& did, Context& context) const { ReadGuard rguard(_rwMutex); - HandlerSnapshot snapshot = getHandlerSnapshot(rguard, b.getBucketSpace()); + auto snap = getHandlerSnapshot(rguard, b.getBucketSpace()); - for (PersistenceHandlerSequence & handlers = snapshot.handlers(); handlers.valid(); handlers.next()) { - IPersistenceHandler::RetrieversSP retrievers = handlers.get()->getDocumentRetrievers(context.getReadConsistency()); + for (;snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler::RetrieversSP retrievers = snap.handlers().get()->getDocumentRetrievers(context.getReadConsistency()); for (size_t i = 0; i < retrievers->size(); ++i) { IDocumentRetriever &retriever = *(*retrievers)[i]; search::DocumentMetaData meta = retriever.getDocumentMetaData(did); @@ -514,17 +518,17 @@ PersistenceEngine::createIterator(const Bucket &bucket, FieldSetSP fields, const IncludedVersions versions, Context &context) { ReadGuard rguard(_rwMutex); - HandlerSnapshot snapshot = getHandlerSnapshot(rguard, bucket.getBucketSpace()); + auto snap = getSafeHandlerSnapshot(rguard, bucket.getBucketSpace()); auto entry = std::make_unique<IteratorEntry>(context.getReadConsistency(), bucket, std::move(fields), selection, versions, _defaultSerializedSize, _ignoreMaxBytes); - for (PersistenceHandlerSequence & handlers = snapshot.handlers(); handlers.valid(); handlers.next()) { - IPersistenceHandler::RetrieversSP retrievers = handlers.get()->getDocumentRetrievers(context.getReadConsistency()); + for (; snap.handlers().valid(); snap.handlers().next()) { + IPersistenceHandler::RetrieversSP retrievers = snap.handlers().get()->getDocumentRetrievers(context.getReadConsistency()); for (const auto & retriever : *retrievers) { entry->it.add(retriever); } } - entry->handler_sequence = HandlerSnapshot::release(std::move(snapshot)); + entry->handler_sequence = HandlerSnapshot::release(std::move(snap)); std::lock_guard<std::mutex> guard(_iterators_lock); static std::atomic<IteratorId::Type> id_counter(0); @@ -591,7 +595,7 @@ PersistenceEngine::createBucketAsync(const Bucket &b, OperationComplete::UP onCo { ReadGuard rguard(_rwMutex); LOG(spam, "createBucket(%s)", b.toString().c_str()); - HandlerSnapshot snap = getHandlerSnapshot(rguard, b.getBucketSpace()); + auto snap = getHandlerSnapshot(rguard, b.getBucketSpace()); auto transportContext = std::make_shared<AsyncTransportContext>(snap.size(), std::move(onComplete)); while (snap.handlers().valid()) { @@ -611,7 +615,7 @@ PersistenceEngine::deleteBucketAsync(const Bucket& b, OperationComplete::UP onCo { ReadGuard rguard(_rwMutex); LOG(spam, "deleteBucket(%s)", b.toString().c_str()); - HandlerSnapshot snap = getHandlerSnapshot(rguard, b.getBucketSpace()); + auto snap = getHandlerSnapshot(rguard, b.getBucketSpace()); auto transportContext = std::make_shared<AsyncTransportContext>(snap.size(), std::move(onComplete)); while (snap.handlers().valid()) { @@ -636,7 +640,7 @@ PersistenceEngine::getModifiedBuckets(BucketSpace bucketSpace) const std::lock_guard<std::mutex> guard(_lock); extraModifiedBuckets.swap(_extraModifiedBuckets[bucketSpace]); } - HandlerSnapshot snap = getHandlerSnapshot(rguard, bucketSpace); + auto snap = getHandlerSnapshot(rguard, bucketSpace); auto catchResult = std::make_unique<storage::spi::CatchResult>(); auto futureResult = catchResult->future_result(); SynchronizedBucketIdListResultHandler resultHandler(snap.size() + extraModifiedBuckets.size(), std::move(catchResult)); @@ -658,7 +662,7 @@ PersistenceEngine::split(const Bucket& source, const Bucket& target1, const Buck LOG(spam, "split(%s, %s, %s)", source.toString().c_str(), target1.toString().c_str(), target2.toString().c_str()); assert(source.getBucketSpace() == target1.getBucketSpace()); assert(source.getBucketSpace() == target2.getBucketSpace()); - HandlerSnapshot snap = getHandlerSnapshot(rguard, source.getBucketSpace()); + auto snap = getHandlerSnapshot(rguard, source.getBucketSpace()); TransportLatch latch(snap.size()); for (; snap.handlers().valid(); snap.handlers().next()) { IPersistenceHandler *handler = snap.handlers().get(); @@ -676,7 +680,7 @@ PersistenceEngine::join(const Bucket& source1, const Bucket& source2, const Buck LOG(spam, "join(%s, %s, %s)", source1.toString().c_str(), source2.toString().c_str(), target.toString().c_str()); assert(source1.getBucketSpace() == target.getBucketSpace()); assert(source2.getBucketSpace() == target.getBucketSpace()); - HandlerSnapshot snap = getHandlerSnapshot(rguard, target.getBucketSpace()); + auto snap = getHandlerSnapshot(rguard, target.getBucketSpace()); TransportLatch latch(snap.size()); for (; snap.handlers().valid(); snap.handlers().next()) { IPersistenceHandler *handler = snap.handlers().get(); diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h index c16cc6e6a83..2581d35064e 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.h @@ -19,8 +19,9 @@ class IDiskMemUsageNotifier; class PersistenceEngine : public storage::spi::AbstractPersistenceProvider, public storage::spi::BucketExecutor { private: - using PersistenceHandlerSequence = PersistenceHandlerMap::PersistenceHandlerSequence; + using PersistenceHandlerSequence = PersistenceHandlerMap::DocTypeToHandlerMap::Snapshot; using HandlerSnapshot = PersistenceHandlerMap::HandlerSnapshot; + using UnsafeHandlerSnapshot = PersistenceHandlerMap::UnsafeHandlerSnapshot; using DocumentUpdate = document::DocumentUpdate; using Bucket = storage::spi::Bucket; using BucketIdListResult = storage::spi::BucketIdListResult; @@ -38,7 +39,6 @@ private: using Result = storage::spi::Result; using Selection = storage::spi::Selection; using Timestamp = storage::spi::Timestamp; - using TimestampList = storage::spi::TimestampList; using UpdateResult = storage::spi::UpdateResult; using OperationComplete = storage::spi::OperationComplete; using BucketExecutor = storage::spi::BucketExecutor; @@ -82,7 +82,8 @@ private: IPersistenceHandler * getHandler(const ReadGuard & guard, document::BucketSpace bucketSpace, const DocTypeName &docType) const; HandlerSnapshot getHandlerSnapshot(const WriteGuard & guard) const; - HandlerSnapshot getHandlerSnapshot(const ReadGuard & guard, document::BucketSpace bucketSpace) const; + HandlerSnapshot getSafeHandlerSnapshot(const ReadGuard & guard, document::BucketSpace bucketSpace) const; + UnsafeHandlerSnapshot getHandlerSnapshot(const ReadGuard & guard, document::BucketSpace bucketSpace) const; HandlerSnapshot getHandlerSnapshot(const WriteGuard & guard, document::BucketSpace bucketSpace) const; void saveClusterState(BucketSpace bucketSpace, const ClusterState &calc); diff --git a/searchlib/src/tests/fef/object_passing/object_passing_test.cpp b/searchlib/src/tests/fef/object_passing/object_passing_test.cpp index 3639da05b9e..d7e5fd2600e 100644 --- a/searchlib/src/tests/fef/object_passing/object_passing_test.cpp +++ b/searchlib/src/tests/fef/object_passing/object_passing_test.cpp @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/util/stringfmt.h> #include <vespa/searchlib/features/valuefeature.h> #include <vespa/searchlib/fef/blueprintfactory.h> #include <vespa/searchlib/fef/test/indexenvironment.h> @@ -102,7 +101,8 @@ struct Fixture { } bool verify(const vespalib::string &feature) { - return verifyFeature(factory, indexEnv, feature, "unit test"); + std::vector<search::fef::Message> errors; + return verifyFeature(factory, indexEnv, feature, "unit test", errors); } }; diff --git a/searchlib/src/tests/ranksetup/verify_feature/verify_feature_test.cpp b/searchlib/src/tests/ranksetup/verify_feature/verify_feature_test.cpp index 24358db8f3a..6d49704e7c1 100644 --- a/searchlib/src/tests/ranksetup/verify_feature/verify_feature_test.cpp +++ b/searchlib/src/tests/ranksetup/verify_feature/verify_feature_test.cpp @@ -4,10 +4,37 @@ #include <vespa/searchlib/fef/test/indexenvironment.h> #include <vespa/searchlib/fef/test/plugin/setup.h> #include <vespa/searchlib/features/valuefeature.h> +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <regex> using namespace search::features; using namespace search::fef::test; using namespace search::fef; +using vespalib::make_string_short::fmt; + +typedef bool (*cmp)(const vespalib::string & a, const vespalib::string &b); + +namespace search::fef { +std::ostream &operator<<(std::ostream &os, Level level) { + if (level == Level::INFO) { + return os << "info"; + } + if (level == Level::WARNING) { + return os << "warning"; + } + return os << "error"; +} +} + +bool equal(const vespalib::string & a, const vespalib::string &b) { + return a == b; +} + +bool regex(const vespalib::string & a, const vespalib::string &b) { + std::regex exp(b.c_str()); + return std::regex_match(a.c_str(), exp); +} struct RankFixture { BlueprintFactory factory; @@ -15,43 +42,89 @@ struct RankFixture { RankFixture() : factory(), indexEnv() { setup_fef_test_plugin(factory); - factory.addPrototype(Blueprint::SP(new ValueBlueprint())); + factory.addPrototype(std::make_shared<ValueBlueprint>()); } - bool verify(const std::string &feature) const { - return verifyFeature(factory, indexEnv, feature, "feature verification test"); + bool verify(const std::string &feature, std::vector<std::pair<cmp, Message>> expected) const { + std::vector<Message> errors; + bool ok = verifyFeature(factory, indexEnv, feature, "feature verification test", errors); + EXPECT_EQUAL(errors.size(), expected.size()); + for (size_t i(0); i < std::min(errors.size(), expected.size()); i++) { + EXPECT_EQUAL(errors[i].first, expected[i].second.first); + EXPECT_TRUE(expected[i].first(errors[i].second, expected[i].second.second)); + } + return ok; } }; TEST_F("verify valid rank feature", RankFixture) { - EXPECT_TRUE(f1.verify("value(1, 2, 3).0")); - EXPECT_TRUE(f1.verify("value(1, 2, 3).1")); - EXPECT_TRUE(f1.verify("value(1, 2, 3).2")); + EXPECT_TRUE(f1.verify("value(1, 2, 3).0", {})); + EXPECT_TRUE(f1.verify("value(1, 2, 3).1", {})); + EXPECT_TRUE(f1.verify("value(1, 2, 3).2", {})); } TEST_F("verify unknown feature", RankFixture) { - EXPECT_FALSE(f1.verify("unknown")); + EXPECT_FALSE(f1.verify("unknown", + {{equal, {Level::WARNING, "invalid rank feature 'unknown': unknown basename: 'unknown'"}}, + {equal, {Level::ERROR, "verification failed: rank feature 'unknown' (feature verification test)"}}})); } TEST_F("verify unknown output", RankFixture) { - EXPECT_FALSE(f1.verify("value(1, 2, 3).3")); + EXPECT_FALSE(f1.verify("value(1, 2, 3).3", + {{equal, {Level::WARNING, "invalid rank feature 'value(1,2,3).3': unknown output: '3'"}}, + {equal, {Level::ERROR, "verification failed: rank feature 'value(1, 2, 3).3' (feature verification test)"}}})); } TEST_F("verify illegal input parameters", RankFixture) { - EXPECT_FALSE(f1.verify("value.0")); + EXPECT_FALSE(f1.verify("value.0", + {{equal, {Level::WARNING, "invalid rank feature 'value.0':" + " The parameter list used for setting up rank feature value is not valid:" + " Expected 1+1x parameter(s), but got 0"}}, + {equal, {Level::ERROR, "verification failed: rank feature 'value.0' (feature verification test)"}}})); } TEST_F("verify illegal feature name", RankFixture) { - EXPECT_FALSE(f1.verify("value(2).")); + EXPECT_FALSE(f1.verify("value(2).", + {{equal, {Level::WARNING, "invalid rank feature 'value(2).': malformed name"}}, + {equal, {Level::ERROR, "verification failed: rank feature 'value(2).' (feature verification test)"}}})); } TEST_F("verify too deep dependency graph", RankFixture) { - EXPECT_TRUE(f1.verify("chain(basic, 255, 4)")); - EXPECT_FALSE(f1.verify("chain(basic, 256, 4)")); + EXPECT_TRUE(f1.verify("chain(basic, 255, 4)", {})); + EXPECT_FALSE(f1.verify("chain(basic, 256, 4)", + {{equal, + {Level::WARNING, + "invalid rank feature 'value(4)': dependency graph too deep\n" + " ... needed by rank feature 'chain(basic,1,4)'\n" + " ... needed by rank feature 'chain(basic,2,4)'\n" + " ... needed by rank feature 'chain(basic,3,4)'\n" + " ... needed by rank feature 'chain(basic,4,4)'\n" + " ... needed by rank feature 'chain(basic,5,4)'\n" + " ... needed by rank feature 'chain(basic,6,4)'\n" + " ... needed by rank feature 'chain(basic,7,4)'\n" + " ... needed by rank feature 'chain(basic,8,4)'\n" + " ... needed by rank feature 'chain(basic,9,4)'\n" + " ... needed by rank feature 'chain(basic,10,4)'\n" + " (skipped 241 entries)\n" + " ... needed by rank feature 'chain(basic,252,4)'\n" + " ... needed by rank feature 'chain(basic,253,4)'\n" + " ... needed by rank feature 'chain(basic,254,4)'\n" + " ... needed by rank feature 'chain(basic,255,4)'\n" + " ... needed by rank feature 'chain(basic,256,4)'\n"}}, + {regex, {Level::WARNING, "high stack usage: [0-9]+ bytes"}}, + {equal, {Level::ERROR, "verification failed: rank feature 'chain(basic, 256, 4)' (feature verification test)"}}})); } TEST_F("verify dependency cycle", RankFixture) { - EXPECT_FALSE(f1.verify("chain(cycle, 4, 2)")); + EXPECT_FALSE(f1.verify("chain(cycle, 4, 2)", + {{equal, + {Level::WARNING, + "invalid rank feature 'chain(cycle,2,2)': dependency cycle detected\n" + " ... needed by rank feature 'chain(cycle,1,2)'\n" + " ... needed by rank feature 'chain(cycle,2,2)'\n" + " ... needed by rank feature 'chain(cycle,3,2)'\n" + " ... needed by rank feature 'chain(cycle,4,2)'\n"}}, + {equal, {Level::ERROR, "verification failed: rank feature 'chain(cycle, 4, 2)' (feature verification test)"}}})); } TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.cpp b/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.cpp index 90386dffd51..4b2d67c933d 100644 --- a/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.cpp +++ b/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.cpp @@ -79,13 +79,13 @@ resolve_attribute_for_field(const fef::IQueryEnvironment& env, } -DistanceCalculatorBundle::Element::Element(fef::TermFieldHandle handle_in) +DistanceCalculatorBundle::Element::Element(fef::TermFieldHandle handle_in) noexcept : handle(handle_in), calc() { } -DistanceCalculatorBundle::Element::Element(fef::TermFieldHandle handle_in, std::unique_ptr<search::tensor::DistanceCalculator> calc_in) +DistanceCalculatorBundle::Element::Element(fef::TermFieldHandle handle_in, std::unique_ptr<search::tensor::DistanceCalculator> calc_in) noexcept : handle(handle_in), calc(std::move(calc_in)) { diff --git a/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.h b/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.h index dd3fc521d96..35295c771a6 100644 --- a/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.h +++ b/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.h @@ -27,8 +27,8 @@ public: fef::TermFieldHandle handle; std::unique_ptr<search::tensor::DistanceCalculator> calc; Element(Element&& rhs) noexcept = default; // Needed as std::vector::reserve() is used. - Element(fef::TermFieldHandle handle_in); - Element(fef::TermFieldHandle handle_in, std::unique_ptr<search::tensor::DistanceCalculator> calc_in); + Element(fef::TermFieldHandle handle_in) noexcept; + Element(fef::TermFieldHandle handle_in, std::unique_ptr<search::tensor::DistanceCalculator> calc_in) noexcept; ~Element(); }; private: diff --git a/searchlib/src/vespa/searchlib/fef/blueprint.cpp b/searchlib/src/vespa/searchlib/fef/blueprint.cpp index 58094121e24..72e4021986a 100644 --- a/searchlib/src/vespa/searchlib/fef/blueprint.cpp +++ b/searchlib/src/vespa/searchlib/fef/blueprint.cpp @@ -5,9 +5,6 @@ #include <cassert> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/log/log.h> -LOG_SETUP(".fef.blueprint"); - namespace search::fef { std::optional<FeatureType> diff --git a/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp b/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp index 731306d1bea..b672d754b72 100644 --- a/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp +++ b/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp @@ -7,14 +7,10 @@ #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/threadstackexecutor.h> -#include <stack> #include <cassert> #include <set> #include <thread> -#include <vespa/log/log.h> -LOG_SETUP(".fef.blueprintresolver"); - using vespalib::make_string_short::fmt; using vespalib::ThreadStackExecutor; using vespalib::makeLambdaTask; @@ -63,6 +59,7 @@ struct Compiler : public Blueprint::DependencyHandler { : spec(std::move(blueprint)), parser(parser_in) {} }; using Stack = std::vector<Frame>; + using Errors = std::vector<vespalib::string>; struct FrameGuard { Stack &stack; @@ -73,15 +70,16 @@ struct Compiler : public Blueprint::DependencyHandler { } }; - const BlueprintFactory &factory; - const IIndexEnvironment &index_env; - Stack resolve_stack; - ExecutorSpecList &spec_list; - FeatureMap &feature_map; - std::set<vespalib::string> setup_set; - std::set<vespalib::string> failed_set; - const char *min_stack; - const char *max_stack; + const BlueprintFactory &factory; + const IIndexEnvironment &index_env; + Stack resolve_stack; + Errors errors; + ExecutorSpecList &spec_list; + FeatureMap &feature_map; + std::set<vespalib::string> setup_set; + std::set<vespalib::string> failed_set; + const char *min_stack; + const char *max_stack; Compiler(const BlueprintFactory &factory_in, const IIndexEnvironment &index_env_in, @@ -90,6 +88,7 @@ struct Compiler : public Blueprint::DependencyHandler { : factory(factory_in), index_env(index_env_in), resolve_stack(), + errors(), spec_list(spec_list_out), feature_map(feature_map_out), setup_set(), @@ -138,11 +137,13 @@ struct Compiler : public Blueprint::DependencyHandler { if (failed_set.count(feature_name) == 0) { failed_set.insert(feature_name); auto trace = make_trace(skip_self); + vespalib::string msg; if (trace.empty()) { - LOG(warning, "invalid %s: %s", describe(feature_name).c_str(), reason.c_str()); + msg = fmt("invalid %s: %s", describe(feature_name).c_str(), reason.c_str()); } else { - LOG(warning, "invalid %s: %s\n%s", describe(feature_name).c_str(), reason.c_str(), trace.c_str()); + msg = fmt("invalid %s: %s\n%s", describe(feature_name).c_str(), reason.c_str(), trace.c_str()); } + errors.emplace_back(msg); } probe_stack(); return FeatureRef(); @@ -264,7 +265,8 @@ BlueprintResolver::BlueprintResolver(const BlueprintFactory &factory, _seeds(), _executorSpecs(), _featureMap(), - _seedMap() + _seedMap(), + _warnings() { } @@ -300,6 +302,7 @@ BlueprintResolver::compile() for (const auto &seed: _seeds) { auto ref = compiler.resolve_feature(seed, Blueprint::AcceptInput::ANY); if (compiler.failed()) { + _warnings = std::move(compiler.errors); return; } _seedMap.emplace(FeatureNameParser(seed).featureName(), ref); @@ -311,27 +314,9 @@ BlueprintResolver::compile() executor.shutdown(); size_t stack_usage = compiler.stack_usage(); if (stack_usage > (128_Ki)) { - LOG(warning, "high stack usage: %zu bytes", stack_usage); + _warnings.emplace_back(fmt("high stack usage: %zu bytes", stack_usage)); } return !compiler.failed(); } -const BlueprintResolver::ExecutorSpecList & -BlueprintResolver::getExecutorSpecs() const -{ - return _executorSpecs; -} - -const BlueprintResolver::FeatureMap & -BlueprintResolver::getFeatureMap() const -{ - return _featureMap; -} - -const BlueprintResolver::FeatureMap & -BlueprintResolver::getSeedMap() const -{ - return _seedMap; -} - } diff --git a/searchlib/src/vespa/searchlib/fef/blueprintresolver.h b/searchlib/src/vespa/searchlib/fef/blueprintresolver.h index 3e3b5879518..459082b4534 100644 --- a/searchlib/src/vespa/searchlib/fef/blueprintresolver.h +++ b/searchlib/src/vespa/searchlib/fef/blueprintresolver.h @@ -24,7 +24,8 @@ class FeatureNameParser; class BlueprintResolver { public: - typedef std::shared_ptr<BlueprintResolver> SP; + using SP = std::shared_ptr<BlueprintResolver>; + using Warnings = std::vector<vespalib::string>; /** * Low-level reference to a single output from a feature @@ -44,7 +45,7 @@ public: : executor(executor_in), output(output_in) {} bool valid() { return (executor != undef); } }; - typedef std::map<vespalib::string, FeatureRef> FeatureMap; + using FeatureMap = std::map<vespalib::string, FeatureRef>; /** * Thin blueprint wrapper with additional information about how @@ -59,7 +60,7 @@ public: ExecutorSpec(Blueprint::SP blueprint_in); ~ExecutorSpec(); }; - typedef std::vector<ExecutorSpec> ExecutorSpecList; + using ExecutorSpecList = std::vector<ExecutorSpec>; /** * The maximum dependency depth. This value is defined to protect @@ -84,6 +85,7 @@ private: ExecutorSpecList _executorSpecs; FeatureMap _featureMap; FeatureMap _seedMap; + Warnings _warnings; public: BlueprintResolver(const BlueprintResolver &) = delete; @@ -134,7 +136,7 @@ public: * * @return feature executor assembly directions **/ - const ExecutorSpecList &getExecutorSpecs() const; + const ExecutorSpecList &getExecutorSpecs() const { return _executorSpecs; } /** * Obtain the location of all named features known to this @@ -145,7 +147,7 @@ public: * * @return feature locations **/ - const FeatureMap &getFeatureMap() const; + const FeatureMap &getFeatureMap() const { return _featureMap; } /** * Obtain the location of all seeds used by this resolver. This @@ -156,7 +158,13 @@ public: * * @return seed locations **/ - const FeatureMap &getSeedMap() const; + const FeatureMap &getSeedMap() const { return _seedMap; } + + /** + * Will return any accumulated warnings during compile + * @return list of warnings + **/ + const Warnings & getWarnings() const { return _warnings; } }; } diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp index ed841ae24b3..30e220b0a34 100644 --- a/searchlib/src/vespa/searchlib/fef/ranksetup.cpp +++ b/searchlib/src/vespa/searchlib/fef/ranksetup.cpp @@ -3,9 +3,10 @@ #include "ranksetup.h" #include "indexproperties.h" #include "featurenameparser.h" +#include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/log/log.h> -LOG_SETUP(".fef.ranksetup"); +using vespalib::make_string_short::fmt; namespace { class VisitorAdapter : public search::fef::IDumpFeatureVisitor @@ -53,6 +54,7 @@ RankSetup::RankSetup(const BlueprintFactory &factory, const IIndexEnvironment &i _match_features(), _summaryFeatures(), _dumpFeatures(), + _warnings(), _feature_rename_map(), _ignoreDefaultRankFeatures(false), _compiled(false), @@ -134,50 +136,59 @@ RankSetup::configure() void RankSetup::setFirstPhaseRank(const vespalib::string &featureName) { - LOG_ASSERT(!_compiled); + assert(!_compiled); _firstPhaseRankFeature = featureName; } void RankSetup::setSecondPhaseRank(const vespalib::string &featureName) { - LOG_ASSERT(!_compiled); + assert(!_compiled); _secondPhaseRankFeature = featureName; } void RankSetup::add_match_feature(const vespalib::string &match_feature) { - LOG_ASSERT(!_compiled); + assert(!_compiled); _match_features.push_back(match_feature); } void RankSetup::addSummaryFeature(const vespalib::string &summaryFeature) { - LOG_ASSERT(!_compiled); + assert(!_compiled); _summaryFeatures.push_back(summaryFeature); } void RankSetup::addDumpFeature(const vespalib::string &dumpFeature) { - LOG_ASSERT(!_compiled); + assert(!_compiled); _dumpFeatures.push_back(dumpFeature); } +void +RankSetup::compileAndCheckForErrors(BlueprintResolver &bpr) { + bool ok = bpr.compile(); + if ( ! ok ) { + _compileError = true; + const auto & warnings = bpr.getWarnings(); + _warnings.insert(_warnings.end(), warnings.begin(), warnings.end()); + } +} bool RankSetup::compile() { - LOG_ASSERT(!_compiled); + assert(!_compiled); if (!_firstPhaseRankFeature.empty()) { FeatureNameParser parser(_firstPhaseRankFeature); if (parser.valid()) { _firstPhaseRankFeature = parser.featureName(); _first_phase_resolver->addSeed(_firstPhaseRankFeature); } else { - LOG(warning, "invalid feature name for initial rank: '%s'", - _firstPhaseRankFeature.c_str()); + vespalib::string e = fmt("invalid feature name for initial rank: '%s'", _firstPhaseRankFeature.c_str()); + _warnings.emplace_back(e); _compileError = true; } } @@ -187,8 +198,8 @@ RankSetup::compile() _secondPhaseRankFeature = parser.featureName(); _second_phase_resolver->addSeed(_secondPhaseRankFeature); } else { - LOG(warning, "invalid feature name for final rank: '%s'", - _secondPhaseRankFeature.c_str()); + vespalib::string e = fmt("invalid feature name for final rank: '%s'", _secondPhaseRankFeature.c_str()); + _warnings.emplace_back(e); _compileError = true; } } @@ -206,12 +217,12 @@ RankSetup::compile() _dumpResolver->addSeed(feature); } _indexEnv.hintFeatureMotivation(IIndexEnvironment::RANK); - _compileError |= !_first_phase_resolver->compile(); - _compileError |= !_second_phase_resolver->compile(); - _compileError |= !_match_resolver->compile(); - _compileError |= !_summary_resolver->compile(); + compileAndCheckForErrors(*_first_phase_resolver); + compileAndCheckForErrors(*_second_phase_resolver); + compileAndCheckForErrors(*_match_resolver); + compileAndCheckForErrors(*_summary_resolver); _indexEnv.hintFeatureMotivation(IIndexEnvironment::DUMP); - _compileError |= !_dumpResolver->compile(); + compileAndCheckForErrors(*_dumpResolver); _compiled = true; return !_compileError; } @@ -219,7 +230,7 @@ RankSetup::compile() void RankSetup::prepareSharedState(const IQueryEnvironment &queryEnv, IObjectStore &objectStore) const { - LOG_ASSERT(_compiled && !_compileError); + assert(_compiled && !_compileError); for (const auto &spec : _first_phase_resolver->getExecutorSpecs()) { spec.blueprint->prepareSharedState(queryEnv, objectStore); } @@ -234,4 +245,13 @@ RankSetup::prepareSharedState(const IQueryEnvironment &queryEnv, IObjectStore &o } } +vespalib::string +RankSetup::getJoinedWarnings() const { + vespalib::asciistream os; + for (const auto & m : _warnings) { + os << m << "\n"; + } + return os.str(); +} + } diff --git a/searchlib/src/vespa/searchlib/fef/ranksetup.h b/searchlib/src/vespa/searchlib/fef/ranksetup.h index 6a2871827ab..1ad2fe0a77a 100644 --- a/searchlib/src/vespa/searchlib/fef/ranksetup.h +++ b/searchlib/src/vespa/searchlib/fef/ranksetup.h @@ -23,6 +23,7 @@ namespace search::fef { class RankSetup { public: + using Warnings = BlueprintResolver::Warnings; struct MutateOperation { public: MutateOperation() : MutateOperation("", "") {} @@ -62,6 +63,7 @@ private: std::vector<vespalib::string> _match_features; std::vector<vespalib::string> _summaryFeatures; std::vector<vespalib::string> _dumpFeatures; + Warnings _warnings; StringStringMap _feature_rename_map; bool _ignoreDefaultRankFeatures; bool _compiled; @@ -80,6 +82,8 @@ private: MutateOperation _mutateOnFirstPhase; MutateOperation _mutateOnSecondPhase; MutateOperation _mutateOnSummary; + + void compileAndCheckForErrors(BlueprintResolver &bp); public: RankSetup(const RankSetup &) = delete; RankSetup &operator=(const RankSetup &) = delete; @@ -438,6 +442,12 @@ public: **/ bool compile(); + /** + * Will return any accumulated warnings during compile + * @return joined string of warnings separated by newline + */ + vespalib::string getJoinedWarnings() const; + // These functions create rank programs for different tasks. Note // that the setup function must be called on rank programs for // them to be ready to use. Also keep in mind that creating a rank diff --git a/searchlib/src/vespa/searchlib/fef/verify_feature.cpp b/searchlib/src/vespa/searchlib/fef/verify_feature.cpp index 85d1daffd01..b012e95c8fc 100644 --- a/searchlib/src/vespa/searchlib/fef/verify_feature.cpp +++ b/searchlib/src/vespa/searchlib/fef/verify_feature.cpp @@ -2,28 +2,31 @@ #include "verify_feature.h" #include "blueprintresolver.h" +#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/log/log.h> -LOG_SETUP(".fef.verify_feature"); +using vespalib::make_string_short::fmt; -namespace search { -namespace fef { +namespace search::fef { bool verifyFeature(const BlueprintFactory &factory, const IIndexEnvironment &indexEnv, const std::string &featureName, - const std::string &desc) + const std::string &desc, + std::vector<Message> & errors) { indexEnv.hintFeatureMotivation(IIndexEnvironment::VERIFY_SETUP); BlueprintResolver resolver(factory, indexEnv); resolver.addSeed(featureName); bool result = resolver.compile(); if (!result) { - LOG(error, "verification failed: %s (%s)", - BlueprintResolver::describe_feature(featureName).c_str(), desc.c_str()); + const BlueprintResolver::Warnings & warnings(resolver.getWarnings()); + for (const auto & msg : warnings) { + errors.emplace_back(Level::WARNING, msg); + } + vespalib::string msg = fmt("verification failed: %s (%s)",BlueprintResolver::describe_feature(featureName).c_str(), desc.c_str()); + errors.emplace_back(Level::ERROR, msg); } return result; } -} // namespace fef -} // namespace search +} diff --git a/searchlib/src/vespa/searchlib/fef/verify_feature.h b/searchlib/src/vespa/searchlib/fef/verify_feature.h index 5c84403fc8a..f6f3924616a 100644 --- a/searchlib/src/vespa/searchlib/fef/verify_feature.h +++ b/searchlib/src/vespa/searchlib/fef/verify_feature.h @@ -4,10 +4,11 @@ #include "blueprintfactory.h" #include "iindexenvironment.h" -#include <string> -namespace search { -namespace fef { +namespace search::fef { + +enum class Level {INFO, WARNING, ERROR}; +using Message = std::pair<Level, vespalib::string>; /** * Verify whether a specific feature can be computed. If the feature @@ -23,8 +24,7 @@ namespace fef { bool verifyFeature(const BlueprintFactory &factory, const IIndexEnvironment &indexEnv, const std::string &featureName, - const std::string &desc); - -} // namespace fef -} // namespace search + const std::string &desc, + std::vector<Message> & errors); +} diff --git a/searchsummary/src/tests/juniper/auxTest.cpp b/searchsummary/src/tests/juniper/auxTest.cpp index 15f5ad1749e..c42cf4ccd83 100644 --- a/searchsummary/src/tests/juniper/auxTest.cpp +++ b/searchsummary/src/tests/juniper/auxTest.cpp @@ -136,15 +136,14 @@ AuxTest::TestDoubleWidth() juniper::QueryParser q("\xef\xbd\x93\xef\xbd\x8f\xef\xbd\x8e\xef\xbd\x99"); juniper::QueryHandle qh(q, nullptr, juniper.getModifier()); - juniper::Result* res = juniper::Analyse(&myConfig, &qh, - input, 17, 0, 0, 0); - _test(res != nullptr); + auto res = juniper::Analyse(myConfig, qh, + input, 17, 0, 0, 0); + _test(static_cast<bool>(res)); - juniper::Summary* sum = juniper::GetTeaser(res, nullptr); + juniper::Summary* sum = juniper::GetTeaser(*res, nullptr); (void) sum; // this should work // _test(sum->Length() != 0); - juniper::ReleaseResult(res); } @@ -175,17 +174,15 @@ AuxTest::TestPartialUTF8() juniper::QueryParser q("ipod"); juniper::QueryHandle qh(q, nullptr, juniper.getModifier()); - juniper::Result* res = juniper::Analyse(&myConfig, &qh, - input, inputSize, 0, 0, 0); - _test(res != nullptr); + auto res = juniper::Analyse(myConfig, qh, + input, inputSize, 0, 0, 0); + _test(static_cast<bool>(res)); - juniper::Summary* sum = juniper::GetTeaser(res, nullptr); + juniper::Summary* sum = juniper::GetTeaser(*res, nullptr); _test(sum->Length() != 0); // check for partial/broken utf-8 _test(countBrokenUTF8(sum->Text(), sum->Length()) == 0); - - juniper::ReleaseResult(res); } void AuxTest::TestLargeBlockChinese() @@ -215,11 +212,11 @@ void AuxTest::TestLargeBlockChinese() juniper::QueryParser q("希望"); juniper::QueryHandle qh(q, nullptr, juniper.getModifier()); - juniper::Result* res = juniper::Analyse(&myConfig, &qh, - input, inputSize, 0, 0, 0); - _test(res != nullptr); + auto res = juniper::Analyse(myConfig, qh, + input, inputSize, 0, 0, 0); + _test(static_cast<bool>(res)); - juniper::Summary* sum = juniper::GetTeaser(res, nullptr); + juniper::Summary* sum = juniper::GetTeaser(*res, nullptr); _test(sum->Length() != 0); // check that the entire block of chinese data is not returned in the summary @@ -227,8 +224,6 @@ void AuxTest::TestLargeBlockChinese() // check for partial/broken utf-8 _test(countBrokenUTF8(sum->Text(), sum->Length()) == 0); - - juniper::ReleaseResult(res); } void AuxTest::TestExample() @@ -241,17 +236,14 @@ void AuxTest::TestExample() "&%#%&! cries the sleepy monkey and jumps down from the tree." "the last token here is split across lines consumed"; int content_len = strlen(content); - juniper::Result* res = - juniper::Analyse(juniper::TestConfig, - &qh, - content, content_len, - 0, 0, 0); - _test(res != nullptr); + auto res = juniper::Analyse(*juniper::TestConfig, qh, + content, content_len, + 0, 0, 0); + _test(static_cast<bool>(res)); res->Scan(); Matcher& m = *res->_matcher; _test(m.TotalMatchCnt(0) == 2 && m.ExactMatchCnt(0) == 0); - juniper::ReleaseResult(res); } @@ -410,8 +402,8 @@ void AuxTest::TestUTF8context() s.append(char_from_u8(u8" beste forekomst av s\u00f8ket med s\u00f8kemotor til brukeren blir det enda bedre. ")); s.append(char_from_u8(u8"Hvis bare UTF8-kodingen virker som den skal for tegn som tar mer enn \u00e9n byte.")); - juniper::Result* res = juniper::Analyse(juniper::TestConfig, &qh, s.c_str(), s.size(), 0, 0, 0); - _test(res != nullptr); + auto res = juniper::Analyse(*juniper::TestConfig, qh, s.c_str(), s.size(), 0, 0, 0); + _test(static_cast<bool>(res)); size_t charsize; Matcher& m = *res->_matcher; @@ -454,7 +446,6 @@ void AuxTest::TestUTF8context() test_dump(s.c_str(), s.size()); m.dump_statistics(); } - juniper::ReleaseResult(res); } @@ -491,10 +482,10 @@ void AuxTest::TestJapanese() const char* content = testjap[i].text; int content_len = strlen(content); - juniper::Result* res = juniper::Analyse(juniper::TestConfig, &qh, - content, content_len, - 0, 0, 0); - _test(res != nullptr); + auto res = juniper::Analyse(*juniper::TestConfig, qh, + content, content_len, + 0, 0, 0); + _test(static_cast<bool>(res)); size_t charsize; Matcher& m = *res->_matcher; @@ -545,7 +536,6 @@ void AuxTest::TestJapanese() default: break; } - juniper::ReleaseResult(res); DeleteSummaryDesc(sumdesc); DeleteSummaryConfig(_sumconf); } @@ -582,15 +572,14 @@ void AuxTest::TestStartHits() " In fact it must be much longer. And then som more text at the end. But this text at " "the end must be much longer than this to trigger the case"; int content_len = strlen(content); - juniper::Result* res = juniper::Analyse(juniper::TestConfig, &qh, - content, content_len, - 0, 0, 0); - _test(res != nullptr); + auto res = juniper::Analyse(*juniper::TestConfig, qh, + content, content_len, + 0, 0, 0); + _test(static_cast<bool>(res)); - juniper::Summary* sum = juniper::GetTeaser(res, nullptr); + juniper::Summary* sum = juniper::GetTeaser(*res, nullptr); (void) sum; // TODO: ReEnable _test(sum->Length() != 0); - juniper::ReleaseResult(res); } @@ -607,14 +596,13 @@ void AuxTest::TestEndHit() "surround_len bytes closer than good towardstheend�����������������������������������"; size_t content_len = strlen(content) - 55; - juniper::Result* res = juniper::Analyse(juniper::TestConfig, &qh, - content, content_len, - 0, 0, 0); - _test(res != nullptr); + auto res = juniper::Analyse(*juniper::TestConfig, qh, + content, content_len, + 0, 0, 0); + _test(static_cast<bool>(res)); - juniper::Summary* sum = juniper::GetTeaser(res, nullptr); + juniper::Summary* sum = juniper::GetTeaser(*res, nullptr); _test(sum->Length() != 0); - juniper::ReleaseResult(res); } void AuxTest::TestJuniperStack() @@ -880,14 +868,13 @@ AuxTest::TestWhiteSpacePreserved() juniper::QueryParser q("best"); juniper::QueryHandle qh(q, nullptr, juniper.getModifier()); - juniper::Result* res = juniper::Analyse(&myConfig, &qh, input.c_str(), input.size(), 0, 0, 0); - _test(res != nullptr); + auto res = juniper::Analyse(myConfig, qh, input.c_str(), input.size(), 0, 0, 0); + _test(static_cast<bool>(res)); - juniper::Summary* sum = juniper::GetTeaser(res, nullptr); + juniper::Summary* sum = juniper::GetTeaser(*res, nullptr); vespalib::string expected = "<hi>best</hi> of \nmetallica"; vespalib::string actual(sum->Text(), sum->Length()); _test(actual == expected); - juniper::ReleaseResult(res); } void AuxTest::Run(MethodContainer::iterator &itr) { diff --git a/searchsummary/src/tests/juniper/matchobjectTest.cpp b/searchsummary/src/tests/juniper/matchobjectTest.cpp index 07e3cf84767..d8b7724d8a5 100644 --- a/searchsummary/src/tests/juniper/matchobjectTest.cpp +++ b/searchsummary/src/tests/juniper/matchobjectTest.cpp @@ -28,10 +28,10 @@ void MatchObjectTest::testTerm() { size_t content_len = strlen(content); // Fetch a result descriptor: - Result* res = juniper::Analyse(juniper::TestConfig, &q._qhandle, - content, content_len, - 0, 0, 0); - _test(res != 0); + auto res = juniper::Analyse(*juniper::TestConfig, q._qhandle, + content, content_len, + 0, 0, 0); + _test(static_cast<bool>(res)); // Do the scanning manually. This calls accept several times res->Scan(); @@ -42,51 +42,39 @@ void MatchObjectTest::testTerm() { _test(ms.size() == 2); - delete res; // printf("%d %d\n", m.TotalHits(),ms.size()); TestQuery q1("t*t"); TestQuery q2("*ea*"); TestQuery q3("*d"); TestQuery q4("*word"); - Result* r1 = juniper::Analyse(juniper::TestConfig, &q1._qhandle, content, content_len, 0, 0, 0); - Result* r2 = juniper::Analyse(juniper::TestConfig, &q2._qhandle, content, content_len, 0, 0, 0); - Result* r3 = juniper::Analyse(juniper::TestConfig, &q3._qhandle, content, content_len, 0, 0, 0); - Result* r4 = juniper::Analyse(juniper::TestConfig, &q4._qhandle, content, content_len, 0, 0, 0); - if (r1 != 0) - { + auto r1 = juniper::Analyse(*juniper::TestConfig, q1._qhandle, content, content_len, 0, 0, 0); + auto r2 = juniper::Analyse(*juniper::TestConfig, q2._qhandle, content, content_len, 0, 0, 0); + auto r3 = juniper::Analyse(*juniper::TestConfig, q3._qhandle, content, content_len, 0, 0, 0); + auto r4 = juniper::Analyse(*juniper::TestConfig, q4._qhandle, content, content_len, 0, 0, 0); + _test(static_cast<bool>(r1)); + if (r1) { r1->Scan(); _test(r1->_matcher->TotalHits() == 1); - delete r1; } - else - _test(r1 != 0); - - if (r2 != 0) - { + _test(static_cast<bool>(r2)); + if (r2) { r2->Scan(); _test(r2->_matcher->TotalHits() == 2); - delete r2; } - else - _test(r2 != 0); - if (r3 != 0) - { + if (r3) { r3->Scan(); _test(r3->_matcher->TotalHits() == 2); - delete r3; + } else { + _test(static_cast<bool>(r3)); } - else - _test(r3 != 0); - if (r4 != 0) - { + if (r4) { r4->Scan(); _test_equal(r4->_matcher->TotalHits(), 2); - delete r4; + } else { + _test(static_cast<bool>(r4)); } - else - _test(r4 != 0); } /** @@ -98,7 +86,7 @@ void MatchObjectTest::testMatch() { juniper::QueryHandle qh(p, NULL, juniper::_Juniper->getModifier()); MatchObject* mo = qh.MatchObj(0); - juniper::Result res(juniper::TestConfig, &qh, "", 0, 0); + juniper::Result res(*juniper::TestConfig, qh, "", 0, 0); unsigned opts = 0; match_iterator mi(mo, &res); ucs4_t ucs4_str[10]; @@ -140,7 +128,7 @@ void MatchObjectTest::testMatch() { "extremelylongwordhit,extremelylongwordhits,extremelylongwordhit," "extremelylongwordhit))"); QueryHandle& qh1(q._qhandle); - juniper::Result res1(juniper::TestConfig, &qh1, + juniper::Result res1(*juniper::TestConfig, qh1, doc.c_str(), doc.size(), 0); juniper::Summary* sum = res1.GetTeaser(NULL); std::string s(sum->Text()); @@ -165,7 +153,7 @@ void MatchObjectTest::testMatchAnnotated() { " stuff"; TestQuery q("AND(big,buy)"); QueryHandle &qh1(q._qhandle); - juniper::Result res1(juniper::TestConfig, &qh1, + juniper::Result res1(*juniper::TestConfig, qh1, doc, strlen(doc), 0); juniper::Summary *sum = res1.GetTeaser(NULL); std::string s(sum->Text()); @@ -205,7 +193,7 @@ void MatchObjectTest::testLangid() std::string doc("see if we can match b or c somewhere in this a3 doc. " "Note that we should not match b1 or c1 or a somewhere.."); - juniper::Result res(juniper::TestConfig, &qh, doc.c_str(), doc.size(),0); + juniper::Result res(*juniper::TestConfig, qh, doc.c_str(), doc.size(),0); juniper::Summary* sum = res.GetTeaser(NULL); std::string s(sum->Text()); @@ -218,7 +206,7 @@ void MatchObjectTest::testLangid() // Do another test with the same query handle (testing reuse of qh with rewriters) std::string doc("Try to run this on another doc just to see if b or c still can be" " matched with the same query handle"); - juniper::Result res(juniper::TestConfig, &qh, + juniper::Result res(*juniper::TestConfig, qh, doc.c_str(), doc.size(), 0); juniper::Summary* sum = res.GetTeaser(NULL); @@ -247,7 +235,7 @@ void MatchObjectTest::testCombined() { std::string doc("see if we can match a3 or c somewhere in this b doc. " "Note that we should not match b1 or c1 or a somewhere.."); - juniper::Result res(juniper::TestConfig, &qh, doc.c_str(), doc.size(), 0); + juniper::Result res(*juniper::TestConfig, qh, doc.c_str(), doc.size(), 0); juniper::Summary* sum = res.GetTeaser(NULL); std::string s(sum->Text()); diff --git a/searchsummary/src/tests/juniper/mcandTest.cpp b/searchsummary/src/tests/juniper/mcandTest.cpp index 5a465275a80..46bd4a5196f 100644 --- a/searchsummary/src/tests/juniper/mcandTest.cpp +++ b/searchsummary/src/tests/juniper/mcandTest.cpp @@ -41,24 +41,23 @@ void MatchCandidateTest::testLog() { TestQuery q(""); std::string content("Here we go hepp and then some words away hoi some silly text here"); - Result* res = juniper::Analyse(juniper::TestConfig, - &q._qhandle, - content.c_str(), content.size(), - 0, 0, 0); - _test(res); // We get a result handle + auto res = juniper::Analyse(*juniper::TestConfig, + q._qhandle, + content.c_str(), content.size(), + 0, 0, 0); + _test(static_cast<bool>(res)); // We get a result handle _test(!res->_mo); // but it is empty - juniper::Summary* sum = juniper::GetTeaser(res); + juniper::Summary* sum = juniper::GetTeaser(*res); std::string s(sum->Text()); _test_equal(s, std::string("")); - long relevance = juniper::GetRelevancy(res); + long relevance = juniper::GetRelevancy(*res); _test_equal(relevance, PROXIMITYBOOST_NOCONSTRAINT_OFFSET); - sum = juniper::GetLog(res); + sum = juniper::GetLog(*res); s = sum->Text(); _test_equal(s, std::string("")); - juniper::ReleaseResult(res); } @@ -70,56 +69,52 @@ void MatchCandidateTest::testDump() { { TestQuery q("NEAR/1(hepp,hoi)"); - Result* res = juniper::Analyse(juniper::TestConfig, - &q._qhandle, - content.c_str(), content.size(), - 0, 0, 0); - _test(res != NULL); - long relevance = juniper::GetRelevancy(res); + auto res = juniper::Analyse(*juniper::TestConfig, + q._qhandle, + content.c_str(), content.size(), + 0, 0, 0); + _test(static_cast<bool>(res)); + long relevance = juniper::GetRelevancy(*res); // zero value since there are no hits and constraints are enabled.. _test_equal(relevance, 0); - juniper::ReleaseResult(res); } { TestQuery q("OR(NEAR/1(hepp,hoi),bananas)"); - Result* res = juniper::Analyse(juniper::TestConfig, - &q._qhandle, - content.c_str(), content.size(), - 0, 0, 0); - _test(res != NULL); - long relevance = juniper::GetRelevancy(res); + auto res = juniper::Analyse(*juniper::TestConfig, + q._qhandle, + content.c_str(), content.size(), + 0, 0, 0); + _test(static_cast<bool>(res)); + long relevance = juniper::GetRelevancy(*res); // Check that X_CONSTR propagates as intended _test_equal(relevance, 0); - juniper::ReleaseResult(res); } { TestQuery q("PHRASE(hepp,hoi)"); - Result* res = juniper::Analyse(juniper::TestConfig, - &q._qhandle, - content.c_str(), content.size(), - 0, 0, 0); - _test(res != NULL); - long relevance = juniper::GetRelevancy(res); + auto res = juniper::Analyse(*juniper::TestConfig, + q._qhandle, + content.c_str(), content.size(), + 0, 0, 0); + _test(static_cast<bool>(res)); + long relevance = juniper::GetRelevancy(*res); // constant value since there are no hits but this is // also not a constrained search.. _test_equal(relevance, PROXIMITYBOOST_NOCONSTRAINT_OFFSET); - juniper::ReleaseResult(res); } { TestQuery q("AND(hepp,hoi)"); - Result* res = juniper::Analyse(juniper::TestConfig, - &q._qhandle, - content.c_str(), content.size(), - 0, 0, 0); - _test(res != NULL); - long relevance = juniper::GetRelevancy(res); + auto res = juniper::Analyse(*juniper::TestConfig, + q._qhandle, + content.c_str(), content.size(), + 0, 0, 0); + _test(static_cast<bool>(res)); + long relevance = juniper::GetRelevancy(*res); // Relevance may change, but nice to discover such changes.. // The important is that we get a nonzero value here as a hit _test_equal(relevance, 4470); - juniper::ReleaseResult(res); } } @@ -135,11 +130,11 @@ void MatchCandidateTest::testorder() { size_t content_len = strlen(content); // Fetch a result descriptor: - Result* res = juniper::Analyse(juniper::TestConfig, - &q._qhandle, - content, content_len, - 0, 0, 0); - _test(res != 0); + auto res = juniper::Analyse(*juniper::TestConfig, + q._qhandle, + content, content_len, + 0, 0, 0); + _test(static_cast<bool>(res)); // Do the scanning manually. Scan calls accept several times res->Scan(); @@ -150,7 +145,6 @@ void MatchCandidateTest::testorder() { match_candidate_set& ms = m.OrderedMatchSet(); _test(ms.size() == 1); - juniper::ReleaseResult(res); } @@ -165,11 +159,11 @@ void MatchCandidateTest::testMatches_limit() { size_t content_len = strlen(content); // Fetch a result descriptor: - Result* res = juniper::Analyse(juniper::TestConfig, - &q._qhandle, - content, content_len, - 0, 0, 0); - _test(res != 0); + auto res = juniper::Analyse(*juniper::TestConfig, + q._qhandle, + content, content_len, + 0, 0, 0); + _test(static_cast<bool>(res)); // Do the scanning manually. This calls accept several times res->Scan(); @@ -182,11 +176,10 @@ void MatchCandidateTest::testMatches_limit() { _test(ms.size() == 2); // The first (complete) match and the second starting at "test" // Check if we get the correct teaser as well.. - juniper::Summary* sum = juniper::GetTeaser(res); + juniper::Summary* sum = juniper::GetTeaser(*res); _test(strcmp(sum->Text(), "This is a simple text where a <b>phrase</b> <b>match</b> can be found not" " quite adjacent to a <b>test</b> <b>word</b>") == 0); - juniper::ReleaseResult(res); } @@ -200,11 +193,11 @@ void MatchCandidateTest::testAccept() { size_t content_len = strlen(content); // Fetch a result descriptor: - Result* res = juniper::Analyse(juniper::TestConfig, - &q._qhandle, - content, content_len, - 0, 0, 0); - _test(res != 0); + auto res = juniper::Analyse(*juniper::TestConfig, + q._qhandle, + content, content_len, + 0, 0, 0); + _test(static_cast<bool>(res)); // Do the scanning manually. This calls accept several times res->Scan(); @@ -218,7 +211,6 @@ void MatchCandidateTest::testAccept() { _test(ms.size() > 0); if (!ms.size()) { - juniper::ReleaseResult(res); return; // No point in continuing.. } @@ -234,10 +226,9 @@ void MatchCandidateTest::testAccept() { _test(mc._klist.size() == 2); // Two occurrence elements in list // Just for the sake of it, verify that we get a proper teaser out of this also.. - juniper::Summary* sum = juniper::GetTeaser(res); + juniper::Summary* sum = juniper::GetTeaser(*res); _test(strcmp(sum->Text(), "This is a <b>simple</b> <b>test</b> where we should get a perfect match") == 0); - juniper::ReleaseResult(res); } @@ -260,11 +251,11 @@ void MatchCandidateTest::testMake_keylist() { size_t content_len = strlen(content); // Fetch a result descriptor: - Result* res = juniper::Analyse(juniper::TestConfig, - &q._qhandle, - content, content_len, - 0, 0, 0); - _test(res != 0); + auto res = juniper::Analyse(*juniper::TestConfig, + q._qhandle, + content, content_len, + 0, 0, 0); + _test(static_cast<bool>(res)); // Do the scanning manually. This calls accept several times res->Scan(); @@ -275,8 +266,6 @@ void MatchCandidateTest::testMake_keylist() { match_candidate_set& ms = m.OrderedMatchSet(); _test_equal(static_cast<size_t>(ms.size()), 6u); - - juniper::ReleaseResult(res); } @@ -291,11 +280,11 @@ void MatchCandidateTest::testAdd_to_keylist() { size_t content_len = strlen(content); // Fetch a result descriptor: - Result* res = juniper::Analyse(juniper::TestConfig, - &q._qhandle, - content, content_len, - 0, 0, 0); - _test(res != 0); + auto res = juniper::Analyse(*juniper::TestConfig, + q._qhandle, + content, content_len, + 0, 0, 0); + _test(static_cast<bool>(res)); // Do the scanning manually. This calls accept several times res->Scan(); @@ -308,13 +297,12 @@ void MatchCandidateTest::testAdd_to_keylist() { _test_equal(static_cast<size_t>(ms.size()), 1u); // Single result // Bug triggered when result is fetched.. - juniper::Summary* sum = juniper::GetTeaser(res); + juniper::Summary* sum = juniper::GetTeaser(*res); std::string s(sum->Text()); _test_equal(s, "connect truende. <b>phr1</b> <b>phr2</b> www www www <b>phr3</b>" " <b>phr4</b> acuicola 8844"); - juniper::ReleaseResult(res); } @@ -331,11 +319,11 @@ void MatchCandidateTest::testLength() { TestQuery q("NEAR/4(pattern,NEAR/1(simple,with),NEAR/2(simple,adjacent))"); // Fetch a result descriptor: - Result* res = juniper::Analyse(juniper::TestConfig, &q._qhandle, - content, content_len, - 0, 0, 0); + auto res = juniper::Analyse(*juniper::TestConfig, q._qhandle, + content, content_len, + 0, 0, 0); - juniper::Summary* sum = juniper::GetTeaser(res); + juniper::Summary* sum = juniper::GetTeaser(*res); Matcher& m = *res->_matcher; match_candidate_set& ms = m.OrderedMatchSet(); _test_equal(static_cast<size_t>(ms.size()), 1u); @@ -345,7 +333,6 @@ void MatchCandidateTest::testLength() { "this <b>simple</b> text <b>with</b> <b>adjacent</b> words of " "a certain <b>pattern</b> must be matched according to specific" " rules to be detailed in this test."); - juniper::ReleaseResult(res); } { @@ -353,17 +340,16 @@ void MatchCandidateTest::testLength() { TestQuery q("ONEAR/4(pattern,NEAR/1(simple,with),NEAR/2(simple,adjacent))"); // Fetch a result descriptor: - Result* res = juniper::Analyse(juniper::TestConfig, - &q._qhandle - ,content, content_len, - 0, 0, 0); + auto res = juniper::Analyse(*juniper::TestConfig, + q._qhandle + ,content, content_len, + 0, 0, 0); res->Scan(); Matcher& m = *res->_matcher; match_candidate_set& ms = m.OrderedMatchSet(); _test_equal(static_cast<size_t>(ms.size()), 0u); - juniper::ReleaseResult(res); } { @@ -372,16 +358,14 @@ void MatchCandidateTest::testLength() { TestQuery q("NEAR/4(pattern,NEAR/1(simple,with),NEAR/1(simple,adjacent))"); // Fetch a result descriptor: - Result* res = juniper::Analyse(juniper::TestConfig, &q._qhandle, - content, content_len, - 0, 0, 0); + auto res = juniper::Analyse(*juniper::TestConfig, q._qhandle, + content, content_len, + 0, 0, 0); res->Scan(); Matcher& m = *res->_matcher; match_candidate_set& ms = m.OrderedMatchSet(); _test_equal(static_cast<size_t>(ms.size()), 0u); - - juniper::ReleaseResult(res); } } @@ -416,11 +400,11 @@ void MatchCandidateTest::requireThatMaxNumberOfMatchCandidatesCanBeControlled() const char *content = "re re re re foo re re re re bar re re re re foo re re re re bar"; size_t content_len = strlen(content); - Result *res = juniper::Analyse(juniper::TestConfig, - &q._qhandle, - content, content_len, - 0, 0, 0); - _test(res != 0); + auto res = juniper::Analyse(*juniper::TestConfig, + q._qhandle, + content, content_len, + 0, 0, 0); + _test(static_cast<bool>(res)); // Deflect tokens to my processor Matcher &m = *res->_matcher; @@ -435,8 +419,6 @@ void MatchCandidateTest::requireThatMaxNumberOfMatchCandidatesCanBeControlled() _test_equal(m.TotalHits(), 20); match_candidate_set& mcs = m.OrderedMatchSet(); _test_equal(static_cast<size_t>(mcs.size()), 2u); - - juniper::ReleaseResult(res); } diff --git a/searchsummary/src/vespa/juniper/Matcher.cpp b/searchsummary/src/vespa/juniper/Matcher.cpp index e286068038b..32f966f4571 100644 --- a/searchsummary/src/vespa/juniper/Matcher.cpp +++ b/searchsummary/src/vespa/juniper/Matcher.cpp @@ -38,7 +38,7 @@ Matcher::Matcher(Result* result) : _log_text("") { _occ.reserve(KEY_OCC_RESERVED); - DocsumParams& dsp = _result->_config->_docsumparams; + const DocsumParams& dsp = _result->_config->_docsumparams; _winsize = _result->WinSize(); _winsizeFallback = static_cast<size_t>(_result->WinSizeFallbackMultiplier() * _winsize); _max_match_candidates = _result->MaxMatchCandidates(); @@ -125,7 +125,6 @@ void Matcher::reset_matches() void Matcher::reset_occurrences() { - delete_all(_occ); _occ.clear(); } @@ -162,10 +161,11 @@ bool Matcher::add_occurrence(off_t pos, off_t tpos, size_t len) LOG(spam, "Match: %s(%" PRId64 ")", mexp->term(), static_cast<int64_t>(tpos)); // Add new occurrence to sequence of all occurrences - key_occ_ptr k = new key_occ(mexp->term(), pos, tpos, len); - if (!k) return false; + auto smart_k = std::make_unique<key_occ>(mexp->term(), pos, tpos, len); + if (!smart_k) return false; - _occ.push_back(k); + auto k = smart_k.get(); + _occ.emplace_back(std::move(smart_k)); if (!(_need_complete_cnt > 0)) { size_t nodeno; diff --git a/searchsummary/src/vespa/juniper/foreach_utils.h b/searchsummary/src/vespa/juniper/foreach_utils.h deleted file mode 100644 index ebbf1f41049..00000000000 --- a/searchsummary/src/vespa/juniper/foreach_utils.h +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once -#include <algorithm> - -/** \if utils - * A simple general deleter object to be passed to for instance std::for_each - * to delete pointer referenced objects - * in STL containers. - * - */ - -struct Deleter -{ - template <typename T> - void operator()(T* t) const - { - delete t; - } -}; - -/* \def Handy macro to delete all pointer objects in a container - * (using \a Deleter) - */ - -#define delete_all(container) \ - std::for_each(container.begin(), container.end(), Deleter()) - - -#define FunctionObj(name, func) \ - struct name \ - { \ - template <typename T> \ - void operator()(T* t) \ - { \ - t->func(); \ - } \ - } - - -#define for_all(container, obj) \ - std::for_each(container.begin(), container.end(), obj()) - -/** \endif (utils) */ - diff --git a/searchsummary/src/vespa/juniper/juniperdebug.h b/searchsummary/src/vespa/juniper/juniperdebug.h index cf6a3c971f8..d475134dae3 100644 --- a/searchsummary/src/vespa/juniper/juniperdebug.h +++ b/searchsummary/src/vespa/juniper/juniperdebug.h @@ -3,6 +3,7 @@ // Include something from STL so that _STLPORT_VERSION gets defined if appropriate #include <string> +#include <algorithm> /* Juniper debug macro */ @@ -43,14 +44,9 @@ extern unsigned debug_level; #endif - -#include "foreach_utils.h" - -FunctionObj(DoDump, dump); - template <class _container> void dump_list(_container& __c) { - for_all(__c, DoDump); + std::for_each(__c.begin(), __c.end(), [](auto& elem) { elem->dump(); }); } diff --git a/searchsummary/src/vespa/juniper/juniperparams.cpp b/searchsummary/src/vespa/juniper/juniperparams.cpp index 4f25b2446ad..2ee0f3c31f6 100644 --- a/searchsummary/src/vespa/juniper/juniperparams.cpp +++ b/searchsummary/src/vespa/juniper/juniperparams.cpp @@ -108,18 +108,12 @@ MatcherParams& MatcherParams::SetWordFolder(Fast_WordFolder* wordfolder) return *this; } -Fast_WordFolder* MatcherParams::WordFolder() { return _wordfolder; } - - MatcherParams& MatcherParams::SetProximityFactor(double factor) { _proximity_factor = factor; return *this; } -double MatcherParams::ProximityFactor() { return _proximity_factor; } - - bool operator==(MatcherParams& mp1, MatcherParams& mp2) { return memcmp(&mp1, &mp2, sizeof(MatcherParams)) == 0; diff --git a/searchsummary/src/vespa/juniper/juniperparams.h b/searchsummary/src/vespa/juniper/juniperparams.h index f4f17779f2d..415c254b3f0 100644 --- a/searchsummary/src/vespa/juniper/juniperparams.h +++ b/searchsummary/src/vespa/juniper/juniperparams.h @@ -48,8 +48,10 @@ class MatcherParams { public: MatcherParams(); - MatcherParams(MatcherParams &) = delete; - MatcherParams &operator=(MatcherParams &) = delete; + MatcherParams(const MatcherParams&) = delete; + MatcherParams(MatcherParams&&) = delete; + MatcherParams &operator=(const MatcherParams&) = delete; + MatcherParams &operator=(MatcherParams&&) = delete; MatcherParams& SetMatchWindowSize(size_t winsize); size_t MatchWindowSize() const; @@ -66,10 +68,10 @@ public: size_t StemMaxExtend() const; MatcherParams& SetWordFolder(Fast_WordFolder* wordfolder); - Fast_WordFolder* WordFolder(); + const Fast_WordFolder* WordFolder() const noexcept { return _wordfolder; } MatcherParams& SetProximityFactor(double factor); - double ProximityFactor(); + double ProximityFactor() const noexcept { return _proximity_factor; }; private: size_t _match_winsize; diff --git a/searchsummary/src/vespa/juniper/keyocc.h b/searchsummary/src/vespa/juniper/keyocc.h index 8c79e51e3a3..f211b5bdf61 100644 --- a/searchsummary/src/vespa/juniper/keyocc.h +++ b/searchsummary/src/vespa/juniper/keyocc.h @@ -3,9 +3,9 @@ #include "matchelem.h" #include "querynode.h" +#include <memory> -typedef key_occ* key_occ_ptr; -typedef std::vector<key_occ_ptr> key_occ_vector; +using key_occ_vector = std::vector<std::unique_ptr<key_occ>>; class key_occ : public MatchElement { diff --git a/searchsummary/src/vespa/juniper/querymodifier.cpp b/searchsummary/src/vespa/juniper/querymodifier.cpp index 797617b605a..da751c52a92 100644 --- a/searchsummary/src/vespa/juniper/querymodifier.cpp +++ b/searchsummary/src/vespa/juniper/querymodifier.cpp @@ -2,7 +2,6 @@ #include "juniperdebug.h" #include "querymodifier.h" -#include "foreach_utils.h" #include "querynode.h" #include <vespa/log/log.h> diff --git a/searchsummary/src/vespa/juniper/result.cpp b/searchsummary/src/vespa/juniper/result.cpp index 15ad9aa2a98..fddc5d65c86 100644 --- a/searchsummary/src/vespa/juniper/result.cpp +++ b/searchsummary/src/vespa/juniper/result.cpp @@ -27,14 +27,14 @@ public: }; -Result::Result(Config* config, QueryHandle* qhandle, +Result::Result(const Config& config, QueryHandle& qhandle, const char* docsum, size_t docsum_len, uint32_t langid) : - _qhandle(qhandle), - _mo(qhandle->MatchObj(langid)), + _qhandle(&qhandle), + _mo(qhandle.MatchObj(langid)), _docsum(docsum), _docsum_len(docsum_len), _langid(langid), - _config(config), + _config(&config), _matcher(), _tokenizer(), _summaries(), @@ -50,8 +50,8 @@ Result::Result(Config* config, QueryHandle* qhandle, { if (!_mo) return; // The empty result.. - MatcherParams& mp = _config->_matcherparams; - Fast_WordFolder* wordfolder = mp.WordFolder(); + const MatcherParams& mp = _config->_matcherparams; + const Fast_WordFolder* wordfolder = mp.WordFolder(); if (_qhandle->_stem_min < 0) _stem_min = mp.StemMinLength(); @@ -87,8 +87,8 @@ Result::Result(Config* config, QueryHandle* qhandle, _registry = std::make_unique<SpecialTokenRegistry>(_matcher->getQuery()); - if (qhandle->_log_mask) - _matcher->set_log(qhandle->_log_mask); + if (qhandle._log_mask) + _matcher->set_log(qhandle._log_mask); _tokenizer->SetSuccessor(_matcher.get()); if (!_registry->getSpecialTokens().empty()) { @@ -98,7 +98,6 @@ Result::Result(Config* config, QueryHandle* qhandle, Result::~Result() { - delete_all(_summaries); } @@ -121,7 +120,7 @@ Summary* Result::GetTeaser(const Config* alt_config) _dynsum_len = dsp.Length(); else _dynsum_len = _qhandle->_dynsum_len; - SummaryImpl *sum = NULL; + std::unique_ptr<SummaryImpl> sum; // Avoid overhead when being called with an empty stack if (_mo && _mo->Query()) { Scan(); @@ -139,13 +138,13 @@ Summary* Result::GetTeaser(const Config* alt_config) if (sdesc) { size_t char_size; - sum = new SummaryImpl(BuildSummary(_docsum, _docsum_len, sdesc, cfg->_sumconf, char_size)); + sum = std::make_unique<SummaryImpl>(BuildSummary(_docsum, _docsum_len, sdesc, cfg->_sumconf, char_size)); DeleteSummaryDesc(sdesc); } } - if (sum == NULL) { - sum = new SummaryImpl(); + if (!sum) { + sum = std::make_unique<SummaryImpl>(); } if (sum->_text.empty() && dsp.Fallback() == DocsumParams::FALLBACK_PREFIX) @@ -157,7 +156,7 @@ Summary* Result::GetTeaser(const Config* alt_config) const char *src_end = _docsum + _docsum_len; ucs4_t *dst = buf; ucs4_t *dst_end = dst + TOKEN_DSTLEN; - Fast_WordFolder *folder = _config->_matcherparams.WordFolder(); + const Fast_WordFolder *folder = _config->_matcherparams.WordFolder(); text.reserve(_dynsum_len*2); if (src_end - src <= _dynsum_len) { @@ -186,25 +185,26 @@ Summary* Result::GetTeaser(const Config* alt_config) } sum->_text = std::string(&text[0], text.size()); } - _summaries.push_back(sum); - return sum; + _summaries.emplace_back(std::move(sum)); + return _summaries.back().get(); } Summary* Result::GetLog() { // Avoid overhead when being called with an empty stack - Summary* sum = NULL; + std::unique_ptr<Summary> sum; if (_mo && _mo->Query()) { LOG(debug, "juniper::GetLog"); Scan(); - sum = new SummaryImpl(_matcher->GetLog()); + sum = std::make_unique<SummaryImpl>(_matcher->GetLog()); } - else - sum = new SummaryImpl(); - _summaries.push_back(sum); - return sum; + else { + sum = std::make_unique<SummaryImpl>(); + } + _summaries.emplace_back(std::move(sum)); + return _summaries.back().get(); } diff --git a/searchsummary/src/vespa/juniper/result.h b/searchsummary/src/vespa/juniper/result.h index f0dcf3d4335..dcbb89fb1dc 100644 --- a/searchsummary/src/vespa/juniper/result.h +++ b/searchsummary/src/vespa/juniper/result.h @@ -14,7 +14,7 @@ namespace juniper class Result { public: - Result(Config* config, QueryHandle* qhandle, + Result(const Config& config, QueryHandle& qhandle, const char* docsum, size_t docsum_len, uint32_t langid); ~Result(); @@ -42,12 +42,12 @@ public: const char* _docsum; size_t _docsum_len; uint32_t _langid; - Config* _config; + const Config* _config; std::unique_ptr<Matcher> _matcher; std::unique_ptr<SpecialTokenRegistry> _registry; std::unique_ptr<JuniperTokenizer> _tokenizer; private: - std::vector<Summary*> _summaries; // Active summaries for this result + std::vector<std::unique_ptr<Summary>> _summaries; // Active summaries for this result bool _scan_done; // State of the result - is text scan done? /* Option storage */ diff --git a/searchsummary/src/vespa/juniper/rpinterface.cpp b/searchsummary/src/vespa/juniper/rpinterface.cpp index f9e91073a9b..de4b4cd3ef0 100644 --- a/searchsummary/src/vespa/juniper/rpinterface.cpp +++ b/searchsummary/src/vespa/juniper/rpinterface.cpp @@ -2,7 +2,6 @@ #include "rpinterface.h" #include "juniperparams.h" -#include "foreach_utils.h" #include "queryvisitor.h" #include "queryhandle.h" #include "propreader.h" @@ -79,9 +78,9 @@ std::unique_ptr<Config> Juniper::CreateConfig(const char* config_name) return std::unique_ptr<Config>(new Config(config_name, *this)); } -QueryHandle* Juniper::CreateQueryHandle(const IQuery& fquery, const char* juniperoptions) +std::unique_ptr<QueryHandle> Juniper::CreateQueryHandle(const IQuery& fquery, const char* juniperoptions) { - return new QueryHandle(fquery, juniperoptions, *_modifier); + return std::make_unique<QueryHandle>(fquery, juniperoptions, *_modifier); } void Juniper::AddRewriter(const char* index_name, IRewriter* rewriter, bool for_query, bool for_document) @@ -95,44 +94,29 @@ void Juniper::FlushRewriters() } -void ReleaseQueryHandle(QueryHandle*& handle) -{ - delete handle; - handle = NULL; -} - - -Result* Analyse(const Config* config, QueryHandle* qhandle, +std::unique_ptr<Result> Analyse(const Config& config, QueryHandle& qhandle, const char* docsum, size_t docsum_len, uint32_t docid, uint32_t /* inputfield_id */, uint32_t langid) { LOG(debug, "juniper::Analyse(): docId(%u), docsumLen(%zu), docsum(%s), langId(%u)", docid, docsum_len, docsum, langid); - Result* res = new Result(const_cast<Config*>(config), qhandle, docsum, docsum_len, langid); - return res; -} - -long GetRelevancy(Result* result_handle) -{ - return result_handle->GetRelevancy(); + return std::make_unique<Result>(config, qhandle, docsum, docsum_len, langid); } -Summary* GetTeaser(Result* result_handle, const Config* alt_config) +long GetRelevancy(Result& result_handle) { - return result_handle->GetTeaser(alt_config); + return result_handle.GetRelevancy(); } -Summary* GetLog(Result* result_handle) +Summary* GetTeaser(Result& result_handle, const Config* alt_config) { - return result_handle->GetLog(); + return result_handle.GetTeaser(alt_config); } -void ReleaseResult(Result*& result_handle) +Summary* GetLog(Result& result_handle) { - LOG(debug, "juniper::ReleaseResult"); - delete result_handle; - result_handle = NULL; + return result_handle.GetLog(); } } // end namespace juniper diff --git a/searchsummary/src/vespa/juniper/rpinterface.h b/searchsummary/src/vespa/juniper/rpinterface.h index 6cda324ae5c..d4ef5c5ebed 100644 --- a/searchsummary/src/vespa/juniper/rpinterface.h +++ b/searchsummary/src/vespa/juniper/rpinterface.h @@ -120,9 +120,9 @@ public: * behaviour such as user customization of teaser parameters, selectively * enabling of Juniper debugging/tracing features and to support Juniper extensions * to the query language. - * @return An allocated handle to be subsequently released by ReleaseQueryHandle() + * @return A unique pointer to a QueryHandle. */ - QueryHandle* CreateQueryHandle(const IQuery& query, const char* juniperoptions); + std::unique_ptr<QueryHandle> CreateQueryHandle(const IQuery& query, const char* juniperoptions); /** Add an rewriter for all terms that are prefixed with the given index. * When Juniper encounter a term in the query tagged with this index, @@ -150,11 +150,6 @@ private: */ bool AnalyseCompatible(Config* conf1, Config* conf2); -/** Release a QueryHandle as previously allocated by CreateQueryHandle. - * @param handle The QueryHandle object to release - */ -void ReleaseQueryHandle(QueryHandle*& handle); - /** Perform initial content analysis on a query/content pair. * Note that the content may either be a simple UTF-8 encoded string or a * more advanced representation including document structure elements, as provided @@ -170,10 +165,9 @@ void ReleaseQueryHandle(QueryHandle*& handle); within the document that contains the provided document summary. * @param langid A unique 32 bit id representing the language which this document summary is to be analysed in context of. - * @return A pointer to an allocated handle to be used in subsequent specific result - * requests (must later be released with ReleaseResult()) + * @return A unique pointer to a Result */ -Result* Analyse(const Config* config, QueryHandle* query, +std::unique_ptr<Result> Analyse(const Config& config, QueryHandle& query, const char* docsum, size_t docsum_len, uint32_t docid, uint32_t inputfield_id, uint32_t langid); @@ -182,17 +176,16 @@ Result* Analyse(const Config* config, QueryHandle* query, * @param result_handle The result to retrieve from * @return The relevancy (proximitymetric) of the processed content. */ -long GetRelevancy(Result* result_handle); +long GetRelevancy(Result& result_handle); /** Generate a teaser based on the provided analysis result * @param result_handle a handle obtained by a previous call to Analyse * @param alt_config An optional alternate config to use for this teaser generation * The purpose of alt_config is to allow generation of multiple teasers * based on the same content and analysis. - * @return The generated Teaser object. This object is valid until ReleaseResult - * is called for result_handle + * @return The generated Teaser object. This object is valid until result_handle is deleted. */ -Summary* GetTeaser(Result* result_handle, const Config* alt_config = NULL); +Summary* GetTeaser(Result& result_handle, const Config* alt_config = NULL); /** Retrieve log information based on the previous calls to this result handle. * Note that for the log to be complete, the juniper log override entry in @@ -200,15 +193,9 @@ Summary* GetTeaser(Result* result_handle, const Config* alt_config = NULL); * @param result_handle a handle obtained by a previous call to Analyse. * @return value: a summary description containing the Juniper log as a text field * if any log information is available, or else an empty summary. - * This object is valid until ReleaseResult is called for result_handle - */ -Summary* GetLog(Result* result_handle); - -/** Release all resources associated with the handle given including the - * summaries created by this result handle. - * @param result_handle The handle to release + * This object is valid until result_handle is deleted. */ -void ReleaseResult(Result*& result_handle); +Summary* GetLog(Result& result_handle); } // end namespace juniper diff --git a/searchsummary/src/vespa/juniper/tokenizer.cpp b/searchsummary/src/vespa/juniper/tokenizer.cpp index db6e1ecfccd..9253f81cf25 100644 --- a/searchsummary/src/vespa/juniper/tokenizer.cpp +++ b/searchsummary/src/vespa/juniper/tokenizer.cpp @@ -7,7 +7,7 @@ #include <vespa/log/log.h> LOG_SETUP(".juniper.tokenizer"); -JuniperTokenizer::JuniperTokenizer(Fast_WordFolder* wordfolder, +JuniperTokenizer::JuniperTokenizer(const Fast_WordFolder* wordfolder, const char* text, size_t len, ITokenProcessor* successor, const juniper::SpecialTokenRegistry * registry) : _wordfolder(wordfolder), _text(text), _len(len), _successor(successor), _registry(registry), diff --git a/searchsummary/src/vespa/juniper/tokenizer.h b/searchsummary/src/vespa/juniper/tokenizer.h index 34ed1dba5bb..bf0c9452665 100644 --- a/searchsummary/src/vespa/juniper/tokenizer.h +++ b/searchsummary/src/vespa/juniper/tokenizer.h @@ -11,7 +11,7 @@ class Fast_WordFolder; class JuniperTokenizer { public: - JuniperTokenizer(Fast_WordFolder* wordfolder, + JuniperTokenizer(const Fast_WordFolder* wordfolder, const char* text, size_t len, ITokenProcessor* = NULL, const juniper::SpecialTokenRegistry * registry = NULL); inline void SetSuccessor(ITokenProcessor* successor) { _successor = successor; } @@ -22,7 +22,7 @@ public: // Scan the input and dispatch to the successor void scan(); private: - Fast_WordFolder* _wordfolder; + const Fast_WordFolder* _wordfolder; const char* _text; // The current input text size_t _len; // Length of the text input ITokenProcessor* _successor; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.cpp index e6c8d0b6ab8..71800eb2128 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.cpp @@ -16,7 +16,7 @@ DocsumStoreDocument::DocsumStoreDocument(std::unique_ptr<document::Document> doc DocsumStoreDocument::~DocsumStoreDocument() = default; -std::unique_ptr<document::FieldValue> +DocsumStoreFieldValue DocsumStoreDocument::get_field_value(const vespalib::string& field_name) const { if (_document) { @@ -24,11 +24,11 @@ DocsumStoreDocument::get_field_value(const vespalib::string& field_name) const auto value(field.getDataType().createFieldValue()); if (value) { if (_document->getValue(field, *value)) { - return value; + return DocsumStoreFieldValue(std::move(value)); } } } - return {}; + return DocsumStoreFieldValue(); } void diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.h b/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.h index 3b0bea6e721..26cccc9970f 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_document.h @@ -17,7 +17,7 @@ class DocsumStoreDocument : public IDocsumStoreDocument public: DocsumStoreDocument(std::unique_ptr<document::Document> document); ~DocsumStoreDocument() override; - std::unique_ptr<document::FieldValue> get_field_value(const vespalib::string& field_name) const override; + DocsumStoreFieldValue get_field_value(const vespalib::string& field_name) const override; void insert_summary_field(const vespalib::string& field_name, vespalib::slime::Inserter& inserter) const override; void insert_document_id(vespalib::slime::Inserter& inserter) const override; }; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_field_value.h b/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_field_value.h new file mode 100644 index 00000000000..d06a2ab8287 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_store_field_value.h @@ -0,0 +1,38 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/document/fieldvalue/fieldvalue.h> +#include <memory> + +namespace search::docsummary { + +/* + * Class containing a field value returned from an IDocsumStoreDocument. + */ +class DocsumStoreFieldValue { + const document::FieldValue* _value; + std::unique_ptr<document::FieldValue> _value_store; +public: + explicit DocsumStoreFieldValue(std::unique_ptr<document::FieldValue> value) noexcept + : _value(value.get()), + _value_store(std::move(value)) + { + } + explicit DocsumStoreFieldValue(const document::FieldValue* value) noexcept + : _value(value), + _value_store() + { + } + DocsumStoreFieldValue() + : DocsumStoreFieldValue(nullptr) + { + } + ~DocsumStoreFieldValue() = default; + const document::FieldValue& operator*() const noexcept { return *_value; } + const document::FieldValue* operator->() const noexcept { return _value; } + const document::FieldValue* get() const noexcept { return _value; } + operator bool () const noexcept { return _value != nullptr; } +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp index 16575a2e9dc..9c810386703 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp @@ -4,6 +4,7 @@ #include "docsum_field_writer_state.h" #include <vespa/juniper/rpinterface.h> #include <vespa/document/datatype/positiondatatype.h> +#include <vespa/juniper/queryhandle.h> #include <vespa/searchcommon/attribute/iattributecontext.h> #include <vespa/searchlib/common/geo_location.h> #include <vespa/searchlib/common/geo_location_parser.h> @@ -36,24 +37,10 @@ GetDocsumsState::GetDocsumsState(GetDocsumsStateCallback &callback) _rankFeatures(nullptr), _matching_elements() { - _dynteaser._docid = static_cast<uint32_t>(-1); - _dynteaser._input = static_cast<uint32_t>(-1); - _dynteaser._lang = static_cast<uint32_t>(-1); - _dynteaser._config = nullptr; - _dynteaser._query = nullptr; - _dynteaser._result = nullptr; } -GetDocsumsState::~GetDocsumsState() -{ - if (_dynteaser._result != nullptr) { - juniper::ReleaseResult(_dynteaser._result); - } - if (_dynteaser._query != nullptr) { - juniper::ReleaseQueryHandle(_dynteaser._query); - } -} +GetDocsumsState::~GetDocsumsState() = default; const MatchingElements & GetDocsumsState::get_matching_elements(const MatchingElementsFields &matching_elems_fields) diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h index 438a8a6d847..adcd47c098c 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h @@ -57,12 +57,7 @@ public: GetDocsumsStateCallback &_callback; struct DynTeaserState { - uint32_t _docid; // document id ('cache key') - uint32_t _input; // input field ('cache key') - uint32_t _lang; // lang field ('cache key') - juniper::Config *_config; // juniper config ('cache key') - juniper::QueryHandle *_query; // juniper query representation - juniper::Result *_result; // juniper analyze result + std::unique_ptr<juniper::QueryHandle> _query; // juniper query representation } _dynteaser; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp index d9878cd2057..58dde39c336 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp @@ -9,6 +9,8 @@ #include <vespa/searchlib/queryeval/split_float.h> #include <vespa/vespalib/objects/hexdump.h> #include <vespa/juniper/config.h> +#include <vespa/juniper/queryhandle.h> +#include <vespa/juniper/result.h> #include <sstream> #include <vespa/log/log.h> @@ -368,27 +370,17 @@ DynamicTeaserDFW::getJuniperInput(GeneralResult *gres) { vespalib::string DynamicTeaserDFW::makeDynamicTeaser(uint32_t docid, vespalib::stringref input, GetDocsumsState *state) { - if (state->_dynteaser._query == nullptr) { + if (!state->_dynteaser._query) { JuniperQueryAdapter iq(state->_kwExtractor, state->_args.getStackDump(), &state->_args.highlightTerms()); state->_dynteaser._query = _juniper->CreateQueryHandle(iq, nullptr); } - LOG(debug, "makeDynamicTeaser: docid (%d,%d), fieldenum (%d,%d), lang (%d,%d) analyse %s", - docid, state->_dynteaser._docid, - _inputFieldEnumValue, state->_dynteaser._input, - _langFieldEnumValue, state->_dynteaser._lang, - (juniper::AnalyseCompatible(_juniperConfig.get(), state->_dynteaser._config) ? "no" : "yes")); + LOG(debug, "makeDynamicTeaser: docid (%d), fieldenum (%d), lang (%d)", + docid, _inputFieldEnumValue, _langFieldEnumValue); - if (state->_dynteaser._result != nullptr) - juniper::ReleaseResult(state->_dynteaser._result); - - state->_dynteaser._docid = docid; - state->_dynteaser._input = _inputFieldEnumValue; - state->_dynteaser._lang = _langFieldEnumValue; - state->_dynteaser._config = _juniperConfig.get(); - state->_dynteaser._result = nullptr; + std::unique_ptr<juniper::Result> result; if (state->_dynteaser._query != nullptr) { @@ -401,13 +393,12 @@ DynamicTeaserDFW::makeDynamicTeaser(uint32_t docid, vespalib::stringref input, G auto langid = static_cast<uint32_t>(-1); - state->_dynteaser._result = - juniper::Analyse(_juniperConfig.get(), state->_dynteaser._query, - input.data(), input.length(), docid, _inputFieldEnumValue, langid); + result = juniper::Analyse(*_juniperConfig, *state->_dynteaser._query, + input.data(), input.length(), docid, _inputFieldEnumValue, langid); } - juniper::Summary *teaser = (state->_dynteaser._result != nullptr) - ? juniper::GetTeaser(state->_dynteaser._result, _juniperConfig.get()) + juniper::Summary *teaser = result + ? juniper::GetTeaser(*result, _juniperConfig.get()) : nullptr; if (LOG_WOULD_LOG(debug)) { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp b/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp index 825c3b39c1b..359959391fd 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/general_result.cpp @@ -62,13 +62,13 @@ GeneralResult::GetPresentEntryFromEnumValue(uint32_t value) return GetPresentEntry(idx); } -std::unique_ptr<document::FieldValue> +DocsumStoreFieldValue GeneralResult::get_field_value(const vespalib::string& field_name) const { if (_document != nullptr) { return _document->get_field_value(field_name); } - return {}; + return DocsumStoreFieldValue(); } bool diff --git a/searchsummary/src/vespa/searchsummary/docsummary/general_result.h b/searchsummary/src/vespa/searchsummary/docsummary/general_result.h index cff27a496e3..8f7a1377502 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/general_result.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/general_result.h @@ -4,10 +4,7 @@ #include "resultclass.h" #include "docsumstorevalue.h" - -namespace document { -class FieldValue; -} +#include "docsum_store_field_value.h" namespace search::docsummary { @@ -42,7 +39,7 @@ public: } ResEntry *GetPresentEntry(const char *name); ResEntry *GetPresentEntryFromEnumValue(uint32_t val); - std::unique_ptr<document::FieldValue> get_field_value(const vespalib::string& field_name) const; + DocsumStoreFieldValue get_field_value(const vespalib::string& field_name) const; bool unpack(const char *buf, const size_t buflen); bool inplaceUnpack(const DocsumStoreValue &value) { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_store_document.h b/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_store_document.h index c177568c467..b9c2ae76e7a 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_store_document.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/i_docsum_store_document.h @@ -2,10 +2,8 @@ #pragma once +#include "docsum_store_field_value.h" #include <vespa/vespalib/stllike/string.h> -#include <memory> - -namespace document { class FieldValue; } namespace vespalib::slime { struct Inserter; } @@ -20,7 +18,7 @@ class IDocsumStoreDocument { public: virtual ~IDocsumStoreDocument() = default; - virtual std::unique_ptr<document::FieldValue> get_field_value(const vespalib::string& field_name) const = 0; + virtual DocsumStoreFieldValue get_field_value(const vespalib::string& field_name) const = 0; virtual void insert_summary_field(const vespalib::string& field_name, vespalib::slime::Inserter& inserter) const = 0; virtual void insert_document_id(vespalib::slime::Inserter& inserter) const = 0; }; diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.h b/storage/src/vespa/storage/bucketdb/bucketmanager.h index eacd0c8ca6a..124acf1864c 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.h +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.h @@ -34,7 +34,6 @@ class BucketManager : public StorageLink, { public: /** Type used for message queues */ - using CommandList = std::list<std::shared_ptr<api::StorageCommand>>; using BucketInfoRequestList = std::list<std::shared_ptr<api::RequestBucketInfoCommand>>; using BucketInfoRequestMap = std::unordered_map<document::BucketSpace, BucketInfoRequestList, document::BucketSpace::hash>; @@ -55,8 +54,7 @@ private: mutable std::mutex _queueProcessingLock; using ReplyQueue = std::vector<api::StorageReply::SP>; - using ConflictingBuckets = std::unordered_set<document::BucketId, - document::BucketId::hash>; + using ConflictingBuckets = std::unordered_set<document::BucketId, document::BucketId::hash>; ReplyQueue _queuedReplies; ConflictingBuckets _conflictingBuckets; /** @@ -80,9 +78,6 @@ private: framework::Thread::UP _thread; std::chrono::milliseconds _simulated_processing_delay; - BucketManager(const BucketManager&); - BucketManager& operator=(const BucketManager&); - class ScopedQueueDispatchGuard { BucketManager& _mgr; public: @@ -94,8 +89,9 @@ private: }; public: - explicit BucketManager(const config::ConfigUri&, - ServiceLayerComponentRegister&); + BucketManager(const config::ConfigUri&, ServiceLayerComponentRegister&); + BucketManager(const BucketManager&) = delete; + BucketManager& operator=(const BucketManager&) = delete; ~BucketManager(); void startWorkerThread(); @@ -127,8 +123,7 @@ private: void updateMinUsedBits(); bool onRequestBucketInfo(const std::shared_ptr<api::RequestBucketInfoCommand>&) override; - bool processRequestBucketInfoCommands(document::BucketSpace bucketSpace, - BucketInfoRequestList &reqs); + bool processRequestBucketInfoCommands(document::BucketSpace bucketSpace, BucketInfoRequestList &reqs); /** * Enqueue reply and add its bucket to the set of conflicting buckets iff @@ -199,42 +194,26 @@ private: * * Not thread safe. */ - bool replyConflictsWithConcurrentOperation( - const api::BucketReply& reply) const; - + bool replyConflictsWithConcurrentOperation(const api::BucketReply& reply) const; bool enqueueIfBucketHasConflicts(const api::BucketReply::SP& reply); - bool onUp(const std::shared_ptr<api::StorageMessage>&) override; - bool onSetSystemState( - const std::shared_ptr<api::SetSystemStateCommand>&) override; - bool onCreateBucket( - const std::shared_ptr<api::CreateBucketCommand>&) override; - bool onMergeBucket( - const std::shared_ptr<api::MergeBucketCommand>&) override; - bool onRemove( - const std::shared_ptr<api::RemoveCommand>&) override; - bool onRemoveReply( - const std::shared_ptr<api::RemoveReply>&) override; - bool onPut( - const std::shared_ptr<api::PutCommand>&) override; - bool onPutReply( - const std::shared_ptr<api::PutReply>&) override; - bool onUpdate( - const std::shared_ptr<api::UpdateCommand>&) override; - bool onUpdateReply( - const std::shared_ptr<api::UpdateReply>&) override; - bool onNotifyBucketChangeReply( - const std::shared_ptr<api::NotifyBucketChangeReply>&) override; + bool onSetSystemState(const std::shared_ptr<api::SetSystemStateCommand>&) override; + bool onCreateBucket(const std::shared_ptr<api::CreateBucketCommand>&) override; + bool onMergeBucket(const std::shared_ptr<api::MergeBucketCommand>&) override; + bool onRemove(const std::shared_ptr<api::RemoveCommand>&) override; + bool onRemoveReply(const std::shared_ptr<api::RemoveReply>&) override; + bool onPut(const std::shared_ptr<api::PutCommand>&) override; + bool onPutReply(const std::shared_ptr<api::PutReply>&) override; + bool onUpdate(const std::shared_ptr<api::UpdateCommand>&) override; + bool onUpdateReply(const std::shared_ptr<api::UpdateReply>&) override; + bool onNotifyBucketChangeReply(const std::shared_ptr<api::NotifyBucketChangeReply>&) override; bool verifyAndUpdateLastModified(api::StorageCommand& cmd, const document::Bucket& bucket, uint64_t lastModified); - bool onSplitBucketReply( - const std::shared_ptr<api::SplitBucketReply>&) override; - bool onJoinBucketsReply( - const std::shared_ptr<api::JoinBucketsReply>&) override; - bool onDeleteBucketReply( - const std::shared_ptr<api::DeleteBucketReply>&) override; + bool onSplitBucketReply(const std::shared_ptr<api::SplitBucketReply>&) override; + bool onJoinBucketsReply(const std::shared_ptr<api::JoinBucketsReply>&) override; + bool onDeleteBucketReply(const std::shared_ptr<api::DeleteBucketReply>&) override; }; } // storage diff --git a/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp b/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp index 13e5ad1c84b..706325a0f7a 100644 --- a/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp @@ -21,6 +21,7 @@ using search::fef::RankSetup; using vsm::VsmfieldsHandle; using vsm::VSMAdapter; using vsm::FieldIdTList; +using vespalib::make_string_short::fmt; namespace streaming { @@ -115,10 +116,10 @@ RankManager::Snapshot::initRankSetup(const BlueprintFactory & factory) for (uint32_t i = 0; i < _indexEnv.size(); ++i) { IndexEnvironment & ie = _indexEnv[i]; - RankSetup::SP rs(new RankSetup(factory, ie)); + auto rs = std::make_shared<RankSetup>(factory, ie); rs->configure(); // reads config values from the property map if (!rs->compile()) { - LOG(warning, "Could not compile rank setup for rank profile '%u'.", i); + LOG(warning, "Could not compile rank setup for rank profile '%u'. Errors = %s", i, rs->getJoinedWarnings().c_str()); return false; } _rankSetup.push_back(rs); @@ -127,7 +128,7 @@ RankManager::Snapshot::initRankSetup(const BlueprintFactory & factory) LOG(debug, "Number of index environments and rank setups: %u", (uint32_t)_indexEnv.size()); LOG_ASSERT(_properties.size() == _rankSetup.size()); for (uint32_t i = 0; i < _properties.size(); ++i) { - vespalib::string number = vespalib::make_string("%u", i); + vespalib::string number = fmt("%u", i); _rpmap[number] = i; } for (uint32_t i = 0; i < _properties.size(); ++i) { diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp index 5bcead79f97..3d0f8ed6e37 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp @@ -133,7 +133,7 @@ class DocsumStoreVsmDocument : public IDocsumStoreDocument public: DocsumStoreVsmDocument(const document::Document* document); ~DocsumStoreVsmDocument() override; - std::unique_ptr<document::FieldValue> get_field_value(const vespalib::string& field_name) const override; + DocsumStoreFieldValue get_field_value(const vespalib::string& field_name) const override; void insert_summary_field(const vespalib::string& field_name, vespalib::slime::Inserter& inserter) const override; void insert_document_id(vespalib::slime::Inserter& inserter) const override; }; @@ -145,7 +145,7 @@ DocsumStoreVsmDocument::DocsumStoreVsmDocument(const document::Document* documen DocsumStoreVsmDocument::~DocsumStoreVsmDocument() = default; -std::unique_ptr<document::FieldValue> +DocsumStoreFieldValue DocsumStoreVsmDocument::get_field_value(const vespalib::string& field_name) const { if (_document != nullptr) { @@ -153,11 +153,11 @@ DocsumStoreVsmDocument::get_field_value(const vespalib::string& field_name) cons auto value(field.getDataType().createFieldValue()); if (value) { if (_document->getValue(field, *value)) { - return value; + return DocsumStoreFieldValue(std::move(value)); } } } - return {}; + return DocsumStoreFieldValue(); } void diff --git a/vespalog/CMakeLists.txt b/vespalog/CMakeLists.txt index 45410a1d29d..cc419681445 100644 --- a/vespalog/CMakeLists.txt +++ b/vespalog/CMakeLists.txt @@ -17,6 +17,5 @@ vespa_define_module( src/test/threads ) -vespa_install_script(src/vespa-logfmt/vespa-logfmt.pl vespa-logfmt bin) install(FILES src/vespa-logfmt/vespa-logfmt.1 DESTINATION man/man1) install(DIRECTORY DESTINATION var/db/vespa/logcontrol) |
