aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMorten Tokle <mortent@yahooinc.com>2023-03-01 08:11:05 +0100
committerGitHub <noreply@github.com>2023-03-01 08:11:05 +0100
commit52625c4410dc46b27fab4061fc2d0f34b3faa9f2 (patch)
tree86985170b3a7ed10e8a82605720d1acb0e832b83
parent6540b004e3a7eea35da0956910331e69bcb87f8c (diff)
parent42f5944056ed2b9d46b1b5538cbcccf0b369ca44 (diff)
Merge branch 'master' into mortent/revert-26208
-rw-r--r--client/go/internal/cli/cmd/root.go5
-rw-r--r--client/go/internal/cli/cmd/visit.go348
-rw-r--r--client/go/internal/cli/cmd/visit_test.go142
-rw-r--r--config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java1
-rw-r--r--config-model/.gitignore1
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java1
-rw-r--r--config-model/src/main/java/com/yahoo/schema/RankProfile.java3
-rw-r--r--config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java11
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java6
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java9
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java24
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/search/RankProfilesEvaluatorComponent.java49
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java2
-rw-r--r--config-model/src/test/derived/globalphase_onnx_inside/files/ax_plus_b.onnx23
-rw-r--r--config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg34
-rw-r--r--config-model/src/test/derived/globalphase_onnx_inside/test.sd42
-rw-r--r--config-model/src/test/derived/rankingexpression/rank-profiles.cfg2
-rw-r--r--config-model/src/test/java/com/yahoo/schema/derived/GlobalPhaseOnnxModelsTestCase.java22
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java2
-rw-r--r--config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java2
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java6
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java3
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java3
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java42
-rw-r--r--configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java2
-rw-r--r--container-core/src/main/resources/configdefinitions/container.qr-searchers.def3
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java32
-rw-r--r--container-search/src/main/java/com/yahoo/search/ranking/Evaluator.java14
-rw-r--r--container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java118
-rw-r--r--container-search/src/main/java/com/yahoo/search/ranking/HitRescorer.java56
-rw-r--r--container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluator.java53
-rw-r--r--container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluatorFactory.java40
-rw-r--r--container-search/src/main/java/com/yahoo/search/ranking/ResultReranker.java91
-rw-r--r--container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java51
-rw-r--r--container-search/src/main/java/com/yahoo/search/ranking/package-info.java (renamed from fastos/src/tests/mazeserver.cpp)6
-rw-r--r--container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java1
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java33
-rw-r--r--dist/vespa.spec5
-rw-r--r--fastos/CMakeLists.txt1
-rw-r--r--fastos/src/tests/.gitignore17
-rw-r--r--fastos/src/tests/CMakeLists.txt15
-rw-r--r--fastos/src/tests/coretest2.cpp41
-rw-r--r--fastos/src/tests/performancetest.cpp54
-rw-r--r--fastos/src/tests/typetest.cpp44
-rw-r--r--fastos/src/vespa/fastos/CMakeLists.txt4
-rw-r--r--fastos/src/vespa/fastos/types.h9
-rw-r--r--flags/src/main/java/com/yahoo/vespa/flags/Flags.java7
-rw-r--r--jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java2
-rw-r--r--jdisc_core/src/test/resources/exportPackages.properties2
-rw-r--r--metrics/src/tests/metricmanagertest.cpp206
-rw-r--r--metrics/src/tests/snapshottest.cpp14
-rw-r--r--metrics/src/vespa/metrics/CMakeLists.txt1
-rw-r--r--metrics/src/vespa/metrics/metricmanager.cpp325
-rw-r--r--metrics/src/vespa/metrics/metricmanager.h25
-rw-r--r--metrics/src/vespa/metrics/updatehook.h3
-rw-r--r--metrics/src/vespa/metrics/xmlwriter.cpp112
-rw-r--r--metrics/src/vespa/metrics/xmlwriter.h30
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java9
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java4
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java45
-rw-r--r--node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java15
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java18
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java16
-rw-r--r--node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java4
-rw-r--r--parent/pom.xml2
-rw-r--r--searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h3
-rw-r--r--searchlib/src/vespa/searchlib/common/bitvector.h2
-rw-r--r--searchlib/src/vespa/searchlib/predicate/predicate_posting_list.h5
-rw-r--r--searchlib/src/vespa/searchlib/queryeval/hitcollector.h1
-rw-r--r--storage/src/tests/common/metricstest.cpp8
-rw-r--r--storage/src/tests/storageserver/statereportertest.cpp40
-rw-r--r--storage/src/vespa/storage/common/statusmetricconsumer.cpp14
-rw-r--r--storage/src/vespa/storage/storageserver/storagenode.cpp62
-rw-r--r--storage/src/vespa/storageapi/mbusprot/serializationhelper.h34
-rw-r--r--vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java2
-rw-r--r--vespalib/CMakeLists.txt18
-rw-r--r--vespalib/src/tests/fastos/CMakeLists.txt9
-rw-r--r--vespalib/src/tests/fastos/file_test.cpp (renamed from fastos/src/tests/filetest.cpp)0
-rw-r--r--vespalib/src/tests/fastos/hello.txt (renamed from fastos/src/tests/hello.txt)0
-rw-r--r--vespalib/src/tests/fastos/tests.h (renamed from fastos/src/tests/tests.h)0
-rw-r--r--vespalib/src/vespa/fastlib/io/CMakeLists.txt2
-rw-r--r--vespalib/src/vespa/fastlib/text/CMakeLists.txt2
-rw-r--r--vespalib/src/vespa/fastos/CMakeLists.txt9
-rw-r--r--vespalib/src/vespa/fastos/file.cpp (renamed from fastos/src/vespa/fastos/file.cpp)0
-rw-r--r--vespalib/src/vespa/fastos/file.h (renamed from fastos/src/vespa/fastos/file.h)3
-rw-r--r--vespalib/src/vespa/fastos/file_rw_ops.cpp (renamed from fastos/src/vespa/fastos/file_rw_ops.cpp)0
-rw-r--r--vespalib/src/vespa/fastos/file_rw_ops.h (renamed from fastos/src/vespa/fastos/file_rw_ops.h)0
-rw-r--r--vespalib/src/vespa/fastos/linux_file.cpp (renamed from fastos/src/vespa/fastos/linux_file.cpp)0
-rw-r--r--vespalib/src/vespa/fastos/linux_file.h (renamed from fastos/src/vespa/fastos/linux_file.h)0
-rw-r--r--vespalib/src/vespa/fastos/unix_file.cpp (renamed from fastos/src/vespa/fastos/unix_file.cpp)0
-rw-r--r--vespalib/src/vespa/fastos/unix_file.h (renamed from fastos/src/vespa/fastos/unix_file.h)0
-rw-r--r--vespalib/src/vespa/vespalib/CMakeLists.txt5
-rw-r--r--vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp1
-rw-r--r--vespalib/src/vespa/vespalib/net/wakeup_pipe.cpp24
-rw-r--r--vespalib/src/vespa/vespalib/net/wakeup_pipe.h7
-rw-r--r--vespalib/src/vespa/vespalib/portal/http_connection.cpp14
-rw-r--r--vespalib/src/vespa/vespalib/portal/http_connection.h6
-rw-r--r--vespalib/src/vespa/vespalib/portal/portal.cpp6
-rw-r--r--vespalib/src/vespa/vespalib/portal/portal.h6
99 files changed, 1656 insertions, 940 deletions
diff --git a/client/go/internal/cli/cmd/root.go b/client/go/internal/cli/cmd/root.go
index faba6bbbfd4..70e0afbcd32 100644
--- a/client/go/internal/cli/cmd/root.go
+++ b/client/go/internal/cli/cmd/root.go
@@ -250,6 +250,7 @@ func (c *CLI) configureCommands() {
rootCmd.AddCommand(statusCmd) // status
rootCmd.AddCommand(newTestCmd(c)) // test
rootCmd.AddCommand(newVersionCmd(c)) // version
+ rootCmd.AddCommand(newVisitCmd(c)) // visit
}
func (c *CLI) printErr(err error, hints ...string) {
@@ -263,6 +264,10 @@ func (c *CLI) printSuccess(msg ...interface{}) {
fmt.Fprintln(c.Stdout, color.GreenString("Success:"), fmt.Sprint(msg...))
}
+func (c *CLI) printDebug(msg ...interface{}) {
+ fmt.Fprintln(c.Stderr, color.CyanString("Debug:"), fmt.Sprint(msg...))
+}
+
func (c *CLI) printWarning(msg interface{}, hints ...string) {
fmt.Fprintln(c.Stderr, color.YellowString("Warning:"), msg)
for _, hint := range hints {
diff --git a/client/go/internal/cli/cmd/visit.go b/client/go/internal/cli/cmd/visit.go
new file mode 100644
index 00000000000..1022d74354d
--- /dev/null
+++ b/client/go/internal/cli/cmd/visit.go
@@ -0,0 +1,348 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// vespa visit command
+// Author: arnej
+
+package cmd
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "strings"
+ "time"
+
+ "github.com/spf13/cobra"
+ "github.com/vespa-engine/vespa/client/go/internal/util"
+ "github.com/vespa-engine/vespa/client/go/internal/vespa"
+)
+
+type visitArgs struct {
+ contentCluster string
+ fieldSet string
+ selection string
+ makeFeed bool
+ jsonLines bool
+ pretty bool
+ debugMode bool
+ chunkCount int
+ cli *CLI
+}
+
+func (v *visitArgs) writeBytes(b []byte) {
+ v.cli.Stdout.Write(b)
+}
+
+func (v *visitArgs) writeString(s string) {
+ v.writeBytes([]byte(s))
+}
+
+func (v *visitArgs) debugPrint(s string) {
+ if v.debugMode {
+ v.cli.printDebug(s)
+ }
+}
+
+func (v *visitArgs) dumpDocuments(documents []DocumentBlob) {
+ comma := false
+ pretty := false
+ if v.makeFeed {
+ comma = true
+ pretty = v.pretty
+ } else if !v.jsonLines {
+ return
+ }
+ for _, value := range documents {
+ if pretty {
+ var prettyJSON bytes.Buffer
+ parseError := json.Indent(&prettyJSON, value.blob, "", " ")
+ if parseError != nil {
+ v.writeBytes(value.blob)
+ } else {
+ v.writeBytes(prettyJSON.Bytes())
+ }
+ } else {
+ v.writeBytes(value.blob)
+ }
+ if comma {
+ v.writeString(",\n")
+ } else {
+ v.writeString("\n")
+ }
+ }
+}
+
+var totalDocCount int
+
+func newVisitCmd(cli *CLI) *cobra.Command {
+ var (
+ vArgs visitArgs
+ )
+ cmd := &cobra.Command{
+ Use: "visit",
+ Short: "Visit and print all documents in a vespa cluster",
+ Long: `Run visiting to retrieve all documents.
+
+By default prints each document received on its own line (JSON-L format).
+`,
+ Example: `$ vespa visit # get documents from any cluster
+$ vespa visit --content-cluster search # get documents from cluster named "search"
+`,
+ Args: cobra.MaximumNArgs(0),
+ DisableAutoGenTag: true,
+ SilenceUsage: true,
+ RunE: func(cmd *cobra.Command, args []string) error {
+ vArgs.cli = cli
+ service, err := documentService(cli)
+ if err != nil {
+ return err
+ }
+ result := probeHandler(service, cli)
+ if result.Success {
+ result = visitClusters(&vArgs, service)
+ }
+ if !result.Success {
+ return fmt.Errorf("visit failed: %s", result.Message)
+ }
+ vArgs.debugPrint(fmt.Sprintf("sum of 'documentCount': %d", totalDocCount))
+ return nil
+ },
+ }
+ cmd.Flags().StringVar(&vArgs.contentCluster, "content-cluster", "*", `Which content cluster to visit documents from`)
+ cmd.Flags().StringVar(&vArgs.fieldSet, "field-set", "", `Which fieldset to ask for`)
+ cmd.Flags().StringVar(&vArgs.selection, "selection", "", `select subset of cluster`)
+ cmd.Flags().BoolVar(&vArgs.debugMode, "debug-mode", false, `print debugging output`)
+ cmd.Flags().BoolVar(&vArgs.jsonLines, "json-lines", true, `output documents as JSON lines`)
+ cmd.Flags().BoolVar(&vArgs.makeFeed, "make-feed", false, `output JSON array suitable for vespa-feeder`)
+ cmd.Flags().BoolVar(&vArgs.pretty, "pretty-json", false, `format pretty JSON`)
+ cmd.Flags().IntVar(&vArgs.chunkCount, "chunk-count", 1000, `chunk by count`)
+ return cmd
+}
+
+type HandlersInfo struct {
+ Handlers []struct {
+ HandlerId string `json:"id"`
+ HandlerClass string `json:"class"`
+ HandlerBundle string `json:"bundle"`
+ ServerBindings []string `json:"serverBindings"`
+ } `json:"handlers"`
+}
+
+func parseHandlersOutput(r io.Reader) (*HandlersInfo, error) {
+ var handlersInfo HandlersInfo
+ codec := json.NewDecoder(r)
+ err := codec.Decode(&handlersInfo)
+ return &handlersInfo, err
+}
+
+func probeHandler(service *vespa.Service, cli *CLI) (res util.OperationResult) {
+ urlPath := service.BaseURL + "/"
+ url, urlParseError := url.Parse(urlPath)
+ if urlParseError != nil {
+ return util.Failure("Invalid request path: '" + urlPath + "': " + urlParseError.Error())
+ }
+ request := &http.Request{
+ URL: url,
+ Method: "GET",
+ }
+ timeout := time.Duration(90) * time.Second
+ response, err := service.Do(request, timeout)
+ if err != nil {
+ return util.Failure("Request failed: " + err.Error())
+ }
+ defer response.Body.Close()
+ if response.StatusCode == 200 {
+ handlersInfo, err := parseHandlersOutput(response.Body)
+ if err != nil || len(handlersInfo.Handlers) == 0 {
+ cli.printWarning("Could not parse JSON response from"+urlPath, err.Error())
+ return util.Failure("Bad endpoint")
+ }
+ for _, h := range handlersInfo.Handlers {
+ if strings.HasSuffix(h.HandlerClass, "DocumentV1ApiHandler") {
+ for _, binding := range h.ServerBindings {
+ if strings.Contains(binding, "/document/v1/") {
+ return util.Success("handler OK")
+ }
+ }
+ w := fmt.Sprintf("expected /document/v1/ binding, but got: %v", h.ServerBindings)
+ cli.printWarning(w)
+ }
+ }
+ cli.printWarning("Missing /document/v1/ API; add <document-api /> to the container cluster delcaration in services.xml")
+ return util.Failure("Missing /document/v1 API")
+ } else {
+ return util.FailureWithPayload(service.Description()+" at "+request.URL.Host+": "+response.Status, util.ReaderToJSON(response.Body))
+ }
+}
+
+func visitClusters(vArgs *visitArgs, service *vespa.Service) (res util.OperationResult) {
+ clusters := []string{
+ vArgs.contentCluster,
+ }
+ if vArgs.contentCluster == "*" {
+ clusters = probeVisit(vArgs, service)
+ }
+ if vArgs.makeFeed {
+ vArgs.writeString("[\n")
+ }
+ for _, c := range clusters {
+ vArgs.contentCluster = c
+ res = runVisit(vArgs, service)
+ if !res.Success {
+ return res
+ }
+ vArgs.debugPrint("Success: " + res.Message)
+ }
+ if vArgs.makeFeed {
+ vArgs.writeString("{}\n]\n")
+ }
+ return res
+}
+
+func probeVisit(vArgs *visitArgs, service *vespa.Service) []string {
+ clusters := make([]string, 0, 3)
+ vvo, _ := runOneVisit(vArgs, service, "")
+ if vvo != nil {
+ msg := vvo.ErrorMsg
+ if strings.Contains(msg, "no content cluster '*'") {
+ for idx, value := range strings.Split(msg, ",") {
+ if idx > 0 {
+ parts := strings.Split(value, "'")
+ if len(parts) == 3 {
+ clusters = append(clusters, parts[1])
+ }
+ }
+ }
+ }
+ }
+ return clusters
+}
+
+func runVisit(vArgs *visitArgs, service *vespa.Service) (res util.OperationResult) {
+ vArgs.debugPrint(fmt.Sprintf("trying to visit: '%s'", vArgs.contentCluster))
+ var totalDocuments int = 0
+ var continuationToken string
+ for {
+ var vvo *VespaVisitOutput
+ vvo, res = runOneVisit(vArgs, service, continuationToken)
+ if !res.Success {
+ if vvo != nil && vvo.ErrorMsg != "" {
+ vArgs.cli.printWarning(vvo.ErrorMsg)
+ }
+ return res
+ }
+ vArgs.dumpDocuments(vvo.Documents)
+ vArgs.debugPrint(fmt.Sprintf("got %d documents", len(vvo.Documents)))
+ totalDocuments += len(vvo.Documents)
+ continuationToken = vvo.Continuation
+ if continuationToken == "" {
+ break
+ }
+ }
+ res.Message = fmt.Sprintf("%s [%d documents visited]", res.Message, totalDocuments)
+ return
+}
+
+func quoteArgForUrl(arg string) string {
+ var buf strings.Builder
+ buf.Grow(len(arg))
+ for _, r := range arg {
+ switch {
+ case 'a' <= r && r <= 'z':
+ buf.WriteRune(r)
+ case 'A' <= r && r <= 'Z':
+ buf.WriteRune(r)
+ case '0' <= r && r <= '9':
+ buf.WriteRune(r)
+ case r <= ' ' || r > '~':
+ buf.WriteRune('+')
+ default:
+ s := fmt.Sprintf("%s%02X", "%", r)
+ buf.WriteString(s)
+ }
+ }
+ return buf.String()
+}
+
+func runOneVisit(vArgs *visitArgs, service *vespa.Service, contToken string) (*VespaVisitOutput, util.OperationResult) {
+ urlPath := service.BaseURL + "/document/v1/?cluster=" + quoteArgForUrl(vArgs.contentCluster)
+ if vArgs.fieldSet != "" {
+ urlPath = urlPath + "&fieldSet=" + quoteArgForUrl(vArgs.fieldSet)
+ }
+ if vArgs.selection != "" {
+ urlPath = urlPath + "&selection=" + quoteArgForUrl(vArgs.selection)
+ }
+ if contToken != "" {
+ urlPath = urlPath + "&continuation=" + contToken
+ }
+ if vArgs.chunkCount > 0 {
+ urlPath = urlPath + fmt.Sprintf("&wantedDocumentCount=%d", vArgs.chunkCount)
+ }
+ url, urlParseError := url.Parse(urlPath)
+ if urlParseError != nil {
+ return nil, util.Failure("Invalid request path: '" + urlPath + "': " + urlParseError.Error())
+ }
+ request := &http.Request{
+ URL: url,
+ Method: "GET",
+ }
+ timeout := time.Duration(900) * time.Second
+ response, err := service.Do(request, timeout)
+ if err != nil {
+ return nil, util.Failure("Request failed: " + err.Error())
+ }
+ defer response.Body.Close()
+ vvo, err := parseVisitOutput(response.Body)
+ if response.StatusCode == 200 {
+ if err == nil {
+ totalDocCount += vvo.DocumentCount
+ if vvo.DocumentCount != len(vvo.Documents) {
+ vArgs.cli.printWarning(fmt.Sprintf("Inconsistent contents from: %v", url))
+ vArgs.cli.printWarning(fmt.Sprintf("claimed count: %d", vvo.DocumentCount))
+ vArgs.cli.printWarning(fmt.Sprintf("document blobs: %d", len(vvo.Documents)))
+ return nil, util.Failure("Inconsistent contents from document API")
+ }
+ return vvo, util.Success("visited " + vArgs.contentCluster)
+ } else {
+ return nil, util.Failure("error reading response: " + err.Error())
+ }
+ } else if response.StatusCode/100 == 4 {
+ return vvo, util.FailureWithPayload("Invalid document operation: "+response.Status, util.ReaderToJSON(response.Body))
+ } else {
+ return vvo, util.FailureWithPayload(service.Description()+" at "+request.URL.Host+": "+response.Status, util.ReaderToJSON(response.Body))
+ }
+}
+
+type DocumentBlob struct {
+ blob []byte
+}
+
+func (d *DocumentBlob) UnmarshalJSON(data []byte) error {
+ d.blob = make([]byte, len(data))
+ copy(d.blob, data)
+ return nil
+}
+
+func (d *DocumentBlob) MarshalJSON() ([]byte, error) {
+ return d.blob, nil
+}
+
+type VespaVisitOutput struct {
+ PathId string `json:"pathId"`
+ Documents []DocumentBlob `json:"documents"`
+ DocumentCount int `json:"documentCount"`
+ Continuation string `json:"continuation"`
+ ErrorMsg string `json:"message"`
+}
+
+func parseVisitOutput(r io.Reader) (*VespaVisitOutput, error) {
+ codec := json.NewDecoder(r)
+ var parsedJson VespaVisitOutput
+ err := codec.Decode(&parsedJson)
+ if err != nil {
+ return nil, fmt.Errorf("could not decode JSON, error: %s", err.Error())
+ }
+ return &parsedJson, nil
+}
diff --git a/client/go/internal/cli/cmd/visit_test.go b/client/go/internal/cli/cmd/visit_test.go
new file mode 100644
index 00000000000..4302680b9d9
--- /dev/null
+++ b/client/go/internal/cli/cmd/visit_test.go
@@ -0,0 +1,142 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package cmd
+
+import (
+ "fmt"
+ "net/http"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/vespa-engine/vespa/client/go/internal/mock"
+ "github.com/vespa-engine/vespa/client/go/internal/vespa"
+)
+
+const (
+ normalpre = `{"pathId":"/document/v1/","documents":[`
+ document1 = `{"id":"id:t:m::1","fields":{"title":"t"}}`
+ document2 = `{"id":"id:t:m::2","fields":{"title":"t2"}}`
+ document3 = `{"id":"id:t:m::3","fields":{"ar":"xyz","w":63,"title":"xyzzy","year":2000}}`
+
+ savedresponse = `{"pathId":"/document/v1/","documents":[{"id":"id:test:music::1921492307","fields":{"title":"song","year":2010}},{"id":"id:test:music::p_try-this-clean-bonus-dvd-_music_1922003403","fields":{"artist":"xyz","weight":600000,"song":"hate","title":"xyz","year":2000}}],"documentCount":2,"continuation":"AAAACAAAAAAAAAAJAAAAAAAAAAgAAAAAAAABAAAAAAEgAAAAAAAAEAAAAAAAAAAA"}`
+
+ saveddoc0 = `{"id":"id:test:music::1921492307","fields":{"title":"song","year":2010}}`
+ saveddoc1 = `{"id":"id:test:music::p_try-this-clean-bonus-dvd-_music_1922003403","fields":{"artist":"xyz","weight":600000,"song":"hate","title":"xyz","year":2000}}`
+ handlersResponse = `{
+ "handlers" : [ {
+ "id" : "com.yahoo.container.usability.BindingsOverviewHandler",
+ "class" : "com.yahoo.container.usability.BindingsOverviewHandler",
+ "bundle" : "container-disc:8.0.0",
+ "serverBindings" : [ "http://*/" ]
+ }, {
+ "id" : "com.yahoo.document.restapi.resource.DocumentV1ApiHandler",
+ "class" : "com.yahoo.document.restapi.resource.DocumentV1ApiHandler",
+ "bundle" : "vespaclient-container-plugin:8.0.0",
+ "serverBindings" : [ "http://*/document/v1/*", "http://*/document/v1/*/" ]
+ } ]
+}`
+ clusterStarResponse = `{"pathId":"/document/v1/","message":"Your Vespa deployment has no content cluster '*', only 'fooCC'"}`
+)
+
+func TestQuoteFunc(t *testing.T) {
+ var buf []byte = make([]byte, 3)
+ buf[0] = 'a'
+ buf[2] = 'z'
+ for i := 0; i < 256; i++ {
+ buf[1] = byte(i)
+ s := string(buf)
+ res := quoteArgForUrl(s)
+ if i < 32 || i > 127 {
+ assert.Equal(t, "a+z", res)
+ } else {
+ fmt.Printf("res %3d => '%s'\n", i, res)
+ }
+ }
+}
+
+// low-level (unit) test
+func TestRunOneVisit(t *testing.T) {
+ withResponse := func(client *mock.HTTPClient) {
+ client.NextResponseString(200, savedresponse)
+ }
+ op := func(service *vespa.Service) {
+ vArgs := visitArgs{
+ contentCluster: "fooCC",
+ }
+ vvo, res := runOneVisit(&vArgs, service, "BBBB")
+ assert.Equal(t, true, res.Success)
+ assert.Equal(t, "visited fooCC", res.Message)
+ assert.Equal(t, "/document/v1/", vvo.PathId)
+ assert.Equal(t, "", vvo.ErrorMsg)
+ assert.Equal(t, "AAAACAAAAAAAAAAJAAAAAAAAAAgAAAAAAAABAAAAAAEgAAAAAAAAEAAAAAAAAAAA", vvo.Continuation)
+ assert.Equal(t, 2, vvo.DocumentCount)
+ assert.Equal(t, 2, len(vvo.Documents))
+ assert.Equal(t, saveddoc0, string(vvo.Documents[0].blob))
+ assert.Equal(t, saveddoc1, string(vvo.Documents[1].blob))
+ }
+ req := withMockClient(t, withResponse, op)
+ assert.Equal(t, "cluster=fooCC&continuation=BBBB", req.URL.RawQuery)
+
+ op = func(service *vespa.Service) {
+ vArgs := visitArgs{
+ contentCluster: "search",
+ fieldSet: "[id]",
+ selection: "music.year>2000",
+ chunkCount: 123,
+ }
+ vvo, res := runOneVisit(&vArgs, service, "asdf")
+ assert.Equal(t, true, res.Success)
+ assert.Equal(t, 2, vvo.DocumentCount)
+ }
+ req = withMockClient(t, withResponse, op)
+ assert.Equal(t, "cluster=search&fieldSet=%5Bid%5D&selection=music%2Eyear%3E2000&continuation=asdf&wantedDocumentCount=123", req.URL.RawQuery)
+}
+
+func withMockClient(t *testing.T, prepCli func(*mock.HTTPClient), runOp func(*vespa.Service)) *http.Request {
+ client := &mock.HTTPClient{}
+ prepCli(client)
+ cli, _, _ := newTestCLI(t)
+ cli.httpClient = client
+ service, _ := documentService(cli)
+ runOp(service)
+ return client.LastRequest
+}
+
+func TestVisitCommand(t *testing.T) {
+ assertVisitResults(
+ []string{
+ "visit",
+ "--json-lines",
+ },
+ t,
+ []string{
+ normalpre +
+ document1 +
+ `],"documentCount":1,"continuation":"CAFE"}`,
+ normalpre +
+ document2 +
+ "," +
+ document3 +
+ `],"documentCount":2}`,
+ },
+ "cluster=fooCC&continuation=CAFE&wantedDocumentCount=1000",
+ document1+"\n"+
+ document2+"\n"+
+ document3+"\n")
+}
+
+func assertVisitResults(arguments []string, t *testing.T, responses []string, queryPart, output string) {
+ client := &mock.HTTPClient{}
+ client.NextResponseString(200, handlersResponse)
+ client.NextResponseString(400, clusterStarResponse)
+ for _, resp := range responses {
+ client.NextResponseString(200, resp)
+ }
+ cli, stdout, stderr := newTestCLI(t)
+ cli.httpClient = client
+ assert.Nil(t, cli.Run(arguments...))
+ assert.Equal(t, output, stdout.String())
+ assert.Equal(t, "", stderr.String())
+ assert.Equal(t, queryPart, client.LastRequest.URL.RawQuery)
+ assert.Equal(t, "/document/v1/", client.LastRequest.URL.Path)
+ assert.Equal(t, "GET", client.LastRequest.Method)
+}
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
index 04de4d2e277..d5c074a191d 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
@@ -111,6 +111,7 @@ public interface ModelContext {
@ModelFeatureFlag(owners = {"tokle"}) default boolean useRestrictedDataPlaneBindings() { return false; }
@ModelFeatureFlag(owners = {"arnej","baldersheim"}, removeAfter = "8.110") default boolean useOldJdiscContainerStartup() { return false; }
@ModelFeatureFlag(owners = {"tokle, bjorncs"}, removeAfter = "8.108") default boolean enableDataPlaneFilter() { return true; }
+ @ModelFeatureFlag(owners = {"arnej, bjorncs"}) default boolean enableGlobalPhase() { return false; }
//Below are all flags that must be kept until 7 is out of the door
@ModelFeatureFlag(owners = {"arnej"}, removeAfter="7.last") default boolean ignoreThreadStackSizes() { return false; }
diff --git a/config-model/.gitignore b/config-model/.gitignore
index 6edd041cbe8..7e7e597675d 100644
--- a/config-model/.gitignore
+++ b/config-model/.gitignore
@@ -4,5 +4,6 @@
/target
/src/test/integration/*/copy/
/src/test/integration/*/models.generated/
+/src/test/derived/*/models.generated/
*.cfg.actual
/var/
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
index 49194a5d1bb..ecbb990f096 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
@@ -137,6 +137,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
@Override public boolean useRestrictedDataPlaneBindings() { return useRestrictedDataPlaneBindings; }
@Override public Optional<CloudAccount> cloudAccount() { return cloudAccount; }
@Override public boolean allowUserFilters() { return allowUserFilters; }
+ @Override public boolean enableGlobalPhase() { return true; } // Enable global-phase by default for unit tests only
public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) {
this.sharedStringRepoNoReclaim = sharedStringRepoNoReclaim;
diff --git a/config-model/src/main/java/com/yahoo/schema/RankProfile.java b/config-model/src/main/java/com/yahoo/schema/RankProfile.java
index ad6eb038058..7cb0a088f5f 100644
--- a/config-model/src/main/java/com/yahoo/schema/RankProfile.java
+++ b/config-model/src/main/java/com/yahoo/schema/RankProfile.java
@@ -1019,6 +1019,9 @@ public class RankProfile implements Cloneable {
var recorder = new InputRecorder(needInputs);
recorder.transform(globalPhaseRanking.function().getBody(), context);
for (String input : needInputs) {
+ if (input.startsWith("constant(") || input.startsWith("query(")) {
+ continue;
+ }
try {
addMatchFeatures(new FeatureList(input));
} catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) {
diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java
index b0f63ebb732..4e7988a2006 100644
--- a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java
+++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java
@@ -3,16 +3,13 @@ package com.yahoo.schema.expressiontransforms;
import com.yahoo.schema.FeatureNames;
import com.yahoo.schema.RankProfile;
-import com.yahoo.searchlib.rankingexpression.RankingExpression;
import com.yahoo.searchlib.rankingexpression.Reference;
-import com.yahoo.searchlib.rankingexpression.parser.ParseException;
import com.yahoo.searchlib.rankingexpression.rule.CompositeNode;
import com.yahoo.searchlib.rankingexpression.rule.ConstantNode;
import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode;
import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode;
import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer;
-import java.io.StringReader;
import java.util.Set;
/**
@@ -86,13 +83,7 @@ public class InputRecorder extends ExpressionTransformer<RankProfileTransformCon
throw new IllegalArgumentException("missing onnx model: " + arg);
}
for (String onnxInput : model.getInputMap().values()) {
- var reader = new StringReader(onnxInput);
- try {
- var asExpression = new RankingExpression(reader);
- transform(asExpression.getRoot(), context);
- } catch (ParseException e) {
- throw new IllegalArgumentException("illegal onnx input '" + onnxInput + "': " + e.getMessage());
- }
+ neededInputs.add(onnxInput);
}
return;
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java
index 3d9a8441ed5..0be3c825614 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java
@@ -26,9 +26,9 @@ public class ContainerModelEvaluation implements
OnnxModelsConfig.Producer,
RankingExpressionsConfig.Producer {
- private final static String EVALUATION_BUNDLE_NAME = "model-evaluation";
- private final static String INTEGRATION_BUNDLE_NAME = "model-integration";
- private final static String ONNXRUNTIME_BUNDLE_NAME = "container-onnxruntime.jar";
+ public final static String EVALUATION_BUNDLE_NAME = "model-evaluation";
+ public final static String INTEGRATION_BUNDLE_NAME = "model-integration";
+ public final static String ONNXRUNTIME_BUNDLE_NAME = "container-onnxruntime.jar";
private final static String EVALUATOR_NAME = ModelsEvaluator.class.getName();
private final static String REST_HANDLER_NAME = "ai.vespa.models.handler.ModelsEvaluationHandler";
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java
index 29b1edc1397..19df9a4064f 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java
@@ -10,6 +10,10 @@ import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import static com.yahoo.vespa.model.container.ContainerModelEvaluation.EVALUATION_BUNDLE_NAME;
+import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATION_BUNDLE_NAME;
+import static com.yahoo.vespa.model.container.ContainerModelEvaluation.ONNXRUNTIME_BUNDLE_NAME;
+
/**
* NOTE: Stable ordering of bundles in config is handled by {@link ContainerCluster#addPlatformBundle(Path)}
*
@@ -53,7 +57,10 @@ public class PlatformBundles {
public static final Set<Path> SEARCH_AND_DOCPROC_BUNDLES = toBundlePaths(
SEARCH_AND_DOCPROC_BUNDLE,
"docprocs",
- "linguistics-components"
+ "linguistics-components",
+ EVALUATION_BUNDLE_NAME,
+ INTEGRATION_BUNDLE_NAME,
+ ONNXRUNTIME_BUNDLE_NAME
);
private static Set<Path> toBundlePaths(String... bundleNames) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java
index 86c48407775..5d4ec598250 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java
@@ -1,12 +1,14 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.container.search;
+import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.container.QrSearchersConfig;
import com.yahoo.prelude.semantics.SemanticRulesConfig;
import com.yahoo.search.config.IndexInfoConfig;
import com.yahoo.search.config.SchemaInfoConfig;
import com.yahoo.search.pagetemplates.PageTemplatesConfig;
import com.yahoo.search.query.profile.config.QueryProfilesConfig;
+import com.yahoo.search.ranking.RankProfilesEvaluatorFactory;
import com.yahoo.schema.derived.SchemaInfo;
import com.yahoo.vespa.configdefinition.IlscriptsConfig;
import com.yahoo.vespa.model.container.ApplicationContainerCluster;
@@ -44,18 +46,22 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains>
private final ApplicationContainerCluster owningCluster;
private final List<SearchCluster> searchClusters = new LinkedList<>();
+ private final boolean globalPhase;
private QueryProfiles queryProfiles;
private SemanticRules semanticRules;
private PageTemplates pageTemplates;
- public ContainerSearch(ApplicationContainerCluster cluster, SearchChains chains) {
+ public ContainerSearch(DeployState deployState, ApplicationContainerCluster cluster, SearchChains chains) {
super(chains);
+ this.globalPhase = deployState.featureFlags().enableGlobalPhase();
this.owningCluster = cluster;
owningCluster.addComponent(Component.fromClassAndBundle(CompiledQueryProfileRegistry.class, SEARCH_AND_DOCPROC_BUNDLE));
owningCluster.addComponent(Component.fromClassAndBundle(com.yahoo.search.schema.SchemaInfo.class, SEARCH_AND_DOCPROC_BUNDLE));
owningCluster.addComponent(Component.fromClassAndBundle(SearchStatusExtension.class, SEARCH_AND_DOCPROC_BUNDLE));
+ owningCluster.addComponent(Component.fromClassAndBundle(RankProfilesEvaluatorFactory.class, SEARCH_AND_DOCPROC_BUNDLE));
+ owningCluster.addComponent(Component.fromClassAndBundle(com.yahoo.search.ranking.GlobalPhaseRanker.class, SEARCH_AND_DOCPROC_BUNDLE));
cluster.addSearchAndDocprocBundles();
}
@@ -68,9 +74,18 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains>
/** Adds a Dispatcher component to the owning container cluster for each search cluster */
private void initializeDispatchers(Collection<SearchCluster> searchClusters) {
for (SearchCluster searchCluster : searchClusters) {
- if ( ! ( searchCluster instanceof IndexedSearchCluster)) continue;
- var dispatcher = new DispatcherComponent((IndexedSearchCluster)searchCluster);
- owningCluster.addComponent(dispatcher);
+ if (searchCluster instanceof IndexedSearchCluster indexed) {
+ var dispatcher = new DispatcherComponent(indexed);
+ owningCluster.addComponent(dispatcher);
+ if (globalPhase) {
+ for (var documentDb : indexed.getDocumentDbs()) {
+ var factory = new RankProfilesEvaluatorComponent(documentDb);
+ if (! owningCluster.getComponentsMap().containsKey(factory.getComponentId())) {
+ owningCluster.addComponent(factory);
+ }
+ }
+ }
+ }
}
}
@@ -143,6 +158,7 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains>
if ( ! (sys instanceof IndexedSearchCluster)) {
scB.storagecluster(new QrSearchersConfig.Searchcluster.Storagecluster.Builder().
routespec(((StreamingSearchCluster)sys).getStorageRouteSpec()));
+ scB.globalphase(globalPhase);
}
builder.searchcluster(scB);
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/RankProfilesEvaluatorComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RankProfilesEvaluatorComponent.java
new file mode 100644
index 00000000000..75a2802ee53
--- /dev/null
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RankProfilesEvaluatorComponent.java
@@ -0,0 +1,49 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.model.container.search;
+
+import com.yahoo.config.model.producer.AnyConfigProducer;
+import com.yahoo.osgi.provider.model.ComponentModel;
+import com.yahoo.search.ranking.RankProfilesEvaluator;
+import com.yahoo.vespa.config.search.RankProfilesConfig;
+import com.yahoo.vespa.config.search.core.OnnxModelsConfig;
+import com.yahoo.vespa.config.search.core.RankingConstantsConfig;
+import com.yahoo.vespa.config.search.core.RankingExpressionsConfig;
+import com.yahoo.vespa.model.container.ContainerModelEvaluation;
+import com.yahoo.vespa.model.container.PlatformBundles;
+import com.yahoo.vespa.model.container.component.Component;
+import com.yahoo.vespa.model.search.DocumentDatabase;
+
+public class RankProfilesEvaluatorComponent
+ extends Component<AnyConfigProducer, ComponentModel>
+ implements
+ RankProfilesConfig.Producer,
+ RankingConstantsConfig.Producer,
+ RankingExpressionsConfig.Producer,
+ OnnxModelsConfig.Producer
+{
+ private final DocumentDatabase ddb;
+
+ public RankProfilesEvaluatorComponent(DocumentDatabase db) {
+ super(toComponentModel(db.getSchemaName()));
+ ddb = db;
+ }
+
+ private static ComponentModel toComponentModel(String p) {
+ String myComponentId = "ranking-expression-evaluator." + p;
+ return new ComponentModel(myComponentId,
+ RankProfilesEvaluator.class.getName(),
+ PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE);
+ }
+
+ @Override
+ public void getConfig(RankProfilesConfig.Builder builder) { ddb.getConfig(builder); }
+
+ @Override
+ public void getConfig(RankingExpressionsConfig.Builder builder) { ddb.getConfig(builder); }
+
+ @Override
+ public void getConfig(RankingConstantsConfig.Builder builder) { ddb.getConfig(builder); }
+
+ @Override
+ public void getConfig(OnnxModelsConfig.Builder builder) { ddb.getConfig(builder); }
+}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
index ceba3864296..a639c158d62 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java
@@ -732,7 +732,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> {
SearchChains searchChains = new DomSearchChainsBuilder()
.build(deployState, containerCluster, producerSpec);
- ContainerSearch containerSearch = new ContainerSearch(containerCluster, searchChains);
+ ContainerSearch containerSearch = new ContainerSearch(deployState, containerCluster, searchChains);
applyApplicationPackageDirectoryConfigs(deployState.getApplicationPackage(), containerSearch);
containerSearch.setQueryProfiles(deployState.getQueryProfiles());
diff --git a/config-model/src/test/derived/globalphase_onnx_inside/files/ax_plus_b.onnx b/config-model/src/test/derived/globalphase_onnx_inside/files/ax_plus_b.onnx
new file mode 100644
index 00000000000..17282d13dc3
--- /dev/null
+++ b/config-model/src/test/derived/globalphase_onnx_inside/files/ax_plus_b.onnx
@@ -0,0 +1,23 @@
+:©
+
+matrix_X
+vector_AXA"MatMul
+
+XA
+vector_Bvector_Y"AddlrZ
+matrix_X
+  
+
+Z
+vector_A
+
+ 
+Z
+vector_B
+
+ 
+b
+vector_Y
+
+ 
+B \ No newline at end of file
diff --git a/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg b/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg
new file mode 100644
index 00000000000..35bb1ccc3d2
--- /dev/null
+++ b/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg
@@ -0,0 +1,34 @@
+rankprofile[].name "default"
+rankprofile[].fef.property[].name "rankingExpression(handicap).rankingScript"
+rankprofile[].fef.property[].value "query(yy)"
+rankprofile[].fef.property[].name "rankingExpression(handicap).type"
+rankprofile[].fef.property[].value "tensor(d0[2])"
+rankprofile[].fef.property[].name "vespa.rank.firstphase"
+rankprofile[].fef.property[].value "rankingExpression(firstphase)"
+rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript"
+rankprofile[].fef.property[].value "reduce(attribute(aa), sum)"
+rankprofile[].fef.property[].name "vespa.rank.globalphase"
+rankprofile[].fef.property[].value "rankingExpression(globalphase)"
+rankprofile[].fef.property[].name "rankingExpression(globalphase).rankingScript"
+rankprofile[].fef.property[].value "reduce(constant(ww) * (onnx(inside).foobar - rankingExpression(handicap)), sum)"
+rankprofile[].fef.property[].name "vespa.match.feature"
+rankprofile[].fef.property[].value "attribute(aa)"
+rankprofile[].fef.property[].name "vespa.globalphase.rerankcount"
+rankprofile[].fef.property[].value "13"
+rankprofile[].fef.property[].name "vespa.type.attribute.aa"
+rankprofile[].fef.property[].value "tensor(d1[3])"
+rankprofile[].fef.property[].name "vespa.type.query.bb"
+rankprofile[].fef.property[].value "tensor(d0[2])"
+rankprofile[].fef.property[].name "vespa.type.query.yy"
+rankprofile[].fef.property[].value "tensor(d0[2])"
+rankprofile[].name "unranked"
+rankprofile[].fef.property[].name "vespa.rank.firstphase"
+rankprofile[].fef.property[].value "value(0)"
+rankprofile[].fef.property[].name "vespa.hitcollector.heapsize"
+rankprofile[].fef.property[].value "0"
+rankprofile[].fef.property[].name "vespa.hitcollector.arraysize"
+rankprofile[].fef.property[].value "0"
+rankprofile[].fef.property[].name "vespa.dump.ignoredefaultfeatures"
+rankprofile[].fef.property[].value "true"
+rankprofile[].fef.property[].name "vespa.type.attribute.aa"
+rankprofile[].fef.property[].value "tensor(d1[3])"
diff --git a/config-model/src/test/derived/globalphase_onnx_inside/test.sd b/config-model/src/test/derived/globalphase_onnx_inside/test.sd
new file mode 100644
index 00000000000..c38e318ce6b
--- /dev/null
+++ b/config-model/src/test/derived/globalphase_onnx_inside/test.sd
@@ -0,0 +1,42 @@
+schema test {
+
+ document test {
+ field aa type tensor(d1[3]) {
+ indexing: attribute
+ }
+ }
+
+ constant xx {
+ file: files/const_xx.json
+ type: tensor(d0[2],d1[3])
+ }
+ constant ww {
+ file: files/const_ww.json
+ type: tensor(d0[2])
+ }
+
+ rank-profile default {
+ inputs {
+ query(bb) tensor(d0[2])
+ query(yy) tensor(d0[2])
+ }
+ onnx-model inside {
+ file: files/ax_plus_b.onnx
+ input vector_A: attribute(aa)
+ input matrix_X: constant(xx)
+ input vector_B: query(bb)
+ output vector_Y: foobar
+ }
+ first-phase {
+ expression: sum(attribute(aa))
+ }
+ function handicap() {
+ expression: query(yy)
+ }
+ global-phase {
+ rerank-count: 13
+ expression: sum(constant(ww) * (onnx(inside).foobar - handicap))
+ }
+ }
+
+}
diff --git a/config-model/src/test/derived/rankingexpression/rank-profiles.cfg b/config-model/src/test/derived/rankingexpression/rank-profiles.cfg
index ea8bc5f77e6..e3947e9e46f 100644
--- a/config-model/src/test/derived/rankingexpression/rank-profiles.cfg
+++ b/config-model/src/test/derived/rankingexpression/rank-profiles.cfg
@@ -410,8 +410,6 @@ rankprofile[].fef.property[].value "rankingExpression(globalphase)"
rankprofile[].fef.property[].name "rankingExpression(globalphase).rankingScript"
rankprofile[].fef.property[].value "rankingExpression(myplus) + reduce(rankingExpression(mymul), sum) + firstPhase"
rankprofile[].fef.property[].name "vespa.match.feature"
-rankprofile[].fef.property[].value "query(fromq)"
-rankprofile[].fef.property[].name "vespa.match.feature"
rankprofile[].fef.property[].value "firstPhase"
rankprofile[].fef.property[].name "vespa.match.feature"
rankprofile[].fef.property[].value "attribute(t1)"
diff --git a/config-model/src/test/java/com/yahoo/schema/derived/GlobalPhaseOnnxModelsTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/GlobalPhaseOnnxModelsTestCase.java
new file mode 100644
index 00000000000..2ff33dd70d8
--- /dev/null
+++ b/config-model/src/test/java/com/yahoo/schema/derived/GlobalPhaseOnnxModelsTestCase.java
@@ -0,0 +1,22 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.schema.derived;
+
+import com.yahoo.schema.parser.ParseException;
+import org.junit.jupiter.api.Test;
+import java.io.IOException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * Tests exporting with global-phase and ONNX models
+ *
+ * @author arnej
+ */
+public class GlobalPhaseOnnxModelsTestCase extends AbstractExportingTestCase {
+
+ @Test
+ void testModelInRankProfile() throws IOException, ParseException {
+ assertCorrectDeriving("globalphase_onnx_inside");
+ }
+
+}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
index ce2d5ca2da5..5973ef56962 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
@@ -485,7 +485,7 @@ public class ContainerClusterTest {
if (isCombinedCluster)
cluster.setHostClusterId("test-content-cluster");
cluster.setMemoryPercentage(memoryPercentage);
- cluster.setSearch(new ContainerSearch(cluster, new SearchChains(cluster, "search-chain")));
+ cluster.setSearch(new ContainerSearch(root.getDeployState(), cluster, new SearchChains(cluster, "search-chain")));
return cluster;
}
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java
new file mode 100644
index 00000000000..cec5fb1a1ed
--- /dev/null
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java
@@ -0,0 +1,2 @@
+package com.yahoo.config.provision;public class ClusterInfo {
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java b/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java
index a614e251dca..89f7755729c 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java
@@ -38,9 +38,15 @@ public class GetConfigContext {
return new GetConfigContext(app, handler, trace);
}
+ public static GetConfigContext empty() {
+ return new GetConfigContext(null, null, null);
+ }
+
public static GetConfigContext testContext(ApplicationId app) {
return new GetConfigContext(app, null, null);
}
+
+ public boolean isEmpty() { return app == null && requestHandler == null && trace == null; }
/**
* Helper to produce a log preamble with the tenant and app id
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
index 1d80740ffe0..d61590c2e84 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
@@ -202,6 +202,7 @@ public class ModelContextImpl implements ModelContext {
private final int rpc_events_before_wakeup;
private final boolean useRestrictedDataPlaneBindings;
private final int heapPercentage;
+ private final boolean enableGlobalPhase;
public FeatureFlags(FlagSource source, ApplicationId appId, Version version) {
this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT);
@@ -246,6 +247,7 @@ public class ModelContextImpl implements ModelContext {
this.queryDispatchWarmup = flagValue(source, appId, version, PermanentFlags.QUERY_DISPATCH_WARMUP);
this.useRestrictedDataPlaneBindings = flagValue(source, appId, version, Flags.RESTRICT_DATA_PLANE_BINDINGS);
this.heapPercentage = flagValue(source, appId, version, PermanentFlags.HEAP_SIZE_PERCENTAGE);
+ this.enableGlobalPhase = flagValue(source, appId, version, PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW);
}
@Override public int heapSizePercentage() { return heapPercentage; }
@@ -298,6 +300,7 @@ public class ModelContextImpl implements ModelContext {
return defVal;
}
@Override public boolean useRestrictedDataPlaneBindings() { return useRestrictedDataPlaneBindings; }
+ @Override public boolean enableGlobalPhase() { return enableGlobalPhase; }
private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) {
return flag.bindTo(source)
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java
index 85b30f4d303..454ab9e490d 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java
@@ -23,6 +23,7 @@ import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
+import java.time.Clock;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.logging.Level;
@@ -153,6 +154,8 @@ public class FileDirectory extends AbstractComponent {
return true;
}
+ // update last modified time so that maintainer deleting unused file references considers this as recently used
+ existingFile.setLastModified(Clock.systemUTC().instant().toEpochMilli());
log.log(Level.FINE, "Directory for file reference '" + fileReference.value() + "' already exists and has all content");
return false;
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java
index 1c419ce047a..5e7fff8c922 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java
@@ -2,7 +2,6 @@
package com.yahoo.vespa.config.server.rpc;
import com.yahoo.cloud.config.SentinelConfig;
-import com.yahoo.collections.Pair;
import com.yahoo.component.Version;
import com.yahoo.config.provision.TenantName;
import com.yahoo.container.di.config.ApplicationBundlesConfig;
@@ -37,12 +36,10 @@ class GetConfigProcessor implements Runnable {
private static final Logger log = Logger.getLogger(GetConfigProcessor.class.getName());
private static final String localHostName = HostName.getLocalhost();
-
private static final PayloadChecksums emptyApplicationBundlesConfigChecksums =
PayloadChecksums.fromPayload(Payload.from(ConfigPayload.fromInstance(new ApplicationBundlesConfig.Builder().build())));
private final JRTServerConfigRequest request;
-
/* True only when this request has expired its server timeout and we need to respond to the client */
private final boolean forceResponse;
private final RpcServer rpcServer;
@@ -75,13 +72,13 @@ class GetConfigProcessor implements Runnable {
}
// TODO: Increment statistics (Metrics) failed counters when requests fail
- public Pair<GetConfigContext, Long> getConfig(JRTServerConfigRequest request) {
+ public Optional<DelayedConfig> resolveConfig(JRTServerConfigRequest request) {
// Request has already been detached
if ( ! request.validateParameters()) {
// Error code is set in verifyParameters if parameters are not OK.
log.log(Level.WARNING, "Parameters for request " + request + " did not validate: " + request.errorCode() + " : " + request.errorMessage());
respond(request);
- return null;
+ return Optional.empty();
}
Trace trace = request.getRequestTrace();
debugLog(trace, "GetConfigProcessor.run() on " + localHostName);
@@ -92,13 +89,13 @@ class GetConfigProcessor implements Runnable {
// fabricate an empty request to cause the sentinel to stop all running services
if (rpcServer.canReturnEmptySentinelConfig() && rpcServer.allTenantsLoaded() && tenant.isEmpty() && isSentinelConfigRequest(request)) {
returnEmpty(request);
- return null;
+ return Optional.empty();
}
GetConfigContext context = rpcServer.createGetConfigContext(tenant, request, trace);
- if (context == null || ! context.requestHandler().hasApplication(context.applicationId(), Optional.empty())) {
+ if (context.isEmpty() || ! context.requestHandler().hasApplication(context.applicationId(), Optional.empty())) {
handleError(request, APPLICATION_NOT_LOADED, "No application exists");
- return null;
+ return Optional.empty();
}
logPre = TenantRepository.logPre(context.applicationId());
@@ -109,7 +106,7 @@ class GetConfigProcessor implements Runnable {
if ( ! context.requestHandler().hasApplication(context.applicationId(), vespaVersion)) {
handleError(request, ErrorCode.UNKNOWN_VESPA_VERSION, "Unknown Vespa version in request: " + printableVespaVersion(vespaVersion));
- return null;
+ return Optional.empty();
}
ConfigResponse config;
@@ -117,14 +114,14 @@ class GetConfigProcessor implements Runnable {
config = rpcServer.resolveConfig(request, context, vespaVersion);
} catch (UnknownConfigDefinitionException e) {
handleError(request, ErrorCode.UNKNOWN_DEFINITION, "Unknown config definition " + request.getConfigKey());
- return null;
+ return Optional.empty();
} catch (UnknownConfigIdException e) {
handleError(request, ErrorCode.ILLEGAL_CONFIGID, "Illegal config id " + request.getConfigKey().getConfigId());
- return null;
+ return Optional.empty();
} catch (Throwable e) {
log.log(Level.SEVERE, "Unexpected error handling config request", e);
handleError(request, ErrorCode.INTERNAL_ERROR, "Internal error " + e.getMessage());
- return null;
+ return Optional.empty();
}
// config == null is not an error, but indicates that the config will be returned later.
@@ -135,7 +132,7 @@ class GetConfigProcessor implements Runnable {
&& ! context.requestHandler().compatibleWith(vespaVersion, context.applicationId()) // ... with a runtime version incompatible with the deploying version ...
&& ! emptyApplicationBundlesConfigChecksums.matches(config.getPayloadChecksums())) { // ... and there actually are incompatible user bundles, then return no config:
handleError(request, ErrorCode.INCOMPATIBLE_VESPA_VERSION, "Version " + printableVespaVersion(vespaVersion) + " is binary incompatible with the latest deployed version");
- return null;
+ return Optional.empty();
}
// debugLog(trace, "config response before encoding:" + config.toString());
@@ -144,23 +141,24 @@ class GetConfigProcessor implements Runnable {
respond(request);
} else {
debugLog(trace, "delaying response " + request.getShortDescription());
- return new Pair<>(context, config != null ? config.getGeneration() : 0);
+ return Optional.of(new DelayedConfig(context, config != null ? config.getGeneration() : 0));
}
- return null;
+ return Optional.empty();
}
@Override
public void run() {
- Pair<GetConfigContext, Long> delayed = getConfig(request);
+ Optional<DelayedConfig> delayed = resolveConfig(request);
- if (delayed != null) {
- rpcServer.delayResponse(request, delayed.getFirst());
- if (rpcServer.hasNewerGeneration(delayed.getFirst().applicationId(), delayed.getSecond())) {
+ delayed.ifPresent(d -> {
+ GetConfigContext context = d.context();
+ rpcServer.delayResponse(request, context);
+ if (rpcServer.hasNewerGeneration(context.applicationId(), d.generation())) {
// This will ensure that if the config activation train left the station while I was boarding,
// another train will immediately be scheduled.
- rpcServer.configActivated(delayed.getFirst().applicationId());
+ rpcServer.configActivated(context.applicationId());
}
- }
+ });
}
private boolean isSentinelConfigRequest(JRTServerConfigRequest request) {
@@ -194,4 +192,6 @@ class GetConfigProcessor implements Runnable {
}
}
+ private record DelayedConfig(GetConfigContext context, long generation) {}
+
}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java
index 2d731370b70..7a1b2d2aeef 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java
@@ -406,7 +406,7 @@ public class RpcServer implements Runnable, ConfigActivationListener, TenantList
metrics.incUnknownHostRequests();
trace.trace(TRACELEVEL, msg);
log.log(Level.WARNING, msg);
- return null;
+ return GetConfigContext.empty();
}
RequestHandler handler = requestHandler.get();
ApplicationId applicationId = handler.resolveApplicationId(request.getClientHostName());
diff --git a/container-core/src/main/resources/configdefinitions/container.qr-searchers.def b/container-core/src/main/resources/configdefinitions/container.qr-searchers.def
index e89ee40d792..034e7fc442b 100644
--- a/container-core/src/main/resources/configdefinitions/container.qr-searchers.def
+++ b/container-core/src/main/resources/configdefinitions/container.qr-searchers.def
@@ -71,3 +71,6 @@ searchcluster[].indexingmode enum { REALTIME, STREAMING } default=REALTIME
## Storage cluster route to use for search cluster if indexingmode is streaming.
searchcluster[].storagecluster.routespec string default=""
+
+## Enable global phase ranking
+searchcluster[].globalphase bool default=false
diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java
index 729aebf2fc2..08d8d54bc53 100644
--- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java
+++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java
@@ -20,6 +20,7 @@ import com.yahoo.search.Searcher;
import com.yahoo.search.config.ClusterConfig;
import com.yahoo.search.dispatch.Dispatcher;
import com.yahoo.search.query.ParameterParser;
+import com.yahoo.search.ranking.GlobalPhaseRanker;
import com.yahoo.search.result.ErrorMessage;
import com.yahoo.search.schema.SchemaInfo;
import com.yahoo.search.searchchain.Execution;
@@ -64,6 +65,8 @@ public class ClusterSearcher extends Searcher {
private final VespaBackEndSearcher server;
private final Executor executor;
+ private final GlobalPhaseRanker globalPhaseHelper;
+ private final boolean enableGlobalPhase;
@Inject
public ClusterSearcher(ComponentId id,
@@ -73,10 +76,12 @@ public class ClusterSearcher extends Searcher {
DocumentdbInfoConfig documentDbConfig,
SchemaInfo schemaInfo,
ComponentRegistry<Dispatcher> dispatchers,
+ GlobalPhaseRanker globalPhaseHelper,
VipStatus vipStatus,
VespaDocumentAccess access) {
super(id);
this.executor = executor;
+ this.globalPhaseHelper = globalPhaseHelper;
int searchClusterIndex = clusterConfig.clusterId();
searchClusterName = clusterConfig.clusterName();
QrSearchersConfig.Searchcluster searchClusterConfig = getSearchClusterConfigFromClusterName(qrsConfig, searchClusterName);
@@ -101,6 +106,7 @@ public class ClusterSearcher extends Searcher {
server = searchDispatch(searchClusterIndex, searchClusterName, uniqueServerId,
docSumParams, documentDbConfig, schemaInfo, dispatchers);
}
+ enableGlobalPhase = searchClusterConfig.globalphase();
}
private static QrSearchersConfig.Searchcluster getSearchClusterConfigFromClusterName(QrSearchersConfig config, String name) {
@@ -159,7 +165,10 @@ public class ClusterSearcher extends Searcher {
maxQueryCacheTimeout = DEFAULT_MAX_QUERY_CACHE_TIMEOUT;
server = searcher;
this.executor = executor;
+ this.globalPhaseHelper = null;
+ this.enableGlobalPhase = false;
}
+
/** Do not use, for internal testing purposes only. **/
ClusterSearcher(Set<String> schemas) {
this(schemas, null, null);
@@ -169,7 +178,7 @@ public class ClusterSearcher extends Searcher {
public void fill(com.yahoo.search.Result result, String summaryClass, Execution execution) {
Query query = result.getQuery();
- VespaBackEndSearcher searcher = server;
+ Searcher searcher = server;
if (searcher != null) {
if (query.getTimeLeft() > 0) {
searcher.fill(result, summaryClass, execution);
@@ -190,7 +199,7 @@ public class ClusterSearcher extends Searcher {
public Result search(Query query, Execution execution) {
validateQueryTimeout(query);
validateQueryCache(query);
- VespaBackEndSearcher searcher = server;
+ Searcher searcher = server;
if (searcher == null) {
return new Result(query, ErrorMessage.createNoBackendsInService("Could not search"));
}
@@ -228,8 +237,21 @@ public class ClusterSearcher extends Searcher {
} else {
String docType = schemas.iterator().next();
query.getModel().setRestrict(docType);
- return searcher.search(query, execution);
+ return perSchemaSearch(searcher, query, execution);
+ }
+ }
+
+ private Result perSchemaSearch(Searcher searcher, Query query, Execution execution) {
+ Set<String> restrict = query.getModel().getRestrict();
+ if (restrict.size() != 1) {
+ throw new IllegalStateException("perSchemaSearch must always be called with 1 schema, got: " + restrict.size());
+ }
+ String schema = restrict.iterator().next();
+ Result result = searcher.search(query, execution);
+ if (globalPhaseHelper != null && enableGlobalPhase) {
+ globalPhaseHelper.process(query, result, schema);
}
+ return result;
}
private static void processResult(Query query, FutureTask<Result> task, Result mergedResult) {
@@ -248,12 +270,12 @@ public class ClusterSearcher extends Searcher {
Set<String> schemas = resolveSchemas(query, execution.context().getIndexFacts());
List<Query> queries = createQueries(query, schemas);
if (queries.size() == 1) {
- return searcher.search(queries.get(0), execution);
+ return perSchemaSearch(searcher, queries.get(0), execution);
} else {
Result mergedResult = new Result(query);
List<FutureTask<Result>> pending = new ArrayList<>(queries.size());
for (Query q : queries) {
- FutureTask<Result> task = new FutureTask<>(() -> searcher.search(q, execution));
+ FutureTask<Result> task = new FutureTask<>(() -> perSchemaSearch(searcher, q, execution));
try {
executor.execute(task);
pending.add(task);
diff --git a/container-search/src/main/java/com/yahoo/search/ranking/Evaluator.java b/container-search/src/main/java/com/yahoo/search/ranking/Evaluator.java
new file mode 100644
index 00000000000..d2edb776c92
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/ranking/Evaluator.java
@@ -0,0 +1,14 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.ranking;
+
+import com.yahoo.tensor.Tensor;
+
+import java.util.Collection;
+
+interface Evaluator {
+ Collection<String> needInputs();
+
+ Evaluator bind(String name, Tensor value);
+
+ double evaluateScore();
+}
diff --git a/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java
new file mode 100644
index 00000000000..87213362acd
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java
@@ -0,0 +1,118 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.ranking;
+
+import ai.vespa.models.evaluation.FunctionEvaluator;
+import ai.vespa.models.evaluation.Model;
+import com.yahoo.component.annotation.Inject;
+import com.yahoo.search.Query;
+import com.yahoo.search.Result;
+import com.yahoo.search.result.Hit;
+import com.yahoo.search.result.HitGroup;
+import com.yahoo.tensor.Tensor;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.logging.Logger;
+import java.util.function.Supplier;
+
+public class GlobalPhaseRanker {
+
+ private static final Logger logger = Logger.getLogger(GlobalPhaseRanker.class.getName());
+ private final RankProfilesEvaluatorFactory factory;
+ private final Set<String> skipProcessing = new HashSet<>();
+ private final Map<String, Supplier<FunctionEvaluator>> scorers = new HashMap<>();
+
+ @Inject
+ public GlobalPhaseRanker(RankProfilesEvaluatorFactory factory) {
+ this.factory = factory;
+ logger.info("using factory: " + factory);
+ }
+
+ public void process(Query query, Result result, String schema) {
+ var functionEvaluatorSource = underlying(query, schema);
+ if (functionEvaluatorSource == null) {
+ return;
+ }
+ var prepared = findFromQuery(query, functionEvaluatorSource.get().function().arguments());
+ Supplier<Evaluator> supplier = () -> {
+ var evaluator = functionEvaluatorSource.get();
+ var simple = new SimpleEvaluator(evaluator);
+ for (var entry : prepared) {
+ simple.bind(entry.name(), entry.value());
+ }
+ return simple;
+ };
+ // TODO need to get rerank-count somehow
+ int rerank = 7;
+ ResultReranker.rerankHits(result, new HitRescorer(supplier), rerank);
+ }
+
+ record NameAndValue(String name, Tensor value) { }
+
+ /* do this only once per query: */
+ List<NameAndValue> findFromQuery(Query query, List<String> needInputs) {
+ List<NameAndValue> result = new ArrayList<>();
+ var ranking = query.getRanking();
+ var rankFeatures = ranking.getFeatures();
+ var rankProps = ranking.getProperties().asMap();
+ for (String needed : needInputs) {
+ var optRef = com.yahoo.searchlib.rankingexpression.Reference.simple(needed);
+ if (optRef.isEmpty()) continue;
+ var ref = optRef.get();
+ if (ref.name().equals("constant")) {
+ // XXX in theory, we should be able to avoid this
+ result.add(new NameAndValue(needed, null));
+ continue;
+ }
+ if (ref.isSimple() && ref.name().equals("query")) {
+ String queryFeatureName = ref.simpleArgument().get();
+ // searchers are recommended to place query features here:
+ var feature = rankFeatures.getTensor(queryFeatureName);
+ if (feature.isPresent()) {
+ result.add(new NameAndValue(needed, feature.get()));
+ } else {
+ // but other ways of setting query features end up in the properties:
+ var objList = rankProps.get(queryFeatureName);
+ if (objList != null && objList.size() == 1 && objList.get(0) instanceof Tensor t) {
+ result.add(new NameAndValue(needed, t));
+ }
+ }
+ }
+ }
+ return result;
+ }
+
+ private Supplier<FunctionEvaluator> underlying(Query query, String schema) {
+ String rankProfile = query.getRanking().getProfile();
+ String key = schema + " with rank profile " + rankProfile;
+ if (skipProcessing.contains(key)) {
+ return null;
+ }
+ Supplier<FunctionEvaluator> supplier = scorers.get(key);
+ if (supplier != null) {
+ return supplier;
+ }
+ try {
+ var proxy = factory.proxyForSchema(schema);
+ var model = proxy.modelForRankProfile(rankProfile);
+ supplier = () -> model.evaluatorOf("globalphase");
+ if (supplier.get() == null) {
+ supplier = null;
+ }
+ } catch (IllegalArgumentException e) {
+ logger.info("no global-phase for " + key + " because: " + e.getMessage());
+ supplier = null;
+ }
+ if (supplier == null) {
+ skipProcessing.add(key);
+ } else {
+ scorers.put(key, supplier);
+ }
+ return supplier;
+ }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/ranking/HitRescorer.java b/container-search/src/main/java/com/yahoo/search/ranking/HitRescorer.java
new file mode 100644
index 00000000000..ebdbbb693f1
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/ranking/HitRescorer.java
@@ -0,0 +1,56 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.ranking;
+
+import com.yahoo.search.result.FeatureData;
+import com.yahoo.search.result.Hit;
+
+import java.util.function.Supplier;
+import java.util.logging.Logger;
+
+class HitRescorer {
+
+ private static final Logger logger = Logger.getLogger(HitRescorer.class.getName());
+
+ private final Supplier<Evaluator> evaluatorSource;
+
+ public HitRescorer(Supplier<Evaluator> evaluatorSource) {
+ this.evaluatorSource = evaluatorSource;
+ }
+
+ boolean rescoreHit(Hit hit) {
+ var features = hit.getField("matchfeatures");
+ if (features instanceof FeatureData matchFeatures) {
+ var scorer = evaluatorSource.get();
+ for (String argName : scorer.needInputs()) {
+ var asTensor = matchFeatures.getTensor(argName);
+ if (asTensor == null) {
+ asTensor = matchFeatures.getTensor(alternate(argName));
+ }
+ if (asTensor != null) {
+ scorer.bind(argName, asTensor);
+ } else {
+ logger.warning("Missing match-feature for Evaluator argument: " + argName);
+ return false;
+ }
+ }
+ double newScore = scorer.evaluateScore();
+ hit.setRelevance(newScore);
+ return true;
+ } else {
+ logger.warning("Hit without match-features: " + hit);
+ return false;
+ }
+ }
+
+ private static final String RE_PREFIX = "rankingExpression(";
+ private static final String RE_SUFFIX = ")";
+ private static final int RE_PRE_LEN = RE_PREFIX.length();
+ private static final int RE_SUF_LEN = RE_SUFFIX.length();
+
+ static String alternate(String argName) {
+ if (argName.startsWith(RE_PREFIX) && argName.endsWith(RE_SUFFIX)) {
+ return argName.substring(RE_PRE_LEN, argName.length() - RE_SUF_LEN);
+ }
+ return argName;
+ }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluator.java b/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluator.java
new file mode 100644
index 00000000000..ccb9b9837fe
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluator.java
@@ -0,0 +1,53 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.search.ranking;
+
+import ai.vespa.models.evaluation.FunctionEvaluator;
+import ai.vespa.models.evaluation.Model;
+import ai.vespa.models.evaluation.ModelsEvaluator;
+import com.yahoo.api.annotations.Beta;
+import com.yahoo.component.AbstractComponent;
+import com.yahoo.component.annotation.Inject;
+import com.yahoo.filedistribution.fileacquirer.FileAcquirer;
+import com.yahoo.vespa.config.search.RankProfilesConfig;
+import com.yahoo.vespa.config.search.core.OnnxModelsConfig;
+import com.yahoo.vespa.config.search.core.RankingConstantsConfig;
+import com.yahoo.vespa.config.search.core.RankingExpressionsConfig;
+
+/**
+ * proxy for model-evaluation components
+ * @author arnej
+ */
+@Beta
+public class RankProfilesEvaluator extends AbstractComponent {
+
+ private final ModelsEvaluator evaluator;
+
+ @Inject
+ public RankProfilesEvaluator(
+ RankProfilesConfig rankProfilesConfig,
+ RankingConstantsConfig constantsConfig,
+ RankingExpressionsConfig expressionsConfig,
+ OnnxModelsConfig onnxModelsConfig,
+ FileAcquirer fileAcquirer)
+ {
+ this.evaluator = new ModelsEvaluator(
+ rankProfilesConfig,
+ constantsConfig,
+ expressionsConfig,
+ onnxModelsConfig,
+ fileAcquirer);
+ }
+
+ public Model modelForRankProfile(String rankProfile) {
+ var m = evaluator.models().get(rankProfile);
+ if (m == null) {
+ throw new IllegalArgumentException("unknown rankprofile: " + rankProfile);
+ }
+ return m;
+ }
+
+ public FunctionEvaluator evaluatorForFunction(String rankProfile, String functionName) {
+ return modelForRankProfile(rankProfile).evaluatorOf(functionName);
+ }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluatorFactory.java b/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluatorFactory.java
new file mode 100644
index 00000000000..edb05ed9788
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluatorFactory.java
@@ -0,0 +1,40 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+package com.yahoo.search.ranking;
+
+import com.yahoo.api.annotations.Beta;
+import com.yahoo.component.annotation.Inject;
+import com.yahoo.component.provider.ComponentRegistry;
+
+/**
+ * factory for model-evaluation proxies
+ * @author arnej
+ */
+@Beta
+public class RankProfilesEvaluatorFactory {
+
+ private final ComponentRegistry<RankProfilesEvaluator> registry;
+
+ @Inject
+ public RankProfilesEvaluatorFactory(ComponentRegistry<RankProfilesEvaluator> registry) {
+ this.registry = registry;
+ }
+
+ public RankProfilesEvaluator proxyForSchema(String schemaName) {
+ var component = registry.getComponent("ranking-expression-evaluator." + schemaName);
+ if (component == null) {
+ throw new IllegalArgumentException("ranking expression evaluator for schema '" + schemaName + "' not found");
+ }
+ return component;
+ }
+
+ public String toString() {
+ var buf = new StringBuilder();
+ buf.append(this.getClass().getName()).append(" containing: [");
+ for (var id : registry.allComponentsById().keySet()) {
+ buf.append(" ").append(id.toString());
+ }
+ buf.append(" ]");
+ return buf.toString();
+ }
+}
diff --git a/container-search/src/main/java/com/yahoo/search/ranking/ResultReranker.java b/container-search/src/main/java/com/yahoo/search/ranking/ResultReranker.java
new file mode 100644
index 00000000000..11b3fa7390a
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/ranking/ResultReranker.java
@@ -0,0 +1,91 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.ranking;
+
+import com.yahoo.search.Result;
+import com.yahoo.search.result.Hit;
+import com.yahoo.search.result.HitGroup;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.logging.Logger;
+
+class ResultReranker {
+
+ private static final Logger logger = Logger.getLogger(ResultReranker.class.getName());
+
+ // scale and adjust the score according to the range
+ // of the original and final score values to avoid that
+ // a score from the backend is larger than finalScores_low
+ static class Ranges {
+ private double initialScores_high = -Double.MAX_VALUE;
+ private double initialScores_low = Double.MAX_VALUE;
+ private double finalScores_high = -Double.MAX_VALUE;
+ private double finalScores_low = Double.MAX_VALUE;
+
+ boolean valid() {
+ return (initialScores_high >= initialScores_low
+ &&
+ finalScores_high >= finalScores_low);
+ }
+ void withInitialScore(double score) {
+ if (score < initialScores_low) initialScores_low = score;
+ if (score > initialScores_high) initialScores_high = score;
+ }
+ void withFinalScore(double score) {
+ if (score < finalScores_low) finalScores_low = score;
+ if (score > finalScores_high) finalScores_high = score;
+ }
+ private double initialRange() {
+ double r = initialScores_high - initialScores_low;
+ if (r < 1.0) r = 1.0;
+ return r;
+ }
+ private double finalRange() {
+ double r = finalScores_high - finalScores_low;
+ if (r < 1.0) r = 1.0;
+ return r;
+ }
+ double scale() { return finalRange() / initialRange(); }
+ double bias() { return finalScores_low - initialScores_low * scale(); }
+ }
+
+ static void rerankHits(Result result, HitRescorer hitRescorer, int rerankCount) {
+ List<Hit> hitsToRescore = new ArrayList<>();
+ // consider doing recursive iteration explicitly instead of using deepIterator?
+ for (var iterator = result.hits().deepIterator(); iterator.hasNext();) {
+ Hit hit = iterator.next();
+ if (hit.isMeta() || hit instanceof HitGroup) {
+ continue;
+ }
+ // what about hits inside grouping results?
+ // they are inside GroupingListHit, we won't recurse into it; so we won't see them.
+ hitsToRescore.add(hit);
+ }
+ // we can't be 100% certain that hits were sorted according to relevance:
+ hitsToRescore.sort(Comparator.naturalOrder());
+ var ranges = new Ranges();
+ for (var iterator = hitsToRescore.iterator(); rerankCount > 0 && iterator.hasNext(); ) {
+ Hit hit = iterator.next();
+ double oldScore = hit.getRelevance().getScore();
+ boolean didRerank = hitRescorer.rescoreHit(hit);
+ if (didRerank) {
+ ranges.withInitialScore(oldScore);
+ ranges.withFinalScore(hit.getRelevance().getScore());
+ --rerankCount;
+ iterator.remove();
+ }
+ }
+ // if any hits are left in the list, they need rescaling:
+ if (ranges.valid()) {
+ double scale = ranges.scale();
+ double bias = ranges.bias();
+ for (Hit hit : hitsToRescore) {
+ double oldScore = hit.getRelevance().getScore();
+ hit.setRelevance(oldScore * scale + bias);
+ }
+ }
+ result.hits().sort();
+ }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java b/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java
new file mode 100644
index 00000000000..f247eab1649
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java
@@ -0,0 +1,51 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.search.ranking;
+
+import ai.vespa.models.evaluation.FunctionEvaluator;
+import com.yahoo.search.result.FeatureData;
+import com.yahoo.search.result.Hit;
+import com.yahoo.tensor.Tensor;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+class SimpleEvaluator implements Evaluator {
+
+ private final FunctionEvaluator evaluator;
+ private final Set<String> neededInputs;
+
+ public SimpleEvaluator(FunctionEvaluator prototype) {
+ this.evaluator = prototype;
+ this.neededInputs = new HashSet<String>(prototype.function().arguments());
+ }
+
+ @Override
+ public Collection<String> needInputs() { return List.copyOf(neededInputs); }
+
+ @Override
+ public SimpleEvaluator bind(String name, Tensor value) {
+ if (value != null) evaluator.bind(name, value);
+ neededInputs.remove(name);
+ return this;
+ }
+
+ @Override
+ public double evaluateScore() {
+ return evaluator.evaluate().asDouble();
+ }
+
+ @Override
+ public String toString() {
+ var buf = new StringBuilder();
+ buf.append("SimpleEvaluator(");
+ buf.append(evaluator.function().toString());
+ buf.append(")[");
+ for (String arg : neededInputs) {
+ buf.append("{").append(arg).append("}");
+ }
+ buf.append("]");
+ return buf.toString();
+ }
+}
diff --git a/fastos/src/tests/mazeserver.cpp b/container-search/src/main/java/com/yahoo/search/ranking/package-info.java
index b62e98645f2..a86a5c1e52f 100644
--- a/fastos/src/tests/mazeserver.cpp
+++ b/container-search/src/main/java/com/yahoo/search/ranking/package-info.java
@@ -1,4 +1,6 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#define DO_MAZE_SERVER 1
-#include "sockettest.cpp"
+@ExportPackage
+package com.yahoo.search.ranking;
+
+import com.yahoo.osgi.annotation.ExportPackage;
diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java
index 5df8d2e5444..06ae9923dae 100644
--- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java
@@ -464,6 +464,7 @@ public class ClusterSearcherTestCase {
documentDbConfig.build(),
new SchemaInfo(List.of(schema.build()), Map.of()),
dispatchers,
+ null,
vipStatus,
null);
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java
index cf9db1517a0..f6029eade37 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java
@@ -1,6 +1,5 @@
package com.yahoo.vespa.hosted.controller.maintenance;
-import com.yahoo.vespa.hosted.controller.Application;
import com.yahoo.vespa.hosted.controller.Controller;
import com.yahoo.vespa.hosted.controller.Instance;
import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
@@ -9,7 +8,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeReposi
import com.yahoo.yolean.Exceptions;
import java.time.Duration;
-import java.util.stream.Stream;
+import java.util.Collection;
/**
* This pulls application deployment information from the node repo on all config servers,
@@ -28,29 +27,35 @@ public class DeploymentInfoMaintainer extends ControllerMaintainer {
@Override
protected double maintain() {
- controller().applications().asList().stream()
- .flatMap(this::mapApplicationToInstances)
- .flatMap(this::mapInstanceToDeployments)
- .forEach(this::updateDeploymentInfo);
- return 1.0;
- }
-
- private Stream<Instance> mapApplicationToInstances(Application application) {
- return application.instances().values().stream();
+ int attempts = 0;
+ int failures = 0;
+ for (var application : controller().applications().asList()) {
+ for (var instance : application.instances().values()) {
+ for (var deployment : instanceDeployments(instance)) {
+ attempts++;
+ if ( ! updateDeploymentInfo(deployment))
+ failures++;
+ }
+ }
+ }
+ return asSuccessFactor(attempts, failures);
}
- private Stream<DeploymentId> mapInstanceToDeployments(Instance instance) {
+ private Collection<DeploymentId> instanceDeployments(Instance instance) {
return instance.deployments().keySet().stream()
.filter(zoneId -> !zoneId.environment().isTest())
- .map(zoneId -> new DeploymentId(instance.id(), zoneId));
+ .map(zoneId -> new DeploymentId(instance.id(), zoneId))
+ .toList();
}
- private void updateDeploymentInfo(DeploymentId id) {
+ private boolean updateDeploymentInfo(DeploymentId id) {
try {
controller().applications().deploymentInfo().put(id, nodeRepository.getApplication(id.zoneId(), id.applicationId()));
+ return true;
}
catch (ConfigServerException e) {
log.info("Could not retrieve deployment info for " + id + ": " + Exceptions.toMessageString(e));
+ return false;
}
}
diff --git a/dist/vespa.spec b/dist/vespa.spec
index 6bed9904124..e880734979a 100644
--- a/dist/vespa.spec
+++ b/dist/vespa.spec
@@ -54,8 +54,7 @@ BuildRequires: gcc-toolset-12-libatomic-devel
BuildRequires: maven
BuildRequires: maven-openjdk17
BuildRequires: vespa-pybind11-devel
-BuildRequires: python3-pytest
-BuildRequires: python36-devel
+BuildRequires: python38-devel
BuildRequires: glibc-langpack-en
%endif
%if 0%{?el9}
@@ -377,7 +376,7 @@ Summary: Vespa - The open big data serving engine - ann-benchmark
Requires: %{name}-base-libs = %{version}-%{release}
Requires: %{name}-libs = %{version}-%{release}
%if 0%{?el8}
-Requires: python36
+Requires: python38
%endif
%if 0%{?el9}
Requires: python3
diff --git a/fastos/CMakeLists.txt b/fastos/CMakeLists.txt
index c17752e234c..60813c569e4 100644
--- a/fastos/CMakeLists.txt
+++ b/fastos/CMakeLists.txt
@@ -4,5 +4,4 @@ vespa_define_module(
src/vespa/fastos
TESTS
- src/tests
)
diff --git a/fastos/src/tests/.gitignore b/fastos/src/tests/.gitignore
deleted file mode 100644
index ccb5f0210ab..00000000000
--- a/fastos/src/tests/.gitignore
+++ /dev/null
@@ -1,17 +0,0 @@
-/Makefile
-/backtracetest
-/backtracetest.log
-/filetest
-/filetest.log
-/processtest
-/processtest.log
-/sockettest
-/sockettest.log
-/threadtest
-/threadtest.log
-/timetest
-/timetest.log
-/typetest
-/typetest.log
-/usecputest
-*test_app
diff --git a/fastos/src/tests/CMakeLists.txt b/fastos/src/tests/CMakeLists.txt
deleted file mode 100644
index 7d4b014b6b3..00000000000
--- a/fastos/src/tests/CMakeLists.txt
+++ /dev/null
@@ -1,15 +0,0 @@
-# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_executable(fastos_filetest_app TEST
- SOURCES
- filetest.cpp
- DEPENDS
- fastos
-)
-vespa_add_test(NAME fastos_filetest_app NO_VALGRIND COMMAND fastos_filetest_app)
-vespa_add_executable(fastos_typetest_app TEST
- SOURCES
- typetest.cpp
- DEPENDS
- fastos
-)
-vespa_add_test(NAME fastos_typetest_app NO_VALGRIND COMMAND fastos_typetest_app)
diff --git a/fastos/src/tests/coretest2.cpp b/fastos/src/tests/coretest2.cpp
deleted file mode 100644
index bd93623922b..00000000000
--- a/fastos/src/tests/coretest2.cpp
+++ /dev/null
@@ -1,41 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-
-
-
-static void
-bomb(void)
-{
- char *p;
-
- p = nullptr;
- *p = 4;
-}
-
-void *BomberRun(void *arg)
-{
- (void) arg;
- bomb();
- return nullptr;
-};
-
-static int
-bombMain(void)
-{
- pthread_t thread;
- void *ret;
-
- (void) pthread_create(&thread, nullptr, BomberRun, nullptr);
- ret = nullptr;
- (void) pthread_join(thread, &ret);
- return (0);
-}
-
-
-int
-main(int argc, char **argv)
-{
- (void) argc;
- (void) argv;
- return bombMain();
-}
diff --git a/fastos/src/tests/performancetest.cpp b/fastos/src/tests/performancetest.cpp
deleted file mode 100644
index f566979957a..00000000000
--- a/fastos/src/tests/performancetest.cpp
+++ /dev/null
@@ -1,54 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-#include <stdlib.h>
-
-#include "tests.h"
-
-void PerformanceTest (char *buffer);
-
-int main (int argc, char **argv)
-{
- (void)argc;
- (void)argv;
-
- void (*test)(char *buffer) = PerformanceTest;
-
- test(nullptr);
- return 0;
-}
-
-void PerformanceTest (char *buffer)
-{
- // Cause exception
- *static_cast<char *>(nullptr) = 'e';
-
-#if 1
- FastOS_File file("test.txt");
-
- if(file.OpenReadOnly())
- {
- file.Read(buffer, 20);
- file.Write2(buffer, 20);
- file.Read(buffer, 20);
- file.Write2(buffer, 20);
- file.Read(buffer, 20);
- file.Write2(buffer, 20);
- }
-#else
-
- int filedes = open("test.txt", O_RDONLY, 0664);
-
- if(filedes != -1)
- {
- write(filedes, buffer, 20);
- read(filedes, buffer, 20);
- write(filedes, buffer, 20);
- read(filedes, buffer, 20);
- write(filedes, buffer, 20);
- read(filedes, buffer, 20);
- write(filedes, buffer, 20);
-
- close(filedes);
- }
-#endif
-}
-
diff --git a/fastos/src/tests/typetest.cpp b/fastos/src/tests/typetest.cpp
deleted file mode 100644
index 7871275e17c..00000000000
--- a/fastos/src/tests/typetest.cpp
+++ /dev/null
@@ -1,44 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "tests.h"
-#include <vespa/fastos/file.h>
-
-class TypeTest : public BaseTest
-{
-private:
-
- void ObjectSizeTest ()
- {
- TestHeader("Object Sizes (bytes)");
-
- Progress(true, "FastOS_DirectoryScan %d", sizeof(FastOS_DirectoryScan));
- Progress(true, "FastOS_File: %d", sizeof(FastOS_File));
- Progress(true, "FastOS_StatInfo %d", sizeof(FastOS_StatInfo));
-
- PrintSeparator();
- }
-
-public:
- virtual ~TypeTest() {};
-
- int Main () override
- {
- printf("grep for the string '%s' to detect failures.\n\n", failString);
-
- ObjectSizeTest();
-
- PrintSeparator();
- printf("END OF TEST (%s)\n", _argv[0]);
-
- return allWasOk() ? 0 : 1;
- }
-};
-
-
-int main (int argc, char **argv)
-{
- setvbuf(stdout, nullptr, _IOLBF, 8192);
- TypeTest app;
- return app.Entry(argc, argv);
-}
-
diff --git a/fastos/src/vespa/fastos/CMakeLists.txt b/fastos/src/vespa/fastos/CMakeLists.txt
index c37eb43b884..29810a4f296 100644
--- a/fastos/src/vespa/fastos/CMakeLists.txt
+++ b/fastos/src/vespa/fastos/CMakeLists.txt
@@ -1,10 +1,6 @@
# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
vespa_add_library(fastos_objects OBJECT
SOURCES
- file.cpp
- file_rw_ops.cpp
- linux_file.cpp
- unix_file.cpp
)
vespa_add_library(fastos
diff --git a/fastos/src/vespa/fastos/types.h b/fastos/src/vespa/fastos/types.h
deleted file mode 100644
index 69dd3e5231c..00000000000
--- a/fastos/src/vespa/fastos/types.h
+++ /dev/null
@@ -1,9 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#define FASTOS_PREFIX(a) FastOS_##a
-
-// New macros to support the new gcc visibility features.
-#define VESPA_DLL_EXPORT __attribute__ ((visibility("default")))
-#define VESPA_DLL_LOCAL __attribute__ ((visibility("hidden")))
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
index 3989b45b9ac..0aec24d4d24 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
@@ -345,6 +345,13 @@ public class Flags {
"Takes effect on the next tick.",
ZONE_ID, NODE_TYPE, HOSTNAME);
+ public static final UnboundBooleanFlag ENABLE_GLOBAL_PHASE = defineFeatureFlag(
+ "enable-global-phase", false,
+ List.of("arnej", "bjorncs"), "2023-02-28", "2024-01-10",
+ "Enable global phase ranking",
+ "Takes effect at redeployment",
+ APPLICATION_ID);
+
/** WARNING: public for testing: All flags should be defined in {@link Flags}. */
public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners,
String createdAt, String expiresAt, String description,
diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java
index 53acd8cbb1b..1ae25aa0cfa 100644
--- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java
+++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java
@@ -45,6 +45,7 @@ public class ExportPackages {
}
}
+ // Make sure to update the junit integration test `ExportPackagesIT.java` if the set of exported packages is modified.
private static String getExportPackages(String[] jars) throws IOException {
StringBuilder out = new StringBuilder();
out.append(getSystemPackages()).append(", ")
@@ -53,6 +54,7 @@ public class ExportPackages {
.append("com.yahoo.jdisc.handler, ")
.append("com.yahoo.jdisc.service, ")
.append("com.yahoo.jdisc.statistics, ")
+ .append("com.yahoo.jdisc.refcount, ")
.append("javax.inject;version=1.0.0, ") // TODO Vespa 9: remove. Included in guice, but not exported. Needed by container-jersey.
.append("org.aopalliance.intercept, ")
diff --git a/jdisc_core/src/test/resources/exportPackages.properties b/jdisc_core/src/test/resources/exportPackages.properties
index 82242b1644b..b49f91842e3 100644
--- a/jdisc_core/src/test/resources/exportPackages.properties
+++ b/jdisc_core/src/test/resources/exportPackages.properties
@@ -1,3 +1,3 @@
#generated by com.yahoo.jdisc.core.ExportPackages
#Wed Jul 20 02:55:26 CEST 2022
-exportPackages=org.osgi.framework; version\="1.10.0", org.osgi.framework.connect; version\="1.0.0", org.osgi.framework.dto; uses\:\="org.osgi.dto"; version\="1.8.0", org.osgi.framework.hooks.bundle; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.resolver; uses\:\="org.osgi.framework.wiring"; version\="1.0.0", org.osgi.framework.hooks.service; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.weaving; uses\:\="org.osgi.framework.wiring"; version\="1.1.0", org.osgi.framework.launch; uses\:\="org.osgi.framework"; version\="1.2.0", org.osgi.framework.namespace; uses\:\="org.osgi.resource"; version\="1.2.0", org.osgi.framework.startlevel; uses\:\="org.osgi.framework"; version\="1.0.0", org.osgi.framework.startlevel.dto; uses\:\="org.osgi.dto"; version\="1.0.0", org.osgi.framework.wiring; uses\:\="org.osgi.framework,org.osgi.resource"; version\="1.2.0", org.osgi.framework.wiring.dto; uses\:\="org.osgi.dto,org.osgi.resource.dto"; version\="1.3.0", org.osgi.resource; version\="1.0.1", org.osgi.resource.dto; uses\:\="org.osgi.dto"; version\="1.0.1", org.osgi.service.packageadmin; uses\:\="org.osgi.framework"; version\="1.2.1", org.osgi.service.startlevel; uses\:\="org.osgi.framework"; version\="1.1.1", org.osgi.service.url; version\="1.0.1", org.osgi.service.resolver; uses\:\="org.osgi.resource"; version\="1.1.1", org.osgi.util.tracker; uses\:\="org.osgi.framework"; version\="1.5.3", org.osgi.dto; version\="1.1.1", org.osgi.service.condition; version\="1.0.0", javax.security.auth.callback; version\="0.0.0.JavaSE_017", javax.security.auth.login; version\="0.0.0.JavaSE_017", javax.net.ssl; version\="0.0.0.JavaSE_017", java.lang.constant; version\="0.0.0.JavaSE_017", java.security.interfaces; version\="0.0.0.JavaSE_017", java.text.spi; version\="0.0.0.JavaSE_017", java.nio.channels.spi; version\="0.0.0.JavaSE_017", java.math; version\="0.0.0.JavaSE_017", java.nio.file; version\="0.0.0.JavaSE_017", java.util.concurrent.atomic; version\="0.0.0.JavaSE_017", java.security.cert; version\="0.0.0.JavaSE_017", java.security.spec; version\="0.0.0.JavaSE_017", java.nio.channels; version\="0.0.0.JavaSE_017", java.time.chrono; version\="0.0.0.JavaSE_017", javax.crypto; version\="0.0.0.JavaSE_017", java.time.zone; version\="0.0.0.JavaSE_017", java.nio.charset; version\="0.0.0.JavaSE_017", java.io; version\="0.0.0.JavaSE_017", java.util.spi; version\="0.0.0.JavaSE_017", java.net; version\="0.0.0.JavaSE_017", javax.security.cert; version\="0.0.0.JavaSE_017", java.lang.annotation; version\="0.0.0.JavaSE_017", javax.security.auth.spi; version\="0.0.0.JavaSE_017", java.util.concurrent; version\="0.0.0.JavaSE_017", java.nio.charset.spi; version\="0.0.0.JavaSE_017", javax.crypto.interfaces; version\="0.0.0.JavaSE_017", java.util; version\="0.0.0.JavaSE_017", java.security; version\="0.0.0.JavaSE_017", java.nio.file.spi; version\="0.0.0.JavaSE_017", java.nio; version\="0.0.0.JavaSE_017", java.util.jar; version\="0.0.0.JavaSE_017", javax.security.auth; version\="0.0.0.JavaSE_017", java.lang.ref; version\="0.0.0.JavaSE_017", java.util.regex; version\="0.0.0.JavaSE_017", java.net.spi; version\="0.0.0.JavaSE_017", java.lang.module; version\="0.0.0.JavaSE_017", java.lang.invoke; version\="0.0.0.JavaSE_017", java.time.format; version\="0.0.0.JavaSE_017", java.util.concurrent.locks; version\="0.0.0.JavaSE_017", java.time.temporal; version\="0.0.0.JavaSE_017", java.util.zip; version\="0.0.0.JavaSE_017", java.nio.file.attribute; version\="0.0.0.JavaSE_017", java.util.random; version\="0.0.0.JavaSE_017", java.text; version\="0.0.0.JavaSE_017", javax.crypto.spec; version\="0.0.0.JavaSE_017", java.util.stream; version\="0.0.0.JavaSE_017", java.time; version\="0.0.0.JavaSE_017", java.lang; version\="0.0.0.JavaSE_017", java.lang.runtime; version\="0.0.0.JavaSE_017", java.util.function; version\="0.0.0.JavaSE_017", javax.net; version\="0.0.0.JavaSE_017", javax.security.auth.x500; version\="0.0.0.JavaSE_017", java.lang.reflect; version\="0.0.0.JavaSE_017", javax.lang.model; version\="0.0.0.JavaSE_017", javax.annotation.processing; version\="0.0.0.JavaSE_017", javax.lang.model.element; version\="0.0.0.JavaSE_017", javax.tools; version\="0.0.0.JavaSE_017", javax.lang.model.type; version\="0.0.0.JavaSE_017", javax.lang.model.util; version\="0.0.0.JavaSE_017", java.awt.datatransfer; version\="0.0.0.JavaSE_017", java.awt.desktop; version\="0.0.0.JavaSE_017", javax.swing.plaf.synth; version\="0.0.0.JavaSE_017", java.beans; version\="0.0.0.JavaSE_017", javax.swing.text.html.parser; version\="0.0.0.JavaSE_017", javax.swing.text.rtf; version\="0.0.0.JavaSE_017", java.awt.font; version\="0.0.0.JavaSE_017", javax.imageio; version\="0.0.0.JavaSE_017", java.awt.im.spi; version\="0.0.0.JavaSE_017", java.applet; version\="0.0.0.JavaSE_017", javax.sound.midi; version\="0.0.0.JavaSE_017", java.awt.dnd; version\="0.0.0.JavaSE_017", javax.swing.text; version\="0.0.0.JavaSE_017", javax.swing.plaf.basic; version\="0.0.0.JavaSE_017", javax.swing.undo; version\="0.0.0.JavaSE_017", javax.swing.plaf; version\="0.0.0.JavaSE_017", javax.swing.filechooser; version\="0.0.0.JavaSE_017", javax.imageio.event; version\="0.0.0.JavaSE_017", javax.sound.sampled; version\="0.0.0.JavaSE_017", javax.print.attribute; version\="0.0.0.JavaSE_017", javax.print; version\="0.0.0.JavaSE_017", javax.swing.plaf.nimbus; version\="0.0.0.JavaSE_017", javax.accessibility; version\="0.0.0.JavaSE_017", java.awt.event; version\="0.0.0.JavaSE_017", javax.swing.text.html; version\="0.0.0.JavaSE_017", javax.imageio.spi; version\="0.0.0.JavaSE_017", javax.swing.border; version\="0.0.0.JavaSE_017", javax.sound.sampled.spi; version\="0.0.0.JavaSE_017", javax.imageio.plugins.bmp; version\="0.0.0.JavaSE_017", java.awt.color; version\="0.0.0.JavaSE_017", java.awt.geom; version\="0.0.0.JavaSE_017", javax.imageio.plugins.jpeg; version\="0.0.0.JavaSE_017", javax.swing.tree; version\="0.0.0.JavaSE_017", javax.imageio.plugins.tiff; version\="0.0.0.JavaSE_017", java.awt.print; version\="0.0.0.JavaSE_017", java.awt.image; version\="0.0.0.JavaSE_017", javax.imageio.metadata; version\="0.0.0.JavaSE_017", javax.swing.table; version\="0.0.0.JavaSE_017", javax.sound.midi.spi; version\="0.0.0.JavaSE_017", javax.print.attribute.standard; version\="0.0.0.JavaSE_017", javax.swing.colorchooser; version\="0.0.0.JavaSE_017", javax.swing; version\="0.0.0.JavaSE_017", java.awt.image.renderable; version\="0.0.0.JavaSE_017", javax.swing.plaf.multi; version\="0.0.0.JavaSE_017", java.awt.im; version\="0.0.0.JavaSE_017", javax.print.event; version\="0.0.0.JavaSE_017", javax.swing.plaf.metal; version\="0.0.0.JavaSE_017", java.beans.beancontext; version\="0.0.0.JavaSE_017", java.awt; version\="0.0.0.JavaSE_017", javax.imageio.stream; version\="0.0.0.JavaSE_017", javax.swing.event; version\="0.0.0.JavaSE_017", java.lang.instrument; version\="0.0.0.JavaSE_017", java.util.logging; version\="0.0.0.JavaSE_017", javax.management.timer; version\="0.0.0.JavaSE_017", javax.management; version\="0.0.0.JavaSE_017", javax.management.relation; version\="0.0.0.JavaSE_017", javax.management.loading; version\="0.0.0.JavaSE_017", javax.management.openmbean; version\="0.0.0.JavaSE_017", java.lang.management; version\="0.0.0.JavaSE_017", javax.management.remote; version\="0.0.0.JavaSE_017", javax.management.monitor; version\="0.0.0.JavaSE_017", javax.management.modelmbean; version\="0.0.0.JavaSE_017", javax.management.remote.rmi; version\="0.0.0.JavaSE_017", javax.naming.event; version\="0.0.0.JavaSE_017", javax.naming.ldap.spi; version\="0.0.0.JavaSE_017", javax.naming; version\="0.0.0.JavaSE_017", javax.naming.spi; version\="0.0.0.JavaSE_017", javax.naming.ldap; version\="0.0.0.JavaSE_017", javax.naming.directory; version\="0.0.0.JavaSE_017", java.net.http; version\="0.0.0.JavaSE_017", java.util.prefs; version\="0.0.0.JavaSE_017", java.rmi.registry; version\="0.0.0.JavaSE_017", javax.rmi.ssl; version\="0.0.0.JavaSE_017", java.rmi.dgc; version\="0.0.0.JavaSE_017", java.rmi; version\="0.0.0.JavaSE_017", java.rmi.server; version\="0.0.0.JavaSE_017", javax.script; version\="0.0.0.JavaSE_017", org.ietf.jgss; version\="0.0.0.JavaSE_017", javax.security.auth.kerberos; version\="0.0.0.JavaSE_017", javax.security.sasl; version\="0.0.0.JavaSE_017", javax.smartcardio; version\="0.0.0.JavaSE_017", java.sql; version\="0.0.0.JavaSE_017", javax.sql; version\="0.0.0.JavaSE_017", javax.sql.rowset.spi; version\="0.0.0.JavaSE_017", javax.sql.rowset.serial; version\="0.0.0.JavaSE_017", javax.sql.rowset; version\="0.0.0.JavaSE_017", javax.transaction.xa; version\="0.0.0.JavaSE_017", javax.xml; version\="0.0.0.JavaSE_017", javax.xml.transform.sax; version\="0.0.0.JavaSE_017", javax.xml.datatype; version\="0.0.0.JavaSE_017", javax.xml.catalog; version\="0.0.0.JavaSE_017", org.w3c.dom.traversal; version\="0.0.0.JavaSE_017", javax.xml.stream.events; version\="0.0.0.JavaSE_017", javax.xml.stream; version\="0.0.0.JavaSE_017", org.xml.sax; version\="0.0.0.JavaSE_017", javax.xml.transform; version\="0.0.0.JavaSE_017", javax.xml.xpath; version\="0.0.0.JavaSE_017", org.w3c.dom.events; version\="0.0.0.JavaSE_017", org.w3c.dom.ranges; version\="0.0.0.JavaSE_017", org.w3c.dom.ls; version\="0.0.0.JavaSE_017", javax.xml.stream.util; version\="0.0.0.JavaSE_017", javax.xml.namespace; version\="0.0.0.JavaSE_017", javax.xml.transform.stax; version\="0.0.0.JavaSE_017", org.xml.sax.helpers; version\="0.0.0.JavaSE_017", org.w3c.dom.views; version\="0.0.0.JavaSE_017", org.w3c.dom.bootstrap; version\="0.0.0.JavaSE_017", org.w3c.dom; version\="0.0.0.JavaSE_017", javax.xml.transform.stream; version\="0.0.0.JavaSE_017", javax.xml.transform.dom; version\="0.0.0.JavaSE_017", javax.xml.validation; version\="0.0.0.JavaSE_017", javax.xml.parsers; version\="0.0.0.JavaSE_017", org.xml.sax.ext; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.spec; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.keyinfo; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig; version\="0.0.0.JavaSE_017", com.sun.java.accessibility.util; version\="0.0.0.JavaSE_017", com.sun.tools.attach.spi; version\="0.0.0.JavaSE_017", com.sun.tools.attach; version\="0.0.0.JavaSE_017", com.sun.source.doctree; version\="0.0.0.JavaSE_017", com.sun.source.tree; version\="0.0.0.JavaSE_017", com.sun.source.util; version\="0.0.0.JavaSE_017", com.sun.tools.javac; version\="0.0.0.JavaSE_017", jdk.dynalink.beans; version\="0.0.0.JavaSE_017", jdk.dynalink.linker.support; version\="0.0.0.JavaSE_017", jdk.dynalink.support; version\="0.0.0.JavaSE_017", jdk.dynalink; version\="0.0.0.JavaSE_017", jdk.dynalink.linker; version\="0.0.0.JavaSE_017", com.sun.net.httpserver; version\="0.0.0.JavaSE_017", com.sun.net.httpserver.spi; version\="0.0.0.JavaSE_017", com.sun.jarsigner; version\="0.0.0.JavaSE_017", jdk.security.jarsigner; version\="0.0.0.JavaSE_017", jdk.javadoc.doclet; version\="0.0.0.JavaSE_017", com.sun.tools.jconsole; version\="0.0.0.JavaSE_017", com.sun.jdi.connect; version\="0.0.0.JavaSE_017", com.sun.jdi.event; version\="0.0.0.JavaSE_017", com.sun.jdi.connect.spi; version\="0.0.0.JavaSE_017", com.sun.jdi; version\="0.0.0.JavaSE_017", com.sun.jdi.request; version\="0.0.0.JavaSE_017", jdk.jfr.consumer; version\="0.0.0.JavaSE_017", jdk.jfr; version\="0.0.0.JavaSE_017", jdk.jshell; version\="0.0.0.JavaSE_017", jdk.jshell.execution; version\="0.0.0.JavaSE_017", jdk.jshell.spi; version\="0.0.0.JavaSE_017", jdk.jshell.tool; version\="0.0.0.JavaSE_017", netscape.javascript; version\="0.0.0.JavaSE_017", com.sun.management; version\="0.0.0.JavaSE_017", jdk.management.jfr; version\="0.0.0.JavaSE_017", jdk.net; version\="0.0.0.JavaSE_017", jdk.nio; version\="0.0.0.JavaSE_017", jdk.nio.mapmode; version\="0.0.0.JavaSE_017", com.sun.nio.sctp; version\="0.0.0.JavaSE_017", com.sun.security.auth.module; version\="0.0.0.JavaSE_017", com.sun.security.auth.login; version\="0.0.0.JavaSE_017", com.sun.security.auth; version\="0.0.0.JavaSE_017", com.sun.security.auth.callback; version\="0.0.0.JavaSE_017", com.sun.security.jgss; version\="0.0.0.JavaSE_017", sun.reflect; version\="0.0.0.JavaSE_017", sun.misc; version\="0.0.0.JavaSE_017", com.sun.nio.file; version\="0.0.0.JavaSE_017", jdk.swing.interop; version\="0.0.0.JavaSE_017", org.w3c.dom.stylesheets; version\="0.0.0.JavaSE_017", org.w3c.dom.html; version\="0.0.0.JavaSE_017", org.w3c.dom.xpath; version\="0.0.0.JavaSE_017", org.w3c.dom.css; version\="0.0.0.JavaSE_017", com.yahoo.jdisc, com.yahoo.jdisc.application, com.yahoo.jdisc.handler, com.yahoo.jdisc.service, com.yahoo.jdisc.statistics, javax.inject;version\=1.0.0, org.aopalliance.intercept, org.aopalliance.aop, com.google.common.annotations;version\="27.1.0",com.google.common.base;version\="27.1.0",com.google.common.cache;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent",com.google.common.collect;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.escape;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.eventbus;version\="27.1.0",com.google.common.graph;version\="27.1.0";uses\:\="com.google.common.collect",com.google.common.hash;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.html;version\="27.1.0";uses\:\="com.google.common.escape",com.google.common.io;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.graph,com.google.common.hash",com.google.common.math;version\="27.1.0",com.google.common.net;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.escape",com.google.common.primitives;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.reflect;version\="27.1.0";uses\:\="com.google.common.collect,com.google.common.io",com.google.common.util.concurrent;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent.internal",com.google.common.xml;version\="27.1.0";uses\:\="com.google.common.escape", com.google.inject;version\="1.4",com.google.inject.binder;version\="1.4",com.google.inject.matcher;version\="1.4",com.google.inject.multibindings;version\="1.4",com.google.inject.name;version\="1.4",com.google.inject.spi;version\="1.4",com.google.inject.util;version\="1.4", org.slf4j;version\=1.7.32, org.slf4j.spi;version\=1.7.32, org.slf4j.helpers;version\=1.7.32, org.slf4j.event;version\=1.7.32, org.slf4j.impl;version\=1.7.32, org.apache.commons.logging;version\=1.2, org.apache.commons.logging.impl;version\=1.2, org.apache.log4j;version\=1.2.17,org.apache.log4j.helpers;version\=1.2.17,org.apache.log4j.spi;version\=1.2.17,org.apache.log4j.xml;version\=1.2.17, com.yahoo.component.annotation;version\="1.0.0", com.yahoo.config;version\=1.0.0, com.yahoo.vespa.defaults;version\=1.0.0, ai.vespa.http;version\=1.0.0,ai.vespa.validation;version\=1.0.0,com.yahoo.binaryprefix;version\=1.0.0,com.yahoo.collections;version\=1.0.0,com.yahoo.compress;version\=1.0.0,com.yahoo.concurrent.classlock;version\=1.0.0,com.yahoo.concurrent.maintenance;version\=1.0.0,com.yahoo.concurrent;version\=1.0.0,com.yahoo.data.access.simple;version\=1.0.0,com.yahoo.data.access.slime;version\=1.0.0,com.yahoo.data.access;version\=1.0.0,com.yahoo.errorhandling;version\=1.0.0,com.yahoo.exception;version\=1.0.0,com.yahoo.geo;version\=1.0.0,com.yahoo.io.reader;version\=1.0.0,com.yahoo.io;version\=1.0.0,com.yahoo.javacc;version\=1.0.0,com.yahoo.lang;version\=1.0.0,com.yahoo.nativec;version\=1.0.0,com.yahoo.net;version\=1.0.0,com.yahoo.path;version\=1.0.0,com.yahoo.protect;version\=1.0.0,com.yahoo.reflection;version\=1.0.0,com.yahoo.slime;version\=1.0.0,com.yahoo.stream;version\=1.0.0,com.yahoo.system.execution;version\=1.0.0,com.yahoo.system;version\=1.0.0,com.yahoo.tensor.evaluation;version\=1.0.0,com.yahoo.tensor.functions;version\=1.0.0,com.yahoo.tensor.serialization;version\=1.0.0,com.yahoo.tensor;version\=1.0.0,com.yahoo.text.internal;version\=1.0.0,com.yahoo.text;version\=1.0.0,com.yahoo.time;version\=1.0.0,com.yahoo.transaction;version\=1.0.0,com.yahoo.vespa.objects;version\=1.0.0,com.yahoo.yolean.chain;version\=1.0.0,com.yahoo.yolean.concurrent;version\=1.0.0,com.yahoo.yolean.function;version\=1.0.0,com.yahoo.yolean.system;version\=1.0.0,com.yahoo.yolean.trace;version\=1.0.0,com.yahoo.yolean;version\=1.0.0, com.yahoo.log.event;version\=1.0.0,com.yahoo.log.impl;version\=1.0.0,com.yahoo.log;version\=1.0.0, javax.xml.bind;version\="2.3";uses\:\="javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.annotation;version\="2.3";uses\:\="javax.xml.bind,javax.xml.parsers,javax.xml.transform,javax.xml.transform.dom,org.w3c.dom",javax.xml.bind.annotation.adapters;version\="2.3",javax.xml.bind.attachment;version\="2.3";uses\:\="javax.activation",javax.xml.bind.helpers;version\="2.3";uses\:\="javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.util;version\="2.3";uses\:\="javax.xml.bind,javax.xml.transform.sax", com.sun.istack;version\="3.0.5";uses\:\="javax.activation,javax.xml.stream,org.xml.sax,org.xml.sax.helpers",com.sun.istack.localization;version\="3.0.5",com.sun.istack.logging;version\="3.0.5",com.sun.xml.bind;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.annotation;version\="2.3.0",com.sun.xml.bind.api;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.api.impl;version\="2.3.0",com.sun.xml.bind.marshaller;uses\:\="javax.xml.parsers,org.w3c.dom,org.xml.sax,org.xml.sax.helpers";version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;version\="2.3.0",com.sun.xml.bind.v2;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.core;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.impl,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.namespace,javax.xml.transform";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.nav";version\="2.3.0",com.sun.xml.bind.v2.model.nav;uses\:\="com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.util;uses\:\="javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.xml.bind.v2.model.annotation,javax.activation,javax.xml.bind,javax.xml.bind.annotation.adapters";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen.episode;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="javax.xml.parsers,javax.xml.transform,javax.xml.validation,javax.xml.xpath";version\="2.3.0",com.sun.xml.txw2;uses\:\="com.sun.xml.txw2.output,javax.xml.namespace";version\="2.3.0",com.sun.xml.txw2.annotation;version\="2.3.0",com.sun.xml.txw2.output;uses\:\="com.sun.xml.txw2,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.dom,javax.xml.transform.sax,javax.xml.transform.stream,org.w3c.dom,org.xml.sax,org.xml.sax.ext,org.xml.sax.helpers";version\="2.3.0", com.sun.xml.bind;uses\:\="com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.datatype,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.api;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.xml.bind,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.marshaller;version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;uses\:\="com.sun.xml.bind,javax.xml.bind.helpers,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,javax.xml.bind";version\="2.3.0",com.sun.xml.bind.v2.bytecode;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.model.runtime;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.namespace,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.istack,com.sun.xml.bind.api,com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.property,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.output;uses\:\="com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.runtime,com.sun.xml.fastinfoset.stax,javax.xml.stream,org.jvnet.staxex,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.property;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,com.sun.xml.bind.v2.runtime.unmarshaller,com.sun.xml.bind.v2.util,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect.opt;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="com.sun.xml.bind,com.sun.xml.bind.api,com.sun.xml.bind.unmarshaller,com.sun.xml.bind.util,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.reflect,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.txw2.output,javax.xml.bind,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.schemagen.xmlschema;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.namespace,javax.xml.transform.stream,org.xml.sax";version\="2.3.0", javax.activation;uses\:\="com.sun.activation.registries";version\="1.2",com.sun.activation.viewers;uses\:\="javax.activation";version\="1.2.0",com.sun.activation.registries;version\="1.2.0"
+exportPackages=org.osgi.framework; version\="1.10.0", org.osgi.framework.connect; version\="1.0.0", org.osgi.framework.dto; uses\:\="org.osgi.dto"; version\="1.8.0", org.osgi.framework.hooks.bundle; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.resolver; uses\:\="org.osgi.framework.wiring"; version\="1.0.0", org.osgi.framework.hooks.service; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.weaving; uses\:\="org.osgi.framework.wiring"; version\="1.1.0", org.osgi.framework.launch; uses\:\="org.osgi.framework"; version\="1.2.0", org.osgi.framework.namespace; uses\:\="org.osgi.resource"; version\="1.2.0", org.osgi.framework.startlevel; uses\:\="org.osgi.framework"; version\="1.0.0", org.osgi.framework.startlevel.dto; uses\:\="org.osgi.dto"; version\="1.0.0", org.osgi.framework.wiring; uses\:\="org.osgi.framework,org.osgi.resource"; version\="1.2.0", org.osgi.framework.wiring.dto; uses\:\="org.osgi.dto,org.osgi.resource.dto"; version\="1.3.0", org.osgi.resource; version\="1.0.1", org.osgi.resource.dto; uses\:\="org.osgi.dto"; version\="1.0.1", org.osgi.service.packageadmin; uses\:\="org.osgi.framework"; version\="1.2.1", org.osgi.service.startlevel; uses\:\="org.osgi.framework"; version\="1.1.1", org.osgi.service.url; version\="1.0.1", org.osgi.service.resolver; uses\:\="org.osgi.resource"; version\="1.1.1", org.osgi.util.tracker; uses\:\="org.osgi.framework"; version\="1.5.3", org.osgi.dto; version\="1.1.1", org.osgi.service.condition; version\="1.0.0", javax.security.auth.callback; version\="0.0.0.JavaSE_017", javax.security.auth.login; version\="0.0.0.JavaSE_017", javax.net.ssl; version\="0.0.0.JavaSE_017", java.lang.constant; version\="0.0.0.JavaSE_017", java.security.interfaces; version\="0.0.0.JavaSE_017", java.text.spi; version\="0.0.0.JavaSE_017", java.nio.channels.spi; version\="0.0.0.JavaSE_017", java.math; version\="0.0.0.JavaSE_017", java.nio.file; version\="0.0.0.JavaSE_017", java.util.concurrent.atomic; version\="0.0.0.JavaSE_017", java.security.cert; version\="0.0.0.JavaSE_017", java.security.spec; version\="0.0.0.JavaSE_017", java.nio.channels; version\="0.0.0.JavaSE_017", java.time.chrono; version\="0.0.0.JavaSE_017", javax.crypto; version\="0.0.0.JavaSE_017", java.time.zone; version\="0.0.0.JavaSE_017", java.nio.charset; version\="0.0.0.JavaSE_017", java.io; version\="0.0.0.JavaSE_017", java.util.spi; version\="0.0.0.JavaSE_017", java.net; version\="0.0.0.JavaSE_017", javax.security.cert; version\="0.0.0.JavaSE_017", java.lang.annotation; version\="0.0.0.JavaSE_017", javax.security.auth.spi; version\="0.0.0.JavaSE_017", java.util.concurrent; version\="0.0.0.JavaSE_017", java.nio.charset.spi; version\="0.0.0.JavaSE_017", javax.crypto.interfaces; version\="0.0.0.JavaSE_017", java.util; version\="0.0.0.JavaSE_017", java.security; version\="0.0.0.JavaSE_017", java.nio.file.spi; version\="0.0.0.JavaSE_017", java.nio; version\="0.0.0.JavaSE_017", java.util.jar; version\="0.0.0.JavaSE_017", javax.security.auth; version\="0.0.0.JavaSE_017", java.lang.ref; version\="0.0.0.JavaSE_017", java.util.regex; version\="0.0.0.JavaSE_017", java.net.spi; version\="0.0.0.JavaSE_017", java.lang.module; version\="0.0.0.JavaSE_017", java.lang.invoke; version\="0.0.0.JavaSE_017", java.time.format; version\="0.0.0.JavaSE_017", java.util.concurrent.locks; version\="0.0.0.JavaSE_017", java.time.temporal; version\="0.0.0.JavaSE_017", java.util.zip; version\="0.0.0.JavaSE_017", java.nio.file.attribute; version\="0.0.0.JavaSE_017", java.util.random; version\="0.0.0.JavaSE_017", java.text; version\="0.0.0.JavaSE_017", javax.crypto.spec; version\="0.0.0.JavaSE_017", java.util.stream; version\="0.0.0.JavaSE_017", java.time; version\="0.0.0.JavaSE_017", java.lang; version\="0.0.0.JavaSE_017", java.lang.runtime; version\="0.0.0.JavaSE_017", java.util.function; version\="0.0.0.JavaSE_017", javax.net; version\="0.0.0.JavaSE_017", javax.security.auth.x500; version\="0.0.0.JavaSE_017", java.lang.reflect; version\="0.0.0.JavaSE_017", javax.lang.model; version\="0.0.0.JavaSE_017", javax.annotation.processing; version\="0.0.0.JavaSE_017", javax.lang.model.element; version\="0.0.0.JavaSE_017", javax.tools; version\="0.0.0.JavaSE_017", javax.lang.model.type; version\="0.0.0.JavaSE_017", javax.lang.model.util; version\="0.0.0.JavaSE_017", java.awt.datatransfer; version\="0.0.0.JavaSE_017", java.awt.desktop; version\="0.0.0.JavaSE_017", javax.swing.plaf.synth; version\="0.0.0.JavaSE_017", java.beans; version\="0.0.0.JavaSE_017", javax.swing.text.html.parser; version\="0.0.0.JavaSE_017", javax.swing.text.rtf; version\="0.0.0.JavaSE_017", java.awt.font; version\="0.0.0.JavaSE_017", javax.imageio; version\="0.0.0.JavaSE_017", java.awt.im.spi; version\="0.0.0.JavaSE_017", java.applet; version\="0.0.0.JavaSE_017", javax.sound.midi; version\="0.0.0.JavaSE_017", java.awt.dnd; version\="0.0.0.JavaSE_017", javax.swing.text; version\="0.0.0.JavaSE_017", javax.swing.plaf.basic; version\="0.0.0.JavaSE_017", javax.swing.undo; version\="0.0.0.JavaSE_017", javax.swing.plaf; version\="0.0.0.JavaSE_017", javax.swing.filechooser; version\="0.0.0.JavaSE_017", javax.imageio.event; version\="0.0.0.JavaSE_017", javax.sound.sampled; version\="0.0.0.JavaSE_017", javax.print.attribute; version\="0.0.0.JavaSE_017", javax.print; version\="0.0.0.JavaSE_017", javax.swing.plaf.nimbus; version\="0.0.0.JavaSE_017", javax.accessibility; version\="0.0.0.JavaSE_017", java.awt.event; version\="0.0.0.JavaSE_017", javax.swing.text.html; version\="0.0.0.JavaSE_017", javax.imageio.spi; version\="0.0.0.JavaSE_017", javax.swing.border; version\="0.0.0.JavaSE_017", javax.sound.sampled.spi; version\="0.0.0.JavaSE_017", javax.imageio.plugins.bmp; version\="0.0.0.JavaSE_017", java.awt.color; version\="0.0.0.JavaSE_017", java.awt.geom; version\="0.0.0.JavaSE_017", javax.imageio.plugins.jpeg; version\="0.0.0.JavaSE_017", javax.swing.tree; version\="0.0.0.JavaSE_017", javax.imageio.plugins.tiff; version\="0.0.0.JavaSE_017", java.awt.print; version\="0.0.0.JavaSE_017", java.awt.image; version\="0.0.0.JavaSE_017", javax.imageio.metadata; version\="0.0.0.JavaSE_017", javax.swing.table; version\="0.0.0.JavaSE_017", javax.sound.midi.spi; version\="0.0.0.JavaSE_017", javax.print.attribute.standard; version\="0.0.0.JavaSE_017", javax.swing.colorchooser; version\="0.0.0.JavaSE_017", javax.swing; version\="0.0.0.JavaSE_017", java.awt.image.renderable; version\="0.0.0.JavaSE_017", javax.swing.plaf.multi; version\="0.0.0.JavaSE_017", java.awt.im; version\="0.0.0.JavaSE_017", javax.print.event; version\="0.0.0.JavaSE_017", javax.swing.plaf.metal; version\="0.0.0.JavaSE_017", java.beans.beancontext; version\="0.0.0.JavaSE_017", java.awt; version\="0.0.0.JavaSE_017", javax.imageio.stream; version\="0.0.0.JavaSE_017", javax.swing.event; version\="0.0.0.JavaSE_017", java.lang.instrument; version\="0.0.0.JavaSE_017", java.util.logging; version\="0.0.0.JavaSE_017", javax.management.timer; version\="0.0.0.JavaSE_017", javax.management; version\="0.0.0.JavaSE_017", javax.management.relation; version\="0.0.0.JavaSE_017", javax.management.loading; version\="0.0.0.JavaSE_017", javax.management.openmbean; version\="0.0.0.JavaSE_017", java.lang.management; version\="0.0.0.JavaSE_017", javax.management.remote; version\="0.0.0.JavaSE_017", javax.management.monitor; version\="0.0.0.JavaSE_017", javax.management.modelmbean; version\="0.0.0.JavaSE_017", javax.management.remote.rmi; version\="0.0.0.JavaSE_017", javax.naming.event; version\="0.0.0.JavaSE_017", javax.naming.ldap.spi; version\="0.0.0.JavaSE_017", javax.naming; version\="0.0.0.JavaSE_017", javax.naming.spi; version\="0.0.0.JavaSE_017", javax.naming.ldap; version\="0.0.0.JavaSE_017", javax.naming.directory; version\="0.0.0.JavaSE_017", java.net.http; version\="0.0.0.JavaSE_017", java.util.prefs; version\="0.0.0.JavaSE_017", java.rmi.registry; version\="0.0.0.JavaSE_017", javax.rmi.ssl; version\="0.0.0.JavaSE_017", java.rmi.dgc; version\="0.0.0.JavaSE_017", java.rmi; version\="0.0.0.JavaSE_017", java.rmi.server; version\="0.0.0.JavaSE_017", javax.script; version\="0.0.0.JavaSE_017", org.ietf.jgss; version\="0.0.0.JavaSE_017", javax.security.auth.kerberos; version\="0.0.0.JavaSE_017", javax.security.sasl; version\="0.0.0.JavaSE_017", javax.smartcardio; version\="0.0.0.JavaSE_017", java.sql; version\="0.0.0.JavaSE_017", javax.sql; version\="0.0.0.JavaSE_017", javax.sql.rowset.spi; version\="0.0.0.JavaSE_017", javax.sql.rowset.serial; version\="0.0.0.JavaSE_017", javax.sql.rowset; version\="0.0.0.JavaSE_017", javax.transaction.xa; version\="0.0.0.JavaSE_017", javax.xml; version\="0.0.0.JavaSE_017", javax.xml.transform.sax; version\="0.0.0.JavaSE_017", javax.xml.datatype; version\="0.0.0.JavaSE_017", javax.xml.catalog; version\="0.0.0.JavaSE_017", org.w3c.dom.traversal; version\="0.0.0.JavaSE_017", javax.xml.stream.events; version\="0.0.0.JavaSE_017", javax.xml.stream; version\="0.0.0.JavaSE_017", org.xml.sax; version\="0.0.0.JavaSE_017", javax.xml.transform; version\="0.0.0.JavaSE_017", javax.xml.xpath; version\="0.0.0.JavaSE_017", org.w3c.dom.events; version\="0.0.0.JavaSE_017", org.w3c.dom.ranges; version\="0.0.0.JavaSE_017", org.w3c.dom.ls; version\="0.0.0.JavaSE_017", javax.xml.stream.util; version\="0.0.0.JavaSE_017", javax.xml.namespace; version\="0.0.0.JavaSE_017", javax.xml.transform.stax; version\="0.0.0.JavaSE_017", org.xml.sax.helpers; version\="0.0.0.JavaSE_017", org.w3c.dom.views; version\="0.0.0.JavaSE_017", org.w3c.dom.bootstrap; version\="0.0.0.JavaSE_017", org.w3c.dom; version\="0.0.0.JavaSE_017", javax.xml.transform.stream; version\="0.0.0.JavaSE_017", javax.xml.transform.dom; version\="0.0.0.JavaSE_017", javax.xml.validation; version\="0.0.0.JavaSE_017", javax.xml.parsers; version\="0.0.0.JavaSE_017", org.xml.sax.ext; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.spec; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.keyinfo; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig; version\="0.0.0.JavaSE_017", com.sun.java.accessibility.util; version\="0.0.0.JavaSE_017", com.sun.tools.attach.spi; version\="0.0.0.JavaSE_017", com.sun.tools.attach; version\="0.0.0.JavaSE_017", com.sun.source.doctree; version\="0.0.0.JavaSE_017", com.sun.source.tree; version\="0.0.0.JavaSE_017", com.sun.source.util; version\="0.0.0.JavaSE_017", com.sun.tools.javac; version\="0.0.0.JavaSE_017", jdk.dynalink.beans; version\="0.0.0.JavaSE_017", jdk.dynalink.linker.support; version\="0.0.0.JavaSE_017", jdk.dynalink.support; version\="0.0.0.JavaSE_017", jdk.dynalink; version\="0.0.0.JavaSE_017", jdk.dynalink.linker; version\="0.0.0.JavaSE_017", com.sun.net.httpserver; version\="0.0.0.JavaSE_017", com.sun.net.httpserver.spi; version\="0.0.0.JavaSE_017", com.sun.jarsigner; version\="0.0.0.JavaSE_017", jdk.security.jarsigner; version\="0.0.0.JavaSE_017", jdk.javadoc.doclet; version\="0.0.0.JavaSE_017", com.sun.tools.jconsole; version\="0.0.0.JavaSE_017", com.sun.jdi.connect; version\="0.0.0.JavaSE_017", com.sun.jdi.event; version\="0.0.0.JavaSE_017", com.sun.jdi.connect.spi; version\="0.0.0.JavaSE_017", com.sun.jdi; version\="0.0.0.JavaSE_017", com.sun.jdi.request; version\="0.0.0.JavaSE_017", jdk.jfr.consumer; version\="0.0.0.JavaSE_017", jdk.jfr; version\="0.0.0.JavaSE_017", jdk.jshell; version\="0.0.0.JavaSE_017", jdk.jshell.execution; version\="0.0.0.JavaSE_017", jdk.jshell.spi; version\="0.0.0.JavaSE_017", jdk.jshell.tool; version\="0.0.0.JavaSE_017", netscape.javascript; version\="0.0.0.JavaSE_017", com.sun.management; version\="0.0.0.JavaSE_017", jdk.management.jfr; version\="0.0.0.JavaSE_017", jdk.net; version\="0.0.0.JavaSE_017", jdk.nio; version\="0.0.0.JavaSE_017", jdk.nio.mapmode; version\="0.0.0.JavaSE_017", com.sun.nio.sctp; version\="0.0.0.JavaSE_017", com.sun.security.auth.module; version\="0.0.0.JavaSE_017", com.sun.security.auth.login; version\="0.0.0.JavaSE_017", com.sun.security.auth; version\="0.0.0.JavaSE_017", com.sun.security.auth.callback; version\="0.0.0.JavaSE_017", com.sun.security.jgss; version\="0.0.0.JavaSE_017", sun.reflect; version\="0.0.0.JavaSE_017", sun.misc; version\="0.0.0.JavaSE_017", com.sun.nio.file; version\="0.0.0.JavaSE_017", jdk.swing.interop; version\="0.0.0.JavaSE_017", org.w3c.dom.stylesheets; version\="0.0.0.JavaSE_017", org.w3c.dom.html; version\="0.0.0.JavaSE_017", org.w3c.dom.xpath; version\="0.0.0.JavaSE_017", org.w3c.dom.css; version\="0.0.0.JavaSE_017", com.yahoo.jdisc, com.yahoo.jdisc.application, com.yahoo.jdisc.handler, com.yahoo.jdisc.service, com.yahoo.jdisc.statistics, com.yahoo.jdisc.refcount, javax.inject;version\=1.0.0, org.aopalliance.intercept, org.aopalliance.aop, com.google.common.annotations;version\="27.1.0",com.google.common.base;version\="27.1.0",com.google.common.cache;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent",com.google.common.collect;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.escape;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.eventbus;version\="27.1.0",com.google.common.graph;version\="27.1.0";uses\:\="com.google.common.collect",com.google.common.hash;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.html;version\="27.1.0";uses\:\="com.google.common.escape",com.google.common.io;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.graph,com.google.common.hash",com.google.common.math;version\="27.1.0",com.google.common.net;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.escape",com.google.common.primitives;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.reflect;version\="27.1.0";uses\:\="com.google.common.collect,com.google.common.io",com.google.common.util.concurrent;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent.internal",com.google.common.xml;version\="27.1.0";uses\:\="com.google.common.escape", com.google.inject;version\="1.4",com.google.inject.binder;version\="1.4",com.google.inject.matcher;version\="1.4",com.google.inject.multibindings;version\="1.4",com.google.inject.name;version\="1.4",com.google.inject.spi;version\="1.4",com.google.inject.util;version\="1.4", org.slf4j;version\=1.7.32, org.slf4j.spi;version\=1.7.32, org.slf4j.helpers;version\=1.7.32, org.slf4j.event;version\=1.7.32, org.slf4j.impl;version\=1.7.32, org.apache.commons.logging;version\=1.2, org.apache.commons.logging.impl;version\=1.2, org.apache.log4j;version\=1.2.17,org.apache.log4j.helpers;version\=1.2.17,org.apache.log4j.spi;version\=1.2.17,org.apache.log4j.xml;version\=1.2.17, com.yahoo.component.annotation;version\="1.0.0", com.yahoo.config;version\=1.0.0, com.yahoo.vespa.defaults;version\=1.0.0, ai.vespa.http;version\=1.0.0,ai.vespa.validation;version\=1.0.0,com.yahoo.binaryprefix;version\=1.0.0,com.yahoo.collections;version\=1.0.0,com.yahoo.compress;version\=1.0.0,com.yahoo.concurrent.classlock;version\=1.0.0,com.yahoo.concurrent.maintenance;version\=1.0.0,com.yahoo.concurrent;version\=1.0.0,com.yahoo.data.access.simple;version\=1.0.0,com.yahoo.data.access.slime;version\=1.0.0,com.yahoo.data.access;version\=1.0.0,com.yahoo.errorhandling;version\=1.0.0,com.yahoo.exception;version\=1.0.0,com.yahoo.geo;version\=1.0.0,com.yahoo.io.reader;version\=1.0.0,com.yahoo.io;version\=1.0.0,com.yahoo.javacc;version\=1.0.0,com.yahoo.lang;version\=1.0.0,com.yahoo.nativec;version\=1.0.0,com.yahoo.net;version\=1.0.0,com.yahoo.path;version\=1.0.0,com.yahoo.protect;version\=1.0.0,com.yahoo.reflection;version\=1.0.0,com.yahoo.slime;version\=1.0.0,com.yahoo.stream;version\=1.0.0,com.yahoo.system.execution;version\=1.0.0,com.yahoo.system;version\=1.0.0,com.yahoo.tensor.evaluation;version\=1.0.0,com.yahoo.tensor.functions;version\=1.0.0,com.yahoo.tensor.serialization;version\=1.0.0,com.yahoo.tensor;version\=1.0.0,com.yahoo.text.internal;version\=1.0.0,com.yahoo.text;version\=1.0.0,com.yahoo.time;version\=1.0.0,com.yahoo.transaction;version\=1.0.0,com.yahoo.vespa.objects;version\=1.0.0,com.yahoo.yolean.chain;version\=1.0.0,com.yahoo.yolean.concurrent;version\=1.0.0,com.yahoo.yolean.function;version\=1.0.0,com.yahoo.yolean.system;version\=1.0.0,com.yahoo.yolean.trace;version\=1.0.0,com.yahoo.yolean;version\=1.0.0, com.yahoo.log.event;version\=1.0.0,com.yahoo.log.impl;version\=1.0.0,com.yahoo.log;version\=1.0.0, javax.xml.bind;version\="2.3";uses\:\="javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.annotation;version\="2.3";uses\:\="javax.xml.bind,javax.xml.parsers,javax.xml.transform,javax.xml.transform.dom,org.w3c.dom",javax.xml.bind.annotation.adapters;version\="2.3",javax.xml.bind.attachment;version\="2.3";uses\:\="javax.activation",javax.xml.bind.helpers;version\="2.3";uses\:\="javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.util;version\="2.3";uses\:\="javax.xml.bind,javax.xml.transform.sax", com.sun.istack;version\="3.0.5";uses\:\="javax.activation,javax.xml.stream,org.xml.sax,org.xml.sax.helpers",com.sun.istack.localization;version\="3.0.5",com.sun.istack.logging;version\="3.0.5",com.sun.xml.bind;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.annotation;version\="2.3.0",com.sun.xml.bind.api;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.api.impl;version\="2.3.0",com.sun.xml.bind.marshaller;uses\:\="javax.xml.parsers,org.w3c.dom,org.xml.sax,org.xml.sax.helpers";version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;version\="2.3.0",com.sun.xml.bind.v2;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.core;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.impl,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.namespace,javax.xml.transform";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.nav";version\="2.3.0",com.sun.xml.bind.v2.model.nav;uses\:\="com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.util;uses\:\="javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.xml.bind.v2.model.annotation,javax.activation,javax.xml.bind,javax.xml.bind.annotation.adapters";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen.episode;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="javax.xml.parsers,javax.xml.transform,javax.xml.validation,javax.xml.xpath";version\="2.3.0",com.sun.xml.txw2;uses\:\="com.sun.xml.txw2.output,javax.xml.namespace";version\="2.3.0",com.sun.xml.txw2.annotation;version\="2.3.0",com.sun.xml.txw2.output;uses\:\="com.sun.xml.txw2,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.dom,javax.xml.transform.sax,javax.xml.transform.stream,org.w3c.dom,org.xml.sax,org.xml.sax.ext,org.xml.sax.helpers";version\="2.3.0", com.sun.xml.bind;uses\:\="com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.datatype,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.api;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.xml.bind,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.marshaller;version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;uses\:\="com.sun.xml.bind,javax.xml.bind.helpers,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,javax.xml.bind";version\="2.3.0",com.sun.xml.bind.v2.bytecode;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.model.runtime;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.namespace,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.istack,com.sun.xml.bind.api,com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.property,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.output;uses\:\="com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.runtime,com.sun.xml.fastinfoset.stax,javax.xml.stream,org.jvnet.staxex,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.property;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,com.sun.xml.bind.v2.runtime.unmarshaller,com.sun.xml.bind.v2.util,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect.opt;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="com.sun.xml.bind,com.sun.xml.bind.api,com.sun.xml.bind.unmarshaller,com.sun.xml.bind.util,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.reflect,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.txw2.output,javax.xml.bind,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.schemagen.xmlschema;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.namespace,javax.xml.transform.stream,org.xml.sax";version\="2.3.0", javax.activation;uses\:\="com.sun.activation.registries";version\="1.2",com.sun.activation.viewers;uses\:\="javax.activation";version\="1.2.0",com.sun.activation.registries;version\="1.2.0"
diff --git a/metrics/src/tests/metricmanagertest.cpp b/metrics/src/tests/metricmanagertest.cpp
index 98d03514de0..6d6f21ea7b0 100644
--- a/metrics/src/tests/metricmanagertest.cpp
+++ b/metrics/src/tests/metricmanagertest.cpp
@@ -5,12 +5,10 @@
#include <vespa/metrics/metricmanager.h>
#include <vespa/metrics/state_api_adapter.h>
#include <vespa/metrics/textwriter.h>
-#include <vespa/metrics/xmlwriter.h>
#include <vespa/vespalib/data/slime/slime.h>
#include <vespa/vespalib/gtest/gtest.h>
#include <vespa/vespalib/stllike/asciistream.h>
#include <vespa/vespalib/util/size_literals.h>
-#include <vespa/vespalib/util/xmlstream.h>
#include <vespa/vespalib/util/time.h>
#include <vespa/vespalib/data/simple_buffer.h>
#include <vespa/vespalib/util/atomic.h>
@@ -57,7 +55,7 @@ SubMetricSet::SubMetricSet(const Metric::String & name, MetricSet* owner)
valsum.addMetricToSum(val1);
valsum.addMetricToSum(val2);
}
-SubMetricSet::~SubMetricSet() { }
+SubMetricSet::~SubMetricSet() = default;
struct MultiSubMetricSet
{
@@ -167,20 +165,15 @@ getMatchedMetrics(const vespalib::string& config)
MetricLockGuard g(mm.getMetricLock());
mm.visit(g, mm.getActiveMetrics(g), visitor, "consumer");
- MetricManager::ConsumerSpec::SP consumerSpec(
- mm.getConsumerSpec(g, "consumer"));
- return std::pair<std::string, std::string>(
- visitor.toString(),
- consumerSpec.get() ? consumerSpec->toString()
- : "Non-existing consumer");
+ MetricManager::ConsumerSpec::SP consumerSpec(mm.getConsumerSpec(g, "consumer"));
+ return { visitor.toString(), consumerSpec ? consumerSpec->toString() : "Non-existing consumer" };
}
}
#define ASSERT_CONSUMER_MATCH(name, expected, config) \
{ \
- std::pair<std::string, std::string> consumerMatch( \
- getMatchedMetrics(config)); \
+ std::pair<std::string, std::string> consumerMatch(getMatchedMetrics(config)); \
EXPECT_EQ("\n" + expected, "\n" + consumerMatch.first) << (name + std::string(": ") + consumerMatch.second); \
}
@@ -371,10 +364,10 @@ class FakeTimer : public MetricManager::Timer {
std::atomic<time_t> _time;
public:
FakeTimer(time_t startTime = 0) : _time(startTime) {}
- time_t getTime() const override { return load_relaxed(_time); }
+ time_point getTime() const override { return time_point(vespalib::from_s(load_relaxed(_time))); }
void set_time(time_t t) noexcept { store_relaxed(_time, t); }
// Not safe for multiple writers, only expected to be called by test.
- void add_time(time_t t) noexcept { set_time(getTime() + t); }
+ void add_time(time_t t) noexcept { set_time(load_relaxed(_time) + t); }
};
struct BriefValuePrinter : public MetricVisitor {
@@ -391,8 +384,7 @@ struct BriefValuePrinter : public MetricVisitor {
}
};
-bool waitForTimeProcessed(const MetricManager& mm,
- time_t processtime, uint32_t timeout = 120)
+bool waitForTimeProcessed(const MetricManager& mm, time_t processtime, uint32_t timeout = 120)
{
uint32_t lastchance = time(0) + timeout;
while (time(0) < lastchance) {
@@ -403,23 +395,20 @@ bool waitForTimeProcessed(const MetricManager& mm,
return false;
}
-std::string dumpAllSnapshots(const MetricManager& mm,
- const std::string& consumer)
+std::string dumpAllSnapshots(const MetricManager& mm, const std::string& consumer)
{
std::ostringstream ost;
ost << "\n";
{
MetricLockGuard metricLock(mm.getMetricLock());
BriefValuePrinter briefValuePrinter;
- mm.visit(metricLock, mm.getActiveMetrics(metricLock),
- briefValuePrinter, consumer);
+ mm.visit(metricLock, mm.getActiveMetrics(metricLock), briefValuePrinter, consumer);
ost << "Current: " << briefValuePrinter.ost.str() << "\n";
}
{
MetricLockGuard metricLock(mm.getMetricLock());
BriefValuePrinter briefValuePrinter;
- mm.visit(metricLock, mm.getTotalMetricSnapshot(metricLock),
- briefValuePrinter, consumer);
+ mm.visit(metricLock, mm.getTotalMetricSnapshot(metricLock), briefValuePrinter, consumer);
ost << "Total: " << briefValuePrinter.ost.str() << "\n";
}
std::vector<uint32_t> periods;
@@ -429,17 +418,15 @@ std::string dumpAllSnapshots(const MetricManager& mm,
}
for (uint32_t i=0; i<periods.size(); ++i) {
MetricLockGuard metricLock(mm.getMetricLock());
- const MetricSnapshotSet& set(mm.getMetricSnapshotSet(
- metricLock, periods[i]));
+ const MetricSnapshotSet& set(mm.getMetricSnapshotSet(metricLock, periods[i]));
ost << set.getName() << "\n";
- uint32_t count = 0;
- for (uint32_t j=0; j<2; ++j) {
+ for (uint32_t count=0,j=0; j<2; ++j) {
if (set.getCount() == 1 && j == 1) continue;
const MetricSnapshot& snap(set.getSnapshot(j == 1));
BriefValuePrinter briefValuePrinter;
mm.visit(metricLock, snap, briefValuePrinter, consumer);
ost << " " << count++ << " " << &snap.getMetrics() << ": "
- << briefValuePrinter.ost.str() << "\n";
+ << briefValuePrinter.ost.str() << "\n";
}
}
return ost.str();
@@ -452,14 +439,11 @@ std::string dumpAllSnapshots(const MetricManager& mm,
MetricLockGuard lockGuard(mm.getMetricLock()); \
BriefValuePrinter briefValuePrinter; \
if (period == -1) { \
- mm.visit(lockGuard, mm.getActiveMetrics(lockGuard), \
- briefValuePrinter, "snapper"); \
+ mm.visit(lockGuard, mm.getActiveMetrics(lockGuard), briefValuePrinter, "snapper"); \
} else if (period == 0) { \
- mm.visit(lockGuard, mm.getTotalMetricSnapshot(lockGuard), \
- briefValuePrinter, "snapper"); \
+ mm.visit(lockGuard, mm.getTotalMetricSnapshot(lockGuard), briefValuePrinter, "snapper"); \
} else { \
- mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, period), \
- briefValuePrinter, "snapper"); \
+ mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, period), briefValuePrinter, "snapper"); \
} \
EXPECT_EQ(std::string(expected), briefValuePrinter.ost.str()) << dumpAllSnapshots(mm, "snapper"); \
}
@@ -474,8 +458,8 @@ std::string dumpAllSnapshots(const MetricManager& mm,
TEST_F(MetricManagerTest, test_snapshots)
{
- FakeTimer* timer = new FakeTimer(1000);
- std::unique_ptr<MetricManager::Timer> timerImpl(timer);
+ auto timerImpl = std::make_unique<FakeTimer>(1000);
+ FakeTimer & timer = *timerImpl;
TestMetricSet mySet;
MetricManager mm(std::move(timerImpl));
{
@@ -522,7 +506,7 @@ TEST_F(MetricManagerTest, test_snapshots)
mySet.val10.a.val1.addValue(7);
mySet.val10.a.val2.addValue(2);
mySet.val10.b.val1.addValue(1);
- timer->add_time(5 * 60);
+ timer.add_time(5 * 60);
ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60);
ASSERT_VALUES(mm, 5 * 60, "2,4,4,1,7,9,1,1,8,2,10");
ASSERT_VALUES(mm, 60 * 60, "");
@@ -536,7 +520,7 @@ TEST_F(MetricManagerTest, test_snapshots)
mySet.val10.a.val1.addValue(8);
mySet.val10.a.val2.addValue(3);
mySet.val10.b.val1.addValue(2);
- timer->add_time(5 * 60);
+ timer.add_time(5 * 60);
ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60 * 2);
ASSERT_VALUES(mm, 5 * 60, "4,5,5,1,8,11,2,2,10,3,13");
ASSERT_VALUES(mm, 60 * 60, "");
@@ -544,7 +528,7 @@ TEST_F(MetricManagerTest, test_snapshots)
// Adding another five minute period where nothing have happened.
// Metric for last 5 minutes should be 0.
- timer->add_time(5 * 60);
+ timer.add_time(5 * 60);
ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60 * 3);
ASSERT_VALUES(mm, 5 * 60, "0,0,0,0,0,0,0,0,0,0,0");
ASSERT_VALUES(mm, 60 * 60, "");
@@ -555,7 +539,7 @@ TEST_F(MetricManagerTest, test_snapshots)
mySet.val6.addValue(6);
for (uint32_t i=0; i<9; ++i) { // 9 x 5 minutes. Avoid snapshot bumping
// due to taking snapshots in the past
- timer->add_time(5 * 60);
+ timer.add_time(5 * 60);
ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60 * (4 + i));
}
ASSERT_VALUES(mm, 5 * 60, "0,0,0,0,0,0,0,0,0,0,0");
@@ -570,86 +554,10 @@ TEST_F(MetricManagerTest, test_snapshots)
ASSERT_VALUES(mm, 0 * 60, "0,0,0,0,0,0,0,0,0,0,0");
}
-TEST_F(MetricManagerTest, test_xml_output)
-{
- FakeTimer* timer = new FakeTimer(1000);
- std::unique_ptr<MetricManager::Timer> timerImpl(timer);
- MetricManager mm(std::move(timerImpl));
- TestMetricSet mySet;
- {
- MetricLockGuard lockGuard(mm.getMetricLock());
- mm.registerMetric(lockGuard, mySet.set);
- }
-
- // Initialize metric manager to get snapshots created.
- mm.init(ConfigUri("raw:"
- "consumer[2]\n"
- "consumer[0].name snapper\n"
- "consumer[0].tags[1]\n"
- "consumer[0].tags[0] snaptest\n"
- "consumer[1].name log\n"
- "consumer[1].tags[1]\n"
- "consumer[1].tags[0] snaptest\n"));
-
- takeSnapshots(mm, 1000);
-
- // Adding metrics to have some values in them
- mySet.val6.addValue(2);
- mySet.val9.val1.addValue(4);
- mySet.val10.count.inc();
- mySet.val10.a.val1.addValue(7);
- mySet.val10.a.val2.addValue(2);
- mySet.val10.b.val1.addValue(1);
-
- timer->set_time(1300);
- takeSnapshots(mm, 1300);
-
- std::string expected(
- "'<snapshot name=\"5 minute\" from=\"1000\" to=\"1300\" period=\"300\">\n"
- " <temp>\n"
- " <val6 average=\"2\" last=\"2\" min=\"2\" max=\"2\" count=\"1\"/>\n"
- " <sub>\n"
- " <val1 average=\"4\" last=\"4\" min=\"4\" max=\"4\" count=\"1\"/>\n"
- " <valsum average=\"4\" last=\"4\" min=\"4\" max=\"4\" count=\"1\"/>\n"
- " </sub>\n"
- " <multisub>\n"
- " <count count=\"1\"/>\n"
- " <a>\n"
- " <val1 average=\"7\" last=\"7\" min=\"7\" max=\"7\" count=\"1\"/>\n"
- " <valsum average=\"9\" last=\"9\"/>\n"
- " </a>\n"
- " <b>\n"
- " <val1 average=\"1\" last=\"1\" min=\"1\" max=\"1\" count=\"1\"/>\n"
- " <valsum average=\"1\" last=\"1\" min=\"1\" max=\"1\" count=\"1\"/>\n"
- " </b>\n"
- " <sum>\n"
- " <val1 average=\"8\" last=\"8\"/>\n"
- " <val2 average=\"2\" last=\"2\" min=\"2\" max=\"2\" count=\"1\"/>\n"
- " <valsum average=\"10\" last=\"10\"/>\n"
- " </sum>\n"
- " </multisub>\n"
- " </temp>\n"
- "</snapshot>'");
-
- std::ostringstream ost;
- vespalib::XmlOutputStream xos(ost, " ");
- XmlWriter writer(xos, 300, 0);
- {
- MetricLockGuard lockGuard(mm.getMetricLock());
- mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, 300, false),
- writer, "snapper");
- }
- std::string actual(ost.str());
- // Not bothering to match all the nitty gritty details as it will test
- // more than it needs to. Just be here in order to check on XML output
- // easily if needed.
- EXPECT_EQ(expected, "'" + actual + "'");
-}
-
TEST_F(MetricManagerTest, test_json_output)
{
- FakeTimer* timer = new FakeTimer(1000);
- std::unique_ptr<MetricManager::Timer> timerImpl(timer);
+ auto timerImpl = std::make_unique<FakeTimer>(1000);
+ FakeTimer & timer = *timerImpl;
MetricManager mm(std::move(timerImpl));
TestMetricSet mySet;
{
@@ -674,7 +582,7 @@ TEST_F(MetricManagerTest, test_json_output)
mySet.val10.a.val2.addValue(2);
mySet.val10.b.val1.addValue(1);
- timer->set_time(1300);
+ timer.set_time(1300);
takeSnapshots(mm, 1300);
// Create json output
@@ -683,8 +591,7 @@ TEST_F(MetricManagerTest, test_json_output)
JsonWriter writer(jsonStream);
{
MetricLockGuard lockGuard(mm.getMetricLock());
- mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, 300, false),
- writer, "snapper");
+ mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, 300, false), writer, "snapper");
}
jsonStream.finalize();
std::string jsonData = as.str();
@@ -773,22 +680,19 @@ struct MetricSnapshotTestFixture
JsonWriter writer(jsonStream);
{
MetricLockGuard lockGuard(manager.getMetricLock());
- manager.visit(lockGuard, manager.getMetricSnapshot(lockGuard, 300, false),
- writer, "snapper");
+ manager.visit(lockGuard, manager.getMetricSnapshot(lockGuard, 300, false), writer, "snapper");
}
jsonStream.finalize();
return as.str();
}
- std::string renderLastSnapshotAsText(
- const std::string& matchPattern = ".*") const
+ std::string renderLastSnapshotAsText(const std::string& matchPattern = ".*") const
{
std::ostringstream ss;
TextWriter writer(ss, 300, matchPattern, true);
{
MetricLockGuard lockGuard(manager.getMetricLock());
- manager.visit(lockGuard, manager.getMetricSnapshot(lockGuard, 300, false),
- writer, "snapper");
+ manager.visit(lockGuard, manager.getMetricSnapshot(lockGuard, 300, false), writer, "snapper");
}
return ss.str();
}
@@ -819,8 +723,7 @@ public:
}
std::string nthMetricDimension(size_t metricIndex, const std::string& key) {
- return nthMetric(metricIndex)["dimensions"][key]
- .asString().make_string();
+ return nthMetric(metricIndex)["dimensions"][key].asString().make_string();
}
// Verify that the nth metric has the given name and the given set of
@@ -843,7 +746,7 @@ JsonMetricWrapper::JsonMetricWrapper(const std::string& jsonText)
{
vespalib::slime::JsonFormat::decode(vespalib::Memory(jsonText), _tree);
}
-JsonMetricWrapper::~JsonMetricWrapper() { }
+JsonMetricWrapper::~JsonMetricWrapper() = default;
struct DimensionTestMetricSet : MetricSet
{
@@ -851,7 +754,7 @@ struct DimensionTestMetricSet : MetricSet
LongCountMetric val2;
DimensionTestMetricSet(MetricSet* owner = nullptr);
- ~DimensionTestMetricSet();
+ ~DimensionTestMetricSet() override;
};
DimensionTestMetricSet::DimensionTestMetricSet(MetricSet* owner)
@@ -859,7 +762,7 @@ DimensionTestMetricSet::DimensionTestMetricSet(MetricSet* owner)
val1("val1", {{"tag1"}}, "val1 desc", this),
val2("val2", {{"baz", "superbaz"}}, "val2 desc", this)
{ }
-DimensionTestMetricSet::~DimensionTestMetricSet() { }
+DimensionTestMetricSet::~DimensionTestMetricSet() = default;
}
@@ -976,9 +879,7 @@ TEST_F(MetricManagerTest, json_output_can_have_multiple_sets_with_same_name)
TEST_F(MetricManagerTest, test_text_output)
{
- FakeTimer* timer = new FakeTimer(1000);
- std::unique_ptr<MetricManager::Timer> timerImpl(timer);
- MetricManager mm(std::move(timerImpl));
+ MetricManager mm(std::make_unique<FakeTimer>(1000));
TestMetricSet mySet;
{
MetricLockGuard lockGuard(mm.getMetricLock());
@@ -1051,10 +952,7 @@ namespace {
std::mutex& _output_mutex;
FakeTimer& _timer;
- MyUpdateHook(std::ostringstream& output,
- std::mutex& output_mutex,
- const char* name,
- FakeTimer& timer)
+ MyUpdateHook(std::ostringstream& output, std::mutex& output_mutex, const char* name, FakeTimer& timer)
: UpdateHook(name),
_output(output),
_output_mutex(output_mutex),
@@ -1064,7 +962,7 @@ namespace {
void updateMetrics(const MetricLockGuard & ) override {
std::lock_guard lock(_output_mutex); // updateMetrics() called from metric manager thread
- _output << _timer.getTime() << ": " << getName() << " called\n";
+ _output << vespalib::count_s(_timer.getTime().time_since_epoch()) << ": " << getName() << " called\n";
}
};
}
@@ -1073,8 +971,8 @@ TEST_F(MetricManagerTest, test_update_hooks)
{
std::mutex output_mutex;
std::ostringstream output;
- FakeTimer* timer = new FakeTimer(1000);
- std::unique_ptr<MetricManager::Timer> timerImpl(timer);
+ auto timerImpl = std::make_unique<FakeTimer>(1000);
+ FakeTimer & timer = *timerImpl;
// Add a metric set just so one exist
TestMetricSet mySet;
MetricManager mm(std::move(timerImpl));
@@ -1083,9 +981,9 @@ TEST_F(MetricManagerTest, test_update_hooks)
mm.registerMetric(lockGuard, mySet.set);
}
- MyUpdateHook preInitShort(output, output_mutex, "BIS", *timer);
- MyUpdateHook preInitLong(output, output_mutex, "BIL", *timer);
- MyUpdateHook preInitInfinite(output, output_mutex, "BII", *timer);
+ MyUpdateHook preInitShort(output, output_mutex, "BIS", timer);
+ MyUpdateHook preInitLong(output, output_mutex, "BIL", timer);
+ MyUpdateHook preInitInfinite(output, output_mutex, "BII", timer);
mm.addMetricUpdateHook(preInitShort, 5);
mm.addMetricUpdateHook(preInitLong, 300);
mm.addMetricUpdateHook(preInitInfinite, 0);
@@ -1104,55 +1002,55 @@ TEST_F(MetricManagerTest, test_update_hooks)
"consumer[1].tags[0] snaptest\n"));
output << "Init done\n";
- MyUpdateHook postInitShort(output, output_mutex, "AIS", *timer);
- MyUpdateHook postInitLong(output, output_mutex, "AIL", *timer);
- MyUpdateHook postInitInfinite(output, output_mutex, "AII", *timer);
+ MyUpdateHook postInitShort(output, output_mutex, "AIS", timer);
+ MyUpdateHook postInitLong(output, output_mutex, "AIL", timer);
+ MyUpdateHook postInitInfinite(output, output_mutex, "AII", timer);
mm.addMetricUpdateHook(postInitShort, 5);
mm.addMetricUpdateHook(postInitLong, 400);
mm.addMetricUpdateHook(postInitInfinite, 0);
// After 5 seconds the short ones should get another.
- timer->set_time(1006);
+ timer.set_time(1006);
waitForTimeProcessed(mm, 1006);
// After 4 more seconds the short ones should get another
// since last update was a second late. (Stable periods, process time
// should not affect how often they are updated)
- timer->set_time(1010);
+ timer.set_time(1010);
waitForTimeProcessed(mm, 1010);
// Bumping considerably ahead, such that next update is in the past,
// we should only get one update called in this period.
- timer->set_time(1200);
+ timer.set_time(1200);
waitForTimeProcessed(mm, 1200);
// No updates at this time.
- timer->set_time(1204);
+ timer.set_time(1204);
waitForTimeProcessed(mm, 1204);
// Give all hooks an update
mm.updateMetrics(true);
// Last update should not have interfered with periods
- timer->set_time(1205);
+ timer.set_time(1205);
waitForTimeProcessed(mm, 1205);
// Time is just ahead of a snapshot.
- timer->set_time(1299);
+ timer.set_time(1299);
waitForTimeProcessed(mm, 1299);
// At time 1300 we are at a 5 minute snapshot bump
// All hooks should thus get an update. The one with matching period
// should only get one
- timer->set_time(1300);
+ timer.set_time(1300);
waitForTimeProcessed(mm, 1300);
// The snapshot time currently doesn't count for the metric at period
// 400. It will get an event at this time.
- timer->set_time(1450);
+ timer.set_time(1450);
waitForTimeProcessed(mm, 1450);
std::string expected(
diff --git a/metrics/src/tests/snapshottest.cpp b/metrics/src/tests/snapshottest.cpp
index b4eb4a1353c..4d2ea96c36d 100644
--- a/metrics/src/tests/snapshottest.cpp
+++ b/metrics/src/tests/snapshottest.cpp
@@ -122,15 +122,15 @@ struct TestMetricSet : public MetricSet {
SubMetricSet set2;
SumMetric<SubMetricSet> setSum;
- TestMetricSet(vespalib::stringref name, MetricSet* owner = 0);
+ TestMetricSet(vespalib::stringref name);
~TestMetricSet();
void incValues();
};
-TestMetricSet::TestMetricSet(vespalib::stringref name, MetricSet* owner)
- : MetricSet(name, {}, "", owner),
+TestMetricSet::TestMetricSet(vespalib::stringref name)
+ : MetricSet(name, {}, "", nullptr),
set1("set1", this),
set2("set2", this),
setSum("setSum", {}, "", this)
@@ -149,7 +149,7 @@ TestMetricSet::incValues() {
struct FakeTimer : public MetricManager::Timer {
uint32_t _timeInSecs;
FakeTimer() : _timeInSecs(1) {}
- time_t getTime() const override { return _timeInSecs; }
+ time_point getTime() const override { return time_point(vespalib::from_s(_timeInSecs)); }
};
void ASSERT_VALUE(int32_t value, const MetricSnapshot & snapshot, const char *name) __attribute__((noinline));
@@ -176,8 +176,7 @@ TEST_F(SnapshotTest, test_snapshot_two_days)
TestMetricSet set("test");
FakeTimer* timer;
- MetricManager mm(
- std::unique_ptr<MetricManager::Timer>(timer = new FakeTimer));
+ MetricManager mm(std::unique_ptr<MetricManager::Timer>(timer = new FakeTimer));
{
MetricLockGuard lockGuard(mm.getMetricLock());
mm.registerMetric(lockGuard, set);
@@ -215,10 +214,9 @@ TEST_F(SnapshotTest, test_snapshot_two_days)
<< "\n";
*/
- const MetricSnapshot* snap = 0;
// active snapshot
MetricLockGuard lockGuard(mm.getMetricLock());
- snap = &mm.getActiveMetrics(lockGuard);
+ const MetricSnapshot* snap = &mm.getActiveMetrics(lockGuard);
ASSERT_VALUE(0, *snap, "test.set1.set1.count1");
ASSERT_VALUE(0, *snap, "test.set1.set1.countSum");
diff --git a/metrics/src/vespa/metrics/CMakeLists.txt b/metrics/src/vespa/metrics/CMakeLists.txt
index 2441bf95c0b..62254a8d620 100644
--- a/metrics/src/vespa/metrics/CMakeLists.txt
+++ b/metrics/src/vespa/metrics/CMakeLists.txt
@@ -18,7 +18,6 @@ vespa_add_library(metrics
updatehook.cpp
valuemetric.cpp
valuemetricvalues.cpp
- xmlwriter.cpp
$<TARGET_OBJECTS:metrics_common>
INSTALL lib64
diff --git a/metrics/src/vespa/metrics/metricmanager.cpp b/metrics/src/vespa/metrics/metricmanager.cpp
index a0e44ddbeac..df83001a4e2 100644
--- a/metrics/src/vespa/metrics/metricmanager.cpp
+++ b/metrics/src/vespa/metrics/metricmanager.cpp
@@ -22,25 +22,30 @@ LOG_SETUP(".metrics.manager");
namespace metrics {
using Config = MetricsmanagerConfig;
+using vespalib::IllegalStateException;
+using vespalib::IllegalArgumentException;
+using vespalib::make_string_short::fmt;
+using vespalib::count_ms;
+using vespalib::count_s;
+using vespalib::from_s;
MetricManager::ConsumerSpec::ConsumerSpec() = default;
MetricManager::ConsumerSpec::~ConsumerSpec() = default;
-time_t
+time_point
MetricManager::Timer::getTime() const {
- return vespalib::count_s(vespalib::system_clock::now().time_since_epoch());
+ return vespalib::system_clock::now();
}
void
MetricManager::assertMetricLockLocked(const MetricLockGuard& g) const {
if ( ! g.owns(_waiter)) {
- throw vespalib::IllegalArgumentException("Given lock does not lock the metric lock.", VESPA_STRLOC);
+ throw IllegalArgumentException("Given lock does not lock the metric lock.", VESPA_STRLOC);
}
}
void
-MetricManager::ConsumerSpec::print(std::ostream& out, bool verbose,
- const std::string& indent) const
+MetricManager::ConsumerSpec::print(std::ostream& out, bool verbose, const std::string& indent) const
{
(void) verbose;
out << "ConsumerSpec(";
@@ -63,6 +68,10 @@ MetricManager::ConsumerSpec::addMemoryUsage(MemoryConsumption& mc) const
}
}
+MetricManager::MetricManager()
+ : MetricManager(std::make_unique<Timer>())
+{ }
+
MetricManager::MetricManager(std::unique_ptr<Timer> timer)
: _activeMetrics("Active metrics showing updates since last snapshot"),
_configSubscriber(),
@@ -70,9 +79,7 @@ MetricManager::MetricManager(std::unique_ptr<Timer> timer)
_config(),
_consumerConfig(),
_snapshots(),
- _totalMetrics(std::make_shared<MetricSnapshot>(
- "Empty metrics before init", 0, _activeMetrics.getMetrics(),
- false)),
+ _totalMetrics(std::make_shared<MetricSnapshot>("Empty metrics before init", 0, _activeMetrics.getMetrics(), false)),
_timer(std::move(timer)),
_lastProcessedTime(0),
_snapshotUnsetMetrics(false),
@@ -114,7 +121,7 @@ MetricManager::addMetricUpdateHook(UpdateHook& hook, uint32_t period)
std::lock_guard sync(_waiter);
// If we've already initialized manager, log period has been set.
// In this case. Call first time after period
- hook._nextCall = _timer->getTime() + period;
+ hook._nextCall = count_s(_timer->getTime().time_since_epoch()) + period;
if (period == 0) {
for (UpdateHook * sHook : _snapshotUpdateHooks) {
if (sHook == &hook) {
@@ -165,9 +172,8 @@ void
MetricManager::init(const config::ConfigUri & uri, bool startThread)
{
if (isInitialized()) {
- throw vespalib::IllegalStateException(
- "The metric manager have already been initialized. "
- "It can only be initialized once.", VESPA_STRLOC);
+ throw IllegalStateException("The metric manager have already been initialized. "
+ "It can only be initialized once.", VESPA_STRLOC);
}
LOG(debug, "Initializing metric manager.");
_configSubscriber = std::make_unique<config::ConfigSubscriber>(uri.getContext());
@@ -233,8 +239,10 @@ struct ConsumerMetricBuilder : public MetricVisitor {
bool nameRemoved;
uint32_t metricCount;
- Result() : tagAdded(false), tagRemoved(false),
- nameAdded(false), nameRemoved(false), metricCount(0) {}
+ Result()
+ : tagAdded(false), tagRemoved(false),
+ nameAdded(false), nameRemoved(false), metricCount(0)
+ {}
};
std::list<Result> result;
@@ -353,9 +361,7 @@ ConsumerMetricBuilder::~ConsumerMetricBuilder() = default;
void
MetricManager::checkMetricsAltered(const MetricLockGuard & guard)
{
- if (_activeMetrics.getMetrics().isRegistrationAltered()
- || _consumerConfigChanged)
- {
+ if (_activeMetrics.getMetrics().isRegistrationAltered() || _consumerConfigChanged) {
handleMetricsAltered(guard);
}
}
@@ -385,8 +391,8 @@ MetricManager::handleMetricsAltered(const MetricLockGuard & guard)
}
LOG(debug, "Recreating snapshots to include altered metrics");
_totalMetrics->recreateSnapshot(_activeMetrics.getMetrics(), _snapshotUnsetMetrics);
- for (uint32_t i=0; i<_snapshots.size(); ++i) {
- _snapshots[i]->recreateSnapshot(_activeMetrics.getMetrics(), _snapshotUnsetMetrics);
+ for (const auto & snapshot: _snapshots) {
+ snapshot->recreateSnapshot(_activeMetrics.getMetrics(), _snapshotUnsetMetrics);
}
LOG(debug, "Setting new consumer config. Clearing dirty flag");
_consumerConfig.swap(configMap);
@@ -394,24 +400,25 @@ MetricManager::handleMetricsAltered(const MetricLockGuard & guard)
}
namespace {
- bool setSnapshotName(std::ostream& out, const char* name, uint32_t length, uint32_t period)
- {
- if (length % period != 0) return false;
- out << (length / period) << ' ' << name;
- if (length / period != 1) out << "s";
- return true;
- }
+
+bool
+setSnapshotName(std::ostream& out, const char* name, uint32_t length, uint32_t period) {
+ if (length % period != 0) return false;
+ out << (length / period) << ' ' << name;
+ if (length / period != 1) out << "s";
+ return true;
+}
+
}
std::vector<MetricManager::SnapSpec>
MetricManager::createSnapshotPeriods(const Config& config)
{
std::vector<SnapSpec> result;
- try{
- for (uint32_t i=0; i<config.snapshot.periods.size(); ++i) {
- uint32_t length = config.snapshot.periods[i];
+ try {
+ for (auto length : config.snapshot.periods) {
if (length < 1)
- throw vespalib::IllegalStateException("Snapshot periods must be positive numbers", VESPA_STRLOC);
+ throw IllegalStateException("Snapshot periods must be positive numbers", VESPA_STRLOC);
std::ostringstream name;
if (setSnapshotName(name, "week", length, 60 * 60 * 24 * 7)) {
} else if (setSnapshotName(name, "day", length, 60 * 60 * 24)) {
@@ -425,16 +432,13 @@ MetricManager::createSnapshotPeriods(const Config& config)
for (uint32_t i=1; i<result.size(); ++i) {
if (result[i].first % result[i-1].first != 0) {
std::ostringstream ost;
- ost << "Period " << result[i].first
- << " is not a multiplum of period "
+ ost << "Period " << result[i].first << " is not a multiplum of period "
<< result[i-1].first << " which is needs to be.";
- throw vespalib::IllegalStateException(
- ost.str(), VESPA_STRLOC);
+ throw IllegalStateException(ost.str(), VESPA_STRLOC);
}
}
} catch (vespalib::Exception& e) {
- LOG(warning, "Invalid snapshot periods specified. Using defaults: %s",
- e.getMessage().c_str());
+ LOG(warning, "Invalid snapshot periods specified. Using defaults: %s", e.getMessage().c_str());
result.clear();
}
if (result.empty()) {
@@ -454,8 +458,7 @@ MetricManager::configure(const MetricLockGuard & , std::unique_ptr<Config> confi
std::ostringstream ost;
config::OstreamConfigWriter w(ost);
w.write(*config);
- LOG(debug, "Received new config for metric manager: %s",
- ost.str().c_str());
+ LOG(debug, "Received new config for metric manager: %s", ost.str().c_str());
}
if (_snapshots.empty()) {
LOG(debug, "Initializing snapshots as this is first configure call");
@@ -463,21 +466,15 @@ MetricManager::configure(const MetricLockGuard & , std::unique_ptr<Config> confi
// Set up snapshots only first time. We don't allow live reconfig
// of snapshot periods.
- time_t currentTime(_timer->getTime());
+ time_t currentTime = count_s(_timer->getTime().time_since_epoch());
_activeMetrics.setFromTime(currentTime);
uint32_t count = 1;
- for (uint32_t i = 0; i< snapshotPeriods.size(); ++i)
- {
+ for (uint32_t i = 0; i< snapshotPeriods.size(); ++i) {
uint32_t nextCount = 1;
if (i + 1 < snapshotPeriods.size()) {
- nextCount = snapshotPeriods[i + 1].first
- / snapshotPeriods[i].first;
- if (snapshotPeriods[i + 1].first
- % snapshotPeriods[i].first != 0)
- {
- throw vespalib::IllegalStateException(
- "Snapshot periods must be multiplum of each other",
- VESPA_STRLOC);
+ nextCount = snapshotPeriods[i + 1].first / snapshotPeriods[i].first;
+ if ((snapshotPeriods[i + 1].first % snapshotPeriods[i].first) != 0) {
+ throw IllegalStateException("Snapshot periods must be multiplum of each other",VESPA_STRLOC);
}
}
_snapshots.push_back(std::make_shared<MetricSnapshotSet>(
@@ -489,9 +486,7 @@ MetricManager::configure(const MetricLockGuard & , std::unique_ptr<Config> confi
_totalMetrics = std::make_shared<MetricSnapshot>("All time snapshot", 0, _activeMetrics.getMetrics(), _snapshotUnsetMetrics);
_totalMetrics->reset(currentTime);
}
- if (_config.get() == 0
- || _config->consumer.size() != config->consumer.size())
- {
+ if (_config.get() == 0 || (_config->consumer.size() != config->consumer.size())) {
_consumerConfigChanged = true;
} else {
for (uint32_t i=0; i<_config->consumer.size(); ++i) {
@@ -519,61 +514,55 @@ MetricManager::getConsumerSpec(const MetricLockGuard &, const Metric::String& co
namespace {
- struct ConsumerMetricVisitor : public MetricVisitor {
- const MetricManager::ConsumerSpec& _metricsToMatch;
- MetricVisitor& _client;
+struct ConsumerMetricVisitor : public MetricVisitor {
+ const MetricManager::ConsumerSpec& _metricsToMatch;
+ MetricVisitor& _client;
#ifdef VERIFY_ALL_METRICS_VISITED
- std::set<Metric::String> _visitedMetrics;
+ std::set<Metric::String> _visitedMetrics;
#endif
- ConsumerMetricVisitor(const MetricManager::ConsumerSpec& spec,
- MetricVisitor& clientVisitor)
- : _metricsToMatch(spec), _client(clientVisitor) {}
+ ConsumerMetricVisitor(const MetricManager::ConsumerSpec& spec, MetricVisitor& clientVisitor)
+ : _metricsToMatch(spec), _client(clientVisitor)
+ {}
- bool visitMetricSet(const MetricSet& metricSet,
- bool autoGenerated) override
- {
- if (metricSet.isTopSet()) return true;
- return (_metricsToMatch.contains(metricSet)
- && _client.visitMetricSet(metricSet, autoGenerated));
- }
- void doneVisitingMetricSet(const MetricSet& metricSet) override {
- if (!metricSet.isTopSet()) {
+ bool visitMetricSet(const MetricSet& metricSet, bool autoGenerated) override {
+ if (metricSet.isTopSet()) return true;
+ return (_metricsToMatch.contains(metricSet)
+ && _client.visitMetricSet(metricSet, autoGenerated));
+ }
+ void doneVisitingMetricSet(const MetricSet& metricSet) override {
+ if (!metricSet.isTopSet()) {
#ifdef VERIFY_ALL_METRICS_VISITED
- _visitedMetrics.insert(metricSet.getPath());
+ _visitedMetrics.insert(metricSet.getPath());
#endif
- _client.doneVisitingMetricSet(metricSet);
- }
+ _client.doneVisitingMetricSet(metricSet);
}
- bool visitCountMetric(const AbstractCountMetric& metric,
- bool autoGenerated) override
- {
- if (_metricsToMatch.contains(metric)) {
+ }
+ bool visitCountMetric(const AbstractCountMetric& metric, bool autoGenerated) override {
+ if (_metricsToMatch.contains(metric)) {
#ifdef VERIFY_ALL_METRICS_VISITED
- _visitedMetrics.insert(metric.getPath());
+ _visitedMetrics.insert(metric.getPath());
#endif
- return _client.visitCountMetric(metric, autoGenerated);
- }
- return true;
+ return _client.visitCountMetric(metric, autoGenerated);
}
- bool visitValueMetric(const AbstractValueMetric& metric,
- bool autoGenerated) override
- {
- if (_metricsToMatch.contains(metric)) {
+ return true;
+ }
+ bool visitValueMetric(const AbstractValueMetric& metric, bool autoGenerated) override {
+ if (_metricsToMatch.contains(metric)) {
#ifdef VERIFY_ALL_METRICS_VISITED
- _visitedMetrics.insert(metric.getPath());
+ _visitedMetrics.insert(metric.getPath());
#endif
- return _client.visitValueMetric(metric, autoGenerated);
- }
- return true;
+ return _client.visitValueMetric(metric, autoGenerated);
}
- };
+ return true;
+ }
+};
}
void
-MetricManager::visit(const MetricLockGuard & guard, const MetricSnapshot& snapshot, MetricVisitor& visitor,
- const std::string& consumer) const
+MetricManager::visit(const MetricLockGuard & guard, const MetricSnapshot& snapshot,
+ MetricVisitor& visitor, const std::string& consumer) const
{
if (visitor.visitSnapshot(snapshot)) {
if (consumer == "") {
@@ -593,9 +582,7 @@ MetricManager::visit(const MetricLockGuard & guard, const MetricSnapshot& snapsh
}
#endif
} else {
- LOGBP(debug,
- "Requested metrics for non-defined consumer '%s'.",
- consumer.c_str());
+ LOGBP(debug, "Requested metrics for non-defined consumer '%s'.", consumer.c_str());
}
}
visitor.doneVisitingSnapshot(snapshot);
@@ -616,39 +603,31 @@ MetricManager::getSnapshotPeriods(const MetricLockGuard& l) const
// Client should have grabbed metrics lock before doing this
const MetricSnapshot&
-MetricManager::getMetricSnapshot(const MetricLockGuard& l,
- uint32_t period, bool getInProgressSet) const
+MetricManager::getMetricSnapshot(const MetricLockGuard& l, uint32_t period, bool getInProgressSet) const
{
assertMetricLockLocked(l);
- for (uint32_t i=0; i<_snapshots.size(); ++i) {
- if (_snapshots[i]->getPeriod() == period) {
- if (_snapshots[i]->getCount() == 1 && getInProgressSet) {
- throw vespalib::IllegalStateException(
- "No temporary snapshot for set "
- + _snapshots[i]->getName(), VESPA_STRLOC);
+ for (const auto & snapshot : _snapshots) {
+ if (snapshot->getPeriod() == period) {
+ if (snapshot->getCount() == 1 && getInProgressSet) {
+ throw IllegalStateException("No temporary snapshot for set " + snapshot->getName(), VESPA_STRLOC);
}
- return _snapshots[i]->getSnapshot(getInProgressSet);
+ return snapshot->getSnapshot(getInProgressSet);
}
}
- std::ostringstream ost;
- ost << "No snapshot for period of length " << period << " exist.";
- throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC);
+ throw IllegalArgumentException(fmt("No snapshot for period of length %u exist.", period), VESPA_STRLOC);
}
// Client should have grabbed metrics lock before doing this
const MetricSnapshotSet&
-MetricManager::getMetricSnapshotSet(const MetricLockGuard& l,
- uint32_t period) const
+MetricManager::getMetricSnapshotSet(const MetricLockGuard& l, uint32_t period) const
{
assertMetricLockLocked(l);
- for (uint32_t i=0; i<_snapshots.size(); ++i) {
- if (_snapshots[i]->getPeriod() == period) {
- return *_snapshots[i];
+ for (const auto & snapshot : _snapshots) {
+ if (snapshot->getPeriod() == period) {
+ return *snapshot;
}
}
- std::ostringstream ost;
- ost << "No snapshot set for period of length " << period << " exist.";
- throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC);
+ throw IllegalArgumentException(fmt("No snapshot set for period of length %u exist.", period), VESPA_STRLOC);
}
void
@@ -661,9 +640,8 @@ MetricManager::timeChangedNotification() const
void
MetricManager::updateMetrics(bool includeSnapshotOnlyHooks)
{
- LOG(debug, "Calling metric update hooks%s.",
- includeSnapshotOnlyHooks ? ", including snapshot hooks" : "");
- // Ensure we're not in the way of the background thread
+ LOG(debug, "Calling metric update hooks%s.", includeSnapshotOnlyHooks ? ", including snapshot hooks" : "");
+ // Ensure we're not in the way of the background thread
MetricLockGuard sync(_waiter);
LOG(debug, "Giving %zu periodic update hooks.", _periodicUpdateHooks.size());
updatePeriodicMetrics(sync, 0, true);
@@ -677,30 +655,29 @@ MetricManager::updateMetrics(bool includeSnapshotOnlyHooks)
time_t
MetricManager::updatePeriodicMetrics(const MetricLockGuard & guard, time_t updateTime, bool outOfSchedule)
{
+ assertMetricLockLocked(guard);
time_t nextUpdateTime = std::numeric_limits<time_t>::max();
- time_t preTime = _timer->getTimeInMilliSecs();
+ time_point preTime = _timer->getTimeInMilliSecs();
for (auto hook : _periodicUpdateHooks) {
if (hook->_nextCall <= updateTime) {
hook->updateMetrics(guard);
if (hook->_nextCall + hook->_period < updateTime) {
if (hook->_nextCall != 0) {
- LOG(debug, "Updated hook %s at time %" PRIu64 ", but next "
- "run in %u seconds have already passed as time"
- " is %" PRIu64 ". Bumping next call to current "
- "time + period.",
+ LOG(debug, "Updated hook %s at time %" PRIu64 ", but next run in %u seconds have already passed as "
+ "time is %" PRIu64 ". Bumping next call to current time + period.",
hook->_name, static_cast<uint64_t>(hook->_nextCall), hook->_period, static_cast<uint64_t>(updateTime));
}
hook->_nextCall = updateTime + hook->_period;
} else {
hook->_nextCall += hook->_period;
}
- time_t postTime = _timer->getTimeInMilliSecs();
- _periodicHookLatency.addValue(postTime - preTime);
+ time_point postTime = _timer->getTimeInMilliSecs();
+ _periodicHookLatency.addValue(count_ms(postTime - preTime));
preTime = postTime;
} else if (outOfSchedule) {
hook->updateMetrics(guard);
- time_t postTime = _timer->getTimeInMilliSecs();
- _periodicHookLatency.addValue(postTime - preTime);
+ time_point postTime = _timer->getTimeInMilliSecs();
+ _periodicHookLatency.addValue(count_ms(postTime - preTime));
preTime = postTime;
}
nextUpdateTime = std::min(nextUpdateTime, hook->_nextCall);
@@ -712,11 +689,12 @@ MetricManager::updatePeriodicMetrics(const MetricLockGuard & guard, time_t updat
void
MetricManager::updateSnapshotMetrics(const MetricLockGuard & guard)
{
- time_t preTime = _timer->getTimeInMilliSecs();
- for (auto it = _snapshotUpdateHooks.begin(); it != _snapshotUpdateHooks.end(); ++it) {
- (**it).updateMetrics(guard);
- time_t postTime = _timer->getTimeInMilliSecs();
- _snapshotHookLatency.addValue(postTime - preTime);
+ assertMetricLockLocked(guard);
+ time_point preTime = _timer->getTimeInMilliSecs();
+ for (const auto & hook : _snapshotUpdateHooks) {
+ hook->updateMetrics(guard);
+ time_point postTime = _timer->getTimeInMilliSecs();
+ _snapshotHookLatency.addValue(count_ms(postTime - preTime));
preTime = postTime;
}
}
@@ -733,17 +711,17 @@ MetricManager::forceEventLogging()
void
MetricManager::reset(time_t currentTime)
{
- time_t preTime = _timer->getTimeInMilliSecs();
+ time_point preTime = _timer->getTimeInMilliSecs();
// Resetting implies visiting metrics, which needs to grab metric lock
// to avoid conflict with adding/removal of metrics
std::lock_guard waiterLock(_waiter);
_activeMetrics.reset(currentTime);
- for (uint32_t i=0; i<_snapshots.size(); ++i) {
- _snapshots[i]->reset(currentTime);
+ for (const auto & snapshot : _snapshots) {
+ snapshot->reset(currentTime);
}
_totalMetrics->reset(currentTime);
- time_t postTime = _timer->getTimeInMilliSecs();
- _resetLatency.addValue(postTime - preTime);
+ time_point postTime = _timer->getTimeInMilliSecs();
+ _resetLatency.addValue(count_ms(postTime - preTime));
}
void
@@ -755,7 +733,7 @@ MetricManager::run()
// we constantly add next time to do something from the last timer.
// For that to work, we need to initialize timers on first iteration
// to set them to current time.
- time_t currentTime = _timer->getTime();
+ time_t currentTime = count_s(_timer->getTime().time_since_epoch());
for (auto & snapshot : _snapshots) {
snapshot->setFromTime(currentTime);
}
@@ -765,12 +743,13 @@ MetricManager::run()
// Ensure correct time for first snapshot
_snapshots[0]->getSnapshot().setToTime(currentTime);
while (!stop_requested()) {
- currentTime = _timer->getTime();
+ time_point now = _timer->getTime();
+ currentTime = count_s(now.time_since_epoch());
time_t next = tick(sync, currentTime);
if (currentTime < next) {
- size_t ms = (next - currentTime) * 1000;
- _cond.wait_for(sync, std::chrono::milliseconds(ms));
- _sleepTimes.addValue(ms);
+ vespalib::duration wait_time = from_s(next - currentTime);
+ _cond.wait_for(sync, wait_time);
+ _sleepTimes.addValue(count_ms(wait_time));
} else {
_sleepTimes.addValue(0);
}
@@ -780,11 +759,10 @@ MetricManager::run()
time_t
MetricManager::tick(const MetricLockGuard & guard, time_t currentTime)
{
- LOG(spam, "Worker thread starting to process for time %" PRIu64 ".",
- static_cast<uint64_t>(currentTime));
+ LOG(spam, "Worker thread starting to process for time %" PRIu64 ".", static_cast<uint64_t>(currentTime));
// Check for new config and reconfigure
- if (_configSubscriber.get() && _configSubscriber->nextConfigNow()) {
+ if (_configSubscriber && _configSubscriber->nextConfigNow()) {
configure(guard, _configHandle->getConfig());
}
@@ -811,8 +789,7 @@ MetricManager::tick(const MetricLockGuard & guard, time_t currentTime)
// Do snapshotting if it is time
if (nextWorkTime <= currentTime) takeSnapshots(guard, nextWorkTime);
- _lastProcessedTime.store(nextWorkTime <= currentTime ? nextWorkTime : currentTime,
- std::memory_order_relaxed);
+ _lastProcessedTime.store(nextWorkTime <= currentTime ? nextWorkTime : currentTime, std::memory_order_relaxed);
LOG(spam, "Worker thread done with processing for time %" PRIu64 ".",
static_cast<uint64_t>(_lastProcessedTime.load(std::memory_order_relaxed)));
time_t next = _snapshots[0]->getPeriod() + _snapshots[0]->getToTime();
@@ -821,8 +798,9 @@ MetricManager::tick(const MetricLockGuard & guard, time_t currentTime)
}
void
-MetricManager::takeSnapshots(const MetricLockGuard &, time_t timeToProcess)
+MetricManager::takeSnapshots(const MetricLockGuard & guard, time_t timeToProcess)
{
+ assertMetricLockLocked(guard);
// If not time to do dump data from active snapshot yet, nothing to do
if (!_snapshots[0]->timeForAnotherSnapshot(timeToProcess)) {
LOG(spam, "Not time to process snapshot %s at time %" PRIu64 ". Current "
@@ -832,7 +810,7 @@ MetricManager::takeSnapshots(const MetricLockGuard &, time_t timeToProcess)
static_cast<uint64_t>(_snapshots[0]->getToTime()));
return;
}
- time_t preTime = _timer->getTimeInMilliSecs();
+ time_point preTime = _timer->getTimeInMilliSecs();
LOG(debug, "Updating %s snapshot and total metrics at time %" PRIu64 ".",
_snapshots[0]->getName().c_str(), static_cast<uint64_t>(timeToProcess));
MetricSnapshot& firstTarget(_snapshots[0]->getNextTarget());
@@ -840,67 +818,54 @@ MetricManager::takeSnapshots(const MetricLockGuard &, time_t timeToProcess)
_activeMetrics.addToSnapshot(firstTarget, false, timeToProcess);
_activeMetrics.addToSnapshot(*_totalMetrics, false, timeToProcess);
_activeMetrics.reset(timeToProcess);
- LOG(debug, "After snapshotting, "
- "active metrics goes from %" PRIu64 " to %" PRIu64", "
+ LOG(debug, "After snapshotting, active metrics goes from %" PRIu64 " to %" PRIu64", "
"and 5 minute metrics goes from %" PRIu64 " to %" PRIu64".",
static_cast<uint64_t>(_activeMetrics.getFromTime()), static_cast<uint64_t>(_activeMetrics.getToTime()),
static_cast<uint64_t>(firstTarget.getFromTime()), static_cast<uint64_t>(firstTarget.getToTime()));
// Update later snapshots if it is time for it
for (uint32_t i=1; i<_snapshots.size(); ++i) {
- LOG(debug, "Adding data from last snapshot to building snapshot of "
- "next period snapshot %s.",
+ LOG(debug, "Adding data from last snapshot to building snapshot of next period snapshot %s.",
_snapshots[i]->getName().c_str());
MetricSnapshot& target(_snapshots[i]->getNextTarget());
- _snapshots[i-1]->getSnapshot().addToSnapshot(
- target, false, timeToProcess);
+ _snapshots[i-1]->getSnapshot().addToSnapshot(target, false, timeToProcess);
target.setToTime(timeToProcess);
if (!_snapshots[i]->haveCompletedNewPeriod(timeToProcess)) {
- LOG(debug, "Not time to roll snapshot %s yet. %u of %u snapshot "
- "taken at time %" PRIu64 ", and period of %u is not up "
- "yet as we're currently processing for time %" PRIu64 ".",
- _snapshots[i]->getName().c_str(),
- _snapshots[i]->getBuilderCount(),
- _snapshots[i]->getCount(),
- static_cast<uint64_t>
- (_snapshots[i]->getBuilderCount() * _snapshots[i]->getPeriod()
- + _snapshots[i]->getFromTime()),
- _snapshots[i]->getPeriod(),
- static_cast<uint64_t>(timeToProcess));
+ LOG(debug, "Not time to roll snapshot %s yet. %u of %u snapshot taken at time %" PRIu64 ", and period of %u "
+ "is not up yet as we're currently processing for time %" PRIu64 ".",
+ _snapshots[i]->getName().c_str(), _snapshots[i]->getBuilderCount(), _snapshots[i]->getCount(),
+ static_cast<uint64_t>(_snapshots[i]->getBuilderCount() * _snapshots[i]->getPeriod() + _snapshots[i]->getFromTime()),
+ _snapshots[i]->getPeriod(), static_cast<uint64_t>(timeToProcess));
break;
} else {
LOG(debug, "Rolled snapshot %s at time %" PRIu64 ".",
- _snapshots[i]->getName().c_str(),
- static_cast<uint64_t>(timeToProcess));
+ _snapshots[i]->getName().c_str(), static_cast<uint64_t>(timeToProcess));
}
}
- time_t postTime = _timer->getTimeInMilliSecs();
- _snapshotLatency.addValue(postTime - preTime);
+ time_point postTime = _timer->getTimeInMilliSecs();
+ _snapshotLatency.addValue(count_ms(postTime - preTime));
}
MemoryConsumption::UP
MetricManager::getMemoryConsumption(const MetricLockGuard & guard) const
{
- (void) guard;
- MemoryConsumption::UP mc(new MemoryConsumption);
+ assertMetricLockLocked(guard);
+ auto mc = std::make_unique<MemoryConsumption>();
mc->_consumerCount += _consumerConfig.size();
- mc->_consumerMeta += (sizeof(ConsumerSpec::SP) + sizeof(ConsumerSpec))
- * _consumerConfig.size();
- for (auto it = _consumerConfig.begin(); it != _consumerConfig.end(); ++it) {
- mc->_consumerId += mc->getStringMemoryUsage(
- it->first, mc->_consumerIdUnique)
- + sizeof(Metric::String);
- it->second->addMemoryUsage(*mc);
+ mc->_consumerMeta += (sizeof(ConsumerSpec::SP) + sizeof(ConsumerSpec)) * _consumerConfig.size();
+ for (const auto & consumer : _consumerConfig) {
+ mc->_consumerId += mc->getStringMemoryUsage(consumer.first, mc->_consumerIdUnique) + sizeof(Metric::String);
+ consumer.second->addMemoryUsage(*mc);
}
uint32_t preTotal = mc->getTotalMemoryUsage();
_activeMetrics.addMemoryUsage(*mc);
uint32_t postTotal = mc->getTotalMemoryUsage();
mc->addSnapShotUsage("active", postTotal - preTotal);
preTotal = postTotal;
- for (uint32_t i=0; i<_snapshots.size(); ++i) {
- _snapshots[i]->addMemoryUsage(*mc);
+ for (const auto & snapshot : _snapshots) {
+ snapshot->addMemoryUsage(*mc);
postTotal = mc->getTotalMemoryUsage();
- mc->addSnapShotUsage(_snapshots[i]->getName(), postTotal - preTotal);
+ mc->addSnapShotUsage(snapshot->getName(), postTotal - preTotal);
preTotal = postTotal;
}
_totalMetrics->addMemoryUsage(*mc);
diff --git a/metrics/src/vespa/metrics/metricmanager.h b/metrics/src/vespa/metrics/metricmanager.h
index 7a34f894282..c3ce37b451f 100644
--- a/metrics/src/vespa/metrics/metricmanager.h
+++ b/metrics/src/vespa/metrics/metricmanager.h
@@ -67,8 +67,8 @@ public:
struct Timer {
virtual ~Timer() = default;
- virtual time_t getTime() const;
- virtual time_t getTimeInMilliSecs() const { return getTime() * 1000; }
+ virtual time_point getTime() const;
+ time_point getTimeInMilliSecs() const { return getTime(); }
};
/**
@@ -88,8 +88,7 @@ public:
return (includedMetrics.find(m.getPath()) != includedMetrics.end());
}
- void print(std::ostream& out, bool verbose,
- const std::string& indent) const override;
+ void print(std::ostream& out, bool verbose, const std::string& indent) const override;
void addMemoryUsage(MemoryConsumption&) const;
};
@@ -126,7 +125,8 @@ private:
bool stop_requested() const { return _stop_requested.load(std::memory_order_relaxed); }
public:
- MetricManager(std::unique_ptr<Timer> timer = std::make_unique<Timer>());
+ MetricManager();
+ MetricManager(std::unique_ptr<Timer> timer);
~MetricManager();
void stop();
@@ -232,19 +232,16 @@ public:
}
/** While accessing the total metrics you should have the metric lock. */
- const MetricSnapshot& getTotalMetricSnapshot(const MetricLockGuard& l) const
- {
+ const MetricSnapshot& getTotalMetricSnapshot(const MetricLockGuard& l) const {
assertMetricLockLocked(l);
return *_totalMetrics;
}
/** While accessing snapshots you should have the metric lock. */
- const MetricSnapshot& getMetricSnapshot(
- const MetricLockGuard&,
- uint32_t period, bool getInProgressSet = false) const;
- const MetricSnapshotSet& getMetricSnapshotSet(
- const MetricLockGuard&, uint32_t period) const;
- bool hasTemporarySnapshot(const MetricLockGuard& l, uint32_t period) const
- { return getMetricSnapshotSet(l, period).hasTemporarySnapshot(); }
+ const MetricSnapshot& getMetricSnapshot( const MetricLockGuard& guard, uint32_t period) const {
+ return getMetricSnapshot(guard, period, false);
+ }
+ const MetricSnapshot& getMetricSnapshot( const MetricLockGuard&, uint32_t period, bool getInProgressSet) const;
+ const MetricSnapshotSet& getMetricSnapshotSet(const MetricLockGuard&, uint32_t period) const;
std::vector<uint32_t> getSnapshotPeriods(const MetricLockGuard& l) const;
diff --git a/metrics/src/vespa/metrics/updatehook.h b/metrics/src/vespa/metrics/updatehook.h
index 9fa0d52027e..58d9ef0d743 100644
--- a/metrics/src/vespa/metrics/updatehook.h
+++ b/metrics/src/vespa/metrics/updatehook.h
@@ -1,10 +1,13 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#pragma once
+#include <vespa/vespalib/util/time.h>
#include <mutex>
namespace metrics {
+using time_point = vespalib::system_time;
+
class MetricLockGuard {
public:
MetricLockGuard(std::mutex & mutex);
diff --git a/metrics/src/vespa/metrics/xmlwriter.cpp b/metrics/src/vespa/metrics/xmlwriter.cpp
deleted file mode 100644
index 11cb450e64d..00000000000
--- a/metrics/src/vespa/metrics/xmlwriter.cpp
+++ /dev/null
@@ -1,112 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#include "xmlwriter.h"
-#include "countmetric.h"
-#include "metricset.h"
-#include "metricsnapshot.h"
-#include "valuemetric.h"
-#include <vespa/vespalib/util/xmlstream.h>
-#include <sstream>
-
-namespace metrics {
-
-XmlWriter::XmlWriter(vespalib::xml::XmlOutputStream& xos,
- [[maybe_unused]] uint32_t period, int verbosity)
- : _xos(xos), _verbosity(verbosity) {}
-
-bool
-XmlWriter::visitSnapshot(const MetricSnapshot& snapshot)
-{
- using namespace vespalib::xml;
- _xos << XmlTag("snapshot") << XmlAttribute("name", snapshot.getName())
- << XmlAttribute("from", snapshot.getFromTime())
- << XmlAttribute("to", snapshot.getToTime())
- << XmlAttribute("period", snapshot.getPeriod());
- return true;
-}
-
-void
-XmlWriter::doneVisitingSnapshot(const MetricSnapshot&)
-{
- using namespace vespalib::xml;
- _xos << XmlEndTag();
-}
-
-bool
-XmlWriter::visitMetricSet(const MetricSet& set, bool)
-{
- using namespace vespalib::xml;
- if (set.used() || _verbosity >= 2) {
- _xos << XmlTag(set.getName(), XmlTagFlags::CONVERT_ILLEGAL_CHARACTERS);
- printCommonXmlParts(set);
- return true;
- }
- return false;
-}
-void
-XmlWriter::doneVisitingMetricSet(const MetricSet&) {
- using namespace vespalib::xml;
- _xos << XmlEndTag();
-}
-
-bool
-XmlWriter::visitCountMetric(const AbstractCountMetric& metric, bool)
-{
- MetricValueClass::UP values(metric.getValues());
- if (!metric.inUse(*values) && _verbosity < 2) return true;
- using namespace vespalib::xml;
- std::ostringstream ost;
- _xos << XmlTag(metric.getName(), XmlTagFlags::CONVERT_ILLEGAL_CHARACTERS)
- << XmlAttribute(metric.sumOnAdd()
- ? "count" : "value", values->toString("count"));
- printCommonXmlParts(metric);
- _xos << XmlEndTag();
- return true;
-}
-
-bool
-XmlWriter::visitValueMetric(const AbstractValueMetric& metric, bool)
-{
- MetricValueClass::UP values(metric.getValues());
- if (!metric.inUse(*values) && _verbosity < 2) return true;
- using namespace vespalib::xml;
- _xos << XmlTag(metric.getName(), XmlTagFlags::CONVERT_ILLEGAL_CHARACTERS)
- << XmlAttribute("average", values->getLongValue("count") == 0
- ? 0 : values->getDoubleValue("total")
- / values->getDoubleValue("count"))
- << XmlAttribute("last", values->toString("last"));
- if (!metric.summedAverage()) {
- if (values->getLongValue("count") > 0) {
- _xos << XmlAttribute("min", values->toString("min"))
- << XmlAttribute("max", values->toString("max"));
- }
- _xos << XmlAttribute("count", values->getLongValue("count"));
- if (_verbosity >= 2) {
- _xos << XmlAttribute("total", values->toString("total"));
- }
- }
- printCommonXmlParts(metric);
- _xos << XmlEndTag();
- return true;
-}
-
-void
-XmlWriter::printCommonXmlParts(const Metric& metric) const
-{
- using namespace vespalib::xml;
- const Metric::Tags& tags(metric.getTags());
- if (_verbosity >= 3 && tags.size() > 0) {
- std::ostringstream ost;
- // XXX print tag values as well
- ost << tags[0].key();
- for (uint32_t i=1; i<tags.size(); ++i) {
- ost << "," << tags[i].key();
- }
- _xos << XmlAttribute("tags", ost.str());
- }
- if (_verbosity >= 1 && !metric.getDescription().empty()) {
- _xos << XmlAttribute("description", metric.getDescription());
- }
-}
-
-} // metrics
diff --git a/metrics/src/vespa/metrics/xmlwriter.h b/metrics/src/vespa/metrics/xmlwriter.h
deleted file mode 100644
index a0e8a3efeea..00000000000
--- a/metrics/src/vespa/metrics/xmlwriter.h
+++ /dev/null
@@ -1,30 +0,0 @@
-// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-
-#pragma once
-
-#include <vespa/metrics/metric.h>
-#include <vespa/vespalib/util/xmlserializable.h>
-
-namespace metrics {
-
-class XmlWriter : public MetricVisitor {
- vespalib::xml::XmlOutputStream& _xos;
- int _verbosity;
-
-public:
- XmlWriter(vespalib::xml::XmlOutputStream& xos,
- uint32_t period, int verbosity);
-
- bool visitSnapshot(const MetricSnapshot&) override;
- void doneVisitingSnapshot(const MetricSnapshot&) override;
- bool visitMetricSet(const MetricSet& set, bool) override;
- void doneVisitingMetricSet(const MetricSet&) override;
- bool visitCountMetric(const AbstractCountMetric&, bool autoGenerated) override;
- bool visitValueMetric(const AbstractValueMetric&, bool autoGenerated) override;
-
-private:
- void printCommonXmlParts(const Metric& metric) const;
-};
-
-}
-
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java
index 27488e4027c..faa360bbcb1 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java
@@ -13,7 +13,6 @@ import com.yahoo.vespa.hosted.provision.persistence.CuratorDb;
import java.time.Duration;
import java.util.Optional;
import java.util.function.Function;
-import java.util.logging.Logger;
/**
* Thread safe class to get and set archive URI for given account and tenants.
@@ -22,7 +21,6 @@ import java.util.logging.Logger;
*/
public class ArchiveUriManager {
- private static final Logger log = Logger.getLogger(ArchiveUriManager.class.getName());
private static final Duration cacheTtl = Duration.ofMinutes(1);
private final CuratorDb db;
@@ -48,7 +46,14 @@ public class ArchiveUriManager {
archiveUris.get().accountArchiveUris().get(node.cloudAccount()) :
archiveUris.get().tenantArchiveUris().get(app.tenant()))
.map(uri -> {
+ // TODO (freva): Remove when all URIs dont have tenant name in them anymore
+ String tenantSuffix = "/" + app.tenant().value() + "/";
+ if (uri.endsWith(tenantSuffix)) return uri.substring(0, uri.length() - tenantSuffix.length() + 1);
+ return uri;
+ })
+ .map(uri -> {
StringBuilder sb = new StringBuilder(100).append(uri)
+ .append(app.tenant().value()).append('/')
.append(app.application().value()).append('/')
.append(app.instance().value()).append('/')
.append(node.allocation().get().membership().cluster().id().value()).append('/');
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java
index 9506bba73e7..2cc43a1eb33 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java
@@ -77,6 +77,10 @@ public class Autoscaling {
return peak.equals(Load.zero());
}
+ public boolean isPresent() {
+ return ! isEmpty();
+ }
+
@Override
public boolean equals(Object o) {
if ( ! (o instanceof Autoscaling other)) return false;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java
index 674c20e25f2..69c03dbf6dc 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java
@@ -2,6 +2,7 @@
package com.yahoo.vespa.hosted.provision.maintenance;
import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationLockException;
import com.yahoo.config.provision.ClusterResources;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.Deployer;
@@ -47,27 +48,30 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer {
@Override
protected double maintain() {
if ( ! nodeRepository().nodes().isWorking()) return 0.0;
-
- if ( ! nodeRepository().zone().environment().isAnyOf(Environment.dev, Environment.perf, Environment.prod)) return 1.0;
-
- activeNodesByApplication().forEach(this::autoscale);
- return 1.0;
- }
-
- private void autoscale(ApplicationId application, NodeList applicationNodes) {
- try {
- nodesByCluster(applicationNodes).forEach((clusterId, clusterNodes) -> autoscale(application, clusterId));
- }
- catch (IllegalArgumentException e) {
- throw new IllegalArgumentException("Illegal arguments for " + application, e);
+ if (nodeRepository().zone().environment().isTest()) return 1.0;
+
+ int attempts = 0;
+ int failures = 0;
+ for (var applicationNodes : activeNodesByApplication().entrySet()) {
+ for (var clusterNodes : nodesByCluster(applicationNodes.getValue()).entrySet()) {
+ attempts++;
+ if ( ! autoscale(applicationNodes.getKey(), clusterNodes.getKey()))
+ failures++;
+ }
}
+ return asSuccessFactor(attempts, failures);
}
- private void autoscale(ApplicationId applicationId, ClusterSpec.Id clusterId) {
+ /**
+ * Autoscales the given cluster.
+ *
+ * @return true if an autoscaling decision was made or nothing should be done, false if there was an error
+ */
+ private boolean autoscale(ApplicationId applicationId, ClusterSpec.Id clusterId) {
try (var lock = nodeRepository().applications().lock(applicationId)) {
Optional<Application> application = nodeRepository().applications().get(applicationId);
- if (application.isEmpty()) return;
- if (application.get().cluster(clusterId).isEmpty()) return;
+ if (application.isEmpty()) return true;
+ if (application.get().cluster(clusterId).isEmpty()) return true;
Cluster cluster = application.get().cluster(clusterId).get();
NodeList clusterNodes = nodeRepository().nodes().list(Node.State.active).owner(applicationId).cluster(clusterId);
@@ -79,7 +83,7 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer {
Autoscaling autoscaling = null;
if (cluster.target().resources().isEmpty() || current.equals(cluster.target().resources().get())) {
autoscaling = autoscaler.autoscale(application.get(), cluster, clusterNodes);
- if ( ! autoscaling.isEmpty()) // Ignore empties we'll get from servers recently started
+ if ( autoscaling.isPresent() || cluster.target().isEmpty()) // Ignore empty from recently started servers
cluster = cluster.withTarget(autoscaling);
}
@@ -94,6 +98,13 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer {
logAutoscaling(current, autoscaling.resources().get(), applicationId, clusterNodes.not().retired());
}
}
+ return true;
+ }
+ catch (ApplicationLockException e) {
+ return false;
+ }
+ catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException("Illegal arguments for " + applicationId + " cluster " + clusterId, e);
}
}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
index 67c1c7359f7..5af74214648 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java
@@ -267,13 +267,20 @@ public class MetricsReporter extends NodeRepositoryMaintainer {
}
private void updateNodeCountMetrics(NodeList nodes) {
- Map<State, List<Node>> nodesByState = nodes.nodeType(NodeType.tenant).asList().stream()
- .collect(Collectors.groupingBy(Node::state));
+ var nodesByState = nodes.nodeType(NodeType.tenant)
+ .asList().stream()
+ .collect(Collectors.groupingBy(Node::state));
+
+ var hostsByState = nodes.nodeType(NodeType.host)
+ .asList().stream()
+ .collect(Collectors.groupingBy(Node::state));
// Count per state
for (State state : State.values()) {
- List<Node> nodesInState = nodesByState.getOrDefault(state, List.of());
- metric.set("hostedVespa." + state.name() + "Hosts", nodesInState.size(), null);
+ var nodesInState = nodesByState.getOrDefault(state, List.of());
+ var hostsInState = hostsByState.getOrDefault(state, List.of());
+ metric.set("hostedVespa." + state.name() + "Nodes", nodesInState.size(), null);
+ metric.set("hostedVespa." + state.name() + "Hosts", hostsInState.size(), null);
}
}
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java
index 8ee72d12f57..44c1c976355 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java
@@ -54,10 +54,22 @@ public class ArchiveUriManagerTest {
assertFalse(archiveUriManager.archiveUriFor(createNode(null, account1)).isPresent()); // URI set for this account, but not allocated
assertFalse(archiveUriManager.archiveUriFor(createNode(null, account2)).isPresent()); // Not allocated
assertFalse(archiveUriManager.archiveUriFor(createNode(app2, null)).isPresent()); // No URI set for this tenant or account
- assertEquals("scheme://tenant-bucket/dir/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, null)).get());
- assertEquals("scheme://account-bucket/dir/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, account1)).get()); // Account has precedence
+ assertEquals("scheme://tenant-bucket/dir/vespa/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, null)).get());
+ assertEquals("scheme://account-bucket/dir/vespa/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, account1)).get()); // Account has precedence
assertFalse(archiveUriManager.archiveUriFor(createNode(app1, account2)).isPresent()); // URI set for this tenant, but is ignored because enclave account
- assertEquals("scheme://tenant-bucket/dir/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, accountSystem)).get()); // URI for tenant because non-enclave acocunt
+ assertEquals("scheme://tenant-bucket/dir/vespa/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, accountSystem)).get()); // URI for tenant because non-enclave acocunt
+ }
+
+ @Test
+ public void handles_uri_with_tenant_name() {
+ ApplicationId app1 = ApplicationId.from("vespa", "music", "main");
+ ArchiveUriManager archiveUriManager = new ProvisioningTester.Builder().build().nodeRepository().archiveUriManager();
+ archiveUriManager.setArchiveUri(app1.tenant(), Optional.of("scheme://tenant-bucket/vespa"));
+ assertEquals("scheme://tenant-bucket/vespa/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, null)).get());
+
+ // Archive URI ends with the tenant name
+ archiveUriManager.setArchiveUri(app1.tenant(), Optional.of("scheme://tenant-vespa/"));
+ assertEquals("scheme://tenant-vespa/vespa/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, null)).get());
}
private Node createNode(ApplicationId appId, CloudAccount account) {
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
index 44050fa747c..d7ffda542ff 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java
@@ -87,7 +87,7 @@ public class MetricsReporterTest {
Map<String, Number> expectedMetrics = new TreeMap<>();
expectedMetrics.put("zone.working", 1);
- expectedMetrics.put("hostedVespa.provisionedHosts", 1);
+ expectedMetrics.put("hostedVespa.provisionedHosts", 0);
expectedMetrics.put("hostedVespa.parkedHosts", 0);
expectedMetrics.put("hostedVespa.readyHosts", 0);
expectedMetrics.put("hostedVespa.reservedHosts", 0);
@@ -97,6 +97,16 @@ public class MetricsReporterTest {
expectedMetrics.put("hostedVespa.failedHosts", 0);
expectedMetrics.put("hostedVespa.deprovisionedHosts", 0);
expectedMetrics.put("hostedVespa.breakfixedHosts", 0);
+ expectedMetrics.put("hostedVespa.provisionedNodes", 1);
+ expectedMetrics.put("hostedVespa.parkedNodes", 0);
+ expectedMetrics.put("hostedVespa.readyNodes", 0);
+ expectedMetrics.put("hostedVespa.reservedNodes", 0);
+ expectedMetrics.put("hostedVespa.activeNodes", 0);
+ expectedMetrics.put("hostedVespa.inactiveNodes", 0);
+ expectedMetrics.put("hostedVespa.dirtyNodes", 0);
+ expectedMetrics.put("hostedVespa.failedNodes", 0);
+ expectedMetrics.put("hostedVespa.deprovisionedNodes", 0);
+ expectedMetrics.put("hostedVespa.breakfixedNodes", 0);
expectedMetrics.put("hostedVespa.pendingRedeployments", 42);
expectedMetrics.put("hostedVespa.docker.totalCapacityDisk", 0.0);
expectedMetrics.put("hostedVespa.docker.totalCapacityMem", 0.0);
@@ -207,8 +217,8 @@ public class MetricsReporterTest {
MetricsReporter metricsReporter = metricsReporter(metric, tester);
metricsReporter.maintain();
- assertEquals(0, metric.values.get("hostedVespa.readyHosts")); // Only tenants counts
- assertEquals(2, metric.values.get("hostedVespa.reservedHosts"));
+ assertEquals(0, metric.values.get("hostedVespa.readyNodes")); // Only tenants counts
+ assertEquals(2, metric.values.get("hostedVespa.reservedNodes"));
assertEquals(120.0, metric.values.get("hostedVespa.docker.totalCapacityDisk"));
assertEquals(100.0, metric.values.get("hostedVespa.docker.totalCapacityMem"));
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java
index 80d79b036e2..03581146b9f 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java
@@ -45,8 +45,8 @@ public class ArchiveApiTest {
"{\"message\":\"Updated archive URI for 777888999000\"}");
- tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "\"archiveUri\":\"ftp://host/dir/application3/instance3/id3/host4/\"", true);
- tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com"), "\"archiveUri\":\"s3://acc-bucket/zoneapp/zoneapp/node-admin/dockerhost2/\"", true);
+ tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "\"archiveUri\":\"ftp://host/dir/tenant3/application3/instance3/id3/host4/\"", true);
+ tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com"), "\"archiveUri\":\"s3://acc-bucket/zoneapp/zoneapp/zoneapp/node-admin/dockerhost2/\"", true);
assertFile(new Request("http://localhost:8080/nodes/v2/archive"), "archives.json");
tester.assertResponse(new Request("http://localhost:8080/nodes/v2/archive/tenant/tenant3", new byte[0], Request.Method.DELETE), "{\"message\":\"Removed archive URI for tenant3\"}");
diff --git a/parent/pom.xml b/parent/pom.xml
index 2fcf5527302..273e9b0ef8f 100644
--- a/parent/pom.xml
+++ b/parent/pom.xml
@@ -1170,7 +1170,7 @@
<prometheus.client.version>0.6.0</prometheus.client.version>
<protobuf.version>3.21.7</protobuf.version>
<spifly.version>1.3.5</spifly.version>
- <surefire.version>2.22.2</surefire.version>
+ <surefire.version>3.0.0-M9</surefire.version>
<wiremock.version>2.35.0</wiremock.version>
<zero-allocation-hashing.version>0.16</zero-allocation-hashing.version>
<zookeeper.client.version>3.8.0</zookeeper.client.version>
diff --git a/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h b/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h
index 177d19b06f9..cd4d1686eeb 100644
--- a/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h
+++ b/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h
@@ -3,13 +3,14 @@
#pragma once
#include <vespa/searchlib/queryeval/begin_and_end_id.h>
-#include <vespa/fastos/types.h>
#include <mutex>
#include <condition_variable>
#include <atomic>
#include <algorithm>
#include <vector>
+#define VESPA_DLL_LOCAL __attribute__ ((visibility("hidden")))
+
namespace proton::matching {
/**
diff --git a/searchlib/src/vespa/searchlib/common/bitvector.h b/searchlib/src/vespa/searchlib/common/bitvector.h
index 67f3e2ad502..149e57390af 100644
--- a/searchlib/src/vespa/searchlib/common/bitvector.h
+++ b/searchlib/src/vespa/searchlib/common/bitvector.h
@@ -3,10 +3,8 @@
#pragma once
#include "bitword.h"
-#include <memory>
#include <vespa/vespalib/util/alloc.h>
#include <vespa/vespalib/util/atomic.h>
-#include <vespa/fastos/types.h>
#include <algorithm>
#include <cassert>
diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_posting_list.h b/searchlib/src/vespa/searchlib/predicate/predicate_posting_list.h
index 0bf33f1d0e5..0de9be332fa 100644
--- a/searchlib/src/vespa/searchlib/predicate/predicate_posting_list.h
+++ b/searchlib/src/vespa/searchlib/predicate/predicate_posting_list.h
@@ -3,7 +3,8 @@
#include <memory>
#include <cstdint>
-#include <vespa/fastos/types.h>
+
+#define VESPA_DLL_LOCAL __attribute__ ((visibility("hidden")))
/**
* Interface for posting lists used by PredicateSearch.
@@ -25,7 +26,7 @@ protected:
public:
using UP = std::unique_ptr<PredicatePostingList>;
- virtual ~PredicatePostingList() {}
+ virtual ~PredicatePostingList() = default;
/*
* Moves to next document after the one supplied.
diff --git a/searchlib/src/vespa/searchlib/queryeval/hitcollector.h b/searchlib/src/vespa/searchlib/queryeval/hitcollector.h
index fe686f3e0bf..e244c856c4a 100644
--- a/searchlib/src/vespa/searchlib/queryeval/hitcollector.h
+++ b/searchlib/src/vespa/searchlib/queryeval/hitcollector.h
@@ -9,7 +9,6 @@
#include <vespa/vespalib/util/sort.h>
#include <algorithm>
#include <vector>
-#include <vespa/fastos/types.h>
namespace search::queryeval {
diff --git a/storage/src/tests/common/metricstest.cpp b/storage/src/tests/common/metricstest.cpp
index 78fa32e24e5..7231a071319 100644
--- a/storage/src/tests/common/metricstest.cpp
+++ b/storage/src/tests/common/metricstest.cpp
@@ -52,8 +52,7 @@ namespace {
{
framework::Clock& _clock;
explicit MetricClock(framework::Clock& c) : _clock(c) {}
- [[nodiscard]] time_t getTime() const override { return vespalib::count_s(_clock.getMonotonicTime().time_since_epoch()); }
- [[nodiscard]] time_t getTimeInMilliSecs() const override { return vespalib::count_ms(_clock.getMonotonicTime().time_since_epoch()); }
+ [[nodiscard]] metrics::time_point getTime() const override { return _clock.getSystemTime(); }
};
}
@@ -85,10 +84,7 @@ void MetricsTest::SetUp() {
_metricManager->registerMetric(guard, *_topSet);
}
- _metricsConsumer = std::make_unique<StatusMetricConsumer>(
- _node->getComponentRegister(),
- *_metricManager,
- "status");
+ _metricsConsumer = std::make_unique<StatusMetricConsumer>(_node->getComponentRegister(), *_metricManager, "status");
_filestorMetrics = std::make_shared<FileStorMetrics>();
_filestorMetrics->initDiskMetrics(1, 1);
diff --git a/storage/src/tests/storageserver/statereportertest.cpp b/storage/src/tests/storageserver/statereportertest.cpp
index d6b528e5a25..3a772c1ddde 100644
--- a/storage/src/tests/storageserver/statereportertest.cpp
+++ b/storage/src/tests/storageserver/statereportertest.cpp
@@ -53,8 +53,7 @@ struct MetricClock : public metrics::MetricManager::Timer
{
framework::Clock& _clock;
explicit MetricClock(framework::Clock& c) : _clock(c) {}
- [[nodiscard]] time_t getTime() const override { return vespalib::count_s(_clock.getMonotonicTime().time_since_epoch()); }
- [[nodiscard]] time_t getTimeInMilliSecs() const override { return vespalib::count_ms(_clock.getMonotonicTime().time_since_epoch()); }
+ [[nodiscard]] metrics::time_point getTime() const override { return _clock.getSystemTime(); }
};
}
@@ -85,11 +84,8 @@ void StateReporterTest::SetUp() {
_metricManager->registerMetric(guard, *_topSet);
}
- _stateReporter = std::make_unique<StateReporter>(
- _node->getComponentRegister(),
- *_metricManager,
- _generationFetcher,
- "status");
+ _stateReporter = std::make_unique<StateReporter>(_node->getComponentRegister(), *_metricManager,
+ _generationFetcher, "status");
_filestorMetrics = std::make_shared<FileStorMetrics>();
_filestorMetrics->initDiskMetrics(1, 1);
@@ -125,20 +121,14 @@ vespalib::Slime slime; \
#define ASSERT_GENERATION(jsonData, component, generation) \
{ \
PARSE_JSON(jsonData); \
- ASSERT_EQ( \
- generation, \
- slime.get()["config"][component]["generation"].asDouble()); \
+ ASSERT_EQ(generation, slime.get()["config"][component]["generation"].asDouble()); \
}
#define ASSERT_NODE_STATUS(jsonData, code, message) \
{ \
PARSE_JSON(jsonData); \
- ASSERT_EQ( \
- vespalib::string(code), \
- slime.get()["status"]["code"].asString().make_string()); \
- ASSERT_EQ( \
- vespalib::string(message), \
- slime.get()["status"]["message"].asString().make_string()); \
+ ASSERT_EQ(vespalib::string(code), slime.get()["status"]["code"].asString().make_string()); \
+ ASSERT_EQ(vespalib::string(message), slime.get()["status"]["message"].asString().make_string()); \
}
#define ASSERT_METRIC_GET_PUT(jsonData, expGetCount, expPutCount) \
@@ -148,16 +138,11 @@ vespalib::Slime slime; \
double putCount = -1; \
size_t metricCount = slime.get()["metrics"]["values"].children(); \
for (size_t j=0; j<metricCount; j++) { \
- const vespalib::string name = slime.get()["metrics"]["values"][j]["name"] \
- .asString().make_string(); \
- if (name.compare("vds.filestor.allthreads.get.count") == 0) \
- { \
- getCount = slime.get()["metrics"]["values"][j]["values"]["count"] \
- .asDouble(); \
- } else if (name.compare("vds.filestor.allthreads.put.count") == 0) \
- { \
- putCount = slime.get()["metrics"]["values"][j]["values"]["count"] \
- .asDouble(); \
+ const vespalib::string name = slime.get()["metrics"]["values"][j]["name"].asString().make_string(); \
+ if (name.compare("vds.filestor.allthreads.get.count") == 0) { \
+ getCount = slime.get()["metrics"]["values"][j]["values"]["count"].asDouble(); \
+ } else if (name.compare("vds.filestor.allthreads.put.count") == 0) { \
+ putCount = slime.get()["metrics"]["values"][j]["values"]["count"].asDouble(); \
} \
} \
ASSERT_EQ(expGetCount, getCount); \
@@ -226,8 +211,7 @@ TEST_F(StateReporterTest, report_metrics) {
for (uint32_t i = 0; i < 6; ++i) {
_clock->addSecondsToTime(60);
_metricManager->timeChangedNotification();
- while (int64_t(_metricManager->getLastProcessedTime()) < vespalib::count_s(_clock->getMonotonicTime().time_since_epoch()))
- {
+ while (int64_t(_metricManager->getLastProcessedTime()) < vespalib::count_s(_clock->getMonotonicTime().time_since_epoch())) {
std::this_thread::sleep_for(1ms);
}
}
diff --git a/storage/src/vespa/storage/common/statusmetricconsumer.cpp b/storage/src/vespa/storage/common/statusmetricconsumer.cpp
index 8eb3e9f3ab6..c6f73540605 100644
--- a/storage/src/vespa/storage/common/statusmetricconsumer.cpp
+++ b/storage/src/vespa/storage/common/statusmetricconsumer.cpp
@@ -5,7 +5,6 @@
#include <boost/lexical_cast.hpp>
#include <vespa/metrics/jsonwriter.h>
#include <vespa/metrics/textwriter.h>
-#include <vespa/metrics/xmlwriter.h>
#include <vespa/metrics/metricmanager.h>
#include <vespa/storageapi/messageapi/storagemessage.h>
#include <vespa/vespalib/stllike/asciistream.h>
@@ -37,10 +36,6 @@ StatusMetricConsumer::getReportContentType(const framework::HttpUrlPath& path) c
return "text/plain";
}
- if (path.getAttribute("format") == "xml") {
- return "application/xml";
- }
-
if (path.getAttribute("format") == "text") {
return "text/plain";
}
@@ -67,7 +62,6 @@ StatusMetricConsumer::reportStatus(std::ostream& out,
LOG(debug, "Not calling update hooks as dontcallupdatehooks option has been given");
}
int64_t currentTimeS(vespalib::count_s(_component.getClock().getMonotonicTime().time_since_epoch()));
- bool xml = (path.getAttribute("format") == "xml");
bool json = (path.getAttribute("format") == "json");
int verbosity(path.get("verbosity", 0));
@@ -131,13 +125,7 @@ StatusMetricConsumer::reportStatus(std::ostream& out,
}
std::string consumer = path.getAttribute("consumer", "");
- if (xml) {
- out << "<?xml version=\"1.0\"?>\n";
- vespalib::XmlOutputStream xos(out);
- metrics::XmlWriter xmlWriter(xos, snapshot->getPeriod(), verbosity);
- _manager.visit(metricLock, *snapshot, xmlWriter, consumer);
- out << "\n";
- } else if (json) {
+ if (json) {
vespalib::asciistream jsonStreamData;
vespalib::JsonStream stream(jsonStreamData, true);
stream << Object() << "metrics";
diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp
index a09abb25f7a..9f8456afc37 100644
--- a/storage/src/vespa/storage/storageserver/storagenode.cpp
+++ b/storage/src/vespa/storage/storageserver/storagenode.cpp
@@ -33,34 +33,36 @@ namespace storage {
namespace {
- using vespalib::getLastErrorString;
+using vespalib::getLastErrorString;
- void writePidFile(const vespalib::string& pidfile)
- {
- ssize_t rv = -1;
- vespalib::string mypid = vespalib::make_string("%d\n", getpid());
- size_t lastSlash = pidfile.rfind('/');
- if (lastSlash != vespalib::string::npos) {
- std::filesystem::create_directories(std::filesystem::path(pidfile.substr(0, lastSlash)));
- }
- int fd = open(pidfile.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644);
- if (fd != -1) {
- rv = write(fd, mypid.c_str(), mypid.size());
- close(fd);
- }
- if (rv < 1) {
- LOG(warning, "Failed to write pidfile '%s': %s",
- pidfile.c_str(), getLastErrorString().c_str());
- }
+void
+writePidFile(const vespalib::string& pidfile)
+{
+ ssize_t rv = -1;
+ vespalib::string mypid = vespalib::make_string("%d\n", getpid());
+ size_t lastSlash = pidfile.rfind('/');
+ if (lastSlash != vespalib::string::npos) {
+ std::filesystem::create_directories(std::filesystem::path(pidfile.substr(0, lastSlash)));
+ }
+ int fd = open(pidfile.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644);
+ if (fd != -1) {
+ rv = write(fd, mypid.c_str(), mypid.size());
+ close(fd);
}
+ if (rv < 1) {
+ LOG(warning, "Failed to write pidfile '%s': %s",
+ pidfile.c_str(), getLastErrorString().c_str());
+ }
+}
- void removePidFile(const vespalib::string& pidfile)
- {
- if (unlink(pidfile.c_str()) != 0) {
- LOG(warning, "Failed to delete pidfile '%s': %s",
- pidfile.c_str(), getLastErrorString().c_str());
- }
+void
+removePidFile(const vespalib::string& pidfile)
+{
+ if (unlink(pidfile.c_str()) != 0) {
+ LOG(warning, "Failed to delete pidfile '%s': %s",
+ pidfile.c_str(), getLastErrorString().c_str());
}
+}
} // End of anonymous namespace
@@ -429,7 +431,8 @@ StorageNode::shutdown()
LOG(debug, "Done shutting down node");
}
-void StorageNode::configure(std::unique_ptr<StorServerConfig> config) {
+void
+StorageNode::configure(std::unique_ptr<StorServerConfig> config) {
log_config_received(*config);
// When we get config, we try to grab the config lock to ensure noone
// else is doing configuration work, and then we write the new config
@@ -445,7 +448,8 @@ void StorageNode::configure(std::unique_ptr<StorServerConfig> config) {
}
}
-void StorageNode::configure(std::unique_ptr<UpgradingConfig> config) {
+void
+StorageNode::configure(std::unique_ptr<UpgradingConfig> config) {
log_config_received(*config);
{
std::lock_guard configLockGuard(_configLock);
@@ -457,7 +461,8 @@ void StorageNode::configure(std::unique_ptr<UpgradingConfig> config) {
}
}
-void StorageNode::configure(std::unique_ptr<StorDistributionConfig> config) {
+void
+StorageNode::configure(std::unique_ptr<StorDistributionConfig> config) {
log_config_received(*config);
{
std::lock_guard configLockGuard(_configLock);
@@ -486,7 +491,8 @@ StorageNode::configure(std::unique_ptr<document::config::DocumenttypesConfig> co
}
}
-void StorageNode::configure(std::unique_ptr<BucketspacesConfig> config) {
+void
+StorageNode::configure(std::unique_ptr<BucketspacesConfig> config) {
log_config_received(*config);
{
std::lock_guard configLockGuard(_configLock);
diff --git a/storage/src/vespa/storageapi/mbusprot/serializationhelper.h b/storage/src/vespa/storageapi/mbusprot/serializationhelper.h
index 457a6178704..671ffbddd6f 100644
--- a/storage/src/vespa/storageapi/mbusprot/serializationhelper.h
+++ b/storage/src/vespa/storageapi/mbusprot/serializationhelper.h
@@ -1,7 +1,6 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
#pragma once
-#include <vespa/fastos/types.h>
#include <vespa/document/base/globalid.h>
#include <vespa/document/fieldvalue/document.h>
#include <vespa/document/util/bytebuffer.h>
@@ -13,12 +12,6 @@ namespace storage::mbusprot {
class SerializationHelper
{
public:
- static int64_t getLong(document::ByteBuffer& buf) {
- int64_t tmp;
- buf.getLongNetwork(tmp);
- return tmp;
- }
-
static int32_t getInt(document::ByteBuffer& buf) {
int32_t tmp;
buf.getIntNetwork(tmp);
@@ -46,26 +39,6 @@ public:
return s;
}
- static bool getBoolean(document::ByteBuffer& buf) {
- uint8_t tmp;
- buf.getByte(tmp);
- return (tmp == 1);
- }
-
- static api::ReturnCode getReturnCode(document::ByteBuffer& buf) {
- api::ReturnCode::Result result = (api::ReturnCode::Result) getInt(buf);
- vespalib::stringref message = getString(buf);
- return api::ReturnCode(result, message);
- }
-
- static void putReturnCode(const api::ReturnCode& code, vespalib::GrowableByteBuffer& buf)
- {
- buf.putInt(code.getResult());
- buf.putString(code.getMessage());
- }
-
- static const uint32_t BUCKET_INFO_SERIALIZED_SIZE = sizeof(uint32_t) * 3;
-
static document::GlobalId getGlobalId(document::ByteBuffer& buf) {
std::vector<char> buffer(getShort(buf));
for (uint32_t i=0; i<buffer.size(); ++i) {
@@ -74,13 +47,6 @@ public:
return document::GlobalId(&buffer[0]);
}
- static void putGlobalId(const document::GlobalId& gid, vespalib::GrowableByteBuffer& buf)
- {
- buf.putShort(document::GlobalId::LENGTH);
- for (uint32_t i=0; i<document::GlobalId::LENGTH; ++i) {
- buf.putByte(gid.get()[i]);
- }
- }
static document::Document::UP getDocument(document::ByteBuffer& buf, const document::DocumentTypeRepo& repo)
{
uint32_t size = getInt(buf);
diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java
index 1e2c0900ff7..33e46ebc75f 100644
--- a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java
+++ b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java
@@ -93,7 +93,7 @@ public abstract class Maintainer implements Runnable {
*
* @return the degree to which the run was successful - a number between 0 (no success), to 1 (complete success).
* Note that this indicates whether something is wrong, so e.g if the call did nothing because it should do
- * nothing, 1.0 should be returned.
+ * nothing, 1.0 should be returned.
*/
protected abstract double maintain();
diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt
index 2720d8786cb..8f1b687449a 100644
--- a/vespalib/CMakeLists.txt
+++ b/vespalib/CMakeLists.txt
@@ -26,6 +26,8 @@ vespa_define_module(
src/apps/vespa-validate-hostname
TESTS
+ ${VESPALIB_DIRECTIO_TESTDIR}
+ ${VESPALIB_PROCESS_MEMORY_STATS_TESTDIR}
src/tests/alloc
src/tests/approx
src/tests/array
@@ -38,9 +40,9 @@ vespa_define_module(
src/tests/bits
src/tests/box
src/tests/btree
- src/tests/btree/btree_store
src/tests/btree/btree-scan-speed
src/tests/btree/btree-stress
+ src/tests/btree/btree_store
src/tests/clock
src/tests/component
src/tests/compress
@@ -73,7 +75,6 @@ vespa_define_module(
src/tests/datastore/unique_store
src/tests/datastore/unique_store_dictionary
src/tests/datastore/unique_store_string_allocator
- ${VESPALIB_DIRECTIO_TESTDIR}
src/tests/detect_type_benchmark
src/tests/dotproduct
src/tests/drop-file-from-cache
@@ -86,6 +87,9 @@ vespa_define_module(
src/tests/executor_idle_tracking
src/tests/explore_modern_cpp
src/tests/false
+ src/tests/fastlib/io
+ src/tests/fastlib/text
+ src/tests/fastos
src/tests/fiddle
src/tests/fileheader
src/tests/floatingpointtype
@@ -95,6 +99,7 @@ vespa_define_module(
src/tests/guard
src/tests/host_name
src/tests/hwaccelrated
+ src/tests/invokeservice
src/tests/io/fileutil
src/tests/io/mapped_file_input
src/tests/issue
@@ -134,7 +139,6 @@ vespa_define_module(
src/tests/printable
src/tests/priority_queue
src/tests/process
- ${VESPALIB_PROCESS_MEMORY_STATS_TESTDIR}
src/tests/programoptions
src/tests/random
src/tests/referencecounter
@@ -144,14 +148,14 @@ vespa_define_module(
src/tests/runnable_pair
src/tests/rusage
src/tests/sequencedtaskexecutor
- src/tests/shutdownguard
- src/tests/singleexecutor
src/tests/sha1
src/tests/shared_operation_throttler
src/tests/shared_string_repo
src/tests/sharedptr
+ src/tests/shutdownguard
src/tests/signalhandler
src/tests/simple_thread_bundle
+ src/tests/singleexecutor
src/tests/slime
src/tests/slime/external_data_value
src/tests/slime/summary-feature-benchmark
@@ -204,17 +208,15 @@ vespa_define_module(
src/tests/util/string_escape
src/tests/valgrind
src/tests/visit_ranges
- src/tests/invokeservice
src/tests/wakeup
src/tests/xmlserializable
src/tests/zcurve
- src/tests/fastlib/io
- src/tests/fastlib/text
LIBS
src/vespa/fastlib/io
src/vespa/fastlib/text
src/vespa/fastlib/text/apps
+ src/vespa/fastos
src/vespa/vespalib
src/vespa/vespalib/btree
src/vespa/vespalib/component
diff --git a/vespalib/src/tests/fastos/CMakeLists.txt b/vespalib/src/tests/fastos/CMakeLists.txt
new file mode 100644
index 00000000000..6ea986ac0b0
--- /dev/null
+++ b/vespalib/src/tests/fastos/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_executable(fastos_file_test_app TEST
+ SOURCES
+ file_test.cpp
+ DEPENDS
+ vespalib
+ GTest::GTest
+)
+vespa_add_test(NAME fastos_file_test_app COMMAND fastos_file_test_app)
diff --git a/fastos/src/tests/filetest.cpp b/vespalib/src/tests/fastos/file_test.cpp
index d0f8bbfd98b..d0f8bbfd98b 100644
--- a/fastos/src/tests/filetest.cpp
+++ b/vespalib/src/tests/fastos/file_test.cpp
diff --git a/fastos/src/tests/hello.txt b/vespalib/src/tests/fastos/hello.txt
index 62a405393a6..62a405393a6 100644
--- a/fastos/src/tests/hello.txt
+++ b/vespalib/src/tests/fastos/hello.txt
diff --git a/fastos/src/tests/tests.h b/vespalib/src/tests/fastos/tests.h
index 9cd7a10ab48..9cd7a10ab48 100644
--- a/fastos/src/tests/tests.h
+++ b/vespalib/src/tests/fastos/tests.h
diff --git a/vespalib/src/vespa/fastlib/io/CMakeLists.txt b/vespalib/src/vespa/fastlib/io/CMakeLists.txt
index f21cf27b21e..9f6211f3e19 100644
--- a/vespalib/src/vespa/fastlib/io/CMakeLists.txt
+++ b/vespalib/src/vespa/fastlib/io/CMakeLists.txt
@@ -1,5 +1,5 @@
# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_library(fastlib_io OBJECT
+vespa_add_library(vespalib_fastlib_io OBJECT
SOURCES
bufferedfile.cpp
DEPENDS
diff --git a/vespalib/src/vespa/fastlib/text/CMakeLists.txt b/vespalib/src/vespa/fastlib/text/CMakeLists.txt
index d6cb8c29305..06573ee814d 100644
--- a/vespalib/src/vespa/fastlib/text/CMakeLists.txt
+++ b/vespalib/src/vespa/fastlib/text/CMakeLists.txt
@@ -1,5 +1,5 @@
# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-vespa_add_library(fastlib_text OBJECT
+vespa_add_library(vespalib_fastlib_text OBJECT
SOURCES
unicodeutil.cpp
normwordfolder.cpp
diff --git a/vespalib/src/vespa/fastos/CMakeLists.txt b/vespalib/src/vespa/fastos/CMakeLists.txt
new file mode 100644
index 00000000000..2b7a2c3b905
--- /dev/null
+++ b/vespalib/src/vespa/fastos/CMakeLists.txt
@@ -0,0 +1,9 @@
+# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+vespa_add_library(vespalib_fastos OBJECT
+ SOURCES
+ file.cpp
+ file_rw_ops.cpp
+ linux_file.cpp
+ unix_file.cpp
+ DEPENDS
+)
diff --git a/fastos/src/vespa/fastos/file.cpp b/vespalib/src/vespa/fastos/file.cpp
index fdbacb570b4..fdbacb570b4 100644
--- a/fastos/src/vespa/fastos/file.cpp
+++ b/vespalib/src/vespa/fastos/file.cpp
diff --git a/fastos/src/vespa/fastos/file.h b/vespalib/src/vespa/fastos/file.h
index 1cf6fee71dd..146127d4fe6 100644
--- a/fastos/src/vespa/fastos/file.h
+++ b/vespalib/src/vespa/fastos/file.h
@@ -10,10 +10,11 @@
#pragma once
-#include "types.h"
#include <cstdint>
#include <string>
+#define FASTOS_PREFIX(a) FastOS_##a
+
constexpr int FASTOS_FILE_OPEN_READ = (1<<0);
constexpr int FASTOS_FILE_OPEN_WRITE = (1<<1);
constexpr int FASTOS_FILE_OPEN_EXISTING = (1<<2);
diff --git a/fastos/src/vespa/fastos/file_rw_ops.cpp b/vespalib/src/vespa/fastos/file_rw_ops.cpp
index 79fe95b21f2..79fe95b21f2 100644
--- a/fastos/src/vespa/fastos/file_rw_ops.cpp
+++ b/vespalib/src/vespa/fastos/file_rw_ops.cpp
diff --git a/fastos/src/vespa/fastos/file_rw_ops.h b/vespalib/src/vespa/fastos/file_rw_ops.h
index 4f7aa6f082f..4f7aa6f082f 100644
--- a/fastos/src/vespa/fastos/file_rw_ops.h
+++ b/vespalib/src/vespa/fastos/file_rw_ops.h
diff --git a/fastos/src/vespa/fastos/linux_file.cpp b/vespalib/src/vespa/fastos/linux_file.cpp
index 6fb29782957..6fb29782957 100644
--- a/fastos/src/vespa/fastos/linux_file.cpp
+++ b/vespalib/src/vespa/fastos/linux_file.cpp
diff --git a/fastos/src/vespa/fastos/linux_file.h b/vespalib/src/vespa/fastos/linux_file.h
index 2481b163210..2481b163210 100644
--- a/fastos/src/vespa/fastos/linux_file.h
+++ b/vespalib/src/vespa/fastos/linux_file.h
diff --git a/fastos/src/vespa/fastos/unix_file.cpp b/vespalib/src/vespa/fastos/unix_file.cpp
index 7c4cde19125..7c4cde19125 100644
--- a/fastos/src/vespa/fastos/unix_file.cpp
+++ b/vespalib/src/vespa/fastos/unix_file.cpp
diff --git a/fastos/src/vespa/fastos/unix_file.h b/vespalib/src/vespa/fastos/unix_file.h
index 31e45f8d2fa..31e45f8d2fa 100644
--- a/fastos/src/vespa/fastos/unix_file.h
+++ b/vespalib/src/vespa/fastos/unix_file.h
diff --git a/vespalib/src/vespa/vespalib/CMakeLists.txt b/vespalib/src/vespa/vespalib/CMakeLists.txt
index 9e31084c067..63876d442ef 100644
--- a/vespalib/src/vespa/vespalib/CMakeLists.txt
+++ b/vespalib/src/vespa/vespalib/CMakeLists.txt
@@ -30,8 +30,9 @@ vespa_add_library(vespalib
$<TARGET_OBJECTS:vespalib_vespalib_time>
$<TARGET_OBJECTS:vespalib_vespalib_trace>
$<TARGET_OBJECTS:vespalib_vespalib_util>
- $<TARGET_OBJECTS:fastlib_io>
- $<TARGET_OBJECTS:fastlib_text>
+ $<TARGET_OBJECTS:vespalib_fastlib_io>
+ $<TARGET_OBJECTS:vespalib_fastlib_text>
+ $<TARGET_OBJECTS:vespalib_fastos>
INSTALL lib64
DEPENDS
${VESPA_GCC_LIB}
diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp b/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp
index f5d87e14802..3bdbb7a81ff 100644
--- a/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp
+++ b/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp
@@ -3,7 +3,6 @@
#pragma once
#include "private_helpers.hpp"
-#include <vespa/fastos/types.h>
namespace vespalib::hwaccelrated::avx {
diff --git a/vespalib/src/vespa/vespalib/net/wakeup_pipe.cpp b/vespalib/src/vespa/vespalib/net/wakeup_pipe.cpp
index 60900289e79..b8025bfcf9f 100644
--- a/vespalib/src/vespa/vespalib/net/wakeup_pipe.cpp
+++ b/vespalib/src/vespa/vespalib/net/wakeup_pipe.cpp
@@ -2,34 +2,40 @@
#include "wakeup_pipe.h"
#include "socket_utils.h"
+#include <vespa/vespalib/util/require.h>
#include <unistd.h>
namespace vespalib {
WakeupPipe::WakeupPipe()
- : _pipe()
+ : _reader(),
+ _writer()
{
- socketutils::nonblocking_pipe(_pipe);
+ int pipe[2];
+ socketutils::nonblocking_pipe(pipe);
+ _reader.reset(pipe[0]);
+ _writer.reset(pipe[1]);
}
-WakeupPipe::~WakeupPipe()
-{
- close(_pipe[0]);
- close(_pipe[1]);
-}
+WakeupPipe::~WakeupPipe() = default;
void
WakeupPipe::write_token()
{
char token = 'T';
- [[maybe_unused]] ssize_t res = write(_pipe[1], &token, 1);
+ ssize_t res = _writer.write(&token, 1);
+ if (res < 0) {
+ res = -errno;
+ }
+ REQUIRE(res > 0 || res == -EAGAIN || res == -EWOULDBLOCK);
}
void
WakeupPipe::read_tokens()
{
char token_trash[128];
- [[maybe_unused]] ssize_t res = read(_pipe[0], token_trash, sizeof(token_trash));
+ ssize_t res = _reader.read(token_trash, sizeof(token_trash));
+ REQUIRE(res > 0);
}
}
diff --git a/vespalib/src/vespa/vespalib/net/wakeup_pipe.h b/vespalib/src/vespa/vespalib/net/wakeup_pipe.h
index b52f7f9e32d..36c88b205c0 100644
--- a/vespalib/src/vespa/vespalib/net/wakeup_pipe.h
+++ b/vespalib/src/vespa/vespalib/net/wakeup_pipe.h
@@ -2,6 +2,8 @@
#pragma once
+#include "socket_handle.h"
+
namespace vespalib {
//-----------------------------------------------------------------------------
@@ -15,11 +17,12 @@ namespace vespalib {
**/
class WakeupPipe {
private:
- int _pipe[2];
+ SocketHandle _reader;
+ SocketHandle _writer;
public:
WakeupPipe();
~WakeupPipe();
- int get_read_fd() const { return _pipe[0]; }
+ int get_read_fd() const { return _reader.get(); }
void write_token();
void read_tokens();
};
diff --git a/vespalib/src/vespa/vespalib/portal/http_connection.cpp b/vespalib/src/vespa/vespalib/portal/http_connection.cpp
index 6ea56e2659c..3d8edf2fc2e 100644
--- a/vespalib/src/vespa/vespalib/portal/http_connection.cpp
+++ b/vespalib/src/vespa/vespalib/portal/http_connection.cpp
@@ -245,14 +245,16 @@ HttpConnection::handle_event(bool, bool)
}
void
-HttpConnection::respond_with_content(const vespalib::string &content_type,
- const vespalib::string &content)
+HttpConnection::respond_with_content(vespalib::stringref content_type,
+ vespalib::stringref content)
{
{
OutputWriter dst(_output, CHUNK_SIZE);
dst.printf("HTTP/1.1 200 OK\r\n");
dst.printf("Connection: close\r\n");
- dst.printf("Content-Type: %s\r\n", content_type.c_str());
+ dst.printf("Content-Type: ");
+ dst.write(content_type.data(), content_type.size());
+ dst.printf("\r\n");
dst.printf("Content-Length: %zu\r\n", content.size());
emit_http_security_headers(dst);
dst.printf("\r\n");
@@ -263,11 +265,13 @@ HttpConnection::respond_with_content(const vespalib::string &content_type,
}
void
-HttpConnection::respond_with_error(int code, const vespalib::string &msg)
+HttpConnection::respond_with_error(int code, vespalib::stringref msg)
{
{
OutputWriter dst(_output, CHUNK_SIZE);
- dst.printf("HTTP/1.1 %d %s\r\n", code, msg.c_str());
+ dst.printf("HTTP/1.1 %d ", code);
+ dst.write(msg.data(), msg.size());
+ dst.printf("\r\n");
dst.printf("Connection: close\r\n");
dst.printf("\r\n");
}
diff --git a/vespalib/src/vespa/vespalib/portal/http_connection.h b/vespalib/src/vespa/vespalib/portal/http_connection.h
index 03d23351e7d..8540cb87e1d 100644
--- a/vespalib/src/vespa/vespalib/portal/http_connection.h
+++ b/vespalib/src/vespa/vespalib/portal/http_connection.h
@@ -53,9 +53,9 @@ public:
// Precondition: handshake must have been completed
const net::ConnectionAuthContext &auth_context() const noexcept { return *_auth_ctx; }
- void respond_with_content(const vespalib::string &content_type,
- const vespalib::string &content);
- void respond_with_error(int code, const vespalib::string &msg);
+ void respond_with_content(vespalib::stringref content_type,
+ vespalib::stringref content);
+ void respond_with_error(int code, const vespalib::stringref msg);
};
} // namespace vespalib::portal
diff --git a/vespalib/src/vespa/vespalib/portal/portal.cpp b/vespalib/src/vespa/vespalib/portal/portal.cpp
index a98562f6504..691ddaef495 100644
--- a/vespalib/src/vespa/vespalib/portal/portal.cpp
+++ b/vespalib/src/vespa/vespalib/portal/portal.cpp
@@ -77,8 +77,8 @@ Portal::GetRequest::export_params() const
}
void
-Portal::GetRequest::respond_with_content(const vespalib::string &content_type,
- const vespalib::string &content)
+Portal::GetRequest::respond_with_content(vespalib::stringref content_type,
+ vespalib::stringref content)
{
assert(active());
_conn->respond_with_content(content_type, content);
@@ -86,7 +86,7 @@ Portal::GetRequest::respond_with_content(const vespalib::string &content_type,
}
void
-Portal::GetRequest::respond_with_error(int code, const vespalib::string &msg)
+Portal::GetRequest::respond_with_error(int code, vespalib::stringref msg)
{
assert(active());
_conn->respond_with_error(code, msg);
diff --git a/vespalib/src/vespa/vespalib/portal/portal.h b/vespalib/src/vespa/vespalib/portal/portal.h
index 314dd6e7de9..6954d800c91 100644
--- a/vespalib/src/vespa/vespalib/portal/portal.h
+++ b/vespalib/src/vespa/vespalib/portal/portal.h
@@ -65,9 +65,9 @@ public:
bool has_param(const vespalib::string &name) const;
const vespalib::string &get_param(const vespalib::string &name) const;
std::map<vespalib::string, vespalib::string> export_params() const;
- void respond_with_content(const vespalib::string &content_type,
- const vespalib::string &content);
- void respond_with_error(int code, const vespalib::string &msg);
+ void respond_with_content(vespalib::stringref content_type,
+ vespalib::stringref content);
+ void respond_with_error(int code, vespalib::stringref msg);
const net::ConnectionAuthContext &auth_context() const noexcept;
~GetRequest();
};