From 1f6a478c42696fcdee5121dc2c737282666c43b7 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Fri, 17 Feb 2023 11:55:14 +0000 Subject: add "visit" as vespa cli command --- client/go/internal/cli/cmd/root.go | 1 + client/go/internal/cli/cmd/visit.go | 331 ++++++++++++++++++++++++++++++++++++ 2 files changed, 332 insertions(+) create mode 100644 client/go/internal/cli/cmd/visit.go diff --git a/client/go/internal/cli/cmd/root.go b/client/go/internal/cli/cmd/root.go index faba6bbbfd4..70a9698c612 100644 --- a/client/go/internal/cli/cmd/root.go +++ b/client/go/internal/cli/cmd/root.go @@ -250,6 +250,7 @@ func (c *CLI) configureCommands() { rootCmd.AddCommand(statusCmd) // status rootCmd.AddCommand(newTestCmd(c)) // test rootCmd.AddCommand(newVersionCmd(c)) // version + rootCmd.AddCommand(newVisitCmd(c)) // visit } func (c *CLI) printErr(err error, hints ...string) { diff --git a/client/go/internal/cli/cmd/visit.go b/client/go/internal/cli/cmd/visit.go new file mode 100644 index 00000000000..d9bacc741ad --- /dev/null +++ b/client/go/internal/cli/cmd/visit.go @@ -0,0 +1,331 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa visit command +// Author: arnej + +package cmd + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "time" + + "github.com/fatih/color" + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +type visitArgs struct { + contentCluster string + fieldSet string + selection string + makeFeed bool + jsonLines bool + pretty bool + quietMode bool + chunkTime string + timeoutTime string + streamFlag bool + chunkCount int +} + +var totalDocCount int + +func newVisitCmd(cli *CLI) *cobra.Command { + var ( + vArgs visitArgs + ) + cmd := &cobra.Command{ + Use: "visit", + Short: "Visit all documents in a content cluster", + Long: `Run visiting of a content cluster to retrieve all documents. + +more explanation here +even more explanation here +`, + Example: `$ vespa visit # get documents from any cluster +$ vespa visit --content-cluster search # get documents from cluster named "search" +`, + Args: cobra.MaximumNArgs(0), + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + vArgs.quietMode = cli.config.isQuiet() + service, err := documentService(cli) + if err != nil { + return err + } + result := probeHandler(service) + if result.Success { + result = visitClusters(vArgs, service) + } + if !result.Success { + return fmt.Errorf("visit failed: %s", result.Message) + } + fmt.Fprintln(os.Stderr, "sum of 'documentCount':", totalDocCount) + return nil + }, + } + cmd.Flags().StringVar(&vArgs.contentCluster, "content-cluster", "*", `Which content cluster to visit documents from`) + cmd.Flags().StringVar(&vArgs.fieldSet, "field-set", "", `Which fieldset to ask for`) + cmd.Flags().StringVar(&vArgs.selection, "selection", "", `select subset of cluster`) + cmd.Flags().BoolVar(&vArgs.jsonLines, "json-lines", false, `output documents as JSON lines`) + cmd.Flags().BoolVar(&vArgs.makeFeed, "make-feed", false, `output JSON array suitable for vespa-feeder`) + cmd.Flags().BoolVar(&vArgs.pretty, "pretty-json", false, `format pretty JSON`) + cmd.Flags().IntVar(&vArgs.chunkCount, "chunk-count", 0, `chunk by count`) + cmd.Flags().BoolVar(&vArgs.streamFlag, "stream", false, `use streaming mode`) + cmd.Flags().StringVar(&vArgs.chunkTime, "chunk-time", "", `chunk by time`) + cmd.Flags().StringVar(&vArgs.timeoutTime, "timeout-time", "", `per-chunk timeout`) + return cmd +} + +type HandlersInfo struct { + Handlers []struct { + HandlerId string `json:"id"` + HandlerClass string `json:"class"` + HandlerBundle string `json:"bundle"` + ServerBindings []string `json:"serverBindings"` + } `json:"handlers"` +} + +func parseHandlersOutput(r io.Reader) (*HandlersInfo, error) { + var handlersInfo HandlersInfo + codec := json.NewDecoder(r) + err := codec.Decode(&handlersInfo) + return &handlersInfo, err +} + +func probeHandler(service *vespa.Service) (res util.OperationResult) { + urlPath := service.BaseURL + "/" + url, urlParseError := url.Parse(urlPath) + if urlParseError != nil { + return util.Failure("Invalid request path: '" + urlPath + "': " + urlParseError.Error()) + } + request := &http.Request{ + URL: url, + Method: "GET", + } + timeout := time.Duration(90) * time.Second + response, err := service.Do(request, timeout) + if err != nil { + return util.Failure("Request failed: " + err.Error()) + } + defer response.Body.Close() + if response.StatusCode == 200 { + handlersInfo, err := parseHandlersOutput(response.Body) + if err != nil || len(handlersInfo.Handlers) == 0 { + fmt.Fprintln(os.Stderr, "Could not parse JSON response from", urlPath, err) + return util.Failure("Bad endpoint") + } + for _, h := range handlersInfo.Handlers { + if strings.HasSuffix(h.HandlerClass, "DocumentV1ApiHandler") { + for _, binding := range h.ServerBindings { + if strings.Contains(binding, "/document/v1/") { + return util.Success("handler OK") + } + } + fmt.Fprintln(os.Stderr, "expected /document/v1/ binding, but got:", h.ServerBindings) + } + } + fmt.Fprintln(os.Stderr, "Missing /document/v1/ API; add to the container cluster delcaration in services.xml") + return util.Failure("Missing /document/v1 API") + } else { + return util.FailureWithPayload(service.Description()+" at "+request.URL.Host+": "+response.Status, util.ReaderToJSON(response.Body)) + } +} + +func visitClusters(vArgs visitArgs, service *vespa.Service) (res util.OperationResult) { + clusters := []string{ + vArgs.contentCluster, + } + if vArgs.contentCluster == "*" { + clusters = probeVisit(vArgs, service) + } + if vArgs.makeFeed { + fmt.Printf("[") + } + for _, c := range clusters { + vArgs.contentCluster = c + res = runVisit(vArgs, service) + if !res.Success { + return res + } + if !vArgs.quietMode { + fmt.Fprintln(os.Stderr, color.GreenString("Success:"), res.Message) + } + } + if vArgs.makeFeed { + fmt.Println("{}\n]") + } + return res +} + +func probeVisit(vArgs visitArgs, service *vespa.Service) []string { + clusters := make([]string, 0, 3) + vvo, _ := runOneVisit(vArgs, service, "") + if vvo != nil { + msg := vvo.ErrorMsg + if strings.Contains(msg, "no content cluster '*'") { + for idx, value := range strings.Split(msg, ",") { + if idx > 0 { + parts := strings.Split(value, "'") + if len(parts) == 3 { + clusters = append(clusters, parts[1]) + } + } + } + } + } + return clusters +} + +func runVisit(vArgs visitArgs, service *vespa.Service) (res util.OperationResult) { + if !vArgs.quietMode { + fmt.Fprintf(os.Stderr, "[debug] trying to visit: '%s'\n", vArgs.contentCluster) + } + var totalDocuments int = 0 + var continuationToken string + for { + var vvo *VespaVisitOutput + vvo, res = runOneVisit(vArgs, service, continuationToken) + if !res.Success { + if vvo != nil && vvo.ErrorMsg != "" { + fmt.Fprintln(os.Stderr, vvo.ErrorMsg) + } + return res + } + if vArgs.makeFeed { + dumpDocuments(vvo.Documents, true, vArgs.pretty) + } else if vArgs.jsonLines { + dumpDocuments(vvo.Documents, false, vArgs.pretty) + } + if !vArgs.quietMode { + fmt.Fprintln(os.Stderr, "got", len(vvo.Documents), "documents") + } + totalDocuments += len(vvo.Documents) + continuationToken = vvo.Continuation + if continuationToken == "" { + break + } + } + res.Message = fmt.Sprintf("%s [%d documents visited]", res.Message, totalDocuments) + return +} + +func runOneVisit(vArgs visitArgs, service *vespa.Service, contToken string) (*VespaVisitOutput, util.OperationResult) { + urlPath := service.BaseURL + "/document/v1/?cluster=" + vArgs.contentCluster + if vArgs.fieldSet != "" { + urlPath = urlPath + "&fieldSet=" + vArgs.fieldSet + } + if vArgs.selection != "" { + urlPath = urlPath + "&selection=" + vArgs.selection + } + if contToken != "" { + urlPath = urlPath + "&continuation=" + contToken + } + if vArgs.streamFlag { + urlPath = urlPath + "&stream=true" + } + if vArgs.chunkTime != "" { + urlPath = urlPath + "&timeChunk=" + vArgs.chunkTime + } + if vArgs.timeoutTime != "" { + urlPath = urlPath + "&timeout=" + vArgs.timeoutTime + } + if vArgs.chunkCount > 0 { + urlPath = urlPath + fmt.Sprintf("&wantedDocumentCount=%d", vArgs.chunkCount) + } + url, urlParseError := url.Parse(urlPath) + if urlParseError != nil { + return nil, util.Failure("Invalid request path: '" + urlPath + "': " + urlParseError.Error()) + } + request := &http.Request{ + URL: url, + Method: "GET", + } + timeout := time.Duration(900) * time.Second + response, err := service.Do(request, timeout) + if err != nil { + return nil, util.Failure("Request failed: " + err.Error()) + } + defer response.Body.Close() + vvo, err := parseVisitOutput(response.Body) + if response.StatusCode == 200 { + if err == nil { + totalDocCount += vvo.DocumentCount + if vvo.DocumentCount != len(vvo.Documents) { + fmt.Fprintln(os.Stderr, "Inconsistent contents from:", url) + fmt.Fprintln(os.Stderr, "claimed count: ", vvo.DocumentCount) + fmt.Fprintln(os.Stderr, "document blobs: ", len(vvo.Documents)) + // return nil, util.Failure("Inconsistent contents from document API") + } + return vvo, util.Success("visited " + vArgs.contentCluster) + } else { + return nil, util.Failure("error reading response: " + err.Error()) + } + } else if response.StatusCode/100 == 4 { + return vvo, util.FailureWithPayload("Invalid document operation: "+response.Status, util.ReaderToJSON(response.Body)) + } else { + return vvo, util.FailureWithPayload(service.Description()+" at "+request.URL.Host+": "+response.Status, util.ReaderToJSON(response.Body)) + } +} + +type DocumentBlob struct { + blob []byte +} + +func (d *DocumentBlob) UnmarshalJSON(data []byte) error { + d.blob = make([]byte, len(data)) + copy(d.blob, data) + return nil +} + +func (d *DocumentBlob) MarshalJSON() ([]byte, error) { + return d.blob, nil +} + +type VespaVisitOutput struct { + PathId string `json:"pathId"` + Documents []DocumentBlob `json:"documents"` + DocumentCount int `json:"documentCount"` + Continuation string `json:"continuation"` + ErrorMsg string `json:"message"` +} + +func parseVisitOutput(r io.Reader) (*VespaVisitOutput, error) { + codec := json.NewDecoder(r) + var parsedJson VespaVisitOutput + err := codec.Decode(&parsedJson) + if err != nil { + fmt.Fprintln(os.Stderr, "could not decode JSON, error:", err) + return nil, err + } + return &parsedJson, nil +} + +func dumpDocuments(documents []DocumentBlob, comma, pretty bool) { + for _, value := range documents { + if pretty { + var prettyJSON bytes.Buffer + parseError := json.Indent(&prettyJSON, value.blob, "", " ") + if parseError != nil { + os.Stdout.Write(value.blob) + } else { + os.Stdout.Write(prettyJSON.Bytes()) + } + } else { + os.Stdout.Write(value.blob) + } + if comma { + fmt.Printf(",\n") + } else { + fmt.Printf("\n") + } + } +} -- cgit v1.2.3 From 58cc508408d72d861f997fae3cb121fcde8b1efa Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Fri, 17 Feb 2023 12:27:16 +0000 Subject: remove experimental options for now --- client/go/internal/cli/cmd/visit.go | 23 +++++------------------ 1 file changed, 5 insertions(+), 18 deletions(-) diff --git a/client/go/internal/cli/cmd/visit.go b/client/go/internal/cli/cmd/visit.go index d9bacc741ad..12442223bae 100644 --- a/client/go/internal/cli/cmd/visit.go +++ b/client/go/internal/cli/cmd/visit.go @@ -29,9 +29,6 @@ type visitArgs struct { jsonLines bool pretty bool quietMode bool - chunkTime string - timeoutTime string - streamFlag bool chunkCount int } @@ -68,7 +65,9 @@ $ vespa visit --content-cluster search # get documents from cluster named "searc if !result.Success { return fmt.Errorf("visit failed: %s", result.Message) } - fmt.Fprintln(os.Stderr, "sum of 'documentCount':", totalDocCount) + if !vArgs.quietMode { + fmt.Fprintln(os.Stderr, "[debug] sum of 'documentCount':", totalDocCount) + } return nil }, } @@ -78,10 +77,7 @@ $ vespa visit --content-cluster search # get documents from cluster named "searc cmd.Flags().BoolVar(&vArgs.jsonLines, "json-lines", false, `output documents as JSON lines`) cmd.Flags().BoolVar(&vArgs.makeFeed, "make-feed", false, `output JSON array suitable for vespa-feeder`) cmd.Flags().BoolVar(&vArgs.pretty, "pretty-json", false, `format pretty JSON`) - cmd.Flags().IntVar(&vArgs.chunkCount, "chunk-count", 0, `chunk by count`) - cmd.Flags().BoolVar(&vArgs.streamFlag, "stream", false, `use streaming mode`) - cmd.Flags().StringVar(&vArgs.chunkTime, "chunk-time", "", `chunk by time`) - cmd.Flags().StringVar(&vArgs.timeoutTime, "timeout-time", "", `per-chunk timeout`) + cmd.Flags().IntVar(&vArgs.chunkCount, "chunk-count", 1000, `chunk by count`) return cmd } @@ -206,7 +202,7 @@ func runVisit(vArgs visitArgs, service *vespa.Service) (res util.OperationResult dumpDocuments(vvo.Documents, false, vArgs.pretty) } if !vArgs.quietMode { - fmt.Fprintln(os.Stderr, "got", len(vvo.Documents), "documents") + fmt.Fprintln(os.Stderr, "[debug] got", len(vvo.Documents), "documents") } totalDocuments += len(vvo.Documents) continuationToken = vvo.Continuation @@ -229,15 +225,6 @@ func runOneVisit(vArgs visitArgs, service *vespa.Service, contToken string) (*Ve if contToken != "" { urlPath = urlPath + "&continuation=" + contToken } - if vArgs.streamFlag { - urlPath = urlPath + "&stream=true" - } - if vArgs.chunkTime != "" { - urlPath = urlPath + "&timeChunk=" + vArgs.chunkTime - } - if vArgs.timeoutTime != "" { - urlPath = urlPath + "&timeout=" + vArgs.timeoutTime - } if vArgs.chunkCount > 0 { urlPath = urlPath + fmt.Sprintf("&wantedDocumentCount=%d", vArgs.chunkCount) } -- cgit v1.2.3 From b9f10138d4a80e1e2cc826809af41e0b3a8bb16d Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Fri, 17 Feb 2023 12:56:35 +0000 Subject: add quoting --- client/go/internal/cli/cmd/visit.go | 27 ++++++++++++++++++++++++--- client/go/internal/cli/cmd/visit_test.go | 25 +++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 client/go/internal/cli/cmd/visit_test.go diff --git a/client/go/internal/cli/cmd/visit.go b/client/go/internal/cli/cmd/visit.go index 12442223bae..219f125534e 100644 --- a/client/go/internal/cli/cmd/visit.go +++ b/client/go/internal/cli/cmd/visit.go @@ -214,13 +214,34 @@ func runVisit(vArgs visitArgs, service *vespa.Service) (res util.OperationResult return } +func quoteArgForUrl(arg string) string { + var buf strings.Builder + buf.Grow(len(arg)) + for _, r := range arg { + switch { + case 'a' <= r && r <= 'z': + buf.WriteRune(r) + case 'A' <= r && r <= 'Z': + buf.WriteRune(r) + case '0' <= r && r <= '9': + buf.WriteRune(r) + case r <= ' ' || r > '~': + buf.WriteRune('+') + default: + s := fmt.Sprintf("%s%02X", "%", r) + buf.WriteString(s) + } + } + return buf.String() +} + func runOneVisit(vArgs visitArgs, service *vespa.Service, contToken string) (*VespaVisitOutput, util.OperationResult) { - urlPath := service.BaseURL + "/document/v1/?cluster=" + vArgs.contentCluster + urlPath := service.BaseURL + "/document/v1/?cluster=" + quoteArgForUrl(vArgs.contentCluster) if vArgs.fieldSet != "" { - urlPath = urlPath + "&fieldSet=" + vArgs.fieldSet + urlPath = urlPath + "&fieldSet=" + quoteArgForUrl(vArgs.fieldSet) } if vArgs.selection != "" { - urlPath = urlPath + "&selection=" + vArgs.selection + urlPath = urlPath + "&selection=" + quoteArgForUrl(vArgs.selection) } if contToken != "" { urlPath = urlPath + "&continuation=" + contToken diff --git a/client/go/internal/cli/cmd/visit_test.go b/client/go/internal/cli/cmd/visit_test.go new file mode 100644 index 00000000000..9f4fbb66e00 --- /dev/null +++ b/client/go/internal/cli/cmd/visit_test.go @@ -0,0 +1,25 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package cmd + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestQuoteFunc(t *testing.T) { + var buf []byte = make([]byte, 3) + buf[0] = 'a' + buf[2] = 'z' + for i := 0; i < 256; i++ { + buf[1] = byte(i) + s := string(buf) + res := quoteArgForUrl(s) + if i < 32 || i > 127 { + assert.Equal(t, "a+z", res) + } else { + fmt.Printf("res %3d => '%s'\n", i, res) + } + } +} -- cgit v1.2.3 From 2b8bb53fa3489a99e054c7036049a252a55783e9 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Mon, 20 Feb 2023 12:09:32 +0000 Subject: add "printDebug" also --- client/go/internal/cli/cmd/root.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/client/go/internal/cli/cmd/root.go b/client/go/internal/cli/cmd/root.go index 70a9698c612..70e0afbcd32 100644 --- a/client/go/internal/cli/cmd/root.go +++ b/client/go/internal/cli/cmd/root.go @@ -264,6 +264,10 @@ func (c *CLI) printSuccess(msg ...interface{}) { fmt.Fprintln(c.Stdout, color.GreenString("Success:"), fmt.Sprint(msg...)) } +func (c *CLI) printDebug(msg ...interface{}) { + fmt.Fprintln(c.Stderr, color.CyanString("Debug:"), fmt.Sprint(msg...)) +} + func (c *CLI) printWarning(msg interface{}, hints ...string) { fmt.Fprintln(c.Stderr, color.YellowString("Warning:"), msg) for _, hint := range hints { -- cgit v1.2.3 From bec5e7ca0bcda7121554e5311b1932df90a8df00 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Mon, 20 Feb 2023 12:11:29 +0000 Subject: restructure to allow unit testing make "json-lines" mode default --- client/go/internal/cli/cmd/visit.go | 134 +++++++++++++++++++----------------- 1 file changed, 72 insertions(+), 62 deletions(-) diff --git a/client/go/internal/cli/cmd/visit.go b/client/go/internal/cli/cmd/visit.go index 219f125534e..a9e13cce444 100644 --- a/client/go/internal/cli/cmd/visit.go +++ b/client/go/internal/cli/cmd/visit.go @@ -11,11 +11,9 @@ import ( "io" "net/http" "net/url" - "os" "strings" "time" - "github.com/fatih/color" "github.com/spf13/cobra" "github.com/vespa-engine/vespa/client/go/internal/util" "github.com/vespa-engine/vespa/client/go/internal/vespa" @@ -28,8 +26,52 @@ type visitArgs struct { makeFeed bool jsonLines bool pretty bool - quietMode bool + debugMode bool chunkCount int + cli *CLI +} + +func (v *visitArgs) writeBytes(b []byte) { + v.cli.Stdout.Write(b) +} + +func (v *visitArgs) writeString(s string) { + v.writeBytes([]byte(s)) +} + +func (v *visitArgs) debugPrint(s string) { + if v.debugMode { + v.cli.printDebug(s) + } +} + +func (v *visitArgs) dumpDocuments(documents []DocumentBlob) { + comma := false + pretty := false + if v.makeFeed { + comma = true + pretty = v.pretty + } else if !v.jsonLines { + return + } + for _, value := range documents { + if pretty { + var prettyJSON bytes.Buffer + parseError := json.Indent(&prettyJSON, value.blob, "", " ") + if parseError != nil { + v.writeBytes(value.blob) + } else { + v.writeBytes(prettyJSON.Bytes()) + } + } else { + v.writeBytes(value.blob) + } + if comma { + v.writeString(",\n") + } else { + v.writeString("\n") + } + } } var totalDocCount int @@ -53,28 +95,27 @@ $ vespa visit --content-cluster search # get documents from cluster named "searc DisableAutoGenTag: true, SilenceUsage: true, RunE: func(cmd *cobra.Command, args []string) error { - vArgs.quietMode = cli.config.isQuiet() + vArgs.cli = cli service, err := documentService(cli) if err != nil { return err } - result := probeHandler(service) + result := probeHandler(service, cli) if result.Success { - result = visitClusters(vArgs, service) + result = visitClusters(&vArgs, service) } if !result.Success { return fmt.Errorf("visit failed: %s", result.Message) } - if !vArgs.quietMode { - fmt.Fprintln(os.Stderr, "[debug] sum of 'documentCount':", totalDocCount) - } + vArgs.debugPrint(fmt.Sprintf("sum of 'documentCount': %d", totalDocCount)) return nil }, } cmd.Flags().StringVar(&vArgs.contentCluster, "content-cluster", "*", `Which content cluster to visit documents from`) cmd.Flags().StringVar(&vArgs.fieldSet, "field-set", "", `Which fieldset to ask for`) cmd.Flags().StringVar(&vArgs.selection, "selection", "", `select subset of cluster`) - cmd.Flags().BoolVar(&vArgs.jsonLines, "json-lines", false, `output documents as JSON lines`) + cmd.Flags().BoolVar(&vArgs.debugMode, "debug-mode", false, `print debugging output`) + cmd.Flags().BoolVar(&vArgs.jsonLines, "json-lines", true, `output documents as JSON lines`) cmd.Flags().BoolVar(&vArgs.makeFeed, "make-feed", false, `output JSON array suitable for vespa-feeder`) cmd.Flags().BoolVar(&vArgs.pretty, "pretty-json", false, `format pretty JSON`) cmd.Flags().IntVar(&vArgs.chunkCount, "chunk-count", 1000, `chunk by count`) @@ -97,7 +138,7 @@ func parseHandlersOutput(r io.Reader) (*HandlersInfo, error) { return &handlersInfo, err } -func probeHandler(service *vespa.Service) (res util.OperationResult) { +func probeHandler(service *vespa.Service, cli *CLI) (res util.OperationResult) { urlPath := service.BaseURL + "/" url, urlParseError := url.Parse(urlPath) if urlParseError != nil { @@ -116,7 +157,7 @@ func probeHandler(service *vespa.Service) (res util.OperationResult) { if response.StatusCode == 200 { handlersInfo, err := parseHandlersOutput(response.Body) if err != nil || len(handlersInfo.Handlers) == 0 { - fmt.Fprintln(os.Stderr, "Could not parse JSON response from", urlPath, err) + cli.printWarning("Could not parse JSON response from"+urlPath, err.Error()) return util.Failure("Bad endpoint") } for _, h := range handlersInfo.Handlers { @@ -126,17 +167,18 @@ func probeHandler(service *vespa.Service) (res util.OperationResult) { return util.Success("handler OK") } } - fmt.Fprintln(os.Stderr, "expected /document/v1/ binding, but got:", h.ServerBindings) + w := fmt.Sprintf("expected /document/v1/ binding, but got: %v", h.ServerBindings) + cli.printWarning(w) } } - fmt.Fprintln(os.Stderr, "Missing /document/v1/ API; add to the container cluster delcaration in services.xml") + cli.printWarning("Missing /document/v1/ API; add to the container cluster delcaration in services.xml") return util.Failure("Missing /document/v1 API") } else { return util.FailureWithPayload(service.Description()+" at "+request.URL.Host+": "+response.Status, util.ReaderToJSON(response.Body)) } } -func visitClusters(vArgs visitArgs, service *vespa.Service) (res util.OperationResult) { +func visitClusters(vArgs *visitArgs, service *vespa.Service) (res util.OperationResult) { clusters := []string{ vArgs.contentCluster, } @@ -144,7 +186,7 @@ func visitClusters(vArgs visitArgs, service *vespa.Service) (res util.OperationR clusters = probeVisit(vArgs, service) } if vArgs.makeFeed { - fmt.Printf("[") + vArgs.writeString("[\n") } for _, c := range clusters { vArgs.contentCluster = c @@ -152,17 +194,15 @@ func visitClusters(vArgs visitArgs, service *vespa.Service) (res util.OperationR if !res.Success { return res } - if !vArgs.quietMode { - fmt.Fprintln(os.Stderr, color.GreenString("Success:"), res.Message) - } + vArgs.debugPrint("Success: " + res.Message) } if vArgs.makeFeed { - fmt.Println("{}\n]") + vArgs.writeString("{}\n]\n") } return res } -func probeVisit(vArgs visitArgs, service *vespa.Service) []string { +func probeVisit(vArgs *visitArgs, service *vespa.Service) []string { clusters := make([]string, 0, 3) vvo, _ := runOneVisit(vArgs, service, "") if vvo != nil { @@ -181,10 +221,8 @@ func probeVisit(vArgs visitArgs, service *vespa.Service) []string { return clusters } -func runVisit(vArgs visitArgs, service *vespa.Service) (res util.OperationResult) { - if !vArgs.quietMode { - fmt.Fprintf(os.Stderr, "[debug] trying to visit: '%s'\n", vArgs.contentCluster) - } +func runVisit(vArgs *visitArgs, service *vespa.Service) (res util.OperationResult) { + vArgs.debugPrint(fmt.Sprintf("trying to visit: '%s'", vArgs.contentCluster)) var totalDocuments int = 0 var continuationToken string for { @@ -192,18 +230,12 @@ func runVisit(vArgs visitArgs, service *vespa.Service) (res util.OperationResult vvo, res = runOneVisit(vArgs, service, continuationToken) if !res.Success { if vvo != nil && vvo.ErrorMsg != "" { - fmt.Fprintln(os.Stderr, vvo.ErrorMsg) + vArgs.cli.printWarning(vvo.ErrorMsg) } return res } - if vArgs.makeFeed { - dumpDocuments(vvo.Documents, true, vArgs.pretty) - } else if vArgs.jsonLines { - dumpDocuments(vvo.Documents, false, vArgs.pretty) - } - if !vArgs.quietMode { - fmt.Fprintln(os.Stderr, "[debug] got", len(vvo.Documents), "documents") - } + vArgs.dumpDocuments(vvo.Documents) + vArgs.debugPrint(fmt.Sprintf("got %d documents", len(vvo.Documents))) totalDocuments += len(vvo.Documents) continuationToken = vvo.Continuation if continuationToken == "" { @@ -235,7 +267,7 @@ func quoteArgForUrl(arg string) string { return buf.String() } -func runOneVisit(vArgs visitArgs, service *vespa.Service, contToken string) (*VespaVisitOutput, util.OperationResult) { +func runOneVisit(vArgs *visitArgs, service *vespa.Service, contToken string) (*VespaVisitOutput, util.OperationResult) { urlPath := service.BaseURL + "/document/v1/?cluster=" + quoteArgForUrl(vArgs.contentCluster) if vArgs.fieldSet != "" { urlPath = urlPath + "&fieldSet=" + quoteArgForUrl(vArgs.fieldSet) @@ -268,10 +300,10 @@ func runOneVisit(vArgs visitArgs, service *vespa.Service, contToken string) (*Ve if err == nil { totalDocCount += vvo.DocumentCount if vvo.DocumentCount != len(vvo.Documents) { - fmt.Fprintln(os.Stderr, "Inconsistent contents from:", url) - fmt.Fprintln(os.Stderr, "claimed count: ", vvo.DocumentCount) - fmt.Fprintln(os.Stderr, "document blobs: ", len(vvo.Documents)) - // return nil, util.Failure("Inconsistent contents from document API") + vArgs.cli.printWarning(fmt.Sprintf("Inconsistent contents from: %v", url)) + vArgs.cli.printWarning(fmt.Sprintf("claimed count: %d", vvo.DocumentCount)) + vArgs.cli.printWarning(fmt.Sprintf("document blobs: %d", len(vvo.Documents))) + return nil, util.Failure("Inconsistent contents from document API") } return vvo, util.Success("visited " + vArgs.contentCluster) } else { @@ -311,29 +343,7 @@ func parseVisitOutput(r io.Reader) (*VespaVisitOutput, error) { var parsedJson VespaVisitOutput err := codec.Decode(&parsedJson) if err != nil { - fmt.Fprintln(os.Stderr, "could not decode JSON, error:", err) - return nil, err + return nil, fmt.Errorf("could not decode JSON, error: %s", err.Error()) } return &parsedJson, nil } - -func dumpDocuments(documents []DocumentBlob, comma, pretty bool) { - for _, value := range documents { - if pretty { - var prettyJSON bytes.Buffer - parseError := json.Indent(&prettyJSON, value.blob, "", " ") - if parseError != nil { - os.Stdout.Write(value.blob) - } else { - os.Stdout.Write(prettyJSON.Bytes()) - } - } else { - os.Stdout.Write(value.blob) - } - if comma { - fmt.Printf(",\n") - } else { - fmt.Printf("\n") - } - } -} -- cgit v1.2.3 From 6d983f7cf92838ace741485f1714cbd8c38709eb Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Mon, 20 Feb 2023 12:12:07 +0000 Subject: add some unit testing --- client/go/internal/cli/cmd/visit_test.go | 101 +++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/client/go/internal/cli/cmd/visit_test.go b/client/go/internal/cli/cmd/visit_test.go index 9f4fbb66e00..0941fa858b2 100644 --- a/client/go/internal/cli/cmd/visit_test.go +++ b/client/go/internal/cli/cmd/visit_test.go @@ -3,9 +3,24 @@ package cmd import ( "fmt" + "net/http" "testing" "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/internal/mock" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +const ( + normalpre = `{"pathId":"/document/v1/","documents":[` + document1 = `{"id":"id:t:m::1","fields":{"title":"t"}}` + document2 = `{"id":"id:t:m::2","fields":{"title":"t2"}}` + document3 = `{"id":"id:t:m::3","fields":{"ar":"xyz","w":63,"title":"xyzzy","year":2000}}` + + savedresponse = `{"pathId":"/document/v1/","documents":[{"id":"id:test:music::1921492307","fields":{"title":"song","year":2010}},{"id":"id:test:music::p_try-this-clean-bonus-dvd-_music_1922003403","fields":{"artist":"xyz","weight":600000,"song":"hate","title":"xyz","year":2000}}],"documentCount":2,"continuation":"AAAACAAAAAAAAAAJAAAAAAAAAAgAAAAAAAABAAAAAAEgAAAAAAAAEAAAAAAAAAAA"}` + + saveddoc0 = `{"id":"id:test:music::1921492307","fields":{"title":"song","year":2010}}` + saveddoc1 = `{"id":"id:test:music::p_try-this-clean-bonus-dvd-_music_1922003403","fields":{"artist":"xyz","weight":600000,"song":"hate","title":"xyz","year":2000}}` ) func TestQuoteFunc(t *testing.T) { @@ -23,3 +38,89 @@ func TestQuoteFunc(t *testing.T) { } } } + +// low-level (unit) test +func TestRunOneVisit(t *testing.T) { + withResponse := func(client *mock.HTTPClient) { + client.NextResponseString(200, savedresponse) + } + op := func(service *vespa.Service) { + vArgs := visitArgs{ + contentCluster: "fooCC", + } + vvo, res := runOneVisit(&vArgs, service, "BBBB") + assert.Equal(t, true, res.Success) + assert.Equal(t, "visited fooCC", res.Message) + assert.Equal(t, "/document/v1/", vvo.PathId) + assert.Equal(t, "", vvo.ErrorMsg) + assert.Equal(t, "AAAACAAAAAAAAAAJAAAAAAAAAAgAAAAAAAABAAAAAAEgAAAAAAAAEAAAAAAAAAAA", vvo.Continuation) + assert.Equal(t, 2, vvo.DocumentCount) + assert.Equal(t, 2, len(vvo.Documents)) + assert.Equal(t, saveddoc0, string(vvo.Documents[0].blob)) + assert.Equal(t, saveddoc1, string(vvo.Documents[1].blob)) + } + req := withMockClient(t, withResponse, op) + assert.Equal(t, "cluster=fooCC&continuation=BBBB", req.URL.RawQuery) +} + +func withMockClient(t *testing.T, prepCli func(*mock.HTTPClient), runOp func(*vespa.Service)) *http.Request { + client := &mock.HTTPClient{} + prepCli(client) + cli, _, _ := newTestCLI(t) + cli.httpClient = client + service, _ := documentService(cli) + runOp(service) + return client.LastRequest +} + +func TestVisitCommand(t *testing.T) { + assertVisitResults( + []string{ + "visit", + "--json-lines", + }, + t, + []string{ + normalpre + + document1 + + `],"documentCount":1,"continuation":"CAFE"}`, + normalpre + + document2 + + "," + + document3 + + `],"documentCount":2}`, + }, + "cluster=fooCC&continuation=CAFE&wantedDocumentCount=1000", + document1+"\n"+ + document2+"\n"+ + document3+"\n") +} + +func assertVisitResults(arguments []string, t *testing.T, responses []string, queryPart, output string) { + client := &mock.HTTPClient{} + client.NextResponseString(200, `{ + "handlers" : [ { + "id" : "com.yahoo.container.usability.BindingsOverviewHandler", + "class" : "com.yahoo.container.usability.BindingsOverviewHandler", + "bundle" : "container-disc:8.0.0", + "serverBindings" : [ "http://*/" ] + }, { + "id" : "com.yahoo.document.restapi.resource.DocumentV1ApiHandler", + "class" : "com.yahoo.document.restapi.resource.DocumentV1ApiHandler", + "bundle" : "vespaclient-container-plugin:8.0.0", + "serverBindings" : [ "http://*/document/v1/*", "http://*/document/v1/*/" ] + }, { + } ] }`) + client.NextResponseString(400, `{"pathId":"/document/v1/","message":"Your Vespa deployment has no content cluster '*', only 'fooCC'"}`) + for _, resp := range responses { + client.NextResponseString(200, resp) + } + cli, stdout, stderr := newTestCLI(t) + cli.httpClient = client + assert.Nil(t, cli.Run(arguments...)) + assert.Equal(t, output, stdout.String()) + assert.Equal(t, "", stderr.String()) + assert.Equal(t, queryPart, client.LastRequest.URL.RawQuery) + assert.Equal(t, "/document/v1/", client.LastRequest.URL.Path) + assert.Equal(t, "GET", client.LastRequest.Method) +} -- cgit v1.2.3 From 61ce79f44d2313180afe327f159806cefd6843b9 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Mon, 20 Feb 2023 12:20:46 +0000 Subject: move long strings to const section --- client/go/internal/cli/cmd/visit_test.go | 48 +++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/client/go/internal/cli/cmd/visit_test.go b/client/go/internal/cli/cmd/visit_test.go index 0941fa858b2..4302680b9d9 100644 --- a/client/go/internal/cli/cmd/visit_test.go +++ b/client/go/internal/cli/cmd/visit_test.go @@ -19,8 +19,22 @@ const ( savedresponse = `{"pathId":"/document/v1/","documents":[{"id":"id:test:music::1921492307","fields":{"title":"song","year":2010}},{"id":"id:test:music::p_try-this-clean-bonus-dvd-_music_1922003403","fields":{"artist":"xyz","weight":600000,"song":"hate","title":"xyz","year":2000}}],"documentCount":2,"continuation":"AAAACAAAAAAAAAAJAAAAAAAAAAgAAAAAAAABAAAAAAEgAAAAAAAAEAAAAAAAAAAA"}` - saveddoc0 = `{"id":"id:test:music::1921492307","fields":{"title":"song","year":2010}}` - saveddoc1 = `{"id":"id:test:music::p_try-this-clean-bonus-dvd-_music_1922003403","fields":{"artist":"xyz","weight":600000,"song":"hate","title":"xyz","year":2000}}` + saveddoc0 = `{"id":"id:test:music::1921492307","fields":{"title":"song","year":2010}}` + saveddoc1 = `{"id":"id:test:music::p_try-this-clean-bonus-dvd-_music_1922003403","fields":{"artist":"xyz","weight":600000,"song":"hate","title":"xyz","year":2000}}` + handlersResponse = `{ + "handlers" : [ { + "id" : "com.yahoo.container.usability.BindingsOverviewHandler", + "class" : "com.yahoo.container.usability.BindingsOverviewHandler", + "bundle" : "container-disc:8.0.0", + "serverBindings" : [ "http://*/" ] + }, { + "id" : "com.yahoo.document.restapi.resource.DocumentV1ApiHandler", + "class" : "com.yahoo.document.restapi.resource.DocumentV1ApiHandler", + "bundle" : "vespaclient-container-plugin:8.0.0", + "serverBindings" : [ "http://*/document/v1/*", "http://*/document/v1/*/" ] + } ] +}` + clusterStarResponse = `{"pathId":"/document/v1/","message":"Your Vespa deployment has no content cluster '*', only 'fooCC'"}` ) func TestQuoteFunc(t *testing.T) { @@ -61,6 +75,20 @@ func TestRunOneVisit(t *testing.T) { } req := withMockClient(t, withResponse, op) assert.Equal(t, "cluster=fooCC&continuation=BBBB", req.URL.RawQuery) + + op = func(service *vespa.Service) { + vArgs := visitArgs{ + contentCluster: "search", + fieldSet: "[id]", + selection: "music.year>2000", + chunkCount: 123, + } + vvo, res := runOneVisit(&vArgs, service, "asdf") + assert.Equal(t, true, res.Success) + assert.Equal(t, 2, vvo.DocumentCount) + } + req = withMockClient(t, withResponse, op) + assert.Equal(t, "cluster=search&fieldSet=%5Bid%5D&selection=music%2Eyear%3E2000&continuation=asdf&wantedDocumentCount=123", req.URL.RawQuery) } func withMockClient(t *testing.T, prepCli func(*mock.HTTPClient), runOp func(*vespa.Service)) *http.Request { @@ -98,20 +126,8 @@ func TestVisitCommand(t *testing.T) { func assertVisitResults(arguments []string, t *testing.T, responses []string, queryPart, output string) { client := &mock.HTTPClient{} - client.NextResponseString(200, `{ - "handlers" : [ { - "id" : "com.yahoo.container.usability.BindingsOverviewHandler", - "class" : "com.yahoo.container.usability.BindingsOverviewHandler", - "bundle" : "container-disc:8.0.0", - "serverBindings" : [ "http://*/" ] - }, { - "id" : "com.yahoo.document.restapi.resource.DocumentV1ApiHandler", - "class" : "com.yahoo.document.restapi.resource.DocumentV1ApiHandler", - "bundle" : "vespaclient-container-plugin:8.0.0", - "serverBindings" : [ "http://*/document/v1/*", "http://*/document/v1/*/" ] - }, { - } ] }`) - client.NextResponseString(400, `{"pathId":"/document/v1/","message":"Your Vespa deployment has no content cluster '*', only 'fooCC'"}`) + client.NextResponseString(200, handlersResponse) + client.NextResponseString(400, clusterStarResponse) for _, resp := range responses { client.NextResponseString(200, resp) } -- cgit v1.2.3 From 542e566de9b06299034fa73bec0a2572bf604c5b Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Mon, 20 Feb 2023 12:23:51 +0000 Subject: minor update to help text --- client/go/internal/cli/cmd/visit.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/client/go/internal/cli/cmd/visit.go b/client/go/internal/cli/cmd/visit.go index a9e13cce444..1022d74354d 100644 --- a/client/go/internal/cli/cmd/visit.go +++ b/client/go/internal/cli/cmd/visit.go @@ -82,11 +82,10 @@ func newVisitCmd(cli *CLI) *cobra.Command { ) cmd := &cobra.Command{ Use: "visit", - Short: "Visit all documents in a content cluster", - Long: `Run visiting of a content cluster to retrieve all documents. + Short: "Visit and print all documents in a vespa cluster", + Long: `Run visiting to retrieve all documents. -more explanation here -even more explanation here +By default prints each document received on its own line (JSON-L format). `, Example: `$ vespa visit # get documents from any cluster $ vespa visit --content-cluster search # get documents from cluster named "search" -- cgit v1.2.3 From 7d2993aad755d93ebba49c967fb51962884486df Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Fri, 24 Feb 2023 14:41:45 +0000 Subject: add new components for global-phase handling --- .../com/yahoo/prelude/cluster/ClusterSearcher.java | 31 ++++- .../java/com/yahoo/search/ranking/Evaluator.java | 14 ++ .../yahoo/search/ranking/GlobalPhaseHelper.java | 152 +++++++++++++++++++++ .../java/com/yahoo/search/ranking/HitRescorer.java | 56 ++++++++ .../ranking/RankingExpressionEvaluatorFactory.java | 40 ++++++ .../ranking/RankingExpressionEvaluatorProxy.java | 53 +++++++ .../com/yahoo/search/ranking/SimpleEvaluator.java | 48 +++++++ .../com/yahoo/search/ranking/package-info.java | 6 + .../prelude/cluster/ClusterSearcherTestCase.java | 1 + 9 files changed, 396 insertions(+), 5 deletions(-) create mode 100644 container-search/src/main/java/com/yahoo/search/ranking/Evaluator.java create mode 100644 container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseHelper.java create mode 100644 container-search/src/main/java/com/yahoo/search/ranking/HitRescorer.java create mode 100644 container-search/src/main/java/com/yahoo/search/ranking/RankingExpressionEvaluatorFactory.java create mode 100644 container-search/src/main/java/com/yahoo/search/ranking/RankingExpressionEvaluatorProxy.java create mode 100644 container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java create mode 100644 container-search/src/main/java/com/yahoo/search/ranking/package-info.java diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 729aebf2fc2..a18250fbcfe 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -20,6 +20,7 @@ import com.yahoo.search.Searcher; import com.yahoo.search.config.ClusterConfig; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.query.ParameterParser; +import com.yahoo.search.ranking.GlobalPhaseHelper; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.searchchain.Execution; @@ -64,6 +65,7 @@ public class ClusterSearcher extends Searcher { private final VespaBackEndSearcher server; private final Executor executor; + private final GlobalPhaseHelper globalPhaseHelper; @Inject public ClusterSearcher(ComponentId id, @@ -73,10 +75,12 @@ public class ClusterSearcher extends Searcher { DocumentdbInfoConfig documentDbConfig, SchemaInfo schemaInfo, ComponentRegistry dispatchers, + GlobalPhaseHelper globalPhaseHelper, VipStatus vipStatus, VespaDocumentAccess access) { super(id); this.executor = executor; + this.globalPhaseHelper = globalPhaseHelper; int searchClusterIndex = clusterConfig.clusterId(); searchClusterName = clusterConfig.clusterName(); QrSearchersConfig.Searchcluster searchClusterConfig = getSearchClusterConfigFromClusterName(qrsConfig, searchClusterName); @@ -159,7 +163,9 @@ public class ClusterSearcher extends Searcher { maxQueryCacheTimeout = DEFAULT_MAX_QUERY_CACHE_TIMEOUT; server = searcher; this.executor = executor; + this.globalPhaseHelper = null; } + /** Do not use, for internal testing purposes only. **/ ClusterSearcher(Set schemas) { this(schemas, null, null); @@ -169,7 +175,7 @@ public class ClusterSearcher extends Searcher { public void fill(com.yahoo.search.Result result, String summaryClass, Execution execution) { Query query = result.getQuery(); - VespaBackEndSearcher searcher = server; + Searcher searcher = server; if (searcher != null) { if (query.getTimeLeft() > 0) { searcher.fill(result, summaryClass, execution); @@ -190,7 +196,7 @@ public class ClusterSearcher extends Searcher { public Result search(Query query, Execution execution) { validateQueryTimeout(query); validateQueryCache(query); - VespaBackEndSearcher searcher = server; + Searcher searcher = server; if (searcher == null) { return new Result(query, ErrorMessage.createNoBackendsInService("Could not search")); } @@ -228,8 +234,23 @@ public class ClusterSearcher extends Searcher { } else { String docType = schemas.iterator().next(); query.getModel().setRestrict(docType); - return searcher.search(query, execution); + return perSchemaSearch(searcher, query, execution); + } + } + + private Result perSchemaSearch(Searcher searcher, Query query, Execution execution) { + Set restrict = query.getModel().getRestrict(); + if (restrict.size() != 1) { + throw new IllegalStateException("perSchemaSearch must always be called with 1 schema, got: " + restrict.size()); + } + for (String schema : restrict) { + Result result = searcher.search(query, execution); + if (globalPhaseHelper != null) { + globalPhaseHelper.process(query, result, schema); + } + return result; } + return null; } private static void processResult(Query query, FutureTask task, Result mergedResult) { @@ -248,12 +269,12 @@ public class ClusterSearcher extends Searcher { Set schemas = resolveSchemas(query, execution.context().getIndexFacts()); List queries = createQueries(query, schemas); if (queries.size() == 1) { - return searcher.search(queries.get(0), execution); + return perSchemaSearch(searcher, queries.get(0), execution); } else { Result mergedResult = new Result(query); List> pending = new ArrayList<>(queries.size()); for (Query q : queries) { - FutureTask task = new FutureTask<>(() -> searcher.search(q, execution)); + FutureTask task = new FutureTask<>(() -> perSchemaSearch(searcher, q, execution)); try { executor.execute(task); pending.add(task); diff --git a/container-search/src/main/java/com/yahoo/search/ranking/Evaluator.java b/container-search/src/main/java/com/yahoo/search/ranking/Evaluator.java new file mode 100644 index 00000000000..d2edb776c92 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/Evaluator.java @@ -0,0 +1,14 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import com.yahoo.tensor.Tensor; + +import java.util.Collection; + +interface Evaluator { + Collection needInputs(); + + Evaluator bind(String name, Tensor value); + + double evaluateScore(); +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseHelper.java b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseHelper.java new file mode 100644 index 00000000000..9810f612e5c --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseHelper.java @@ -0,0 +1,152 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import ai.vespa.models.evaluation.FunctionEvaluator; +import ai.vespa.models.evaluation.Model; +import com.yahoo.component.annotation.Inject; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.tensor.Tensor; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.logging.Logger; +import java.util.function.Supplier; + +public class GlobalPhaseHelper { + + private static final Logger logger = Logger.getLogger(GlobalPhaseHelper.class.getName()); + private final RankingExpressionEvaluatorFactory factory; + private final Set skipProcessing = new HashSet<>(); + private final Map> scorers = new HashMap<>(); + + @Inject + public GlobalPhaseHelper(RankingExpressionEvaluatorFactory factory) { + this.factory = factory; + logger.info("using factory: " + factory); + } + + public void process(Query query, Result result, String schema) { + var functionEvaluatorSource = underlying(query, schema); + if (functionEvaluatorSource == null) { + return; + } + var prepared = findFromQuery(query, functionEvaluatorSource.get().function().arguments()); + Supplier supplier = () -> { + var evaluator = functionEvaluatorSource.get(); + var simple = new SimpleEvaluator(evaluator); + for (var entry : prepared) { + simple.bind(entry.name(), entry.value()); + } + return simple; + }; + // TODO need to get rerank-count somehow + int rerank = 7; + rerankHits(query, result, new HitRescorer(supplier), rerank); + } + + record NameAndValue(String name, Tensor value) { } + + /* do this only once per query: */ + List findFromQuery(Query query, List needInputs) { + List result = new ArrayList<>(); + var ranking = query.getRanking(); + var rankFeatures = ranking.getFeatures(); + var rankProps = ranking.getProperties().asMap(); + for (String needed : needInputs) { + var optRef = com.yahoo.searchlib.rankingexpression.Reference.simple(needed); + if (optRef.isEmpty()) continue; + var ref = optRef.get(); + if (ref.name().equals("constant")) { + // XXX in theory, we should be able to avoid this + result.add(new NameAndValue(needed, null)); + continue; + } + if (ref.isSimple() && ref.name().equals("query")) { + String queryFeatureName = ref.simpleArgument().get(); + // searchers are recommended to place query features here: + var feature = rankFeatures.getTensor(queryFeatureName); + if (feature.isPresent()) { + result.add(new NameAndValue(needed, feature.get())); + } else { + // but other ways of setting query features end up in the properties: + var objList = rankProps.get(queryFeatureName); + if (objList != null && objList.size() == 1 && objList.get(0) instanceof Tensor t) { + result.add(new NameAndValue(needed, t)); + } + } + } + } + return result; + } + + void rerankHits(Query query, Result result, HitRescorer hitRescorer, int rerank) { + double worstRerankedScore = Double.MAX_VALUE; + double worstRerankedOldScore = Double.MAX_VALUE; + // TODO consider doing recursive iteration instead of deepIterator + for (var iterator = result.hits().deepIterator(); iterator.hasNext();) { + Hit hit = iterator.next(); + if (hit.isMeta() || hit instanceof HitGroup) { + continue; + } + // what about hits inside grouping results? + if (rerank > 0) { + double oldScore = hit.getRelevance().getScore(); + boolean didRerank = hitRescorer.rescoreHit(hit); + if (didRerank) { + double newScore = hit.getRelevance().getScore(); + if (oldScore < worstRerankedOldScore) worstRerankedOldScore = oldScore; + if (newScore < worstRerankedScore) worstRerankedScore = newScore; + --rerank; + } else { + // failed to rescore this hit, what should we do? + hit.setRelevance(-Double.MAX_VALUE); + } + } else { + // too low quality + if (worstRerankedOldScore > worstRerankedScore) { + double penalty = worstRerankedOldScore - worstRerankedScore; + double oldScore = hit.getRelevance().getScore(); + hit.setRelevance(oldScore - penalty); + } + } + } + result.hits().sort(); + } + + private Supplier underlying(Query query, String schema) { + String rankProfile = query.getRanking().getProfile(); + String key = schema + " with rank profile " + rankProfile; + if (skipProcessing.contains(key)) { + return null; + } + Supplier supplier = scorers.get(key); + if (supplier != null) { + return supplier; + } + try { + var proxy = factory.proxyForSchema(schema); + var model = proxy.modelForRankProfile(rankProfile); + supplier = () -> model.evaluatorOf("globalphase"); + if (supplier.get() == null) { + supplier = null; + } + } catch (IllegalArgumentException e) { + logger.info("no global-phase for " + key + " because: " + e.getMessage()); + supplier = null; + } + if (supplier == null) { + skipProcessing.add(key); + } else { + scorers.put(key, supplier); + } + return supplier; + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/HitRescorer.java b/container-search/src/main/java/com/yahoo/search/ranking/HitRescorer.java new file mode 100644 index 00000000000..b92f9da2e2a --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/HitRescorer.java @@ -0,0 +1,56 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import com.yahoo.search.result.FeatureData; +import com.yahoo.search.result.Hit; + +import java.util.function.Supplier; +import java.util.logging.Logger; + +public class HitRescorer { + + private static final Logger logger = Logger.getLogger(HitRescorer.class.getName()); + + private final Supplier evaluatorSource; + + public HitRescorer(Supplier evaluatorSource) { + this.evaluatorSource = evaluatorSource; + } + + boolean rescoreHit(Hit hit) { + var features = hit.getField("matchfeatures"); + if (features instanceof FeatureData matchFeatures) { + var scorer = evaluatorSource.get(); + for (String argName : scorer.needInputs()) { + var asTensor = matchFeatures.getTensor(argName); + if (asTensor == null) { + asTensor = matchFeatures.getTensor(alternate(argName)); + } + if (asTensor != null) { + scorer.bind(argName, asTensor); + } else { + logger.warning("Missing match-feature for Evaluator argument: " + argName); + return false; + } + } + double newScore = scorer.evaluateScore(); + hit.setRelevance(newScore); + return true; + } else { + logger.warning("Hit without match-features: " + hit); + return false; + } + } + + private static final String RE_PREFIX = "rankingExpression("; + private static final String RE_SUFFIX = ")"; + private static final int RE_PRE_LEN = RE_PREFIX.length(); + private static final int RE_SUF_LEN = RE_SUFFIX.length(); + + static String alternate(String argName) { + if (argName.startsWith(RE_PREFIX) && argName.endsWith(RE_SUFFIX)) { + return argName.substring(RE_PRE_LEN, argName.length() - RE_SUF_LEN); + } + return argName; + } +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/RankingExpressionEvaluatorFactory.java b/container-search/src/main/java/com/yahoo/search/ranking/RankingExpressionEvaluatorFactory.java new file mode 100644 index 00000000000..8ec3fc919db --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/RankingExpressionEvaluatorFactory.java @@ -0,0 +1,40 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.search.ranking; + +import com.yahoo.api.annotations.Beta; +import com.yahoo.component.annotation.Inject; +import com.yahoo.component.provider.ComponentRegistry; + +/** + * factory for model-evaluation proxies + * @author arnej + */ +@Beta +public class RankingExpressionEvaluatorFactory { + + private final ComponentRegistry registry; + + @Inject + public RankingExpressionEvaluatorFactory(ComponentRegistry registry) { + this.registry = registry; + } + + public RankingExpressionEvaluatorProxy proxyForSchema(String schemaName) { + var component = registry.getComponent("ranking-expression-evaluator." + schemaName); + if (component == null) { + throw new IllegalArgumentException("ranking expression evaluator for schema '" + schemaName + "' not found"); + } + return component; + } + + public String toString() { + var buf = new StringBuilder(); + buf.append(this.getClass().getName()).append(" containing: ["); + for (var id : registry.allComponentsById().keySet()) { + buf.append(" ").append(id.toString()); + } + buf.append(" ]"); + return buf.toString(); + } +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/RankingExpressionEvaluatorProxy.java b/container-search/src/main/java/com/yahoo/search/ranking/RankingExpressionEvaluatorProxy.java new file mode 100644 index 00000000000..b4ee33263f1 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/RankingExpressionEvaluatorProxy.java @@ -0,0 +1,53 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.search.ranking; + +import ai.vespa.models.evaluation.FunctionEvaluator; +import ai.vespa.models.evaluation.Model; +import ai.vespa.models.evaluation.ModelsEvaluator; +import com.yahoo.api.annotations.Beta; +import com.yahoo.component.AbstractComponent; +import com.yahoo.component.annotation.Inject; +import com.yahoo.filedistribution.fileacquirer.FileAcquirer; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; + +/** + * proxy for model-evaluation components + * @author arnej + */ +@Beta +public class RankingExpressionEvaluatorProxy extends AbstractComponent { + + private final ModelsEvaluator evaluator; + + @Inject + public RankingExpressionEvaluatorProxy( + RankProfilesConfig rankProfilesConfig, + RankingConstantsConfig constantsConfig, + RankingExpressionsConfig expressionsConfig, + OnnxModelsConfig onnxModelsConfig, + FileAcquirer fileAcquirer) + { + this.evaluator = new ModelsEvaluator( + rankProfilesConfig, + constantsConfig, + expressionsConfig, + onnxModelsConfig, + fileAcquirer); + } + + public Model modelForRankProfile(String rankProfile) { + var m = evaluator.models().get(rankProfile); + if (m == null) { + throw new IllegalArgumentException("unknown rankprofile: " + rankProfile); + } + return m; + } + + public FunctionEvaluator evaluatorForFunction(String rankProfile, String functionName) { + return modelForRankProfile(rankProfile).evaluatorOf(functionName); + } +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java b/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java new file mode 100644 index 00000000000..39abacc17f1 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java @@ -0,0 +1,48 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import ai.vespa.models.evaluation.FunctionEvaluator; +import com.yahoo.search.result.FeatureData; +import com.yahoo.search.result.Hit; +import com.yahoo.tensor.Tensor; + +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class SimpleEvaluator implements Evaluator { + + private final FunctionEvaluator evaluator; + private final Set neededInputs; + + + public SimpleEvaluator(FunctionEvaluator prototype) { + this.evaluator = prototype; + this.neededInputs = new HashSet(prototype.function().arguments()); + } + + public Collection needInputs() { return List.copyOf(neededInputs); } + + public SimpleEvaluator bind(String name, Tensor value) { + if (value != null) evaluator.bind(name, value); + neededInputs.remove(name); + return this; + } + + public double evaluateScore() { + return evaluator.evaluate().asDouble(); + } + + public String toString() { + var buf = new StringBuilder(); + buf.append("SimpleEvaluator("); + buf.append(evaluator.function().toString()); + buf.append(")["); + for (String arg : neededInputs) { + buf.append("{").append(arg).append("}"); + } + buf.append("]"); + return buf.toString(); + } +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/package-info.java b/container-search/src/main/java/com/yahoo/search/ranking/package-info.java new file mode 100644 index 00000000000..a86a5c1e52f --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/package-info.java @@ -0,0 +1,6 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +@ExportPackage +package com.yahoo.search.ranking; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java index 5df8d2e5444..06ae9923dae 100644 --- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java @@ -464,6 +464,7 @@ public class ClusterSearcherTestCase { documentDbConfig.build(), new SchemaInfo(List.of(schema.build()), Map.of()), dispatchers, + null, vipStatus, null); } -- cgit v1.2.3 From 9c495646db8acadc02f698b5326c67dc8a59d283 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Thu, 23 Feb 2023 10:02:11 +0000 Subject: Revert "deeper processing of onnx inputs" This reverts commit 8d8f523c32fab2bd396f960496459eb079b03872. --- .../com/yahoo/schema/expressiontransforms/InputRecorder.java | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java index b0f63ebb732..4e7988a2006 100644 --- a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java @@ -3,16 +3,13 @@ package com.yahoo.schema.expressiontransforms; import com.yahoo.schema.FeatureNames; import com.yahoo.schema.RankProfile; -import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.Reference; -import com.yahoo.searchlib.rankingexpression.parser.ParseException; import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; -import java.io.StringReader; import java.util.Set; /** @@ -86,13 +83,7 @@ public class InputRecorder extends ExpressionTransformer Date: Fri, 24 Feb 2023 15:09:43 +0000 Subject: add component modeling --- .../main/java/com/yahoo/schema/RankProfile.java | 3 ++ .../model/container/search/ContainerSearch.java | 16 +++++-- .../RankingExpressionEvaluatorProxyComponent.java | 49 ++++++++++++++++++++++ .../globalphase_onnx_inside/files/ax_plus_b.onnx | 23 ++++++++++ .../globalphase_onnx_inside/rank-profiles.cfg | 34 +++++++++++++++ .../test/derived/globalphase_onnx_inside/test.sd | 42 +++++++++++++++++++ .../derived/rankingexpression/rank-profiles.cfg | 2 - .../derived/GlobalPhaseOnnxModelsTestCase.java | 22 ++++++++++ 8 files changed, 186 insertions(+), 5 deletions(-) create mode 100644 config-model/src/main/java/com/yahoo/vespa/model/container/search/RankingExpressionEvaluatorProxyComponent.java create mode 100644 config-model/src/test/derived/globalphase_onnx_inside/files/ax_plus_b.onnx create mode 100644 config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg create mode 100644 config-model/src/test/derived/globalphase_onnx_inside/test.sd create mode 100644 config-model/src/test/java/com/yahoo/schema/derived/GlobalPhaseOnnxModelsTestCase.java diff --git a/config-model/src/main/java/com/yahoo/schema/RankProfile.java b/config-model/src/main/java/com/yahoo/schema/RankProfile.java index ad6eb038058..7cb0a088f5f 100644 --- a/config-model/src/main/java/com/yahoo/schema/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/RankProfile.java @@ -1019,6 +1019,9 @@ public class RankProfile implements Cloneable { var recorder = new InputRecorder(needInputs); recorder.transform(globalPhaseRanking.function().getBody(), context); for (String input : needInputs) { + if (input.startsWith("constant(") || input.startsWith("query(")) { + continue; + } try { addMatchFeatures(new FeatureList(input)); } catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index 86c48407775..e3ba40c9037 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -7,6 +7,7 @@ import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.search.pagetemplates.PageTemplatesConfig; import com.yahoo.search.query.profile.config.QueryProfilesConfig; +import com.yahoo.search.ranking.RankingExpressionEvaluatorFactory; import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; @@ -56,6 +57,8 @@ public class ContainerSearch extends ContainerSubsystem owningCluster.addComponent(Component.fromClassAndBundle(CompiledQueryProfileRegistry.class, SEARCH_AND_DOCPROC_BUNDLE)); owningCluster.addComponent(Component.fromClassAndBundle(com.yahoo.search.schema.SchemaInfo.class, SEARCH_AND_DOCPROC_BUNDLE)); owningCluster.addComponent(Component.fromClassAndBundle(SearchStatusExtension.class, SEARCH_AND_DOCPROC_BUNDLE)); + owningCluster.addComponent(Component.fromClassAndBundle(RankingExpressionEvaluatorFactory.class, SEARCH_AND_DOCPROC_BUNDLE)); + owningCluster.addComponent(Component.fromClassAndBundle(com.yahoo.search.ranking.GlobalPhaseHelper.class, SEARCH_AND_DOCPROC_BUNDLE)); cluster.addSearchAndDocprocBundles(); } @@ -68,9 +71,16 @@ public class ContainerSearch extends ContainerSubsystem /** Adds a Dispatcher component to the owning container cluster for each search cluster */ private void initializeDispatchers(Collection searchClusters) { for (SearchCluster searchCluster : searchClusters) { - if ( ! ( searchCluster instanceof IndexedSearchCluster)) continue; - var dispatcher = new DispatcherComponent((IndexedSearchCluster)searchCluster); - owningCluster.addComponent(dispatcher); + if (searchCluster instanceof IndexedSearchCluster indexed) { + var dispatcher = new DispatcherComponent(indexed); + owningCluster.addComponent(dispatcher); + for (var documentDb : indexed.getDocumentDbs()) { + var factory = new RankingExpressionEvaluatorProxyComponent(documentDb); + if (! owningCluster.getComponentsMap().containsKey(factory.getComponentId())) { + owningCluster.addComponent(factory); + } + } + } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/RankingExpressionEvaluatorProxyComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RankingExpressionEvaluatorProxyComponent.java new file mode 100644 index 00000000000..6e0f2249d06 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RankingExpressionEvaluatorProxyComponent.java @@ -0,0 +1,49 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search; + +import com.yahoo.config.model.producer.AnyConfigProducer; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.search.ranking.RankingExpressionEvaluatorProxy; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; +import com.yahoo.vespa.model.container.ContainerModelEvaluation; +import com.yahoo.vespa.model.container.PlatformBundles; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.search.DocumentDatabase; + +public class RankingExpressionEvaluatorProxyComponent + extends Component + implements + RankProfilesConfig.Producer, + RankingConstantsConfig.Producer, + RankingExpressionsConfig.Producer, + OnnxModelsConfig.Producer +{ + private final DocumentDatabase ddb; + + public RankingExpressionEvaluatorProxyComponent(DocumentDatabase db) { + super(toComponentModel(db.getSchemaName())); + ddb = db; + } + + private static ComponentModel toComponentModel(String p) { + String myComponentId = "ranking-expression-evaluator." + p; + return new ComponentModel(myComponentId, + RankingExpressionEvaluatorProxy.class.getName(), + PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE); + } + + @Override + public void getConfig(RankProfilesConfig.Builder builder) { ddb.getConfig(builder); } + + @Override + public void getConfig(RankingExpressionsConfig.Builder builder) { ddb.getConfig(builder); } + + @Override + public void getConfig(RankingConstantsConfig.Builder builder) { ddb.getConfig(builder); } + + @Override + public void getConfig(OnnxModelsConfig.Builder builder) { ddb.getConfig(builder); } +} diff --git a/config-model/src/test/derived/globalphase_onnx_inside/files/ax_plus_b.onnx b/config-model/src/test/derived/globalphase_onnx_inside/files/ax_plus_b.onnx new file mode 100644 index 00000000000..17282d13dc3 --- /dev/null +++ b/config-model/src/test/derived/globalphase_onnx_inside/files/ax_plus_b.onnx @@ -0,0 +1,23 @@ +:© + +matrix_X +vector_AXA"MatMul + +XA +vector_Bvector_Y"AddlrZ +matrix_X +   + +Z +vector_A + +  +Z +vector_B + +  +b +vector_Y + +  +B \ No newline at end of file diff --git a/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg b/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg new file mode 100644 index 00000000000..35bb1ccc3d2 --- /dev/null +++ b/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg @@ -0,0 +1,34 @@ +rankprofile[].name "default" +rankprofile[].fef.property[].name "rankingExpression(handicap).rankingScript" +rankprofile[].fef.property[].value "query(yy)" +rankprofile[].fef.property[].name "rankingExpression(handicap).type" +rankprofile[].fef.property[].value "tensor(d0[2])" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "rankingExpression(firstphase)" +rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" +rankprofile[].fef.property[].value "reduce(attribute(aa), sum)" +rankprofile[].fef.property[].name "vespa.rank.globalphase" +rankprofile[].fef.property[].value "rankingExpression(globalphase)" +rankprofile[].fef.property[].name "rankingExpression(globalphase).rankingScript" +rankprofile[].fef.property[].value "reduce(constant(ww) * (onnx(inside).foobar - rankingExpression(handicap)), sum)" +rankprofile[].fef.property[].name "vespa.match.feature" +rankprofile[].fef.property[].value "attribute(aa)" +rankprofile[].fef.property[].name "vespa.globalphase.rerankcount" +rankprofile[].fef.property[].value "13" +rankprofile[].fef.property[].name "vespa.type.attribute.aa" +rankprofile[].fef.property[].value "tensor(d1[3])" +rankprofile[].fef.property[].name "vespa.type.query.bb" +rankprofile[].fef.property[].value "tensor(d0[2])" +rankprofile[].fef.property[].name "vespa.type.query.yy" +rankprofile[].fef.property[].value "tensor(d0[2])" +rankprofile[].name "unranked" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "value(0)" +rankprofile[].fef.property[].name "vespa.hitcollector.heapsize" +rankprofile[].fef.property[].value "0" +rankprofile[].fef.property[].name "vespa.hitcollector.arraysize" +rankprofile[].fef.property[].value "0" +rankprofile[].fef.property[].name "vespa.dump.ignoredefaultfeatures" +rankprofile[].fef.property[].value "true" +rankprofile[].fef.property[].name "vespa.type.attribute.aa" +rankprofile[].fef.property[].value "tensor(d1[3])" diff --git a/config-model/src/test/derived/globalphase_onnx_inside/test.sd b/config-model/src/test/derived/globalphase_onnx_inside/test.sd new file mode 100644 index 00000000000..c38e318ce6b --- /dev/null +++ b/config-model/src/test/derived/globalphase_onnx_inside/test.sd @@ -0,0 +1,42 @@ +schema test { + + document test { + field aa type tensor(d1[3]) { + indexing: attribute + } + } + + constant xx { + file: files/const_xx.json + type: tensor(d0[2],d1[3]) + } + constant ww { + file: files/const_ww.json + type: tensor(d0[2]) + } + + rank-profile default { + inputs { + query(bb) tensor(d0[2]) + query(yy) tensor(d0[2]) + } + onnx-model inside { + file: files/ax_plus_b.onnx + input vector_A: attribute(aa) + input matrix_X: constant(xx) + input vector_B: query(bb) + output vector_Y: foobar + } + first-phase { + expression: sum(attribute(aa)) + } + function handicap() { + expression: query(yy) + } + global-phase { + rerank-count: 13 + expression: sum(constant(ww) * (onnx(inside).foobar - handicap)) + } + } + +} diff --git a/config-model/src/test/derived/rankingexpression/rank-profiles.cfg b/config-model/src/test/derived/rankingexpression/rank-profiles.cfg index ea8bc5f77e6..e3947e9e46f 100644 --- a/config-model/src/test/derived/rankingexpression/rank-profiles.cfg +++ b/config-model/src/test/derived/rankingexpression/rank-profiles.cfg @@ -410,8 +410,6 @@ rankprofile[].fef.property[].value "rankingExpression(globalphase)" rankprofile[].fef.property[].name "rankingExpression(globalphase).rankingScript" rankprofile[].fef.property[].value "rankingExpression(myplus) + reduce(rankingExpression(mymul), sum) + firstPhase" rankprofile[].fef.property[].name "vespa.match.feature" -rankprofile[].fef.property[].value "query(fromq)" -rankprofile[].fef.property[].name "vespa.match.feature" rankprofile[].fef.property[].value "firstPhase" rankprofile[].fef.property[].name "vespa.match.feature" rankprofile[].fef.property[].value "attribute(t1)" diff --git a/config-model/src/test/java/com/yahoo/schema/derived/GlobalPhaseOnnxModelsTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/GlobalPhaseOnnxModelsTestCase.java new file mode 100644 index 00000000000..2ff33dd70d8 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/GlobalPhaseOnnxModelsTestCase.java @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.parser.ParseException; +import org.junit.jupiter.api.Test; +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Tests exporting with global-phase and ONNX models + * + * @author arnej + */ +public class GlobalPhaseOnnxModelsTestCase extends AbstractExportingTestCase { + + @Test + void testModelInRankProfile() throws IOException, ParseException { + assertCorrectDeriving("globalphase_onnx_inside"); + } + +} -- cgit v1.2.3 From c32d266e593f48a4c1c63e600d865a40d749cc62 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Mon, 27 Feb 2023 14:19:11 +0000 Subject: add Override annotations --- .../src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java b/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java index 39abacc17f1..c8aa2b6b776 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java @@ -15,25 +15,28 @@ public class SimpleEvaluator implements Evaluator { private final FunctionEvaluator evaluator; private final Set neededInputs; - public SimpleEvaluator(FunctionEvaluator prototype) { this.evaluator = prototype; this.neededInputs = new HashSet(prototype.function().arguments()); } + @Override public Collection needInputs() { return List.copyOf(neededInputs); } + @Override public SimpleEvaluator bind(String name, Tensor value) { if (value != null) evaluator.bind(name, value); neededInputs.remove(name); return this; } + @Override public double evaluateScore() { return evaluator.evaluate().asDouble(); } + @Override public String toString() { var buf = new StringBuilder(); buf.append("SimpleEvaluator("); -- cgit v1.2.3 From c095c1061088bae7e6f5f26ec595325fc79be43c Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Mon, 27 Feb 2023 14:20:11 +0000 Subject: remove loop which was guaranteed to run exactly once --- .../main/java/com/yahoo/prelude/cluster/ClusterSearcher.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index a18250fbcfe..c29c7d12f9a 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -243,14 +243,12 @@ public class ClusterSearcher extends Searcher { if (restrict.size() != 1) { throw new IllegalStateException("perSchemaSearch must always be called with 1 schema, got: " + restrict.size()); } - for (String schema : restrict) { - Result result = searcher.search(query, execution); - if (globalPhaseHelper != null) { - globalPhaseHelper.process(query, result, schema); - } - return result; + String schema = restrict.iterator().next(); + Result result = searcher.search(query, execution); + if (globalPhaseHelper != null) { + globalPhaseHelper.process(query, result, schema); } - return null; + return result; } private static void processResult(Query query, FutureTask task, Result mergedResult) { -- cgit v1.2.3 From f723c2ca9252e6aab036a3ee21a24603f6fffb85 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Mon, 27 Feb 2023 14:34:36 +0000 Subject: split out rerankHits and add adjust of scores just like second-phase adjustment --- .../yahoo/search/ranking/GlobalPhaseHelper.java | 36 +-------- .../com/yahoo/search/ranking/ResultReranker.java | 91 ++++++++++++++++++++++ 2 files changed, 92 insertions(+), 35 deletions(-) create mode 100644 container-search/src/main/java/com/yahoo/search/ranking/ResultReranker.java diff --git a/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseHelper.java b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseHelper.java index 9810f612e5c..be4ba5444fe 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseHelper.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseHelper.java @@ -48,7 +48,7 @@ public class GlobalPhaseHelper { }; // TODO need to get rerank-count somehow int rerank = 7; - rerankHits(query, result, new HitRescorer(supplier), rerank); + ResultReranker.rerankHits(result, new HitRescorer(supplier), rerank); } record NameAndValue(String name, Tensor value) { } @@ -86,40 +86,6 @@ public class GlobalPhaseHelper { return result; } - void rerankHits(Query query, Result result, HitRescorer hitRescorer, int rerank) { - double worstRerankedScore = Double.MAX_VALUE; - double worstRerankedOldScore = Double.MAX_VALUE; - // TODO consider doing recursive iteration instead of deepIterator - for (var iterator = result.hits().deepIterator(); iterator.hasNext();) { - Hit hit = iterator.next(); - if (hit.isMeta() || hit instanceof HitGroup) { - continue; - } - // what about hits inside grouping results? - if (rerank > 0) { - double oldScore = hit.getRelevance().getScore(); - boolean didRerank = hitRescorer.rescoreHit(hit); - if (didRerank) { - double newScore = hit.getRelevance().getScore(); - if (oldScore < worstRerankedOldScore) worstRerankedOldScore = oldScore; - if (newScore < worstRerankedScore) worstRerankedScore = newScore; - --rerank; - } else { - // failed to rescore this hit, what should we do? - hit.setRelevance(-Double.MAX_VALUE); - } - } else { - // too low quality - if (worstRerankedOldScore > worstRerankedScore) { - double penalty = worstRerankedOldScore - worstRerankedScore; - double oldScore = hit.getRelevance().getScore(); - hit.setRelevance(oldScore - penalty); - } - } - } - result.hits().sort(); - } - private Supplier underlying(Query query, String schema) { String rankProfile = query.getRanking().getProfile(); String key = schema + " with rank profile " + rankProfile; diff --git a/container-search/src/main/java/com/yahoo/search/ranking/ResultReranker.java b/container-search/src/main/java/com/yahoo/search/ranking/ResultReranker.java new file mode 100644 index 00000000000..e700276b151 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/ResultReranker.java @@ -0,0 +1,91 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.logging.Logger; + +public class ResultReranker { + + private static final Logger logger = Logger.getLogger(ResultReranker.class.getName()); + + // scale and adjust the score according to the range + // of the original and final score values to avoid that + // a score from the backend is larger than finalScores_low + static class Ranges { + private double initialScores_high = -Double.MAX_VALUE; + private double initialScores_low = Double.MAX_VALUE; + private double finalScores_high = -Double.MAX_VALUE; + private double finalScores_low = Double.MAX_VALUE; + + boolean valid() { + return (initialScores_high >= initialScores_low + && + finalScores_high >= finalScores_low); + } + void withInitialScore(double score) { + if (score < initialScores_low) initialScores_low = score; + if (score > initialScores_high) initialScores_high = score; + } + void withFinalScore(double score) { + if (score < finalScores_low) finalScores_low = score; + if (score > finalScores_high) finalScores_high = score; + } + private double initialRange() { + double r = initialScores_high - initialScores_low; + if (r < 1.0) r = 1.0; + return r; + } + private double finalRange() { + double r = finalScores_high - finalScores_low; + if (r < 1.0) r = 1.0; + return r; + } + double scale() { return finalRange() / initialRange(); } + double bias() { return finalScores_low - initialScores_low * scale(); } + } + + static void rerankHits(Result result, HitRescorer hitRescorer, int rerankCount) { + List hitsToRescore = new ArrayList<>(); + // consider doing recursive iteration explicitly instead of using deepIterator? + for (var iterator = result.hits().deepIterator(); iterator.hasNext();) { + Hit hit = iterator.next(); + if (hit.isMeta() || hit instanceof HitGroup) { + continue; + } + // what about hits inside grouping results? + // they are inside GroupingListHit, we won't recurse into it; so we won't see them. + hitsToRescore.add(hit); + } + // we can't be 100% certain that hits were sorted according to relevance: + hitsToRescore.sort(Comparator.naturalOrder()); + var ranges = new Ranges(); + for (var iterator = hitsToRescore.iterator(); rerankCount > 0 && iterator.hasNext(); ) { + Hit hit = iterator.next(); + double oldScore = hit.getRelevance().getScore(); + boolean didRerank = hitRescorer.rescoreHit(hit); + if (didRerank) { + ranges.withInitialScore(oldScore); + ranges.withFinalScore(hit.getRelevance().getScore()); + --rerankCount; + iterator.remove(); + } + } + // if any hits are left in the list, they need rescaling: + if (ranges.valid()) { + double scale = ranges.scale(); + double bias = ranges.bias(); + for (Hit hit : hitsToRescore) { + double oldScore = hit.getRelevance().getScore(); + hit.setRelevance(oldScore * scale + bias); + } + } + result.hits().sort(); + } + +} -- cgit v1.2.3 From c9c6360578182ea06fa19f45c8ddc51f00b3f0c0 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Mon, 27 Feb 2023 16:31:20 +0100 Subject: Handle archive URI without tenant name dir --- .../vespa/hosted/provision/archive/ArchiveUriManager.java | 9 +++++++-- .../hosted/provision/archive/ArchiveUriManagerTest.java | 15 ++++++++++++--- .../vespa/hosted/provision/restapi/ArchiveApiTest.java | 4 ++-- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java index 27488e4027c..3ff6999f082 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java @@ -13,7 +13,6 @@ import com.yahoo.vespa.hosted.provision.persistence.CuratorDb; import java.time.Duration; import java.util.Optional; import java.util.function.Function; -import java.util.logging.Logger; /** * Thread safe class to get and set archive URI for given account and tenants. @@ -22,7 +21,6 @@ import java.util.logging.Logger; */ public class ArchiveUriManager { - private static final Logger log = Logger.getLogger(ArchiveUriManager.class.getName()); private static final Duration cacheTtl = Duration.ofMinutes(1); private final CuratorDb db; @@ -47,8 +45,15 @@ public class ArchiveUriManager { return Optional.ofNullable(node.cloudAccount().isEnclave(zone) ? archiveUris.get().accountArchiveUris().get(node.cloudAccount()) : archiveUris.get().tenantArchiveUris().get(app.tenant())) + .map(uri -> { + // TODO (freva): Remove when all URIs dont have tenant name in them anymore + String tenantSuffix = app.tenant().value() + "/"; + if (uri.endsWith(tenantSuffix)) return uri.substring(0, uri.length() - tenantSuffix.length()); + return uri; + }) .map(uri -> { StringBuilder sb = new StringBuilder(100).append(uri) + .append(app.tenant().value()).append('/') .append(app.application().value()).append('/') .append(app.instance().value()).append('/') .append(node.allocation().get().membership().cluster().id().value()).append('/'); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java index 8ee72d12f57..e759a0b0a7f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java @@ -54,10 +54,19 @@ public class ArchiveUriManagerTest { assertFalse(archiveUriManager.archiveUriFor(createNode(null, account1)).isPresent()); // URI set for this account, but not allocated assertFalse(archiveUriManager.archiveUriFor(createNode(null, account2)).isPresent()); // Not allocated assertFalse(archiveUriManager.archiveUriFor(createNode(app2, null)).isPresent()); // No URI set for this tenant or account - assertEquals("scheme://tenant-bucket/dir/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, null)).get()); - assertEquals("scheme://account-bucket/dir/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, account1)).get()); // Account has precedence + assertEquals("scheme://tenant-bucket/dir/vespa/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, null)).get()); + assertEquals("scheme://account-bucket/dir/vespa/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, account1)).get()); // Account has precedence assertFalse(archiveUriManager.archiveUriFor(createNode(app1, account2)).isPresent()); // URI set for this tenant, but is ignored because enclave account - assertEquals("scheme://tenant-bucket/dir/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, accountSystem)).get()); // URI for tenant because non-enclave acocunt + assertEquals("scheme://tenant-bucket/dir/vespa/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, accountSystem)).get()); // URI for tenant because non-enclave acocunt + } + + @Test + public void handles_uri_with_tenant_name() { + ApplicationId app1 = ApplicationId.from("vespa", "music", "main"); + ArchiveUriManager archiveUriManager = new ProvisioningTester.Builder().build().nodeRepository().archiveUriManager(); + archiveUriManager.setArchiveUri(app1.tenant(), Optional.of("scheme://tenant-bucket/vespa")); + + assertEquals("scheme://tenant-bucket/vespa/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, null)).get()); } private Node createNode(ApplicationId appId, CloudAccount account) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java index 80d79b036e2..03581146b9f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java @@ -45,8 +45,8 @@ public class ArchiveApiTest { "{\"message\":\"Updated archive URI for 777888999000\"}"); - tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "\"archiveUri\":\"ftp://host/dir/application3/instance3/id3/host4/\"", true); - tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com"), "\"archiveUri\":\"s3://acc-bucket/zoneapp/zoneapp/node-admin/dockerhost2/\"", true); + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "\"archiveUri\":\"ftp://host/dir/tenant3/application3/instance3/id3/host4/\"", true); + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com"), "\"archiveUri\":\"s3://acc-bucket/zoneapp/zoneapp/zoneapp/node-admin/dockerhost2/\"", true); assertFile(new Request("http://localhost:8080/nodes/v2/archive"), "archives.json"); tester.assertResponse(new Request("http://localhost:8080/nodes/v2/archive/tenant/tenant3", new byte[0], Request.Method.DELETE), "{\"message\":\"Removed archive URI for tenant3\"}"); -- cgit v1.2.3 From 3458357e094c88c0562b7561f01698944a2998f6 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Mon, 27 Feb 2023 22:04:54 +0100 Subject: Use python 3.8 on CentOS Stream 8. --- dist/vespa.spec | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dist/vespa.spec b/dist/vespa.spec index 6bed9904124..e880734979a 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -54,8 +54,7 @@ BuildRequires: gcc-toolset-12-libatomic-devel BuildRequires: maven BuildRequires: maven-openjdk17 BuildRequires: vespa-pybind11-devel -BuildRequires: python3-pytest -BuildRequires: python36-devel +BuildRequires: python38-devel BuildRequires: glibc-langpack-en %endif %if 0%{?el9} @@ -377,7 +376,7 @@ Summary: Vespa - The open big data serving engine - ann-benchmark Requires: %{name}-base-libs = %{version}-%{release} Requires: %{name}-libs = %{version}-%{release} %if 0%{?el8} -Requires: python36 +Requires: python38 %endif %if 0%{?el9} Requires: python3 -- cgit v1.2.3 From 453342085e6fb785971e3e35f2466bf8b4061adf Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Tue, 28 Feb 2023 10:28:29 +0100 Subject: Add empty() method to GetConfigContext --- .../main/java/com/yahoo/vespa/config/server/GetConfigContext.java | 6 ++++++ .../java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java | 2 +- .../src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java b/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java index a614e251dca..89f7755729c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java @@ -38,9 +38,15 @@ public class GetConfigContext { return new GetConfigContext(app, handler, trace); } + public static GetConfigContext empty() { + return new GetConfigContext(null, null, null); + } + public static GetConfigContext testContext(ApplicationId app) { return new GetConfigContext(app, null, null); } + + public boolean isEmpty() { return app == null && requestHandler == null && trace == null; } /** * Helper to produce a log preamble with the tenant and app id diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java index 1c419ce047a..6aafe5e0261 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java @@ -96,7 +96,7 @@ class GetConfigProcessor implements Runnable { } GetConfigContext context = rpcServer.createGetConfigContext(tenant, request, trace); - if (context == null || ! context.requestHandler().hasApplication(context.applicationId(), Optional.empty())) { + if (context.isEmpty() || ! context.requestHandler().hasApplication(context.applicationId(), Optional.empty())) { handleError(request, APPLICATION_NOT_LOADED, "No application exists"); return null; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index 2d731370b70..7a1b2d2aeef 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -406,7 +406,7 @@ public class RpcServer implements Runnable, ConfigActivationListener, TenantList metrics.incUnknownHostRequests(); trace.trace(TRACELEVEL, msg); log.log(Level.WARNING, msg); - return null; + return GetConfigContext.empty(); } RequestHandler handler = requestHandler.get(); ApplicationId applicationId = handler.resolveApplicationId(request.getClientHostName()); -- cgit v1.2.3 From 62a2290b49a10e5efda6522b4b26ed8dd7c295c3 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Tue, 28 Feb 2023 10:51:25 +0100 Subject: Make code clearer by using a new record DelayedConfig --- .../config/server/rpc/GetConfigProcessor.java | 40 +++++++++++----------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java index 6aafe5e0261..5e7fff8c922 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.config.server.rpc; import com.yahoo.cloud.config.SentinelConfig; -import com.yahoo.collections.Pair; import com.yahoo.component.Version; import com.yahoo.config.provision.TenantName; import com.yahoo.container.di.config.ApplicationBundlesConfig; @@ -37,12 +36,10 @@ class GetConfigProcessor implements Runnable { private static final Logger log = Logger.getLogger(GetConfigProcessor.class.getName()); private static final String localHostName = HostName.getLocalhost(); - private static final PayloadChecksums emptyApplicationBundlesConfigChecksums = PayloadChecksums.fromPayload(Payload.from(ConfigPayload.fromInstance(new ApplicationBundlesConfig.Builder().build()))); private final JRTServerConfigRequest request; - /* True only when this request has expired its server timeout and we need to respond to the client */ private final boolean forceResponse; private final RpcServer rpcServer; @@ -75,13 +72,13 @@ class GetConfigProcessor implements Runnable { } // TODO: Increment statistics (Metrics) failed counters when requests fail - public Pair getConfig(JRTServerConfigRequest request) { + public Optional resolveConfig(JRTServerConfigRequest request) { // Request has already been detached if ( ! request.validateParameters()) { // Error code is set in verifyParameters if parameters are not OK. log.log(Level.WARNING, "Parameters for request " + request + " did not validate: " + request.errorCode() + " : " + request.errorMessage()); respond(request); - return null; + return Optional.empty(); } Trace trace = request.getRequestTrace(); debugLog(trace, "GetConfigProcessor.run() on " + localHostName); @@ -92,13 +89,13 @@ class GetConfigProcessor implements Runnable { // fabricate an empty request to cause the sentinel to stop all running services if (rpcServer.canReturnEmptySentinelConfig() && rpcServer.allTenantsLoaded() && tenant.isEmpty() && isSentinelConfigRequest(request)) { returnEmpty(request); - return null; + return Optional.empty(); } GetConfigContext context = rpcServer.createGetConfigContext(tenant, request, trace); if (context.isEmpty() || ! context.requestHandler().hasApplication(context.applicationId(), Optional.empty())) { handleError(request, APPLICATION_NOT_LOADED, "No application exists"); - return null; + return Optional.empty(); } logPre = TenantRepository.logPre(context.applicationId()); @@ -109,7 +106,7 @@ class GetConfigProcessor implements Runnable { if ( ! context.requestHandler().hasApplication(context.applicationId(), vespaVersion)) { handleError(request, ErrorCode.UNKNOWN_VESPA_VERSION, "Unknown Vespa version in request: " + printableVespaVersion(vespaVersion)); - return null; + return Optional.empty(); } ConfigResponse config; @@ -117,14 +114,14 @@ class GetConfigProcessor implements Runnable { config = rpcServer.resolveConfig(request, context, vespaVersion); } catch (UnknownConfigDefinitionException e) { handleError(request, ErrorCode.UNKNOWN_DEFINITION, "Unknown config definition " + request.getConfigKey()); - return null; + return Optional.empty(); } catch (UnknownConfigIdException e) { handleError(request, ErrorCode.ILLEGAL_CONFIGID, "Illegal config id " + request.getConfigKey().getConfigId()); - return null; + return Optional.empty(); } catch (Throwable e) { log.log(Level.SEVERE, "Unexpected error handling config request", e); handleError(request, ErrorCode.INTERNAL_ERROR, "Internal error " + e.getMessage()); - return null; + return Optional.empty(); } // config == null is not an error, but indicates that the config will be returned later. @@ -135,7 +132,7 @@ class GetConfigProcessor implements Runnable { && ! context.requestHandler().compatibleWith(vespaVersion, context.applicationId()) // ... with a runtime version incompatible with the deploying version ... && ! emptyApplicationBundlesConfigChecksums.matches(config.getPayloadChecksums())) { // ... and there actually are incompatible user bundles, then return no config: handleError(request, ErrorCode.INCOMPATIBLE_VESPA_VERSION, "Version " + printableVespaVersion(vespaVersion) + " is binary incompatible with the latest deployed version"); - return null; + return Optional.empty(); } // debugLog(trace, "config response before encoding:" + config.toString()); @@ -144,23 +141,24 @@ class GetConfigProcessor implements Runnable { respond(request); } else { debugLog(trace, "delaying response " + request.getShortDescription()); - return new Pair<>(context, config != null ? config.getGeneration() : 0); + return Optional.of(new DelayedConfig(context, config != null ? config.getGeneration() : 0)); } - return null; + return Optional.empty(); } @Override public void run() { - Pair delayed = getConfig(request); + Optional delayed = resolveConfig(request); - if (delayed != null) { - rpcServer.delayResponse(request, delayed.getFirst()); - if (rpcServer.hasNewerGeneration(delayed.getFirst().applicationId(), delayed.getSecond())) { + delayed.ifPresent(d -> { + GetConfigContext context = d.context(); + rpcServer.delayResponse(request, context); + if (rpcServer.hasNewerGeneration(context.applicationId(), d.generation())) { // This will ensure that if the config activation train left the station while I was boarding, // another train will immediately be scheduled. - rpcServer.configActivated(delayed.getFirst().applicationId()); + rpcServer.configActivated(context.applicationId()); } - } + }); } private boolean isSentinelConfigRequest(JRTServerConfigRequest request) { @@ -194,4 +192,6 @@ class GetConfigProcessor implements Runnable { } } + private record DelayedConfig(GetConfigContext context, long generation) {} + } -- cgit v1.2.3 From 841ddc2ca6f3f311e3cea83dccdcabdddd348266 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Tue, 28 Feb 2023 10:08:04 +0000 Subject: make implementation details package private --- .../src/main/java/com/yahoo/search/ranking/HitRescorer.java | 2 +- .../src/main/java/com/yahoo/search/ranking/ResultReranker.java | 2 +- .../src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/container-search/src/main/java/com/yahoo/search/ranking/HitRescorer.java b/container-search/src/main/java/com/yahoo/search/ranking/HitRescorer.java index b92f9da2e2a..ebdbbb693f1 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/HitRescorer.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/HitRescorer.java @@ -7,7 +7,7 @@ import com.yahoo.search.result.Hit; import java.util.function.Supplier; import java.util.logging.Logger; -public class HitRescorer { +class HitRescorer { private static final Logger logger = Logger.getLogger(HitRescorer.class.getName()); diff --git a/container-search/src/main/java/com/yahoo/search/ranking/ResultReranker.java b/container-search/src/main/java/com/yahoo/search/ranking/ResultReranker.java index e700276b151..11b3fa7390a 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/ResultReranker.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/ResultReranker.java @@ -10,7 +10,7 @@ import java.util.Comparator; import java.util.List; import java.util.logging.Logger; -public class ResultReranker { +class ResultReranker { private static final Logger logger = Logger.getLogger(ResultReranker.class.getName()); diff --git a/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java b/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java index c8aa2b6b776..f247eab1649 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java @@ -11,7 +11,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -public class SimpleEvaluator implements Evaluator { +class SimpleEvaluator implements Evaluator { private final FunctionEvaluator evaluator; private final Set neededInputs; -- cgit v1.2.3 From d18281c03338be2c53ccaffc0b849a9e86c2f488 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Tue, 28 Feb 2023 11:17:45 +0100 Subject: Handle bucket name ending with tenant name --- .../com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java | 4 ++-- .../yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java index 3ff6999f082..faa360bbcb1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java @@ -47,8 +47,8 @@ public class ArchiveUriManager { archiveUris.get().tenantArchiveUris().get(app.tenant())) .map(uri -> { // TODO (freva): Remove when all URIs dont have tenant name in them anymore - String tenantSuffix = app.tenant().value() + "/"; - if (uri.endsWith(tenantSuffix)) return uri.substring(0, uri.length() - tenantSuffix.length()); + String tenantSuffix = "/" + app.tenant().value() + "/"; + if (uri.endsWith(tenantSuffix)) return uri.substring(0, uri.length() - tenantSuffix.length() + 1); return uri; }) .map(uri -> { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java index e759a0b0a7f..44c1c976355 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java @@ -65,8 +65,11 @@ public class ArchiveUriManagerTest { ApplicationId app1 = ApplicationId.from("vespa", "music", "main"); ArchiveUriManager archiveUriManager = new ProvisioningTester.Builder().build().nodeRepository().archiveUriManager(); archiveUriManager.setArchiveUri(app1.tenant(), Optional.of("scheme://tenant-bucket/vespa")); - assertEquals("scheme://tenant-bucket/vespa/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, null)).get()); + + // Archive URI ends with the tenant name + archiveUriManager.setArchiveUri(app1.tenant(), Optional.of("scheme://tenant-vespa/")); + assertEquals("scheme://tenant-vespa/vespa/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, null)).get()); } private Node createNode(ApplicationId appId, CloudAccount account) { -- cgit v1.2.3 From f3eea7be512f6714cc9aeacf931a75afb9be455d Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Tue, 28 Feb 2023 10:28:11 +0000 Subject: rename classes after review --- .../model/container/search/ContainerSearch.java | 4 +- .../search/RankProfilesEvaluatorComponent.java | 49 +++++++++ .../RankingExpressionEvaluatorProxyComponent.java | 49 --------- .../com/yahoo/prelude/cluster/ClusterSearcher.java | 6 +- .../yahoo/search/ranking/GlobalPhaseHelper.java | 118 --------------------- .../yahoo/search/ranking/GlobalPhaseRanker.java | 118 +++++++++++++++++++++ .../search/ranking/RankProfilesEvaluator.java | 53 +++++++++ .../ranking/RankingExpressionEvaluatorFactory.java | 6 +- .../ranking/RankingExpressionEvaluatorProxy.java | 53 --------- 9 files changed, 228 insertions(+), 228 deletions(-) create mode 100644 config-model/src/main/java/com/yahoo/vespa/model/container/search/RankProfilesEvaluatorComponent.java delete mode 100644 config-model/src/main/java/com/yahoo/vespa/model/container/search/RankingExpressionEvaluatorProxyComponent.java delete mode 100644 container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseHelper.java create mode 100644 container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java create mode 100644 container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluator.java delete mode 100644 container-search/src/main/java/com/yahoo/search/ranking/RankingExpressionEvaluatorProxy.java diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index e3ba40c9037..b3f0959029c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -58,7 +58,7 @@ public class ContainerSearch extends ContainerSubsystem owningCluster.addComponent(Component.fromClassAndBundle(com.yahoo.search.schema.SchemaInfo.class, SEARCH_AND_DOCPROC_BUNDLE)); owningCluster.addComponent(Component.fromClassAndBundle(SearchStatusExtension.class, SEARCH_AND_DOCPROC_BUNDLE)); owningCluster.addComponent(Component.fromClassAndBundle(RankingExpressionEvaluatorFactory.class, SEARCH_AND_DOCPROC_BUNDLE)); - owningCluster.addComponent(Component.fromClassAndBundle(com.yahoo.search.ranking.GlobalPhaseHelper.class, SEARCH_AND_DOCPROC_BUNDLE)); + owningCluster.addComponent(Component.fromClassAndBundle(com.yahoo.search.ranking.GlobalPhaseRanker.class, SEARCH_AND_DOCPROC_BUNDLE)); cluster.addSearchAndDocprocBundles(); } @@ -75,7 +75,7 @@ public class ContainerSearch extends ContainerSubsystem var dispatcher = new DispatcherComponent(indexed); owningCluster.addComponent(dispatcher); for (var documentDb : indexed.getDocumentDbs()) { - var factory = new RankingExpressionEvaluatorProxyComponent(documentDb); + var factory = new RankProfilesEvaluatorComponent(documentDb); if (! owningCluster.getComponentsMap().containsKey(factory.getComponentId())) { owningCluster.addComponent(factory); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/RankProfilesEvaluatorComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RankProfilesEvaluatorComponent.java new file mode 100644 index 00000000000..75a2802ee53 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RankProfilesEvaluatorComponent.java @@ -0,0 +1,49 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search; + +import com.yahoo.config.model.producer.AnyConfigProducer; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.search.ranking.RankProfilesEvaluator; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; +import com.yahoo.vespa.model.container.ContainerModelEvaluation; +import com.yahoo.vespa.model.container.PlatformBundles; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.search.DocumentDatabase; + +public class RankProfilesEvaluatorComponent + extends Component + implements + RankProfilesConfig.Producer, + RankingConstantsConfig.Producer, + RankingExpressionsConfig.Producer, + OnnxModelsConfig.Producer +{ + private final DocumentDatabase ddb; + + public RankProfilesEvaluatorComponent(DocumentDatabase db) { + super(toComponentModel(db.getSchemaName())); + ddb = db; + } + + private static ComponentModel toComponentModel(String p) { + String myComponentId = "ranking-expression-evaluator." + p; + return new ComponentModel(myComponentId, + RankProfilesEvaluator.class.getName(), + PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE); + } + + @Override + public void getConfig(RankProfilesConfig.Builder builder) { ddb.getConfig(builder); } + + @Override + public void getConfig(RankingExpressionsConfig.Builder builder) { ddb.getConfig(builder); } + + @Override + public void getConfig(RankingConstantsConfig.Builder builder) { ddb.getConfig(builder); } + + @Override + public void getConfig(OnnxModelsConfig.Builder builder) { ddb.getConfig(builder); } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/RankingExpressionEvaluatorProxyComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RankingExpressionEvaluatorProxyComponent.java deleted file mode 100644 index 6e0f2249d06..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/RankingExpressionEvaluatorProxyComponent.java +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.container.search; - -import com.yahoo.config.model.producer.AnyConfigProducer; -import com.yahoo.osgi.provider.model.ComponentModel; -import com.yahoo.search.ranking.RankingExpressionEvaluatorProxy; -import com.yahoo.vespa.config.search.RankProfilesConfig; -import com.yahoo.vespa.config.search.core.OnnxModelsConfig; -import com.yahoo.vespa.config.search.core.RankingConstantsConfig; -import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; -import com.yahoo.vespa.model.container.ContainerModelEvaluation; -import com.yahoo.vespa.model.container.PlatformBundles; -import com.yahoo.vespa.model.container.component.Component; -import com.yahoo.vespa.model.search.DocumentDatabase; - -public class RankingExpressionEvaluatorProxyComponent - extends Component - implements - RankProfilesConfig.Producer, - RankingConstantsConfig.Producer, - RankingExpressionsConfig.Producer, - OnnxModelsConfig.Producer -{ - private final DocumentDatabase ddb; - - public RankingExpressionEvaluatorProxyComponent(DocumentDatabase db) { - super(toComponentModel(db.getSchemaName())); - ddb = db; - } - - private static ComponentModel toComponentModel(String p) { - String myComponentId = "ranking-expression-evaluator." + p; - return new ComponentModel(myComponentId, - RankingExpressionEvaluatorProxy.class.getName(), - PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE); - } - - @Override - public void getConfig(RankProfilesConfig.Builder builder) { ddb.getConfig(builder); } - - @Override - public void getConfig(RankingExpressionsConfig.Builder builder) { ddb.getConfig(builder); } - - @Override - public void getConfig(RankingConstantsConfig.Builder builder) { ddb.getConfig(builder); } - - @Override - public void getConfig(OnnxModelsConfig.Builder builder) { ddb.getConfig(builder); } -} diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index c29c7d12f9a..7787d7d7702 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -20,7 +20,7 @@ import com.yahoo.search.Searcher; import com.yahoo.search.config.ClusterConfig; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.query.ParameterParser; -import com.yahoo.search.ranking.GlobalPhaseHelper; +import com.yahoo.search.ranking.GlobalPhaseRanker; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.searchchain.Execution; @@ -65,7 +65,7 @@ public class ClusterSearcher extends Searcher { private final VespaBackEndSearcher server; private final Executor executor; - private final GlobalPhaseHelper globalPhaseHelper; + private final GlobalPhaseRanker globalPhaseHelper; @Inject public ClusterSearcher(ComponentId id, @@ -75,7 +75,7 @@ public class ClusterSearcher extends Searcher { DocumentdbInfoConfig documentDbConfig, SchemaInfo schemaInfo, ComponentRegistry dispatchers, - GlobalPhaseHelper globalPhaseHelper, + GlobalPhaseRanker globalPhaseHelper, VipStatus vipStatus, VespaDocumentAccess access) { super(id); diff --git a/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseHelper.java b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseHelper.java deleted file mode 100644 index be4ba5444fe..00000000000 --- a/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseHelper.java +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.search.ranking; - -import ai.vespa.models.evaluation.FunctionEvaluator; -import ai.vespa.models.evaluation.Model; -import com.yahoo.component.annotation.Inject; -import com.yahoo.search.Query; -import com.yahoo.search.Result; -import com.yahoo.search.result.Hit; -import com.yahoo.search.result.HitGroup; -import com.yahoo.tensor.Tensor; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.logging.Logger; -import java.util.function.Supplier; - -public class GlobalPhaseHelper { - - private static final Logger logger = Logger.getLogger(GlobalPhaseHelper.class.getName()); - private final RankingExpressionEvaluatorFactory factory; - private final Set skipProcessing = new HashSet<>(); - private final Map> scorers = new HashMap<>(); - - @Inject - public GlobalPhaseHelper(RankingExpressionEvaluatorFactory factory) { - this.factory = factory; - logger.info("using factory: " + factory); - } - - public void process(Query query, Result result, String schema) { - var functionEvaluatorSource = underlying(query, schema); - if (functionEvaluatorSource == null) { - return; - } - var prepared = findFromQuery(query, functionEvaluatorSource.get().function().arguments()); - Supplier supplier = () -> { - var evaluator = functionEvaluatorSource.get(); - var simple = new SimpleEvaluator(evaluator); - for (var entry : prepared) { - simple.bind(entry.name(), entry.value()); - } - return simple; - }; - // TODO need to get rerank-count somehow - int rerank = 7; - ResultReranker.rerankHits(result, new HitRescorer(supplier), rerank); - } - - record NameAndValue(String name, Tensor value) { } - - /* do this only once per query: */ - List findFromQuery(Query query, List needInputs) { - List result = new ArrayList<>(); - var ranking = query.getRanking(); - var rankFeatures = ranking.getFeatures(); - var rankProps = ranking.getProperties().asMap(); - for (String needed : needInputs) { - var optRef = com.yahoo.searchlib.rankingexpression.Reference.simple(needed); - if (optRef.isEmpty()) continue; - var ref = optRef.get(); - if (ref.name().equals("constant")) { - // XXX in theory, we should be able to avoid this - result.add(new NameAndValue(needed, null)); - continue; - } - if (ref.isSimple() && ref.name().equals("query")) { - String queryFeatureName = ref.simpleArgument().get(); - // searchers are recommended to place query features here: - var feature = rankFeatures.getTensor(queryFeatureName); - if (feature.isPresent()) { - result.add(new NameAndValue(needed, feature.get())); - } else { - // but other ways of setting query features end up in the properties: - var objList = rankProps.get(queryFeatureName); - if (objList != null && objList.size() == 1 && objList.get(0) instanceof Tensor t) { - result.add(new NameAndValue(needed, t)); - } - } - } - } - return result; - } - - private Supplier underlying(Query query, String schema) { - String rankProfile = query.getRanking().getProfile(); - String key = schema + " with rank profile " + rankProfile; - if (skipProcessing.contains(key)) { - return null; - } - Supplier supplier = scorers.get(key); - if (supplier != null) { - return supplier; - } - try { - var proxy = factory.proxyForSchema(schema); - var model = proxy.modelForRankProfile(rankProfile); - supplier = () -> model.evaluatorOf("globalphase"); - if (supplier.get() == null) { - supplier = null; - } - } catch (IllegalArgumentException e) { - logger.info("no global-phase for " + key + " because: " + e.getMessage()); - supplier = null; - } - if (supplier == null) { - skipProcessing.add(key); - } else { - scorers.put(key, supplier); - } - return supplier; - } - -} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java new file mode 100644 index 00000000000..fefe8583f3f --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java @@ -0,0 +1,118 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import ai.vespa.models.evaluation.FunctionEvaluator; +import ai.vespa.models.evaluation.Model; +import com.yahoo.component.annotation.Inject; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.tensor.Tensor; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.logging.Logger; +import java.util.function.Supplier; + +public class GlobalPhaseRanker { + + private static final Logger logger = Logger.getLogger(GlobalPhaseRanker.class.getName()); + private final RankingExpressionEvaluatorFactory factory; + private final Set skipProcessing = new HashSet<>(); + private final Map> scorers = new HashMap<>(); + + @Inject + public GlobalPhaseRanker(RankingExpressionEvaluatorFactory factory) { + this.factory = factory; + logger.info("using factory: " + factory); + } + + public void process(Query query, Result result, String schema) { + var functionEvaluatorSource = underlying(query, schema); + if (functionEvaluatorSource == null) { + return; + } + var prepared = findFromQuery(query, functionEvaluatorSource.get().function().arguments()); + Supplier supplier = () -> { + var evaluator = functionEvaluatorSource.get(); + var simple = new SimpleEvaluator(evaluator); + for (var entry : prepared) { + simple.bind(entry.name(), entry.value()); + } + return simple; + }; + // TODO need to get rerank-count somehow + int rerank = 7; + ResultReranker.rerankHits(result, new HitRescorer(supplier), rerank); + } + + record NameAndValue(String name, Tensor value) { } + + /* do this only once per query: */ + List findFromQuery(Query query, List needInputs) { + List result = new ArrayList<>(); + var ranking = query.getRanking(); + var rankFeatures = ranking.getFeatures(); + var rankProps = ranking.getProperties().asMap(); + for (String needed : needInputs) { + var optRef = com.yahoo.searchlib.rankingexpression.Reference.simple(needed); + if (optRef.isEmpty()) continue; + var ref = optRef.get(); + if (ref.name().equals("constant")) { + // XXX in theory, we should be able to avoid this + result.add(new NameAndValue(needed, null)); + continue; + } + if (ref.isSimple() && ref.name().equals("query")) { + String queryFeatureName = ref.simpleArgument().get(); + // searchers are recommended to place query features here: + var feature = rankFeatures.getTensor(queryFeatureName); + if (feature.isPresent()) { + result.add(new NameAndValue(needed, feature.get())); + } else { + // but other ways of setting query features end up in the properties: + var objList = rankProps.get(queryFeatureName); + if (objList != null && objList.size() == 1 && objList.get(0) instanceof Tensor t) { + result.add(new NameAndValue(needed, t)); + } + } + } + } + return result; + } + + private Supplier underlying(Query query, String schema) { + String rankProfile = query.getRanking().getProfile(); + String key = schema + " with rank profile " + rankProfile; + if (skipProcessing.contains(key)) { + return null; + } + Supplier supplier = scorers.get(key); + if (supplier != null) { + return supplier; + } + try { + var proxy = factory.proxyForSchema(schema); + var model = proxy.modelForRankProfile(rankProfile); + supplier = () -> model.evaluatorOf("globalphase"); + if (supplier.get() == null) { + supplier = null; + } + } catch (IllegalArgumentException e) { + logger.info("no global-phase for " + key + " because: " + e.getMessage()); + supplier = null; + } + if (supplier == null) { + skipProcessing.add(key); + } else { + scorers.put(key, supplier); + } + return supplier; + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluator.java b/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluator.java new file mode 100644 index 00000000000..ccb9b9837fe --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluator.java @@ -0,0 +1,53 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.search.ranking; + +import ai.vespa.models.evaluation.FunctionEvaluator; +import ai.vespa.models.evaluation.Model; +import ai.vespa.models.evaluation.ModelsEvaluator; +import com.yahoo.api.annotations.Beta; +import com.yahoo.component.AbstractComponent; +import com.yahoo.component.annotation.Inject; +import com.yahoo.filedistribution.fileacquirer.FileAcquirer; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; + +/** + * proxy for model-evaluation components + * @author arnej + */ +@Beta +public class RankProfilesEvaluator extends AbstractComponent { + + private final ModelsEvaluator evaluator; + + @Inject + public RankProfilesEvaluator( + RankProfilesConfig rankProfilesConfig, + RankingConstantsConfig constantsConfig, + RankingExpressionsConfig expressionsConfig, + OnnxModelsConfig onnxModelsConfig, + FileAcquirer fileAcquirer) + { + this.evaluator = new ModelsEvaluator( + rankProfilesConfig, + constantsConfig, + expressionsConfig, + onnxModelsConfig, + fileAcquirer); + } + + public Model modelForRankProfile(String rankProfile) { + var m = evaluator.models().get(rankProfile); + if (m == null) { + throw new IllegalArgumentException("unknown rankprofile: " + rankProfile); + } + return m; + } + + public FunctionEvaluator evaluatorForFunction(String rankProfile, String functionName) { + return modelForRankProfile(rankProfile).evaluatorOf(functionName); + } +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/RankingExpressionEvaluatorFactory.java b/container-search/src/main/java/com/yahoo/search/ranking/RankingExpressionEvaluatorFactory.java index 8ec3fc919db..b7b3020392f 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/RankingExpressionEvaluatorFactory.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/RankingExpressionEvaluatorFactory.java @@ -13,14 +13,14 @@ import com.yahoo.component.provider.ComponentRegistry; @Beta public class RankingExpressionEvaluatorFactory { - private final ComponentRegistry registry; + private final ComponentRegistry registry; @Inject - public RankingExpressionEvaluatorFactory(ComponentRegistry registry) { + public RankingExpressionEvaluatorFactory(ComponentRegistry registry) { this.registry = registry; } - public RankingExpressionEvaluatorProxy proxyForSchema(String schemaName) { + public RankProfilesEvaluator proxyForSchema(String schemaName) { var component = registry.getComponent("ranking-expression-evaluator." + schemaName); if (component == null) { throw new IllegalArgumentException("ranking expression evaluator for schema '" + schemaName + "' not found"); diff --git a/container-search/src/main/java/com/yahoo/search/ranking/RankingExpressionEvaluatorProxy.java b/container-search/src/main/java/com/yahoo/search/ranking/RankingExpressionEvaluatorProxy.java deleted file mode 100644 index b4ee33263f1..00000000000 --- a/container-search/src/main/java/com/yahoo/search/ranking/RankingExpressionEvaluatorProxy.java +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.search.ranking; - -import ai.vespa.models.evaluation.FunctionEvaluator; -import ai.vespa.models.evaluation.Model; -import ai.vespa.models.evaluation.ModelsEvaluator; -import com.yahoo.api.annotations.Beta; -import com.yahoo.component.AbstractComponent; -import com.yahoo.component.annotation.Inject; -import com.yahoo.filedistribution.fileacquirer.FileAcquirer; -import com.yahoo.vespa.config.search.RankProfilesConfig; -import com.yahoo.vespa.config.search.core.OnnxModelsConfig; -import com.yahoo.vespa.config.search.core.RankingConstantsConfig; -import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; - -/** - * proxy for model-evaluation components - * @author arnej - */ -@Beta -public class RankingExpressionEvaluatorProxy extends AbstractComponent { - - private final ModelsEvaluator evaluator; - - @Inject - public RankingExpressionEvaluatorProxy( - RankProfilesConfig rankProfilesConfig, - RankingConstantsConfig constantsConfig, - RankingExpressionsConfig expressionsConfig, - OnnxModelsConfig onnxModelsConfig, - FileAcquirer fileAcquirer) - { - this.evaluator = new ModelsEvaluator( - rankProfilesConfig, - constantsConfig, - expressionsConfig, - onnxModelsConfig, - fileAcquirer); - } - - public Model modelForRankProfile(String rankProfile) { - var m = evaluator.models().get(rankProfile); - if (m == null) { - throw new IllegalArgumentException("unknown rankprofile: " + rankProfile); - } - return m; - } - - public FunctionEvaluator evaluatorForFunction(String rankProfile, String functionName) { - return modelForRankProfile(rankProfile).evaluatorOf(functionName); - } -} -- cgit v1.2.3 From b1014f00fff138b1171af85745885984eeec7b45 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Tue, 28 Feb 2023 12:06:04 +0100 Subject: Deopend on component to get AbstractComponent --- model-integration/pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/model-integration/pom.xml b/model-integration/pom.xml index 9bb60827a68..fcc45d63252 100644 --- a/model-integration/pom.xml +++ b/model-integration/pom.xml @@ -50,6 +50,12 @@ ${project.version} provided + + com.yahoo.vespa + component + ${project.version} + provided + com.yahoo.vespa linguistics -- cgit v1.2.3 From 0df29582f6d9d5e373cca7a5b71b2fd1e5d75179 Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Tue, 28 Feb 2023 11:10:14 +0000 Subject: GC xml output of metrics. --- metrics/src/tests/metricmanagertest.cpp | 153 +++--------- metrics/src/vespa/metrics/CMakeLists.txt | 1 - metrics/src/vespa/metrics/metricmanager.cpp | 277 +++++++++------------ metrics/src/vespa/metrics/metricmanager.h | 21 +- metrics/src/vespa/metrics/xmlwriter.cpp | 112 --------- metrics/src/vespa/metrics/xmlwriter.h | 30 --- .../vespa/storage/common/statusmetricconsumer.cpp | 14 +- 7 files changed, 157 insertions(+), 451 deletions(-) delete mode 100644 metrics/src/vespa/metrics/xmlwriter.cpp delete mode 100644 metrics/src/vespa/metrics/xmlwriter.h diff --git a/metrics/src/tests/metricmanagertest.cpp b/metrics/src/tests/metricmanagertest.cpp index 98d03514de0..1ba1cad9463 100644 --- a/metrics/src/tests/metricmanagertest.cpp +++ b/metrics/src/tests/metricmanagertest.cpp @@ -5,12 +5,10 @@ #include #include #include -#include #include #include #include #include -#include #include #include #include @@ -57,7 +55,7 @@ SubMetricSet::SubMetricSet(const Metric::String & name, MetricSet* owner) valsum.addMetricToSum(val1); valsum.addMetricToSum(val2); } -SubMetricSet::~SubMetricSet() { } +SubMetricSet::~SubMetricSet() = default; struct MultiSubMetricSet { @@ -167,20 +165,15 @@ getMatchedMetrics(const vespalib::string& config) MetricLockGuard g(mm.getMetricLock()); mm.visit(g, mm.getActiveMetrics(g), visitor, "consumer"); - MetricManager::ConsumerSpec::SP consumerSpec( - mm.getConsumerSpec(g, "consumer")); - return std::pair( - visitor.toString(), - consumerSpec.get() ? consumerSpec->toString() - : "Non-existing consumer"); + MetricManager::ConsumerSpec::SP consumerSpec(mm.getConsumerSpec(g, "consumer")); + return { visitor.toString(), consumerSpec ? consumerSpec->toString() : "Non-existing consumer" }; } } #define ASSERT_CONSUMER_MATCH(name, expected, config) \ { \ - std::pair consumerMatch( \ - getMatchedMetrics(config)); \ + std::pair consumerMatch(getMatchedMetrics(config)); \ EXPECT_EQ("\n" + expected, "\n" + consumerMatch.first) << (name + std::string(": ") + consumerMatch.second); \ } @@ -391,8 +384,7 @@ struct BriefValuePrinter : public MetricVisitor { } }; -bool waitForTimeProcessed(const MetricManager& mm, - time_t processtime, uint32_t timeout = 120) +bool waitForTimeProcessed(const MetricManager& mm, time_t processtime, uint32_t timeout = 120) { uint32_t lastchance = time(0) + timeout; while (time(0) < lastchance) { @@ -403,23 +395,20 @@ bool waitForTimeProcessed(const MetricManager& mm, return false; } -std::string dumpAllSnapshots(const MetricManager& mm, - const std::string& consumer) +std::string dumpAllSnapshots(const MetricManager& mm, const std::string& consumer) { std::ostringstream ost; ost << "\n"; { MetricLockGuard metricLock(mm.getMetricLock()); BriefValuePrinter briefValuePrinter; - mm.visit(metricLock, mm.getActiveMetrics(metricLock), - briefValuePrinter, consumer); + mm.visit(metricLock, mm.getActiveMetrics(metricLock), briefValuePrinter, consumer); ost << "Current: " << briefValuePrinter.ost.str() << "\n"; } { MetricLockGuard metricLock(mm.getMetricLock()); BriefValuePrinter briefValuePrinter; - mm.visit(metricLock, mm.getTotalMetricSnapshot(metricLock), - briefValuePrinter, consumer); + mm.visit(metricLock, mm.getTotalMetricSnapshot(metricLock), briefValuePrinter, consumer); ost << "Total: " << briefValuePrinter.ost.str() << "\n"; } std::vector periods; @@ -429,17 +418,15 @@ std::string dumpAllSnapshots(const MetricManager& mm, } for (uint32_t i=0; i timerImpl(timer); + auto timerImpl = std::make_unique(1000); + FakeTimer & timer = *timerImpl; TestMetricSet mySet; MetricManager mm(std::move(timerImpl)); { @@ -522,7 +506,7 @@ TEST_F(MetricManagerTest, test_snapshots) mySet.val10.a.val1.addValue(7); mySet.val10.a.val2.addValue(2); mySet.val10.b.val1.addValue(1); - timer->add_time(5 * 60); + timer.add_time(5 * 60); ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60); ASSERT_VALUES(mm, 5 * 60, "2,4,4,1,7,9,1,1,8,2,10"); ASSERT_VALUES(mm, 60 * 60, ""); @@ -536,7 +520,7 @@ TEST_F(MetricManagerTest, test_snapshots) mySet.val10.a.val1.addValue(8); mySet.val10.a.val2.addValue(3); mySet.val10.b.val1.addValue(2); - timer->add_time(5 * 60); + timer.add_time(5 * 60); ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60 * 2); ASSERT_VALUES(mm, 5 * 60, "4,5,5,1,8,11,2,2,10,3,13"); ASSERT_VALUES(mm, 60 * 60, ""); @@ -544,7 +528,7 @@ TEST_F(MetricManagerTest, test_snapshots) // Adding another five minute period where nothing have happened. // Metric for last 5 minutes should be 0. - timer->add_time(5 * 60); + timer.add_time(5 * 60); ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60 * 3); ASSERT_VALUES(mm, 5 * 60, "0,0,0,0,0,0,0,0,0,0,0"); ASSERT_VALUES(mm, 60 * 60, ""); @@ -555,7 +539,7 @@ TEST_F(MetricManagerTest, test_snapshots) mySet.val6.addValue(6); for (uint32_t i=0; i<9; ++i) { // 9 x 5 minutes. Avoid snapshot bumping // due to taking snapshots in the past - timer->add_time(5 * 60); + timer.add_time(5 * 60); ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60 * (4 + i)); } ASSERT_VALUES(mm, 5 * 60, "0,0,0,0,0,0,0,0,0,0,0"); @@ -570,82 +554,6 @@ TEST_F(MetricManagerTest, test_snapshots) ASSERT_VALUES(mm, 0 * 60, "0,0,0,0,0,0,0,0,0,0,0"); } -TEST_F(MetricManagerTest, test_xml_output) -{ - FakeTimer* timer = new FakeTimer(1000); - std::unique_ptr timerImpl(timer); - MetricManager mm(std::move(timerImpl)); - TestMetricSet mySet; - { - MetricLockGuard lockGuard(mm.getMetricLock()); - mm.registerMetric(lockGuard, mySet.set); - } - - // Initialize metric manager to get snapshots created. - mm.init(ConfigUri("raw:" - "consumer[2]\n" - "consumer[0].name snapper\n" - "consumer[0].tags[1]\n" - "consumer[0].tags[0] snaptest\n" - "consumer[1].name log\n" - "consumer[1].tags[1]\n" - "consumer[1].tags[0] snaptest\n")); - - takeSnapshots(mm, 1000); - - // Adding metrics to have some values in them - mySet.val6.addValue(2); - mySet.val9.val1.addValue(4); - mySet.val10.count.inc(); - mySet.val10.a.val1.addValue(7); - mySet.val10.a.val2.addValue(2); - mySet.val10.b.val1.addValue(1); - - timer->set_time(1300); - takeSnapshots(mm, 1300); - - std::string expected( - "'\n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - " \n" - "'"); - - std::ostringstream ost; - vespalib::XmlOutputStream xos(ost, " "); - XmlWriter writer(xos, 300, 0); - { - MetricLockGuard lockGuard(mm.getMetricLock()); - mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, 300, false), - writer, "snapper"); - } - std::string actual(ost.str()); - // Not bothering to match all the nitty gritty details as it will test - // more than it needs to. Just be here in order to check on XML output - // easily if needed. - EXPECT_EQ(expected, "'" + actual + "'"); -} - TEST_F(MetricManagerTest, test_json_output) { FakeTimer* timer = new FakeTimer(1000); @@ -683,8 +591,7 @@ TEST_F(MetricManagerTest, test_json_output) JsonWriter writer(jsonStream); { MetricLockGuard lockGuard(mm.getMetricLock()); - mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, 300, false), - writer, "snapper"); + mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, 300, false), writer, "snapper"); } jsonStream.finalize(); std::string jsonData = as.str(); @@ -773,22 +680,19 @@ struct MetricSnapshotTestFixture JsonWriter writer(jsonStream); { MetricLockGuard lockGuard(manager.getMetricLock()); - manager.visit(lockGuard, manager.getMetricSnapshot(lockGuard, 300, false), - writer, "snapper"); + manager.visit(lockGuard, manager.getMetricSnapshot(lockGuard, 300, false), writer, "snapper"); } jsonStream.finalize(); return as.str(); } - std::string renderLastSnapshotAsText( - const std::string& matchPattern = ".*") const + std::string renderLastSnapshotAsText(const std::string& matchPattern = ".*") const { std::ostringstream ss; TextWriter writer(ss, 300, matchPattern, true); { MetricLockGuard lockGuard(manager.getMetricLock()); - manager.visit(lockGuard, manager.getMetricSnapshot(lockGuard, 300, false), - writer, "snapper"); + manager.visit(lockGuard, manager.getMetricSnapshot(lockGuard, 300, false), writer, "snapper"); } return ss.str(); } @@ -819,8 +723,7 @@ public: } std::string nthMetricDimension(size_t metricIndex, const std::string& key) { - return nthMetric(metricIndex)["dimensions"][key] - .asString().make_string(); + return nthMetric(metricIndex)["dimensions"][key].asString().make_string(); } // Verify that the nth metric has the given name and the given set of @@ -843,7 +746,7 @@ JsonMetricWrapper::JsonMetricWrapper(const std::string& jsonText) { vespalib::slime::JsonFormat::decode(vespalib::Memory(jsonText), _tree); } -JsonMetricWrapper::~JsonMetricWrapper() { } +JsonMetricWrapper::~JsonMetricWrapper() = default; struct DimensionTestMetricSet : MetricSet { @@ -851,7 +754,7 @@ struct DimensionTestMetricSet : MetricSet LongCountMetric val2; DimensionTestMetricSet(MetricSet* owner = nullptr); - ~DimensionTestMetricSet(); + ~DimensionTestMetricSet() override; }; DimensionTestMetricSet::DimensionTestMetricSet(MetricSet* owner) @@ -859,7 +762,7 @@ DimensionTestMetricSet::DimensionTestMetricSet(MetricSet* owner) val1("val1", {{"tag1"}}, "val1 desc", this), val2("val2", {{"baz", "superbaz"}}, "val2 desc", this) { } -DimensionTestMetricSet::~DimensionTestMetricSet() { } +DimensionTestMetricSet::~DimensionTestMetricSet() = default; } diff --git a/metrics/src/vespa/metrics/CMakeLists.txt b/metrics/src/vespa/metrics/CMakeLists.txt index 2441bf95c0b..62254a8d620 100644 --- a/metrics/src/vespa/metrics/CMakeLists.txt +++ b/metrics/src/vespa/metrics/CMakeLists.txt @@ -18,7 +18,6 @@ vespa_add_library(metrics updatehook.cpp valuemetric.cpp valuemetricvalues.cpp - xmlwriter.cpp $ INSTALL lib64 diff --git a/metrics/src/vespa/metrics/metricmanager.cpp b/metrics/src/vespa/metrics/metricmanager.cpp index a0e44ddbeac..da7b8f71fdf 100644 --- a/metrics/src/vespa/metrics/metricmanager.cpp +++ b/metrics/src/vespa/metrics/metricmanager.cpp @@ -22,6 +22,9 @@ LOG_SETUP(".metrics.manager"); namespace metrics { using Config = MetricsmanagerConfig; +using vespalib::IllegalStateException; +using vespalib::IllegalArgumentException; +using vespalib::make_string_short::fmt; MetricManager::ConsumerSpec::ConsumerSpec() = default; MetricManager::ConsumerSpec::~ConsumerSpec() = default; @@ -34,13 +37,12 @@ MetricManager::Timer::getTime() const { void MetricManager::assertMetricLockLocked(const MetricLockGuard& g) const { if ( ! g.owns(_waiter)) { - throw vespalib::IllegalArgumentException("Given lock does not lock the metric lock.", VESPA_STRLOC); + throw IllegalArgumentException("Given lock does not lock the metric lock.", VESPA_STRLOC); } } void -MetricManager::ConsumerSpec::print(std::ostream& out, bool verbose, - const std::string& indent) const +MetricManager::ConsumerSpec::print(std::ostream& out, bool verbose, const std::string& indent) const { (void) verbose; out << "ConsumerSpec("; @@ -63,6 +65,10 @@ MetricManager::ConsumerSpec::addMemoryUsage(MemoryConsumption& mc) const } } +MetricManager::MetricManager() + : MetricManager(std::make_unique()) +{ } + MetricManager::MetricManager(std::unique_ptr timer) : _activeMetrics("Active metrics showing updates since last snapshot"), _configSubscriber(), @@ -70,9 +76,7 @@ MetricManager::MetricManager(std::unique_ptr timer) _config(), _consumerConfig(), _snapshots(), - _totalMetrics(std::make_shared( - "Empty metrics before init", 0, _activeMetrics.getMetrics(), - false)), + _totalMetrics(std::make_shared("Empty metrics before init", 0, _activeMetrics.getMetrics(), false)), _timer(std::move(timer)), _lastProcessedTime(0), _snapshotUnsetMetrics(false), @@ -165,9 +169,8 @@ void MetricManager::init(const config::ConfigUri & uri, bool startThread) { if (isInitialized()) { - throw vespalib::IllegalStateException( - "The metric manager have already been initialized. " - "It can only be initialized once.", VESPA_STRLOC); + throw IllegalStateException("The metric manager have already been initialized. " + "It can only be initialized once.", VESPA_STRLOC); } LOG(debug, "Initializing metric manager."); _configSubscriber = std::make_unique(uri.getContext()); @@ -233,8 +236,10 @@ struct ConsumerMetricBuilder : public MetricVisitor { bool nameRemoved; uint32_t metricCount; - Result() : tagAdded(false), tagRemoved(false), - nameAdded(false), nameRemoved(false), metricCount(0) {} + Result() + : tagAdded(false), tagRemoved(false), + nameAdded(false), nameRemoved(false), metricCount(0) + {} }; std::list result; @@ -353,9 +358,7 @@ ConsumerMetricBuilder::~ConsumerMetricBuilder() = default; void MetricManager::checkMetricsAltered(const MetricLockGuard & guard) { - if (_activeMetrics.getMetrics().isRegistrationAltered() - || _consumerConfigChanged) - { + if (_activeMetrics.getMetrics().isRegistrationAltered() || _consumerConfigChanged) { handleMetricsAltered(guard); } } @@ -385,8 +388,8 @@ MetricManager::handleMetricsAltered(const MetricLockGuard & guard) } LOG(debug, "Recreating snapshots to include altered metrics"); _totalMetrics->recreateSnapshot(_activeMetrics.getMetrics(), _snapshotUnsetMetrics); - for (uint32_t i=0; i<_snapshots.size(); ++i) { - _snapshots[i]->recreateSnapshot(_activeMetrics.getMetrics(), _snapshotUnsetMetrics); + for (const auto & snapshot: _snapshots) { + snapshot->recreateSnapshot(_activeMetrics.getMetrics(), _snapshotUnsetMetrics); } LOG(debug, "Setting new consumer config. Clearing dirty flag"); _consumerConfig.swap(configMap); @@ -394,24 +397,25 @@ MetricManager::handleMetricsAltered(const MetricLockGuard & guard) } namespace { - bool setSnapshotName(std::ostream& out, const char* name, uint32_t length, uint32_t period) - { - if (length % period != 0) return false; - out << (length / period) << ' ' << name; - if (length / period != 1) out << "s"; - return true; - } + +bool +setSnapshotName(std::ostream& out, const char* name, uint32_t length, uint32_t period) { + if (length % period != 0) return false; + out << (length / period) << ' ' << name; + if (length / period != 1) out << "s"; + return true; +} + } std::vector MetricManager::createSnapshotPeriods(const Config& config) { std::vector result; - try{ - for (uint32_t i=0; i confi std::ostringstream ost; config::OstreamConfigWriter w(ost); w.write(*config); - LOG(debug, "Received new config for metric manager: %s", - ost.str().c_str()); + LOG(debug, "Received new config for metric manager: %s", ost.str().c_str()); } if (_snapshots.empty()) { LOG(debug, "Initializing snapshots as this is first configure call"); @@ -466,18 +466,12 @@ MetricManager::configure(const MetricLockGuard & , std::unique_ptr confi time_t currentTime(_timer->getTime()); _activeMetrics.setFromTime(currentTime); uint32_t count = 1; - for (uint32_t i = 0; i< snapshotPeriods.size(); ++i) - { + for (uint32_t i = 0; i< snapshotPeriods.size(); ++i) { uint32_t nextCount = 1; if (i + 1 < snapshotPeriods.size()) { - nextCount = snapshotPeriods[i + 1].first - / snapshotPeriods[i].first; - if (snapshotPeriods[i + 1].first - % snapshotPeriods[i].first != 0) - { - throw vespalib::IllegalStateException( - "Snapshot periods must be multiplum of each other", - VESPA_STRLOC); + nextCount = snapshotPeriods[i + 1].first / snapshotPeriods[i].first; + if ((snapshotPeriods[i + 1].first % snapshotPeriods[i].first) != 0) { + throw IllegalStateException("Snapshot periods must be multiplum of each other",VESPA_STRLOC); } } _snapshots.push_back(std::make_shared( @@ -489,9 +483,7 @@ MetricManager::configure(const MetricLockGuard & , std::unique_ptr confi _totalMetrics = std::make_shared("All time snapshot", 0, _activeMetrics.getMetrics(), _snapshotUnsetMetrics); _totalMetrics->reset(currentTime); } - if (_config.get() == 0 - || _config->consumer.size() != config->consumer.size()) - { + if (_config.get() == 0 || (_config->consumer.size() != config->consumer.size())) { _consumerConfigChanged = true; } else { for (uint32_t i=0; i<_config->consumer.size(); ++i) { @@ -519,61 +511,55 @@ MetricManager::getConsumerSpec(const MetricLockGuard &, const Metric::String& co namespace { - struct ConsumerMetricVisitor : public MetricVisitor { - const MetricManager::ConsumerSpec& _metricsToMatch; - MetricVisitor& _client; +struct ConsumerMetricVisitor : public MetricVisitor { + const MetricManager::ConsumerSpec& _metricsToMatch; + MetricVisitor& _client; #ifdef VERIFY_ALL_METRICS_VISITED - std::set _visitedMetrics; + std::set _visitedMetrics; #endif - ConsumerMetricVisitor(const MetricManager::ConsumerSpec& spec, - MetricVisitor& clientVisitor) - : _metricsToMatch(spec), _client(clientVisitor) {} + ConsumerMetricVisitor(const MetricManager::ConsumerSpec& spec, MetricVisitor& clientVisitor) + : _metricsToMatch(spec), _client(clientVisitor) + {} - bool visitMetricSet(const MetricSet& metricSet, - bool autoGenerated) override - { - if (metricSet.isTopSet()) return true; - return (_metricsToMatch.contains(metricSet) - && _client.visitMetricSet(metricSet, autoGenerated)); - } - void doneVisitingMetricSet(const MetricSet& metricSet) override { - if (!metricSet.isTopSet()) { + bool visitMetricSet(const MetricSet& metricSet, bool autoGenerated) override { + if (metricSet.isTopSet()) return true; + return (_metricsToMatch.contains(metricSet) + && _client.visitMetricSet(metricSet, autoGenerated)); + } + void doneVisitingMetricSet(const MetricSet& metricSet) override { + if (!metricSet.isTopSet()) { #ifdef VERIFY_ALL_METRICS_VISITED - _visitedMetrics.insert(metricSet.getPath()); + _visitedMetrics.insert(metricSet.getPath()); #endif - _client.doneVisitingMetricSet(metricSet); - } + _client.doneVisitingMetricSet(metricSet); } - bool visitCountMetric(const AbstractCountMetric& metric, - bool autoGenerated) override - { - if (_metricsToMatch.contains(metric)) { + } + bool visitCountMetric(const AbstractCountMetric& metric, bool autoGenerated) override { + if (_metricsToMatch.contains(metric)) { #ifdef VERIFY_ALL_METRICS_VISITED - _visitedMetrics.insert(metric.getPath()); + _visitedMetrics.insert(metric.getPath()); #endif - return _client.visitCountMetric(metric, autoGenerated); - } - return true; + return _client.visitCountMetric(metric, autoGenerated); } - bool visitValueMetric(const AbstractValueMetric& metric, - bool autoGenerated) override - { - if (_metricsToMatch.contains(metric)) { + return true; + } + bool visitValueMetric(const AbstractValueMetric& metric, bool autoGenerated) override { + if (_metricsToMatch.contains(metric)) { #ifdef VERIFY_ALL_METRICS_VISITED - _visitedMetrics.insert(metric.getPath()); + _visitedMetrics.insert(metric.getPath()); #endif - return _client.visitValueMetric(metric, autoGenerated); - } - return true; + return _client.visitValueMetric(metric, autoGenerated); } - }; + return true; + } +}; } void -MetricManager::visit(const MetricLockGuard & guard, const MetricSnapshot& snapshot, MetricVisitor& visitor, - const std::string& consumer) const +MetricManager::visit(const MetricLockGuard & guard, const MetricSnapshot& snapshot, + MetricVisitor& visitor, const std::string& consumer) const { if (visitor.visitSnapshot(snapshot)) { if (consumer == "") { @@ -593,9 +579,7 @@ MetricManager::visit(const MetricLockGuard & guard, const MetricSnapshot& snapsh } #endif } else { - LOGBP(debug, - "Requested metrics for non-defined consumer '%s'.", - consumer.c_str()); + LOGBP(debug, "Requested metrics for non-defined consumer '%s'.", consumer.c_str()); } } visitor.doneVisitingSnapshot(snapshot); @@ -616,39 +600,31 @@ MetricManager::getSnapshotPeriods(const MetricLockGuard& l) const // Client should have grabbed metrics lock before doing this const MetricSnapshot& -MetricManager::getMetricSnapshot(const MetricLockGuard& l, - uint32_t period, bool getInProgressSet) const +MetricManager::getMetricSnapshot(const MetricLockGuard& l, uint32_t period, bool getInProgressSet) const { assertMetricLockLocked(l); - for (uint32_t i=0; i<_snapshots.size(); ++i) { - if (_snapshots[i]->getPeriod() == period) { - if (_snapshots[i]->getCount() == 1 && getInProgressSet) { - throw vespalib::IllegalStateException( - "No temporary snapshot for set " - + _snapshots[i]->getName(), VESPA_STRLOC); + for (const auto & snapshot : _snapshots) { + if (snapshot->getPeriod() == period) { + if (snapshot->getCount() == 1 && getInProgressSet) { + throw IllegalStateException("No temporary snapshot for set " + snapshot->getName(), VESPA_STRLOC); } - return _snapshots[i]->getSnapshot(getInProgressSet); + return snapshot->getSnapshot(getInProgressSet); } } - std::ostringstream ost; - ost << "No snapshot for period of length " << period << " exist."; - throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC); + throw IllegalArgumentException(fmt("No snapshot for period of length %u exist.", period), VESPA_STRLOC); } // Client should have grabbed metrics lock before doing this const MetricSnapshotSet& -MetricManager::getMetricSnapshotSet(const MetricLockGuard& l, - uint32_t period) const +MetricManager::getMetricSnapshotSet(const MetricLockGuard& l, uint32_t period) const { assertMetricLockLocked(l); - for (uint32_t i=0; i<_snapshots.size(); ++i) { - if (_snapshots[i]->getPeriod() == period) { - return *_snapshots[i]; + for (const auto & snapshot : _snapshots) { + if (snapshot->getPeriod() == period) { + return *snapshot; } } - std::ostringstream ost; - ost << "No snapshot set for period of length " << period << " exist."; - throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC); + throw IllegalArgumentException(fmt("No snapshot set for period of length %u exist.", period), VESPA_STRLOC); } void @@ -661,9 +637,8 @@ MetricManager::timeChangedNotification() const void MetricManager::updateMetrics(bool includeSnapshotOnlyHooks) { - LOG(debug, "Calling metric update hooks%s.", - includeSnapshotOnlyHooks ? ", including snapshot hooks" : ""); - // Ensure we're not in the way of the background thread + LOG(debug, "Calling metric update hooks%s.", includeSnapshotOnlyHooks ? ", including snapshot hooks" : ""); + // Ensure we're not in the way of the background thread MetricLockGuard sync(_waiter); LOG(debug, "Giving %zu periodic update hooks.", _periodicUpdateHooks.size()); updatePeriodicMetrics(sync, 0, true); @@ -677,6 +652,7 @@ MetricManager::updateMetrics(bool includeSnapshotOnlyHooks) time_t MetricManager::updatePeriodicMetrics(const MetricLockGuard & guard, time_t updateTime, bool outOfSchedule) { + assertMetricLockLocked(guard); time_t nextUpdateTime = std::numeric_limits::max(); time_t preTime = _timer->getTimeInMilliSecs(); for (auto hook : _periodicUpdateHooks) { @@ -684,10 +660,8 @@ MetricManager::updatePeriodicMetrics(const MetricLockGuard & guard, time_t updat hook->updateMetrics(guard); if (hook->_nextCall + hook->_period < updateTime) { if (hook->_nextCall != 0) { - LOG(debug, "Updated hook %s at time %" PRIu64 ", but next " - "run in %u seconds have already passed as time" - " is %" PRIu64 ". Bumping next call to current " - "time + period.", + LOG(debug, "Updated hook %s at time %" PRIu64 ", but next run in %u seconds have already passed as " + "time is %" PRIu64 ". Bumping next call to current time + period.", hook->_name, static_cast(hook->_nextCall), hook->_period, static_cast(updateTime)); } hook->_nextCall = updateTime + hook->_period; @@ -712,9 +686,10 @@ MetricManager::updatePeriodicMetrics(const MetricLockGuard & guard, time_t updat void MetricManager::updateSnapshotMetrics(const MetricLockGuard & guard) { + assertMetricLockLocked(guard); time_t preTime = _timer->getTimeInMilliSecs(); - for (auto it = _snapshotUpdateHooks.begin(); it != _snapshotUpdateHooks.end(); ++it) { - (**it).updateMetrics(guard); + for (const auto & hook : _snapshotUpdateHooks) { + hook->updateMetrics(guard); time_t postTime = _timer->getTimeInMilliSecs(); _snapshotHookLatency.addValue(postTime - preTime); preTime = postTime; @@ -738,8 +713,8 @@ MetricManager::reset(time_t currentTime) // to avoid conflict with adding/removal of metrics std::lock_guard waiterLock(_waiter); _activeMetrics.reset(currentTime); - for (uint32_t i=0; i<_snapshots.size(); ++i) { - _snapshots[i]->reset(currentTime); + for (const auto & snapshot : _snapshots) { + snapshot->reset(currentTime); } _totalMetrics->reset(currentTime); time_t postTime = _timer->getTimeInMilliSecs(); @@ -780,11 +755,10 @@ MetricManager::run() time_t MetricManager::tick(const MetricLockGuard & guard, time_t currentTime) { - LOG(spam, "Worker thread starting to process for time %" PRIu64 ".", - static_cast(currentTime)); + LOG(spam, "Worker thread starting to process for time %" PRIu64 ".", static_cast(currentTime)); // Check for new config and reconfigure - if (_configSubscriber.get() && _configSubscriber->nextConfigNow()) { + if (_configSubscriber && _configSubscriber->nextConfigNow()) { configure(guard, _configHandle->getConfig()); } @@ -811,8 +785,7 @@ MetricManager::tick(const MetricLockGuard & guard, time_t currentTime) // Do snapshotting if it is time if (nextWorkTime <= currentTime) takeSnapshots(guard, nextWorkTime); - _lastProcessedTime.store(nextWorkTime <= currentTime ? nextWorkTime : currentTime, - std::memory_order_relaxed); + _lastProcessedTime.store(nextWorkTime <= currentTime ? nextWorkTime : currentTime, std::memory_order_relaxed); LOG(spam, "Worker thread done with processing for time %" PRIu64 ".", static_cast(_lastProcessedTime.load(std::memory_order_relaxed))); time_t next = _snapshots[0]->getPeriod() + _snapshots[0]->getToTime(); @@ -821,8 +794,9 @@ MetricManager::tick(const MetricLockGuard & guard, time_t currentTime) } void -MetricManager::takeSnapshots(const MetricLockGuard &, time_t timeToProcess) +MetricManager::takeSnapshots(const MetricLockGuard & guard, time_t timeToProcess) { + assertMetricLockLocked(guard); // If not time to do dump data from active snapshot yet, nothing to do if (!_snapshots[0]->timeForAnotherSnapshot(timeToProcess)) { LOG(spam, "Not time to process snapshot %s at time %" PRIu64 ". Current " @@ -840,67 +814,54 @@ MetricManager::takeSnapshots(const MetricLockGuard &, time_t timeToProcess) _activeMetrics.addToSnapshot(firstTarget, false, timeToProcess); _activeMetrics.addToSnapshot(*_totalMetrics, false, timeToProcess); _activeMetrics.reset(timeToProcess); - LOG(debug, "After snapshotting, " - "active metrics goes from %" PRIu64 " to %" PRIu64", " + LOG(debug, "After snapshotting, active metrics goes from %" PRIu64 " to %" PRIu64", " "and 5 minute metrics goes from %" PRIu64 " to %" PRIu64".", static_cast(_activeMetrics.getFromTime()), static_cast(_activeMetrics.getToTime()), static_cast(firstTarget.getFromTime()), static_cast(firstTarget.getToTime())); // Update later snapshots if it is time for it for (uint32_t i=1; i<_snapshots.size(); ++i) { - LOG(debug, "Adding data from last snapshot to building snapshot of " - "next period snapshot %s.", + LOG(debug, "Adding data from last snapshot to building snapshot of next period snapshot %s.", _snapshots[i]->getName().c_str()); MetricSnapshot& target(_snapshots[i]->getNextTarget()); - _snapshots[i-1]->getSnapshot().addToSnapshot( - target, false, timeToProcess); + _snapshots[i-1]->getSnapshot().addToSnapshot(target, false, timeToProcess); target.setToTime(timeToProcess); if (!_snapshots[i]->haveCompletedNewPeriod(timeToProcess)) { - LOG(debug, "Not time to roll snapshot %s yet. %u of %u snapshot " - "taken at time %" PRIu64 ", and period of %u is not up " - "yet as we're currently processing for time %" PRIu64 ".", - _snapshots[i]->getName().c_str(), - _snapshots[i]->getBuilderCount(), - _snapshots[i]->getCount(), - static_cast - (_snapshots[i]->getBuilderCount() * _snapshots[i]->getPeriod() - + _snapshots[i]->getFromTime()), - _snapshots[i]->getPeriod(), - static_cast(timeToProcess)); + LOG(debug, "Not time to roll snapshot %s yet. %u of %u snapshot taken at time %" PRIu64 ", and period of %u " + "is not up yet as we're currently processing for time %" PRIu64 ".", + _snapshots[i]->getName().c_str(), _snapshots[i]->getBuilderCount(), _snapshots[i]->getCount(), + static_cast(_snapshots[i]->getBuilderCount() * _snapshots[i]->getPeriod() + _snapshots[i]->getFromTime()), + _snapshots[i]->getPeriod(), static_cast(timeToProcess)); break; } else { LOG(debug, "Rolled snapshot %s at time %" PRIu64 ".", - _snapshots[i]->getName().c_str(), - static_cast(timeToProcess)); + _snapshots[i]->getName().c_str(), static_cast(timeToProcess)); } } time_t postTime = _timer->getTimeInMilliSecs(); - _snapshotLatency.addValue(postTime - preTime); + _snapshotLatency.addValue(postTime - preTime); } MemoryConsumption::UP MetricManager::getMemoryConsumption(const MetricLockGuard & guard) const { - (void) guard; - MemoryConsumption::UP mc(new MemoryConsumption); + assertMetricLockLocked(guard); + auto mc = std::make_unique(); mc->_consumerCount += _consumerConfig.size(); - mc->_consumerMeta += (sizeof(ConsumerSpec::SP) + sizeof(ConsumerSpec)) - * _consumerConfig.size(); - for (auto it = _consumerConfig.begin(); it != _consumerConfig.end(); ++it) { - mc->_consumerId += mc->getStringMemoryUsage( - it->first, mc->_consumerIdUnique) - + sizeof(Metric::String); - it->second->addMemoryUsage(*mc); + mc->_consumerMeta += (sizeof(ConsumerSpec::SP) + sizeof(ConsumerSpec)) * _consumerConfig.size(); + for (const auto & consumer : _consumerConfig) { + mc->_consumerId += mc->getStringMemoryUsage(consumer.first, mc->_consumerIdUnique) + sizeof(Metric::String); + consumer.second->addMemoryUsage(*mc); } uint32_t preTotal = mc->getTotalMemoryUsage(); _activeMetrics.addMemoryUsage(*mc); uint32_t postTotal = mc->getTotalMemoryUsage(); mc->addSnapShotUsage("active", postTotal - preTotal); preTotal = postTotal; - for (uint32_t i=0; i<_snapshots.size(); ++i) { - _snapshots[i]->addMemoryUsage(*mc); + for (const auto & snapshot : _snapshots) { + snapshot->addMemoryUsage(*mc); postTotal = mc->getTotalMemoryUsage(); - mc->addSnapShotUsage(_snapshots[i]->getName(), postTotal - preTotal); + mc->addSnapShotUsage(snapshot->getName(), postTotal - preTotal); preTotal = postTotal; } _totalMetrics->addMemoryUsage(*mc); diff --git a/metrics/src/vespa/metrics/metricmanager.h b/metrics/src/vespa/metrics/metricmanager.h index 7a34f894282..2c595186ea4 100644 --- a/metrics/src/vespa/metrics/metricmanager.h +++ b/metrics/src/vespa/metrics/metricmanager.h @@ -88,8 +88,7 @@ public: return (includedMetrics.find(m.getPath()) != includedMetrics.end()); } - void print(std::ostream& out, bool verbose, - const std::string& indent) const override; + void print(std::ostream& out, bool verbose, const std::string& indent) const override; void addMemoryUsage(MemoryConsumption&) const; }; @@ -126,7 +125,8 @@ private: bool stop_requested() const { return _stop_requested.load(std::memory_order_relaxed); } public: - MetricManager(std::unique_ptr timer = std::make_unique()); + MetricManager(); + MetricManager(std::unique_ptr timer); ~MetricManager(); void stop(); @@ -232,19 +232,16 @@ public: } /** While accessing the total metrics you should have the metric lock. */ - const MetricSnapshot& getTotalMetricSnapshot(const MetricLockGuard& l) const - { + const MetricSnapshot& getTotalMetricSnapshot(const MetricLockGuard& l) const { assertMetricLockLocked(l); return *_totalMetrics; } /** While accessing snapshots you should have the metric lock. */ - const MetricSnapshot& getMetricSnapshot( - const MetricLockGuard&, - uint32_t period, bool getInProgressSet = false) const; - const MetricSnapshotSet& getMetricSnapshotSet( - const MetricLockGuard&, uint32_t period) const; - bool hasTemporarySnapshot(const MetricLockGuard& l, uint32_t period) const - { return getMetricSnapshotSet(l, period).hasTemporarySnapshot(); } + const MetricSnapshot& getMetricSnapshot( const MetricLockGuard& guard, uint32_t period) const { + return getMetricSnapshot(guard, period, false); + } + const MetricSnapshot& getMetricSnapshot( const MetricLockGuard&, uint32_t period, bool getInProgressSet) const; + const MetricSnapshotSet& getMetricSnapshotSet(const MetricLockGuard&, uint32_t period) const; std::vector getSnapshotPeriods(const MetricLockGuard& l) const; diff --git a/metrics/src/vespa/metrics/xmlwriter.cpp b/metrics/src/vespa/metrics/xmlwriter.cpp deleted file mode 100644 index 11cb450e64d..00000000000 --- a/metrics/src/vespa/metrics/xmlwriter.cpp +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "xmlwriter.h" -#include "countmetric.h" -#include "metricset.h" -#include "metricsnapshot.h" -#include "valuemetric.h" -#include -#include - -namespace metrics { - -XmlWriter::XmlWriter(vespalib::xml::XmlOutputStream& xos, - [[maybe_unused]] uint32_t period, int verbosity) - : _xos(xos), _verbosity(verbosity) {} - -bool -XmlWriter::visitSnapshot(const MetricSnapshot& snapshot) -{ - using namespace vespalib::xml; - _xos << XmlTag("snapshot") << XmlAttribute("name", snapshot.getName()) - << XmlAttribute("from", snapshot.getFromTime()) - << XmlAttribute("to", snapshot.getToTime()) - << XmlAttribute("period", snapshot.getPeriod()); - return true; -} - -void -XmlWriter::doneVisitingSnapshot(const MetricSnapshot&) -{ - using namespace vespalib::xml; - _xos << XmlEndTag(); -} - -bool -XmlWriter::visitMetricSet(const MetricSet& set, bool) -{ - using namespace vespalib::xml; - if (set.used() || _verbosity >= 2) { - _xos << XmlTag(set.getName(), XmlTagFlags::CONVERT_ILLEGAL_CHARACTERS); - printCommonXmlParts(set); - return true; - } - return false; -} -void -XmlWriter::doneVisitingMetricSet(const MetricSet&) { - using namespace vespalib::xml; - _xos << XmlEndTag(); -} - -bool -XmlWriter::visitCountMetric(const AbstractCountMetric& metric, bool) -{ - MetricValueClass::UP values(metric.getValues()); - if (!metric.inUse(*values) && _verbosity < 2) return true; - using namespace vespalib::xml; - std::ostringstream ost; - _xos << XmlTag(metric.getName(), XmlTagFlags::CONVERT_ILLEGAL_CHARACTERS) - << XmlAttribute(metric.sumOnAdd() - ? "count" : "value", values->toString("count")); - printCommonXmlParts(metric); - _xos << XmlEndTag(); - return true; -} - -bool -XmlWriter::visitValueMetric(const AbstractValueMetric& metric, bool) -{ - MetricValueClass::UP values(metric.getValues()); - if (!metric.inUse(*values) && _verbosity < 2) return true; - using namespace vespalib::xml; - _xos << XmlTag(metric.getName(), XmlTagFlags::CONVERT_ILLEGAL_CHARACTERS) - << XmlAttribute("average", values->getLongValue("count") == 0 - ? 0 : values->getDoubleValue("total") - / values->getDoubleValue("count")) - << XmlAttribute("last", values->toString("last")); - if (!metric.summedAverage()) { - if (values->getLongValue("count") > 0) { - _xos << XmlAttribute("min", values->toString("min")) - << XmlAttribute("max", values->toString("max")); - } - _xos << XmlAttribute("count", values->getLongValue("count")); - if (_verbosity >= 2) { - _xos << XmlAttribute("total", values->toString("total")); - } - } - printCommonXmlParts(metric); - _xos << XmlEndTag(); - return true; -} - -void -XmlWriter::printCommonXmlParts(const Metric& metric) const -{ - using namespace vespalib::xml; - const Metric::Tags& tags(metric.getTags()); - if (_verbosity >= 3 && tags.size() > 0) { - std::ostringstream ost; - // XXX print tag values as well - ost << tags[0].key(); - for (uint32_t i=1; i= 1 && !metric.getDescription().empty()) { - _xos << XmlAttribute("description", metric.getDescription()); - } -} - -} // metrics diff --git a/metrics/src/vespa/metrics/xmlwriter.h b/metrics/src/vespa/metrics/xmlwriter.h deleted file mode 100644 index a0e8a3efeea..00000000000 --- a/metrics/src/vespa/metrics/xmlwriter.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include -#include - -namespace metrics { - -class XmlWriter : public MetricVisitor { - vespalib::xml::XmlOutputStream& _xos; - int _verbosity; - -public: - XmlWriter(vespalib::xml::XmlOutputStream& xos, - uint32_t period, int verbosity); - - bool visitSnapshot(const MetricSnapshot&) override; - void doneVisitingSnapshot(const MetricSnapshot&) override; - bool visitMetricSet(const MetricSet& set, bool) override; - void doneVisitingMetricSet(const MetricSet&) override; - bool visitCountMetric(const AbstractCountMetric&, bool autoGenerated) override; - bool visitValueMetric(const AbstractValueMetric&, bool autoGenerated) override; - -private: - void printCommonXmlParts(const Metric& metric) const; -}; - -} - diff --git a/storage/src/vespa/storage/common/statusmetricconsumer.cpp b/storage/src/vespa/storage/common/statusmetricconsumer.cpp index 8eb3e9f3ab6..c6f73540605 100644 --- a/storage/src/vespa/storage/common/statusmetricconsumer.cpp +++ b/storage/src/vespa/storage/common/statusmetricconsumer.cpp @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -37,10 +36,6 @@ StatusMetricConsumer::getReportContentType(const framework::HttpUrlPath& path) c return "text/plain"; } - if (path.getAttribute("format") == "xml") { - return "application/xml"; - } - if (path.getAttribute("format") == "text") { return "text/plain"; } @@ -67,7 +62,6 @@ StatusMetricConsumer::reportStatus(std::ostream& out, LOG(debug, "Not calling update hooks as dontcallupdatehooks option has been given"); } int64_t currentTimeS(vespalib::count_s(_component.getClock().getMonotonicTime().time_since_epoch())); - bool xml = (path.getAttribute("format") == "xml"); bool json = (path.getAttribute("format") == "json"); int verbosity(path.get("verbosity", 0)); @@ -131,13 +125,7 @@ StatusMetricConsumer::reportStatus(std::ostream& out, } std::string consumer = path.getAttribute("consumer", ""); - if (xml) { - out << "\n"; - vespalib::XmlOutputStream xos(out); - metrics::XmlWriter xmlWriter(xos, snapshot->getPeriod(), verbosity); - _manager.visit(metricLock, *snapshot, xmlWriter, consumer); - out << "\n"; - } else if (json) { + if (json) { vespalib::asciistream jsonStreamData; vespalib::JsonStream stream(jsonStreamData, true); stream << Object() << "metrics"; -- cgit v1.2.3 From 625d08a9d21a09d8be69158572dcbf09fa090c0d Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Tue, 28 Feb 2023 12:12:40 +0100 Subject: Track autoscaling success --- .../maintenance/AutoscalingMaintainer.java | 43 ++++++++++++++-------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index 674c20e25f2..3fdb088097c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Deployer; @@ -47,27 +48,30 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { @Override protected double maintain() { if ( ! nodeRepository().nodes().isWorking()) return 0.0; - - if ( ! nodeRepository().zone().environment().isAnyOf(Environment.dev, Environment.perf, Environment.prod)) return 1.0; - - activeNodesByApplication().forEach(this::autoscale); - return 1.0; - } - - private void autoscale(ApplicationId application, NodeList applicationNodes) { - try { - nodesByCluster(applicationNodes).forEach((clusterId, clusterNodes) -> autoscale(application, clusterId)); - } - catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Illegal arguments for " + application, e); + if (nodeRepository().zone().environment().isTest()) return 1.0; + + int attempts = 0; + int failures = 0; + for (var applicationNodes : activeNodesByApplication().entrySet()) { + for (var clusterNodes : nodesByCluster(applicationNodes.getValue()).entrySet()) { + attempts++; + if ( ! autoscale(applicationNodes.getKey(), clusterNodes.getKey())) + failures++; + } } + return asSuccessFactor(attempts, failures); } - private void autoscale(ApplicationId applicationId, ClusterSpec.Id clusterId) { + /** + * Autoscales the given cluster. + * + * @return true if an autoscaling decision was made or nothing should be done, false if there was an error + */ + private boolean autoscale(ApplicationId applicationId, ClusterSpec.Id clusterId) { try (var lock = nodeRepository().applications().lock(applicationId)) { Optional application = nodeRepository().applications().get(applicationId); - if (application.isEmpty()) return; - if (application.get().cluster(clusterId).isEmpty()) return; + if (application.isEmpty()) return true; + if (application.get().cluster(clusterId).isEmpty()) return true; Cluster cluster = application.get().cluster(clusterId).get(); NodeList clusterNodes = nodeRepository().nodes().list(Node.State.active).owner(applicationId).cluster(clusterId); @@ -94,6 +98,13 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { logAutoscaling(current, autoscaling.resources().get(), applicationId, clusterNodes.not().retired()); } } + return true; + } + catch (ApplicationLockException e) { + return false; + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Illegal arguments for " + applicationId + " cluster " + clusterId, e); } } -- cgit v1.2.3 From 7a0c9c66710b2bec22a7f2999b4ce938475b8a2c Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Tue, 28 Feb 2023 12:28:21 +0100 Subject: Track DeploymentInfoMaintainer success --- .../maintenance/DeploymentInfoMaintainer.java | 33 +++++++++++++--------- .../yahoo/concurrent/maintenance/Maintainer.java | 2 +- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java index cf9db1517a0..f6029eade37 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java @@ -1,6 +1,5 @@ package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; @@ -9,7 +8,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeReposi import com.yahoo.yolean.Exceptions; import java.time.Duration; -import java.util.stream.Stream; +import java.util.Collection; /** * This pulls application deployment information from the node repo on all config servers, @@ -28,29 +27,35 @@ public class DeploymentInfoMaintainer extends ControllerMaintainer { @Override protected double maintain() { - controller().applications().asList().stream() - .flatMap(this::mapApplicationToInstances) - .flatMap(this::mapInstanceToDeployments) - .forEach(this::updateDeploymentInfo); - return 1.0; - } - - private Stream mapApplicationToInstances(Application application) { - return application.instances().values().stream(); + int attempts = 0; + int failures = 0; + for (var application : controller().applications().asList()) { + for (var instance : application.instances().values()) { + for (var deployment : instanceDeployments(instance)) { + attempts++; + if ( ! updateDeploymentInfo(deployment)) + failures++; + } + } + } + return asSuccessFactor(attempts, failures); } - private Stream mapInstanceToDeployments(Instance instance) { + private Collection instanceDeployments(Instance instance) { return instance.deployments().keySet().stream() .filter(zoneId -> !zoneId.environment().isTest()) - .map(zoneId -> new DeploymentId(instance.id(), zoneId)); + .map(zoneId -> new DeploymentId(instance.id(), zoneId)) + .toList(); } - private void updateDeploymentInfo(DeploymentId id) { + private boolean updateDeploymentInfo(DeploymentId id) { try { controller().applications().deploymentInfo().put(id, nodeRepository.getApplication(id.zoneId(), id.applicationId())); + return true; } catch (ConfigServerException e) { log.info("Could not retrieve deployment info for " + id + ": " + Exceptions.toMessageString(e)); + return false; } } diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java index 1e2c0900ff7..33e46ebc75f 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java @@ -93,7 +93,7 @@ public abstract class Maintainer implements Runnable { * * @return the degree to which the run was successful - a number between 0 (no success), to 1 (complete success). * Note that this indicates whether something is wrong, so e.g if the call did nothing because it should do - * nothing, 1.0 should be returned. + * nothing, 1.0 should be returned. */ protected abstract double maintain(); -- cgit v1.2.3 From b57e0dc6255b69370db80b88fd4be68be96ca6a5 Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Tue, 28 Feb 2023 12:34:12 +0100 Subject: Revert "Deopend on component to get AbstractComponent" --- model-integration/pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/model-integration/pom.xml b/model-integration/pom.xml index fcc45d63252..9bb60827a68 100644 --- a/model-integration/pom.xml +++ b/model-integration/pom.xml @@ -50,12 +50,6 @@ ${project.version} provided - - com.yahoo.vespa - component - ${project.version} - provided - com.yahoo.vespa linguistics -- cgit v1.2.3 From 662aa224142b8b8f57d49d2cc6b3592784833a7d Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Tue, 28 Feb 2023 12:57:10 +0100 Subject: Export refcount package --- jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java | 1 + 1 file changed, 1 insertion(+) diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java index 53acd8cbb1b..e6c273fafde 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java @@ -53,6 +53,7 @@ public class ExportPackages { .append("com.yahoo.jdisc.handler, ") .append("com.yahoo.jdisc.service, ") .append("com.yahoo.jdisc.statistics, ") + .append("com.yahoo.jdisc.refcount, ") .append("javax.inject;version=1.0.0, ") // TODO Vespa 9: remove. Included in guice, but not exported. Needed by container-jersey. .append("org.aopalliance.intercept, ") -- cgit v1.2.3 From 28b7b010f18a223afe51538ffc09f1aabdb29cac Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Tue, 28 Feb 2023 12:05:51 +0000 Subject: Use vespalib::steady_time for getMilliSecTime to ensure no wraps around and safer code. --- metrics/src/vespa/metrics/metricmanager.cpp | 29 +++++----- metrics/src/vespa/metrics/metricmanager.h | 2 +- metrics/src/vespa/metrics/updatehook.h | 3 ++ storage/src/tests/common/metricstest.cpp | 6 +-- .../src/tests/storageserver/statereportertest.cpp | 38 ++++--------- .../vespa/storage/storageserver/storagenode.cpp | 62 ++++++++++++---------- 6 files changed, 65 insertions(+), 75 deletions(-) diff --git a/metrics/src/vespa/metrics/metricmanager.cpp b/metrics/src/vespa/metrics/metricmanager.cpp index da7b8f71fdf..9135dd86a16 100644 --- a/metrics/src/vespa/metrics/metricmanager.cpp +++ b/metrics/src/vespa/metrics/metricmanager.cpp @@ -25,6 +25,7 @@ using Config = MetricsmanagerConfig; using vespalib::IllegalStateException; using vespalib::IllegalArgumentException; using vespalib::make_string_short::fmt; +using vespalib::count_ms; MetricManager::ConsumerSpec::ConsumerSpec() = default; MetricManager::ConsumerSpec::~ConsumerSpec() = default; @@ -654,7 +655,7 @@ MetricManager::updatePeriodicMetrics(const MetricLockGuard & guard, time_t updat { assertMetricLockLocked(guard); time_t nextUpdateTime = std::numeric_limits::max(); - time_t preTime = _timer->getTimeInMilliSecs(); + time_point preTime = _timer->getTimeInMilliSecs(); for (auto hook : _periodicUpdateHooks) { if (hook->_nextCall <= updateTime) { hook->updateMetrics(guard); @@ -668,13 +669,13 @@ MetricManager::updatePeriodicMetrics(const MetricLockGuard & guard, time_t updat } else { hook->_nextCall += hook->_period; } - time_t postTime = _timer->getTimeInMilliSecs(); - _periodicHookLatency.addValue(postTime - preTime); + time_point postTime = _timer->getTimeInMilliSecs(); + _periodicHookLatency.addValue(count_ms(postTime - preTime)); preTime = postTime; } else if (outOfSchedule) { hook->updateMetrics(guard); - time_t postTime = _timer->getTimeInMilliSecs(); - _periodicHookLatency.addValue(postTime - preTime); + time_point postTime = _timer->getTimeInMilliSecs(); + _periodicHookLatency.addValue(count_ms(postTime - preTime)); preTime = postTime; } nextUpdateTime = std::min(nextUpdateTime, hook->_nextCall); @@ -687,11 +688,11 @@ void MetricManager::updateSnapshotMetrics(const MetricLockGuard & guard) { assertMetricLockLocked(guard); - time_t preTime = _timer->getTimeInMilliSecs(); + time_point preTime = _timer->getTimeInMilliSecs(); for (const auto & hook : _snapshotUpdateHooks) { hook->updateMetrics(guard); - time_t postTime = _timer->getTimeInMilliSecs(); - _snapshotHookLatency.addValue(postTime - preTime); + time_point postTime = _timer->getTimeInMilliSecs(); + _snapshotHookLatency.addValue(count_ms(postTime - preTime)); preTime = postTime; } } @@ -708,7 +709,7 @@ MetricManager::forceEventLogging() void MetricManager::reset(time_t currentTime) { - time_t preTime = _timer->getTimeInMilliSecs(); + time_point preTime = _timer->getTimeInMilliSecs(); // Resetting implies visiting metrics, which needs to grab metric lock // to avoid conflict with adding/removal of metrics std::lock_guard waiterLock(_waiter); @@ -717,8 +718,8 @@ MetricManager::reset(time_t currentTime) snapshot->reset(currentTime); } _totalMetrics->reset(currentTime); - time_t postTime = _timer->getTimeInMilliSecs(); - _resetLatency.addValue(postTime - preTime); + time_point postTime = _timer->getTimeInMilliSecs(); + _resetLatency.addValue(count_ms(postTime - preTime)); } void @@ -806,7 +807,7 @@ MetricManager::takeSnapshots(const MetricLockGuard & guard, time_t timeToProcess static_cast(_snapshots[0]->getToTime())); return; } - time_t preTime = _timer->getTimeInMilliSecs(); + time_point preTime = _timer->getTimeInMilliSecs(); LOG(debug, "Updating %s snapshot and total metrics at time %" PRIu64 ".", _snapshots[0]->getName().c_str(), static_cast(timeToProcess)); MetricSnapshot& firstTarget(_snapshots[0]->getNextTarget()); @@ -838,8 +839,8 @@ MetricManager::takeSnapshots(const MetricLockGuard & guard, time_t timeToProcess _snapshots[i]->getName().c_str(), static_cast(timeToProcess)); } } - time_t postTime = _timer->getTimeInMilliSecs(); - _snapshotLatency.addValue(postTime - preTime); + time_point postTime = _timer->getTimeInMilliSecs(); + _snapshotLatency.addValue(count_ms(postTime - preTime)); } MemoryConsumption::UP diff --git a/metrics/src/vespa/metrics/metricmanager.h b/metrics/src/vespa/metrics/metricmanager.h index 2c595186ea4..3c81d9abd6a 100644 --- a/metrics/src/vespa/metrics/metricmanager.h +++ b/metrics/src/vespa/metrics/metricmanager.h @@ -68,7 +68,7 @@ public: struct Timer { virtual ~Timer() = default; virtual time_t getTime() const; - virtual time_t getTimeInMilliSecs() const { return getTime() * 1000; } + time_point getTimeInMilliSecs() const { return time_point(vespalib::from_s(getTime())); } }; /** diff --git a/metrics/src/vespa/metrics/updatehook.h b/metrics/src/vespa/metrics/updatehook.h index 9fa0d52027e..f355197bbb7 100644 --- a/metrics/src/vespa/metrics/updatehook.h +++ b/metrics/src/vespa/metrics/updatehook.h @@ -1,10 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include #include namespace metrics { +using time_point = vespalib::steady_time; + class MetricLockGuard { public: MetricLockGuard(std::mutex & mutex); diff --git a/storage/src/tests/common/metricstest.cpp b/storage/src/tests/common/metricstest.cpp index 78fa32e24e5..76c6a9cf991 100644 --- a/storage/src/tests/common/metricstest.cpp +++ b/storage/src/tests/common/metricstest.cpp @@ -53,7 +53,6 @@ namespace { framework::Clock& _clock; explicit MetricClock(framework::Clock& c) : _clock(c) {} [[nodiscard]] time_t getTime() const override { return vespalib::count_s(_clock.getMonotonicTime().time_since_epoch()); } - [[nodiscard]] time_t getTimeInMilliSecs() const override { return vespalib::count_ms(_clock.getMonotonicTime().time_since_epoch()); } }; } @@ -85,10 +84,7 @@ void MetricsTest::SetUp() { _metricManager->registerMetric(guard, *_topSet); } - _metricsConsumer = std::make_unique( - _node->getComponentRegister(), - *_metricManager, - "status"); + _metricsConsumer = std::make_unique(_node->getComponentRegister(), *_metricManager, "status"); _filestorMetrics = std::make_shared(); _filestorMetrics->initDiskMetrics(1, 1); diff --git a/storage/src/tests/storageserver/statereportertest.cpp b/storage/src/tests/storageserver/statereportertest.cpp index d6b528e5a25..2570b5232eb 100644 --- a/storage/src/tests/storageserver/statereportertest.cpp +++ b/storage/src/tests/storageserver/statereportertest.cpp @@ -54,7 +54,6 @@ struct MetricClock : public metrics::MetricManager::Timer framework::Clock& _clock; explicit MetricClock(framework::Clock& c) : _clock(c) {} [[nodiscard]] time_t getTime() const override { return vespalib::count_s(_clock.getMonotonicTime().time_since_epoch()); } - [[nodiscard]] time_t getTimeInMilliSecs() const override { return vespalib::count_ms(_clock.getMonotonicTime().time_since_epoch()); } }; } @@ -85,11 +84,8 @@ void StateReporterTest::SetUp() { _metricManager->registerMetric(guard, *_topSet); } - _stateReporter = std::make_unique( - _node->getComponentRegister(), - *_metricManager, - _generationFetcher, - "status"); + _stateReporter = std::make_unique(_node->getComponentRegister(), *_metricManager, + _generationFetcher, "status"); _filestorMetrics = std::make_shared(); _filestorMetrics->initDiskMetrics(1, 1); @@ -125,20 +121,14 @@ vespalib::Slime slime; \ #define ASSERT_GENERATION(jsonData, component, generation) \ { \ PARSE_JSON(jsonData); \ - ASSERT_EQ( \ - generation, \ - slime.get()["config"][component]["generation"].asDouble()); \ + ASSERT_EQ(generation, slime.get()["config"][component]["generation"].asDouble()); \ } #define ASSERT_NODE_STATUS(jsonData, code, message) \ { \ PARSE_JSON(jsonData); \ - ASSERT_EQ( \ - vespalib::string(code), \ - slime.get()["status"]["code"].asString().make_string()); \ - ASSERT_EQ( \ - vespalib::string(message), \ - slime.get()["status"]["message"].asString().make_string()); \ + ASSERT_EQ(vespalib::string(code), slime.get()["status"]["code"].asString().make_string()); \ + ASSERT_EQ(vespalib::string(message), slime.get()["status"]["message"].asString().make_string()); \ } #define ASSERT_METRIC_GET_PUT(jsonData, expGetCount, expPutCount) \ @@ -148,16 +138,11 @@ vespalib::Slime slime; \ double putCount = -1; \ size_t metricCount = slime.get()["metrics"]["values"].children(); \ for (size_t j=0; jaddSecondsToTime(60); _metricManager->timeChangedNotification(); - while (int64_t(_metricManager->getLastProcessedTime()) < vespalib::count_s(_clock->getMonotonicTime().time_since_epoch())) - { + while (int64_t(_metricManager->getLastProcessedTime()) < vespalib::count_s(_clock->getMonotonicTime().time_since_epoch())) { std::this_thread::sleep_for(1ms); } } diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp index a09abb25f7a..9f8456afc37 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.cpp +++ b/storage/src/vespa/storage/storageserver/storagenode.cpp @@ -33,34 +33,36 @@ namespace storage { namespace { - using vespalib::getLastErrorString; +using vespalib::getLastErrorString; - void writePidFile(const vespalib::string& pidfile) - { - ssize_t rv = -1; - vespalib::string mypid = vespalib::make_string("%d\n", getpid()); - size_t lastSlash = pidfile.rfind('/'); - if (lastSlash != vespalib::string::npos) { - std::filesystem::create_directories(std::filesystem::path(pidfile.substr(0, lastSlash))); - } - int fd = open(pidfile.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); - if (fd != -1) { - rv = write(fd, mypid.c_str(), mypid.size()); - close(fd); - } - if (rv < 1) { - LOG(warning, "Failed to write pidfile '%s': %s", - pidfile.c_str(), getLastErrorString().c_str()); - } +void +writePidFile(const vespalib::string& pidfile) +{ + ssize_t rv = -1; + vespalib::string mypid = vespalib::make_string("%d\n", getpid()); + size_t lastSlash = pidfile.rfind('/'); + if (lastSlash != vespalib::string::npos) { + std::filesystem::create_directories(std::filesystem::path(pidfile.substr(0, lastSlash))); + } + int fd = open(pidfile.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd != -1) { + rv = write(fd, mypid.c_str(), mypid.size()); + close(fd); } + if (rv < 1) { + LOG(warning, "Failed to write pidfile '%s': %s", + pidfile.c_str(), getLastErrorString().c_str()); + } +} - void removePidFile(const vespalib::string& pidfile) - { - if (unlink(pidfile.c_str()) != 0) { - LOG(warning, "Failed to delete pidfile '%s': %s", - pidfile.c_str(), getLastErrorString().c_str()); - } +void +removePidFile(const vespalib::string& pidfile) +{ + if (unlink(pidfile.c_str()) != 0) { + LOG(warning, "Failed to delete pidfile '%s': %s", + pidfile.c_str(), getLastErrorString().c_str()); } +} } // End of anonymous namespace @@ -429,7 +431,8 @@ StorageNode::shutdown() LOG(debug, "Done shutting down node"); } -void StorageNode::configure(std::unique_ptr config) { +void +StorageNode::configure(std::unique_ptr config) { log_config_received(*config); // When we get config, we try to grab the config lock to ensure noone // else is doing configuration work, and then we write the new config @@ -445,7 +448,8 @@ void StorageNode::configure(std::unique_ptr config) { } } -void StorageNode::configure(std::unique_ptr config) { +void +StorageNode::configure(std::unique_ptr config) { log_config_received(*config); { std::lock_guard configLockGuard(_configLock); @@ -457,7 +461,8 @@ void StorageNode::configure(std::unique_ptr config) { } } -void StorageNode::configure(std::unique_ptr config) { +void +StorageNode::configure(std::unique_ptr config) { log_config_received(*config); { std::lock_guard configLockGuard(_configLock); @@ -486,7 +491,8 @@ StorageNode::configure(std::unique_ptr co } } -void StorageNode::configure(std::unique_ptr config) { +void +StorageNode::configure(std::unique_ptr config) { log_config_received(*config); { std::lock_guard configLockGuard(_configLock); -- cgit v1.2.3 From 719d3f42b486e6d847cec812dcddc4b4d16ab3f8 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Tue, 28 Feb 2023 13:36:01 +0100 Subject: Revert "Export refcount package" --- jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java | 1 - 1 file changed, 1 deletion(-) diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java index e6c273fafde..53acd8cbb1b 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java @@ -53,7 +53,6 @@ public class ExportPackages { .append("com.yahoo.jdisc.handler, ") .append("com.yahoo.jdisc.service, ") .append("com.yahoo.jdisc.statistics, ") - .append("com.yahoo.jdisc.refcount, ") .append("javax.inject;version=1.0.0, ") // TODO Vespa 9: remove. Included in guice, but not exported. Needed by container-jersey. .append("org.aopalliance.intercept, ") -- cgit v1.2.3 From 18aa303fe0959786838aa63ec8f0aca092be2d99 Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Tue, 28 Feb 2023 12:42:43 +0000 Subject: rename factory class also --- .../model/container/search/ContainerSearch.java | 4 +-- .../yahoo/search/ranking/GlobalPhaseRanker.java | 4 +-- .../ranking/RankProfilesEvaluatorFactory.java | 40 ++++++++++++++++++++++ .../ranking/RankingExpressionEvaluatorFactory.java | 40 ---------------------- 4 files changed, 44 insertions(+), 44 deletions(-) create mode 100644 container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluatorFactory.java delete mode 100644 container-search/src/main/java/com/yahoo/search/ranking/RankingExpressionEvaluatorFactory.java diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index b3f0959029c..14c25ee7452 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -7,7 +7,7 @@ import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.search.pagetemplates.PageTemplatesConfig; import com.yahoo.search.query.profile.config.QueryProfilesConfig; -import com.yahoo.search.ranking.RankingExpressionEvaluatorFactory; +import com.yahoo.search.ranking.RankProfilesEvaluatorFactory; import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; @@ -57,7 +57,7 @@ public class ContainerSearch extends ContainerSubsystem owningCluster.addComponent(Component.fromClassAndBundle(CompiledQueryProfileRegistry.class, SEARCH_AND_DOCPROC_BUNDLE)); owningCluster.addComponent(Component.fromClassAndBundle(com.yahoo.search.schema.SchemaInfo.class, SEARCH_AND_DOCPROC_BUNDLE)); owningCluster.addComponent(Component.fromClassAndBundle(SearchStatusExtension.class, SEARCH_AND_DOCPROC_BUNDLE)); - owningCluster.addComponent(Component.fromClassAndBundle(RankingExpressionEvaluatorFactory.class, SEARCH_AND_DOCPROC_BUNDLE)); + owningCluster.addComponent(Component.fromClassAndBundle(RankProfilesEvaluatorFactory.class, SEARCH_AND_DOCPROC_BUNDLE)); owningCluster.addComponent(Component.fromClassAndBundle(com.yahoo.search.ranking.GlobalPhaseRanker.class, SEARCH_AND_DOCPROC_BUNDLE)); cluster.addSearchAndDocprocBundles(); } diff --git a/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java index fefe8583f3f..87213362acd 100644 --- a/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java +++ b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java @@ -22,12 +22,12 @@ import java.util.function.Supplier; public class GlobalPhaseRanker { private static final Logger logger = Logger.getLogger(GlobalPhaseRanker.class.getName()); - private final RankingExpressionEvaluatorFactory factory; + private final RankProfilesEvaluatorFactory factory; private final Set skipProcessing = new HashSet<>(); private final Map> scorers = new HashMap<>(); @Inject - public GlobalPhaseRanker(RankingExpressionEvaluatorFactory factory) { + public GlobalPhaseRanker(RankProfilesEvaluatorFactory factory) { this.factory = factory; logger.info("using factory: " + factory); } diff --git a/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluatorFactory.java b/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluatorFactory.java new file mode 100644 index 00000000000..edb05ed9788 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluatorFactory.java @@ -0,0 +1,40 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.search.ranking; + +import com.yahoo.api.annotations.Beta; +import com.yahoo.component.annotation.Inject; +import com.yahoo.component.provider.ComponentRegistry; + +/** + * factory for model-evaluation proxies + * @author arnej + */ +@Beta +public class RankProfilesEvaluatorFactory { + + private final ComponentRegistry registry; + + @Inject + public RankProfilesEvaluatorFactory(ComponentRegistry registry) { + this.registry = registry; + } + + public RankProfilesEvaluator proxyForSchema(String schemaName) { + var component = registry.getComponent("ranking-expression-evaluator." + schemaName); + if (component == null) { + throw new IllegalArgumentException("ranking expression evaluator for schema '" + schemaName + "' not found"); + } + return component; + } + + public String toString() { + var buf = new StringBuilder(); + buf.append(this.getClass().getName()).append(" containing: ["); + for (var id : registry.allComponentsById().keySet()) { + buf.append(" ").append(id.toString()); + } + buf.append(" ]"); + return buf.toString(); + } +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/RankingExpressionEvaluatorFactory.java b/container-search/src/main/java/com/yahoo/search/ranking/RankingExpressionEvaluatorFactory.java deleted file mode 100644 index b7b3020392f..00000000000 --- a/container-search/src/main/java/com/yahoo/search/ranking/RankingExpressionEvaluatorFactory.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -package com.yahoo.search.ranking; - -import com.yahoo.api.annotations.Beta; -import com.yahoo.component.annotation.Inject; -import com.yahoo.component.provider.ComponentRegistry; - -/** - * factory for model-evaluation proxies - * @author arnej - */ -@Beta -public class RankingExpressionEvaluatorFactory { - - private final ComponentRegistry registry; - - @Inject - public RankingExpressionEvaluatorFactory(ComponentRegistry registry) { - this.registry = registry; - } - - public RankProfilesEvaluator proxyForSchema(String schemaName) { - var component = registry.getComponent("ranking-expression-evaluator." + schemaName); - if (component == null) { - throw new IllegalArgumentException("ranking expression evaluator for schema '" + schemaName + "' not found"); - } - return component; - } - - public String toString() { - var buf = new StringBuilder(); - buf.append(this.getClass().getName()).append(" containing: ["); - for (var id : registry.allComponentsById().keySet()) { - buf.append(" ").append(id.toString()); - } - buf.append(" ]"); - return buf.toString(); - } -} -- cgit v1.2.3 From 9ce9ed480a5248ffb14439473f6498c6769d74ee Mon Sep 17 00:00:00 2001 From: HÃ¥vard Pettersen Date: Tue, 28 Feb 2023 12:44:11 +0000 Subject: move fastos file code to vespalib --- fastos/CMakeLists.txt | 1 - fastos/src/tests/.gitignore | 17 - fastos/src/tests/CMakeLists.txt | 15 - fastos/src/tests/coretest2.cpp | 41 -- fastos/src/tests/filetest.cpp | 555 ------------------ fastos/src/tests/hello.txt | 1 - fastos/src/tests/mazeserver.cpp | 4 - fastos/src/tests/performancetest.cpp | 54 -- fastos/src/tests/tests.h | 149 ----- fastos/src/tests/typetest.cpp | 44 -- fastos/src/vespa/fastos/CMakeLists.txt | 4 - fastos/src/vespa/fastos/file.cpp | 357 ------------ fastos/src/vespa/fastos/file.h | 745 ------------------------- fastos/src/vespa/fastos/file_rw_ops.cpp | 13 - fastos/src/vespa/fastos/file_rw_ops.h | 33 -- fastos/src/vespa/fastos/linux_file.cpp | 436 --------------- fastos/src/vespa/fastos/linux_file.h | 57 -- fastos/src/vespa/fastos/types.h | 9 - fastos/src/vespa/fastos/unix_file.cpp | 590 -------------------- fastos/src/vespa/fastos/unix_file.h | 130 ----- vespalib/CMakeLists.txt | 18 +- vespalib/src/tests/fastos/CMakeLists.txt | 9 + vespalib/src/tests/fastos/file_test.cpp | 555 ++++++++++++++++++ vespalib/src/tests/fastos/hello.txt | 1 + vespalib/src/tests/fastos/tests.h | 149 +++++ vespalib/src/vespa/fastlib/io/CMakeLists.txt | 2 +- vespalib/src/vespa/fastlib/text/CMakeLists.txt | 2 +- vespalib/src/vespa/fastos/CMakeLists.txt | 9 + vespalib/src/vespa/fastos/file.cpp | 357 ++++++++++++ vespalib/src/vespa/fastos/file.h | 745 +++++++++++++++++++++++++ vespalib/src/vespa/fastos/file_rw_ops.cpp | 13 + vespalib/src/vespa/fastos/file_rw_ops.h | 33 ++ vespalib/src/vespa/fastos/linux_file.cpp | 436 +++++++++++++++ vespalib/src/vespa/fastos/linux_file.h | 57 ++ vespalib/src/vespa/fastos/types.h | 9 + vespalib/src/vespa/fastos/unix_file.cpp | 590 ++++++++++++++++++++ vespalib/src/vespa/fastos/unix_file.h | 130 +++++ vespalib/src/vespa/vespalib/CMakeLists.txt | 5 +- 38 files changed, 3108 insertions(+), 3267 deletions(-) delete mode 100644 fastos/src/tests/.gitignore delete mode 100644 fastos/src/tests/CMakeLists.txt delete mode 100644 fastos/src/tests/coretest2.cpp delete mode 100644 fastos/src/tests/filetest.cpp delete mode 100644 fastos/src/tests/hello.txt delete mode 100644 fastos/src/tests/mazeserver.cpp delete mode 100644 fastos/src/tests/performancetest.cpp delete mode 100644 fastos/src/tests/tests.h delete mode 100644 fastos/src/tests/typetest.cpp delete mode 100644 fastos/src/vespa/fastos/file.cpp delete mode 100644 fastos/src/vespa/fastos/file.h delete mode 100644 fastos/src/vespa/fastos/file_rw_ops.cpp delete mode 100644 fastos/src/vespa/fastos/file_rw_ops.h delete mode 100644 fastos/src/vespa/fastos/linux_file.cpp delete mode 100644 fastos/src/vespa/fastos/linux_file.h delete mode 100644 fastos/src/vespa/fastos/types.h delete mode 100644 fastos/src/vespa/fastos/unix_file.cpp delete mode 100644 fastos/src/vespa/fastos/unix_file.h create mode 100644 vespalib/src/tests/fastos/CMakeLists.txt create mode 100644 vespalib/src/tests/fastos/file_test.cpp create mode 100644 vespalib/src/tests/fastos/hello.txt create mode 100644 vespalib/src/tests/fastos/tests.h create mode 100644 vespalib/src/vespa/fastos/CMakeLists.txt create mode 100644 vespalib/src/vespa/fastos/file.cpp create mode 100644 vespalib/src/vespa/fastos/file.h create mode 100644 vespalib/src/vespa/fastos/file_rw_ops.cpp create mode 100644 vespalib/src/vespa/fastos/file_rw_ops.h create mode 100644 vespalib/src/vespa/fastos/linux_file.cpp create mode 100644 vespalib/src/vespa/fastos/linux_file.h create mode 100644 vespalib/src/vespa/fastos/types.h create mode 100644 vespalib/src/vespa/fastos/unix_file.cpp create mode 100644 vespalib/src/vespa/fastos/unix_file.h diff --git a/fastos/CMakeLists.txt b/fastos/CMakeLists.txt index c17752e234c..60813c569e4 100644 --- a/fastos/CMakeLists.txt +++ b/fastos/CMakeLists.txt @@ -4,5 +4,4 @@ vespa_define_module( src/vespa/fastos TESTS - src/tests ) diff --git a/fastos/src/tests/.gitignore b/fastos/src/tests/.gitignore deleted file mode 100644 index ccb5f0210ab..00000000000 --- a/fastos/src/tests/.gitignore +++ /dev/null @@ -1,17 +0,0 @@ -/Makefile -/backtracetest -/backtracetest.log -/filetest -/filetest.log -/processtest -/processtest.log -/sockettest -/sockettest.log -/threadtest -/threadtest.log -/timetest -/timetest.log -/typetest -/typetest.log -/usecputest -*test_app diff --git a/fastos/src/tests/CMakeLists.txt b/fastos/src/tests/CMakeLists.txt deleted file mode 100644 index 7d4b014b6b3..00000000000 --- a/fastos/src/tests/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(fastos_filetest_app TEST - SOURCES - filetest.cpp - DEPENDS - fastos -) -vespa_add_test(NAME fastos_filetest_app NO_VALGRIND COMMAND fastos_filetest_app) -vespa_add_executable(fastos_typetest_app TEST - SOURCES - typetest.cpp - DEPENDS - fastos -) -vespa_add_test(NAME fastos_typetest_app NO_VALGRIND COMMAND fastos_typetest_app) diff --git a/fastos/src/tests/coretest2.cpp b/fastos/src/tests/coretest2.cpp deleted file mode 100644 index bd93623922b..00000000000 --- a/fastos/src/tests/coretest2.cpp +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - - - - -static void -bomb(void) -{ - char *p; - - p = nullptr; - *p = 4; -} - -void *BomberRun(void *arg) -{ - (void) arg; - bomb(); - return nullptr; -}; - -static int -bombMain(void) -{ - pthread_t thread; - void *ret; - - (void) pthread_create(&thread, nullptr, BomberRun, nullptr); - ret = nullptr; - (void) pthread_join(thread, &ret); - return (0); -} - - -int -main(int argc, char **argv) -{ - (void) argc; - (void) argv; - return bombMain(); -} diff --git a/fastos/src/tests/filetest.cpp b/fastos/src/tests/filetest.cpp deleted file mode 100644 index d0f8bbfd98b..00000000000 --- a/fastos/src/tests/filetest.cpp +++ /dev/null @@ -1,555 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "tests.h" -#include -#include -#include -#include -#include - -class FileTest : public BaseTest -{ -public: - const std::string srcDir = getenv("SOURCE_DIRECTORY") ? getenv("SOURCE_DIRECTORY") : "."; - const std::string roFilename = srcDir + "/hello.txt"; - const std::string woFilename = "generated/writeonlytest.txt"; - const std::string rwFilename = "generated/readwritetest.txt"; - - void GetCurrentDirTest () - { - TestHeader ("Get Current Directory Test"); - - std::string currentDir = FastOS_File::getCurrentDirectory(); - - Progress(!currentDir.empty(), - "Current dir: %s", !currentDir.empty() ? - currentDir.c_str() : ""); - - bool dirrc = FastOS_File::SetCurrentDirectory(".."); - - std::string parentDir; - - if (dirrc) { - parentDir = FastOS_File::getCurrentDirectory(); - } - - Progress(dirrc && strcmp(currentDir.c_str(), parentDir.c_str()) != 0, - "Parent dir: %s", !parentDir.empty() ? - parentDir.c_str() : ""); - - dirrc = FastOS_File::SetCurrentDirectory(currentDir.c_str()); - - Progress(dirrc, "Changed back to working directory."); - - PrintSeparator(); - } - - void MemoryMapTest (int mmap_flags) - { - TestHeader ("Memory Map Test"); - - int i; - const int bufSize = 1000; - - std::filesystem::create_directory(std::filesystem::path("generated")); - FastOS_File file("generated/memorymaptest"); - - bool rc = file.OpenReadWrite(); - Progress(rc, "Opening file 'generated/memorymaptest'"); - - if (rc) { - char *buffer = new char [bufSize]; - for (i = 0; i < bufSize; i++) { - buffer[i] = i % 256; - } - ssize_t wroteB = file.Write2(buffer, bufSize); - Progress(wroteB == bufSize, "Writing %d bytes to file", bufSize); - - bool close_ok = file.Close(); - assert(close_ok); - file.enableMemoryMap(mmap_flags); - - rc = file.OpenReadOnly(); - - Progress(rc, "Opening file 'generated/memorymaptest' read-only"); - if (rc) { - bool mmapEnabled; - char *mmapBuffer = nullptr; - - mmapEnabled = file.IsMemoryMapped(); - mmapBuffer = static_cast(file.MemoryMapPtr(0)); - - Progress(rc, "Memory mapping %s", - mmapEnabled ? "enabled" : "disabled"); - Progress(rc, "Map address: 0x%p", mmapBuffer); - - if (mmapEnabled) { - rc = 0; - for (i = 0; i < bufSize; i++) { - rc |= (mmapBuffer[i] == i % 256); - } - Progress(rc, "Reading %d bytes from memory map", bufSize); - } - } - delete [] buffer; - } - std::filesystem::remove_all(std::filesystem::path("generated")); - PrintSeparator(); - } - - void DirectIOTest () - { - TestHeader ("Direct Disk IO Test"); - - int i; - const int bufSize = 40000; - - std::filesystem::create_directory(std::filesystem::path("generated")); - FastOS_File file("generated/diotest"); - - bool rc = file.OpenWriteOnly(); - Progress(rc, "Opening file 'generated/diotest' write-only"); - - if (rc) { - char *buffer = new char [bufSize]; - - for (i=0; i - (reinterpret_cast(alignPtr) & - (memoryAlignment - 1)); - if (align != 0) { - alignPtr = &alignPtr[memoryAlignment - align]; - } - int residue = bufSize; - int pos=0; - while (residue > 0) { - int readThisTime = eachRead * transferGranularity; - if (readThisTime > residue) { - readThisTime = residue; - } - file.ReadBuf(alignPtr, readThisTime, pos); - - for (i=0; iOpenReadOnly()) { - int64_t filesize; - filesize = myFile->GetSize(); - ProgressI64((filesize == 27), "File size: %ld", filesize); - - char dummyData[6] = "Dummy"; - bool writeResult = myFile->CheckedWrite(dummyData, 6); - - if (writeResult) { - Progress(false, "Should not be able to write a file opened for read-only access."); - } else { - char dummyData2[28]; - Progress(true, "Write failed with read-only access."); - - bool rc = myFile->SetPosition(1); - Progress(rc, "Setting position to 1"); - - if (rc) { - ssize_t readBytes; - int64_t filePosition; - readBytes = myFile->Read(dummyData2, 28); - - Progress(readBytes == 26, "Attempting to read 28 bytes, should get 26. Got: %d", readBytes); - - filePosition = myFile->GetPosition(); - Progress(filePosition == 27, "File position should now be 27. Was: %d", int(filePosition)); - - readBytes = myFile->Read(dummyData2, 6); - Progress(readBytes == 0, "We should now get 0 bytes. Read: %d bytes", readBytes); - - filePosition = myFile->GetPosition(); - Progress(filePosition == 27, "File position should now be 27. Was: %d", int(filePosition)); - } - } - } else { - Progress(false, "Unable to open file '%s'.", roFilename.c_str()); - } - delete(myFile); - PrintSeparator(); - } - - void WriteOnlyTest () - { - TestHeader("Write-Only Test"); - std::filesystem::create_directory(std::filesystem::path("generated")); - - FastOS_File *myFile = new FastOS_File(woFilename.c_str()); - - if (myFile->OpenWriteOnly()) { - int64_t filesize; - filesize = myFile->GetSize(); - - ProgressI64((filesize == 0), "File size: %ld", filesize); - - char dummyData[6] = "Dummy"; - bool writeResult = myFile->CheckedWrite(dummyData, 6); - - if (!writeResult) { - Progress(false, "Should be able to write to file opened for write-only access."); - } else { - Progress(true, "Write 6 bytes ok."); - - int64_t filePosition = myFile->GetPosition(); - if (filePosition == 6) { - Progress(true, "Fileposition is now 6."); - - if (myFile->SetPosition(0)) { - Progress(true, "SetPosition(0) success."); - filePosition = myFile->GetPosition(); - - if (filePosition == 0) { - Progress(true, "Fileposition is now 0."); - - int readBytes = myFile->Read(dummyData, 6); - - if (readBytes != 6) { - Progress(true, "Trying to read a write-only file should fail and it did."); - Progress(true, "Return code was: %d.", readBytes); - } else { - Progress(false, "Read on a file with write-only access should fail, but it didn't."); - } - } else { - ProgressI64(false, "Fileposition should be 6, but was %ld.", filePosition); - } - } else { - Progress(false, "SetPosition(0) failed"); - } - } else { - ProgressI64(false, "Fileposition should be 6, but was %ld.", filePosition); - } - } - bool closeResult = myFile->Close(); - Progress(closeResult, "Close file."); - } else { - Progress(false, "Unable to open file '%s'.", woFilename.c_str()); - } - - - bool deleteResult = myFile->Delete(); - Progress(deleteResult, "Delete file '%s'.", woFilename.c_str()); - - delete(myFile); - std::filesystem::remove_all(std::filesystem::path("generated")); - PrintSeparator(); - } - - void ReadWriteTest () - { - TestHeader("Read/Write Test"); - std::filesystem::create_directory(std::filesystem::path("generated")); - - FastOS_File *myFile = new FastOS_File(rwFilename.c_str()); - - if (myFile->OpenExisting()) { - Progress(false, "OpenExisting() should not work when '%s' does not exist.", rwFilename.c_str()); - bool close_ok = myFile->Close(); - assert(close_ok); - } else { - Progress(true, "OpenExisting() should fail when '%s' does not exist, and it did.", rwFilename.c_str()); - } - - if (myFile->OpenReadWrite()) { - int64_t filesize; - - filesize = myFile->GetSize(); - - ProgressI64((filesize == 0), "File size: %ld", filesize); - - char dummyData[6] = "Dummy"; - - bool writeResult = myFile->CheckedWrite(dummyData, 6); - - if (!writeResult) { - Progress(false, "Should be able to write to file opened for read/write access."); - } else { - Progress(true, "Write 6 bytes ok."); - - int64_t filePosition = myFile->GetPosition(); - - if (filePosition == 6) { - Progress(true, "Fileposition is now 6."); - - if (myFile->SetPosition(0)) { - Progress(true, "SetPosition(0) success."); - filePosition = myFile->GetPosition(); - - if (filePosition == 0) { - Progress(true, "Fileposition is now 0."); - - char dummyData2[7]; - int readBytes = myFile->Read(dummyData2, 6); - - if (readBytes == 6) { - Progress(true, "Reading 6 bytes worked."); - - int cmpResult = memcmp(dummyData, dummyData2, 6); - Progress((cmpResult == 0), "Comparing the written and read result.\n"); - - bool rc = myFile->SetPosition(1); - Progress(rc, "Setting position to 1"); - - if (rc) { - readBytes = myFile->Read(dummyData2, 7); - - Progress(readBytes == 5, "Attempting to read 7 bytes, should get 5. Got: %d", readBytes); - - filePosition = myFile->GetPosition(); - Progress(filePosition == 6, "File position should now be 6. Was: %d", int(filePosition)); - - readBytes = myFile->Read(dummyData2, 6); - Progress(readBytes == 0, "We should not be able to read any more. Read: %d bytes", readBytes); - - filePosition = myFile->GetPosition(); - Progress(filePosition == 6, "File position should now be 6. Was: %d", int(filePosition)); - } - } else { - Progress(false, "Reading 6 bytes failed."); - } - } else { - ProgressI64(false, "Fileposition should be 6, but was %ld.", filePosition); - } - } else { - Progress(false, "SetPosition(0) failed"); - } - } else { - ProgressI64(false, "Fileposition should be 6, but was %ld.", filePosition); - } - } - - bool closeResult = myFile->Close(); - Progress(closeResult, "Close file."); - } else { - Progress(false, "Unable to open file '%s'.", rwFilename.c_str()); - } - bool deleteResult = myFile->Delete(); - Progress(deleteResult, "Delete file '%s'.", rwFilename.c_str()); - - delete(myFile); - std::filesystem::remove_all(std::filesystem::path("generated")); - PrintSeparator(); - } - - void ScanDirectoryTest() - { - TestHeader("DirectoryScan Test"); - - FastOS_DirectoryScan *scanDir = new FastOS_DirectoryScan("."); - - while (scanDir->ReadNext()) { - const char *name = scanDir->GetName(); - bool isDirectory = scanDir->IsDirectory(); - bool isRegular = scanDir->IsRegular(); - - printf("%-30s %s\n", name, - isDirectory ? "DIR" : (isRegular ? "FILE" : "UNKN")); - } - - delete(scanDir); - PrintSeparator(); - } - - void ReadBufTest () - { - TestHeader("ReadBuf Test"); - - FastOS_File file(roFilename.c_str()); - - char buffer[20]; - - if (file.OpenReadOnly()) { - int64_t position = file.GetPosition(); - Progress(position == 0, "File pointer should be 0 after opening file"); - - ssize_t has_read = file.Read(buffer, 4); - Progress(has_read == 4, "Must read 4 bytes"); - buffer[4] = '\0'; - position = file.GetPosition(); - Progress(position == 4, "File pointer should be 4 after reading 4 bytes"); - Progress(strcmp(buffer, "This") == 0, "[This]=[%s]", buffer); - - file.ReadBuf(buffer, 6, 8); - buffer[6] = '\0'; - position = file.GetPosition(); - Progress(position == 4, "File pointer should still be 4 after ReadBuf"); - Progress(strcmp(buffer, "a test") == 0, "[a test]=[%s]", buffer); - } - - PrintSeparator(); - } - - void DiskFreeSpaceTest () - { - TestHeader("DiskFreeSpace Test"); - - int64_t freeSpace = FastOS_File::GetFreeDiskSpace(roFilename.c_str()); - ProgressI64(freeSpace != -1, "DiskFreeSpace using file ('hello.txt'): %ld MB.", freeSpace == -1 ? -1 : freeSpace/(1024*1024)); - freeSpace = FastOS_File::GetFreeDiskSpace("."); - ProgressI64(freeSpace != -1, "DiskFreeSpace using dir (.): %ld MB.", freeSpace == -1 ? -1 : freeSpace/(1024*1024)); - PrintSeparator(); - } - - void MaxLengthTest () - { - TestHeader ("Max Lengths Test"); - - int maxval = FastOS_File::GetMaximumFilenameLength("."); - Progress(maxval > 5 && maxval < (512*1024), - "Maximum filename length = %d", maxval); - - maxval = FastOS_File::GetMaximumPathLength("."); - Progress(maxval > 5 && maxval < (512*1024), - "Maximum path length = %d", maxval); - - PrintSeparator(); - } - - int Main () override - { - printf("This test should be run in the 'tests' directory.\n\n"); - printf("grep for the string '%s' to detect failures.\n\n", failString); - - GetCurrentDirTest(); - DirectIOTest(); - MaxLengthTest(); - DiskFreeSpaceTest(); - ReadOnlyTest(); - WriteOnlyTest(); - ReadWriteTest(); - ScanDirectoryTest(); - ReadBufTest(); - MemoryMapTest(0); -#ifdef __linux__ - MemoryMapTest(MAP_HUGETLB); -#endif - - PrintSeparator(); - printf("END OF TEST (%s)\n", _argv[0]); - - return allWasOk() ? 0 : 1; - } - FileTest(); - ~FileTest(); -}; - -FileTest::FileTest() { } -FileTest::~FileTest() { } - - -int main (int argc, char **argv) -{ - FileTest app; - - setvbuf(stdout, nullptr, _IOLBF, 8192); - return app.Entry(argc, argv); -} diff --git a/fastos/src/tests/hello.txt b/fastos/src/tests/hello.txt deleted file mode 100644 index 62a405393a6..00000000000 --- a/fastos/src/tests/hello.txt +++ /dev/null @@ -1 +0,0 @@ -This is a test input file. diff --git a/fastos/src/tests/mazeserver.cpp b/fastos/src/tests/mazeserver.cpp deleted file mode 100644 index b62e98645f2..00000000000 --- a/fastos/src/tests/mazeserver.cpp +++ /dev/null @@ -1,4 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#define DO_MAZE_SERVER 1 - -#include "sockettest.cpp" diff --git a/fastos/src/tests/performancetest.cpp b/fastos/src/tests/performancetest.cpp deleted file mode 100644 index f566979957a..00000000000 --- a/fastos/src/tests/performancetest.cpp +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include - -#include "tests.h" - -void PerformanceTest (char *buffer); - -int main (int argc, char **argv) -{ - (void)argc; - (void)argv; - - void (*test)(char *buffer) = PerformanceTest; - - test(nullptr); - return 0; -} - -void PerformanceTest (char *buffer) -{ - // Cause exception - *static_cast(nullptr) = 'e'; - -#if 1 - FastOS_File file("test.txt"); - - if(file.OpenReadOnly()) - { - file.Read(buffer, 20); - file.Write2(buffer, 20); - file.Read(buffer, 20); - file.Write2(buffer, 20); - file.Read(buffer, 20); - file.Write2(buffer, 20); - } -#else - - int filedes = open("test.txt", O_RDONLY, 0664); - - if(filedes != -1) - { - write(filedes, buffer, 20); - read(filedes, buffer, 20); - write(filedes, buffer, 20); - read(filedes, buffer, 20); - write(filedes, buffer, 20); - read(filedes, buffer, 20); - write(filedes, buffer, 20); - - close(filedes); - } -#endif -} - diff --git a/fastos/src/tests/tests.h b/fastos/src/tests/tests.h deleted file mode 100644 index 9cd7a10ab48..00000000000 --- a/fastos/src/tests/tests.h +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include -#include -#include -#include - -class BaseTest -{ -private: - BaseTest(const BaseTest&); - BaseTest &operator=(const BaseTest&); - - int totallen; - bool _allOkFlag; -public: - int _argc; - char **_argv; - - const char *okString; - const char *failString; - - BaseTest () - : totallen(70), - _allOkFlag(true), - _argc(0), - _argv(nullptr), - okString("SUCCESS"), - failString("FAILURE") - { - } - - virtual int Main() = 0; - - int Entry(int argc, char **argv) { - _argc = argc; - _argv = argv; - return Main(); - } - - virtual ~BaseTest() {}; - - bool allWasOk() const { return _allOkFlag; } - - void PrintSeparator () - { - for(int i=0; i - -class TypeTest : public BaseTest -{ -private: - - void ObjectSizeTest () - { - TestHeader("Object Sizes (bytes)"); - - Progress(true, "FastOS_DirectoryScan %d", sizeof(FastOS_DirectoryScan)); - Progress(true, "FastOS_File: %d", sizeof(FastOS_File)); - Progress(true, "FastOS_StatInfo %d", sizeof(FastOS_StatInfo)); - - PrintSeparator(); - } - -public: - virtual ~TypeTest() {}; - - int Main () override - { - printf("grep for the string '%s' to detect failures.\n\n", failString); - - ObjectSizeTest(); - - PrintSeparator(); - printf("END OF TEST (%s)\n", _argv[0]); - - return allWasOk() ? 0 : 1; - } -}; - - -int main (int argc, char **argv) -{ - setvbuf(stdout, nullptr, _IOLBF, 8192); - TypeTest app; - return app.Entry(argc, argv); -} - diff --git a/fastos/src/vespa/fastos/CMakeLists.txt b/fastos/src/vespa/fastos/CMakeLists.txt index c37eb43b884..29810a4f296 100644 --- a/fastos/src/vespa/fastos/CMakeLists.txt +++ b/fastos/src/vespa/fastos/CMakeLists.txt @@ -1,10 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(fastos_objects OBJECT SOURCES - file.cpp - file_rw_ops.cpp - linux_file.cpp - unix_file.cpp ) vespa_add_library(fastos diff --git a/fastos/src/vespa/fastos/file.cpp b/fastos/src/vespa/fastos/file.cpp deleted file mode 100644 index fdbacb570b4..00000000000 --- a/fastos/src/vespa/fastos/file.cpp +++ /dev/null @@ -1,357 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -//************************************************************************ -/** - * Implementation of FastOS_FileInterface methods. - * - * @author Div, Oivind H. Danielsen - */ - -#include "file.h" -#include -#include -#include -#include -#include - -DirectIOException::DirectIOException(const char * fileName, const void * buffer, size_t length, int64_t offset) : - std::exception(), - _what(), - _fileName(fileName), - _buffer(buffer), - _length(length), - _offset(offset) -{ - std::ostringstream os; - os << "DirectIO failed for file '" << fileName << "' buffer=0x" << std::hex << reinterpret_cast(buffer); - os << " length=0x" << length << " offset=0x" << offset; - _what = os.str(); -} - -DirectIOException::~DirectIOException() {} - -#ifdef __linux__ -int FastOS_FileInterface::_defaultFAdviseOptions = POSIX_FADV_NORMAL; -#else -int FastOS_FileInterface::_defaultFAdviseOptions = 0; -#endif - -static const size_t MAX_CHUNK_SIZE = 0x4000000; // 64 MB - -FastOS_FileInterface::FastOS_FileInterface(const char *filename) - : _fAdviseOptions(_defaultFAdviseOptions), - _chunkSize(MAX_CHUNK_SIZE), - _filename(), - _openFlags(0), - _directIOEnabled(false), - _syncWritesEnabled(false) -{ - if (filename != nullptr) - SetFileName(filename); -} - - -FastOS_FileInterface::~FastOS_FileInterface() = default; - -void -FastOS_FileInterface::ReadBuf(void *buffer, size_t length) -{ - ssize_t readResult = Read(buffer, length); - - if ((readResult == -1) || (static_cast(readResult) != length)) { - std::string errorString = readResult != -1 ? - std::string("short read") : - FastOS_FileInterface::getLastErrorString(); - std::ostringstream os; - os << "Fatal: Reading " << length << " bytes from '" << GetFileName() << "' failed: " << errorString; - throw std::runtime_error(os.str()); - } -} - -void -FastOS_FileInterface::WriteBuf(const void *buffer, size_t length) -{ - WriteBufInternal(buffer, length); -} - -void -FastOS_FileInterface::WriteBufInternal(const void *buffer, size_t length) -{ - ssize_t writeResult = Write2(buffer, length); - if (length - writeResult != 0) { - std::string errorString = writeResult != -1 ? - std::string("short write") : - FastOS_FileInterface::getLastErrorString(); - std::ostringstream os; - os << "Fatal: Writing " << length << " bytes to '" << GetFileName() << "' failed (wrote " << writeResult << "): " << errorString; - throw std::runtime_error(os.str()); - } -} - -bool -FastOS_FileInterface::CheckedWrite(const void *buffer, size_t len) -{ - ssize_t writeResult = Write2(buffer, len); - if (writeResult < 0) { - std::string errorString = FastOS_FileInterface::getLastErrorString(); - fprintf(stderr, "Writing %lu bytes to '%s' failed: %s\n", - static_cast(len), - GetFileName(), - errorString.c_str()); - return false; - } - if (writeResult != (ssize_t)len) { - fprintf(stderr, "Short write, tried to write %lu bytes to '%s', only wrote %lu bytes\n", - static_cast(len), - GetFileName(), - static_cast(writeResult)); - return false; - } - return true; -} - - -void -FastOS_FileInterface::ReadBuf(void *buffer, size_t length, int64_t readOffset) -{ - if (!SetPosition(readOffset)) { - std::string errorString = FastOS_FileInterface::getLastErrorString(); - std::ostringstream os; - os << "Fatal: Setting fileoffset to " << readOffset << " in '" << GetFileName() << "' : " << errorString; - throw std::runtime_error(os.str()); - } - ReadBuf(buffer, length); -} - - -void -FastOS_FileInterface::EnableDirectIO() -{ - // Only subclasses with support for DirectIO do something here. -} - - -void -FastOS_FileInterface::EnableSyncWrites() -{ - if (!IsOpened()) - _syncWritesEnabled = true; -} - - -bool -FastOS_FileInterface:: -GetDirectIORestrictions(size_t &memoryAlignment, - size_t &transferGranularity, - size_t &transferMaximum) -{ - memoryAlignment = 1; - transferGranularity = 1; - transferMaximum = 0x7FFFFFFF; - return false; -} - -bool -FastOS_FileInterface::DirectIOPadding(int64_t offset, - size_t buflen, - size_t &padBefore, - size_t &padAfter) -{ - (void)offset; - (void)buflen; - padBefore = 0; - padAfter = 0; - return false; -} - - -void * -FastOS_FileInterface::allocateGenericDirectIOBuffer(size_t byteSize, void *&realPtr) -{ - realPtr = malloc(byteSize); // Default - use malloc allignment - return realPtr; -} - -size_t -FastOS_FileInterface::getMaxDirectIOMemAlign() -{ - return 1u; -} - -void * -FastOS_FileInterface::AllocateDirectIOBuffer(size_t byteSize, void *&realPtr) -{ - return allocateGenericDirectIOBuffer(byteSize, realPtr); -} - -void -FastOS_FileInterface::enableMemoryMap(int mmapFlags) -{ - // Only subclases with support for memory mapping do something here. - (void) mmapFlags; -} - - -void * -FastOS_FileInterface::MemoryMapPtr(int64_t position) const -{ - // Only subclases with support for memory mapping do something here. - (void) position; - return nullptr; -} - - -bool -FastOS_FileInterface::IsMemoryMapped() const -{ - // Only subclases with support for memory mapping do something here. - return false; -} - -void -FastOS_FileInterface::SetFileName(const char *filename) -{ - _filename = filename; -} - - -const char * -FastOS_FileInterface::GetFileName() const -{ - return _filename.c_str(); -} - - -bool -FastOS_FileInterface::OpenReadWrite(const char *filename) -{ - return Open(FASTOS_FILE_OPEN_READ | - FASTOS_FILE_OPEN_WRITE, filename); -} - - -bool -FastOS_FileInterface::OpenExisting(bool abortIfNotExist, - const char *filename) -{ - bool rc = Open(FASTOS_FILE_OPEN_READ | - FASTOS_FILE_OPEN_WRITE | - FASTOS_FILE_OPEN_EXISTING, - filename); - - if (abortIfNotExist && (!rc)) { - std::string errorString = - FastOS_FileInterface::getLastErrorString(); - fprintf(stderr, - "Cannot open %s: %s\n", - filename, - errorString.c_str()); - abort(); - } - - return rc; -} - - -bool -FastOS_FileInterface::OpenReadOnlyExisting(bool abortIfNotExist, - const char *filename) -{ - bool rc = Open(FASTOS_FILE_OPEN_READ | - FASTOS_FILE_OPEN_EXISTING, - filename); - - if (abortIfNotExist && (!rc)) { - std::string errorString = - FastOS_FileInterface::getLastErrorString(); - fprintf(stderr, - "Cannot open %s: %s\n", - filename, - errorString.c_str()); - abort(); - } - - return rc; -} - - -bool -FastOS_FileInterface::OpenWriteOnlyTruncate(const char *filename) -{ - // printf("********* OpenWriteOnlyTruncate %s\n", filename); - return Open(FASTOS_FILE_OPEN_WRITE | - FASTOS_FILE_OPEN_CREATE | - FASTOS_FILE_OPEN_TRUNCATE, - filename); -} - - -bool -FastOS_FileInterface::OpenWriteOnlyExisting(bool abortIfNotExist, - const char *filename) -{ - bool rc = Open(FASTOS_FILE_OPEN_WRITE | - FASTOS_FILE_OPEN_EXISTING, - filename); - - if (abortIfNotExist && (!rc)) { - std::string errorString = - FastOS_FileInterface::getLastErrorString(); - fprintf(stderr, - "Cannot open %s: %s\n", - filename, - errorString.c_str()); - abort(); - } - - return rc; -} - -bool -FastOS_FileInterface::OpenReadOnly(const char *filename) -{ - return Open(FASTOS_FILE_OPEN_READ | - FASTOS_FILE_OPEN_EXISTING, - filename); -} - - -bool -FastOS_FileInterface::OpenWriteOnly(const char *filename) -{ - return Open(FASTOS_FILE_OPEN_WRITE, filename); -} - -FastOS_File::Error -FastOS_FileInterface::GetLastError() -{ - return FastOS_File::TranslateError(FastOS_File::GetLastOSError()); -} - - -std::string -FastOS_FileInterface::getLastErrorString() -{ - int err = FastOS_File::GetLastOSError(); - return FastOS_File::getErrorString(err); -} - -bool FastOS_FileInterface::Rename (const char *newFileName) -{ - bool rc=false; - if (FastOS_File::Rename(GetFileName(), newFileName)) { - SetFileName(newFileName); - rc = true; - } - return rc; -} - -void FastOS_FileInterface::dropFromCache() const -{ -} - -FastOS_DirectoryScanInterface::FastOS_DirectoryScanInterface(const char *path) - : _searchPath(path) -{ -} - -FastOS_DirectoryScanInterface::~FastOS_DirectoryScanInterface() = default; diff --git a/fastos/src/vespa/fastos/file.h b/fastos/src/vespa/fastos/file.h deleted file mode 100644 index 1cf6fee71dd..00000000000 --- a/fastos/src/vespa/fastos/file.h +++ /dev/null @@ -1,745 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -//************************************************************************ -/** - * @file - * Class definitions for FastOS_File, FastOS_DirectoryScan and - * FastOS_StatInfo. - * - * @author Div, Oivind H. Danielsen - */ - -#pragma once - -#include "types.h" -#include -#include - -constexpr int FASTOS_FILE_OPEN_READ = (1<<0); -constexpr int FASTOS_FILE_OPEN_WRITE = (1<<1); -constexpr int FASTOS_FILE_OPEN_EXISTING = (1<<2); -constexpr int FASTOS_FILE_OPEN_CREATE = (1<<3); -constexpr int FASTOS_FILE_OPEN_TRUNCATE = (1<<4); -constexpr int FASTOS_FILE_OPEN_STDOUT = (2<<5); -constexpr int FASTOS_FILE_OPEN_STDERR = (3<<5); -constexpr int FASTOS_FILE_OPEN_STDFLAGS = (3<<5); -constexpr int FASTOS_FILE_OPEN_DIRECTIO = (1<<7); -constexpr int FASTOS_FILE_OPEN_SYNCWRITES = (1<<9); // synchronous writes - -/** - * This class contains regular file-access functionality. - * - * Example (how to read 10 bytes from the file 'hello.txt'): - * - * @code - * void Foo::Bar () - * { - * FastOS_File file("hello.txt"); - * - * if(file.OpenReadOnly()) - * { - * char buffer[10]; - * - * if(file.Read(buffer, 10) == 10) - * { - * // Read success - * } - * else - * { - * // Read failure - * } - * } - * else - * { - * // Unable to open file 'hello.txt' - * } - * } - * @endcode - */ - -class DirectIOException : public std::exception -{ -public: - DirectIOException(const char * fileName, const void * buffer, size_t length, int64_t offset); - ~DirectIOException(); - const char* what() const noexcept override { return _what.c_str(); } - const void * getBuffer() const { return _buffer; } - size_t getLength() const { return _length; } - int64_t getOffset() const { return _offset; } - const std::string & getFileName() const { return _fileName; } -private: - std::string _what; - std::string _fileName; - const void * _buffer; - size_t _length; - int64_t _offset; -}; - -class FastOS_FileInterface -{ -private: - FastOS_FileInterface (const FastOS_FileInterface&); - FastOS_FileInterface& operator=(const FastOS_FileInterface&); - - // Default options for madvise used on every file opening. Default is FADV_NORMAL - // Set with setDefaultFAdviseOptions() application wide. - // And setFAdviseOptions() per file. - static int _defaultFAdviseOptions; - int _fAdviseOptions; - size_t _chunkSize; - void WriteBufInternal(const void *buffer, size_t length); - -protected: - std::string _filename; - unsigned int _openFlags; - bool _directIOEnabled; - bool _syncWritesEnabled; - -public: - static void setDefaultFAdviseOptions(int options) { _defaultFAdviseOptions = options; } - int getFAdviseOptions() const { return _fAdviseOptions; } - void setFAdviseOptions(int options) { _fAdviseOptions = options; } - - /** - * Return path separator string. This will yield "/" on UNIX systems. - * @return pointer to path separator character string - */ - static const char *GetPathSeparator() { return "/"; } - - /** - * Constructor. A filename could be supplied at this point, or specified - * later using @ref SetFileName() or @ref Open(). - * @param filename a filename (optional) - */ - FastOS_FileInterface(const char *filename=nullptr); - - /** - * Destructor. If the current file is open, the destructor will close - * it for you. - */ - virtual ~FastOS_FileInterface(); - - /** - * Associate a new filename with this object. This filename will be - * used when performing @ref Open() (unless @ref Open() specifies a - * different filename). - * @param filename filename character string (will be copied internally) - */ - virtual void SetFileName(const char *filename); - - /** - * Return the filename associated with the File object. If no filename - * has been set, an empty string is returned instead. - * @return The filename associated with the File object - */ - virtual const char *GetFileName() const; - - /** - * Open a file. - * @param openFlags A combination of the flags: FASTOS_FILE_OPEN_READ, - * (Is read-access desired?), FASTOS_FILE_OPEN_WRITE - * (Is write-access desired?), and - * FASTOS_FILE_OPEN_EXISTING (The file to be opened - * should already exist. If the file does not exist, - * the call will fail.). - * @param filename You may optionally specify a filename here. This - * will replace the currently associated filename - * (if any) set using either the constructor, - * @ref SetFileName() or a previous call to - * @ref Open(). - * @return Boolean success/failure - */ - virtual bool Open(unsigned int openFlags, const char *filename=nullptr) = 0; - - /** - * Open a file for read/write access. The file will be created if it does - * not already exist. - * @param filename You may optionally specify a filename here. This - * will replace the currently associated filename - * (if any) set using either the constructor, - * @ref SetFileName() or a previous call to @ref Open(). - * @return Boolean success/failure - */ - bool OpenReadWrite(const char *filename=nullptr); - - /** - * Open a file for read/write access. This method fails if the file does - * not already exist. - * @param abortIfNotExist Abort the program if the file does not exist. - * @param filename You may optionally specify a filename here. This - * will replace the currently associated filename - * (if any) set using either the constructor, - * @ref SetFileName() or a previous call to - * @ref Open(). - * @return Boolean success/failure - */ - bool OpenExisting(bool abortIfNotExist=false, const char *filename=nullptr); - - /** - * Open a file for read access. This method fails if the file does - * not already exist. - * @param abortIfNotExist Abort the program if the file does not exist. - * @param filename You may optionally specify a filename here. This - * will replace the currently associated filename - * (if any) set using either the constructor, - * @ref SetFileName() or a previous call to - * @ref Open(). - * @return Boolean success/failure - */ - bool OpenReadOnlyExisting (bool abortIfNotExist=false, const char *filename=nullptr); - - /** - * Open a file for write access. If the file does not exist, it is created. - * If the file exists, it is truncated to 0 bytes. - * @param filename You may optionally specify a filename here. This - * will replace the currently associated filename - * (if any) set using either the constructor, - * @ref SetFileName() or a previous call to - * @ref Open(). - * @return Boolean success/failure - */ - bool OpenWriteOnlyTruncate(const char *filename=nullptr); - - /** - * Open a file for write access. This method fails if the file does - * not already exist. - * @param abortIfNotExist Abort the program if the file does not exist. - * @param filename You may optionally specify a filename here. This - * will replace the currently associated filename - * (if any) set using either the constructor, - * @ref SetFileName() or a previous call to - * @ref Open(). - * @return Boolean success/failure - */ - bool OpenWriteOnlyExisting (bool abortIfNotExist=false, const char *filename=nullptr); - - /** - * Open a file for read-access only. This method fails if the file does - * not already exist. - * @param filename You may optionally specify a filename here. This - * will replace the currently associated filename - * (if any) set using either the constructor, - * @ref SetFileName() or a previous call to @ref Open(). - * @return Boolean success/failure - */ - bool OpenReadOnly(const char *filename=nullptr); - - /** - * Open a file for write-access only. The file will be created if it does - * not already exist. - * @param filename You may optionally specify a filename here. This - * will replace the currently associated filename - * (if any) set using either the constructor, - * @ref SetFileName() or a previous call to @ref Open(). - * @return Boolean success/failure - */ - bool OpenWriteOnly(const char *filename=nullptr); - - /** - * Close the file. The call will successfully do nothing if the file - * already is closed. - * @return Boolean success/failure - */ - [[nodiscard]] virtual bool Close() = 0; - - /** - * Is the file currently opened? - * @return true if the file is opened, else false - */ - virtual bool IsOpened() const = 0; - - /** - * Read [length] bytes into [buffer]. - * @param buffer buffer pointer - * @param length number of bytes to read - * @return The number of bytes which was actually read, - * or -1 on error. - */ - [[nodiscard]] virtual ssize_t Read(void *buffer, size_t length) = 0; - - /** - * Write [len] bytes from [buffer]. This is just a wrapper for - * Write2, which does the same thing but returns the number of - * bytes written instead of just a bool. - * @param buffer buffer pointer - * @param len number of bytes to write - * @return Boolean success/failure - */ - [[nodiscard]] bool CheckedWrite(const void *buffer, size_t len); - - /** - * Write [len] bytes from [buffer]. - * @param buffer buffer pointer - * @param len number of bytes to write - * @return The number of bytes actually written, or -1 on error - */ - [[nodiscard]] virtual ssize_t Write2(const void *buffer, size_t len) = 0; - - /** - * Read [length] bytes into [buffer]. Caution! If the actual number - * of bytes read != [length], an error message is printed to stderr - * and the program aborts. - * - * The method is included for backwards compatibility reasons. - * @param buffer buffer pointer - * @param length number of bytes to read - */ - virtual void ReadBuf(void *buffer, size_t length); - - /** - * Write [length] bytes from [buffer] in chunks. Caution! If the write fails, - * an error message is printed to stderr and the program aborts. - * - * The method is included for backwards compatibility reasons. - * @param buffer buffer pointer - * @param length number of bytes to write - */ - virtual void WriteBuf(const void *buffer, size_t length); - - - /** - * Read [length] bytes at file offset [readOffset] into [buffer]. - * Only thread-safe if an OS-specific implementation exists. - * - * Caution! If the actual number of bytes read != [length], an - * error message is printed to stderr and the program aborts. - * - * @param buffer buffer pointer - * @param length number of bytes to read - * @param readOffset file offset where the read operation starts - */ - virtual void ReadBuf(void *buffer, size_t length, int64_t readOffset); - - /** - * Set the filepointer. The next @ref Read() or @ref Write() will - * continue from this position. - * @param position position of the new file pointer (in bytes) - * @return Boolean success/failure - */ - virtual bool SetPosition(int64_t position) = 0; - - /** - * Get the filepointer. -1 is returned if the operation fails. - * @return current position of file pointer (in bytes) - */ - virtual int64_t GetPosition() = 0; - - /** - * const version of @link GetPosition - */ - int64_t getPosition() const { return const_cast(this)->GetPosition(); } - - /** - * Return the file size. This method requires that the file is - * currently opened. If you wish to determine the file size - * without opening the file, use @ref Stat(). - * If an error occurs, the returned value is -1. - * @return file size (in bytes) - */ - virtual int64_t GetSize() = 0; - - /** - * const version of @link GetSize - */ - int64_t getSize() const { return const_cast(this)->GetSize(); } - - /** - * Return the time when file was last modified. - * @return time of last modification - */ - virtual time_t GetModificationTime() = 0; - - /** - * Delete the file. This method requires that the file is - * currently not opened. - * @return Boolean success/failure - */ - virtual bool Delete() = 0; - - /** - * Rename/move a file or directory. This method requires that - * the file is currently not opened. A move operation is - * supported as long as the source and destination reside - * on the same volume/device. - * The method fails if the destination already exists. - * @param newFileName New file name - * @return Boolean success/failure - */ - virtual bool Rename (const char *newFileName); - - /** - * Force completion of pending disk writes (flush cache). - */ - [[nodiscard]] virtual bool Sync() = 0; - - /** - * Are we in some kind of file read mode? - */ - bool IsReadMode() { - return ((_openFlags & FASTOS_FILE_OPEN_READ) != 0); - } - - /** - * Are we in some kind of file write mode? - */ - bool IsWriteMode() { - return ((_openFlags & FASTOS_FILE_OPEN_WRITE) != 0); - } - - /** - * Truncate or extend the file to the new size. - * The file pointer is also set to [newSize]. - * @ref SetSize() requires write access to the file. - * @param newSize New file size - * @return Boolean success/failure. - */ - virtual bool SetSize(int64_t newSize) = 0; - - /** - * Enable direct disk I/O (disable OS buffering & cache). Reads - * and writes will be performed directly to or from the user - * program buffer, provided appropriate size and alignment - * restrictions are met. If the restrictions are not met, a - * normal read or write is performed as a fallback. - * Call this before opening a file, and query - * @ref GetDirectIORestrictions() after a file is opened to get the - * neccessary alignment restrictions. It is possible that direct - * disk I/O could not be enabled. In that case - * @ref GetDirectIORestrictions will return false. - */ - virtual void EnableDirectIO(); - - virtual void EnableSyncWrites(); - - bool useSyncWrites() const { return _syncWritesEnabled; } - - /** - * Set the write chunk size used in WriteBuf. - */ - void setChunkSize(size_t chunkSize) { _chunkSize = chunkSize; } - size_t getChunkSize() const { return _chunkSize; } - - /** - * Get restrictions for direct disk I/O. The file should be opened - * before this method is called. - * Even though direct disk I/O is enabled through @ref EnableDirectIO(), - * this method could still return false, indicating that direct disk I/O - * is not being used for this file. This could be caused by either: no - * OS support for direct disk I/O, direct disk I/O might only be implemented - * for certain access-modes, the file is on a network drive or other - * partition where direct disk I/O is disallowed. - * - * The restriction-arguments are always filled with valid data, independant - * of the return code and direct disk I/O availability. - * - * @param memoryAlignment Buffer alignment restriction - * @param transferGranularity All transfers must be a multiple of - * [transferGranularity] bytes. All - * file offsets for these transfers must - * also be a multiple of [transferGranularity] - * bytes. - * @param transferMaximum All transfers must be <= [transferMaximum] - * bytes. - * @return True if direct disk I/O is being used for this file, else false. - */ - virtual bool - GetDirectIORestrictions(size_t &memoryAlignment, - size_t &transferGranularity, - size_t &transferMaximum); - - /** - * Retrieve the required padding for direct I/O to be used. - * - * @param offset File offset - * @param buflen Buffer length - * @param padBefore Number of pad bytes needed in front of the buffer - * @param padAfter Number of pad bytes needed after the buffer - * - * @return True if the file access can be accomplished with - * direct I/O, else false. - */ - virtual bool DirectIOPadding(int64_t offset, - size_t buflen, - size_t &padBefore, - size_t &padAfter); - - /** - * Allocate a buffer properly alligned with regards to direct io - * access restrictions. - * @param byteSize Number of bytes to be allocated - * @param realPtr Reference where the actual pointer returned - * from malloc will be saved. Use free() with - * this pointer to deallocate the buffer. - * This value is always set. - * @return Alligned pointer value or nullptr if out of memory - */ - static void *allocateGenericDirectIOBuffer(size_t byteSize, void *&realPtr); - - /** - * Get maximum memory alignment for directio buffers. - * @return maximum memory alignment for directio buffers. - */ - static size_t getMaxDirectIOMemAlign(); - - /** - * Allocate a buffer properly alligned with regards to direct io - * access restrictions. - * @param byteSize Number of bytes to be allocated - * @param realPtr Reference where the actual pointer returned - * from malloc will be saved. Use free() with - * this pointer to deallocate the buffer. - * This value is always set. - * @return Alligned pointer value or nullptr if out of memory - */ - virtual void *AllocateDirectIOBuffer(size_t byteSize, void *&realPtr); - - /** - * Enable mapping of complete file contents into the address space of the - * running process. This only works for small files. This operation - * will be ignored on some file types. - */ - virtual void enableMemoryMap(int mmapFlags); - - /** - * Inquiry about where in memory file data is located. - * @return location of file data in memory. If the file is not mapped, - * nullptr is returned. - */ - virtual void *MemoryMapPtr(int64_t position) const; - - /** - * Inquiry if file content is mapped into memory. - * @return true if file is mapped in memory, false otherwise. - */ - virtual bool IsMemoryMapped() const; - - /** - * Will drop whatever is in the FS cache when called. Does not have effect in the future. - **/ - virtual void dropFromCache() const; - - enum Error - { - ERR_ZERO = 1, // No error New style - ERR_NOENT, // No such file or directory - ERR_NOMEM, // Not enough memory - ERR_ACCES, // Permission denied - ERR_EXIST, // File exists - ERR_INVAL, // Invalid argument - ERR_NFILE, // File table overflow - ERR_MFILE, // Too many open files - ERR_NOSPC, // No space left on device - ERR_INTR, // interrupt - ERR_AGAIN, // Resource unavailable, try again - ERR_BUSY, // Device or resource busy - ERR_IO, // I/O error - ERR_PERM, // Not owner - ERR_NODEV, // No such device - ERR_NXIO, // Device not configured - ERR_UNKNOWN, // Unknown - - ERR_EZERO = 1, // No error Old style - ERR_ENOENT, // No such file or directory - ERR_ENOMEM, // Not enough memory - ERR_EACCES, // Permission denied - ERR_EEXIST, // File exists - ERR_EINVAL, // Invalid argument - ERR_ENFILE, // File table overflow - ERR_EMFILE, // Too many open files - ERR_ENOSPC, // No space left on device - ERR_EINTR, // interrupt - ERR_EAGAIN, // Resource unavailable, try again - ERR_EBUSY, // Device or resource busy - ERR_EIO, // I/O error - ERR_EPERM, // Not owner - ERR_ENODEV, // No such device - ERR_ENXIO // Device not configured - }; - - - /** - * If a file operation fails, the error code can be retrieved - * via this method. See @ref Error for possible error codes. - * @return Error code - */ - static Error GetLastError(); - - /** - * Similar to @ref GetLastError(), but this method returns a string - * instead of an error code. - * @return String describing the last error - */ - static std::string getLastErrorString(); -}; - - -/** - * The class serves as a container for information returned by - * @ref FastOS_File::Stat(). - */ -class FastOS_StatInfo -{ -public: - /** - * Possible error codes. - */ - enum StatError - { - Ok, //!< ok - Unknown, //!< unknown error - FileNotFound //!< file not found error - }; - - StatError _error; - - /** - * Is it a regular file? This field is only valid if @ref _error is - * @ref Ok. - */ - bool _isRegular; - - /** - * Is it a directory? This field is only valid if @ref _error is - * @ref Ok. - */ - bool _isDirectory; - - /** - * File size. This field is only valid if @ref _error is - * @ref Ok. - */ - int64_t _size; - - /** - * Time of last modification in seconds. - */ - time_t _modifiedTime; - - /** - * Time of last modification in seconds. - */ - uint64_t _modifiedTimeNS; -}; - - -/** - * This class enumerates the contents of a given directory. - * - * Example: - * @code - * void Foo::Bar() - * { - * // Scan and print the contents of the directory '/usr/src/include' - * - * FastOS_DirectoryScan dirScan("/usr/src/include"); - * int numEntries = 0; - * - * while(dirScan.ReadNext()) - * { - * const char *name = dirScan.GetName(); - * bool isDirectory = dirScan.IsDirectory(); - * bool isRegular = dirScan.IsRegular(); - * - * printf("%-30s %s\n", name, - * isDirectory ? "DIR" : (isRegular ? "FILE" : "UNKN")); - * - * numEntries++; - * } - * - * printf("The directory contained %d entries.\n", numEntries); - * } - * @endcode - */ -class FastOS_DirectoryScanInterface -{ -private: - FastOS_DirectoryScanInterface(const FastOS_DirectoryScanInterface&); - FastOS_DirectoryScanInterface& operator= (const FastOS_DirectoryScanInterface&); - -protected: - std::string _searchPath; - -public: - - /** - * Constructor. - * - * @param path Path of the directory to be scanned. The path string - * is copied internally. - */ - FastOS_DirectoryScanInterface(const char *path); - - /** - * Destructor. - * - * Frees operating system resources related to the directory scan. - */ - virtual ~FastOS_DirectoryScanInterface(); - - /** - * Get search path. - * This is an internal copy of the path specified in the constructor. - * @return Search path string. - */ - const char *GetSearchPath () { return _searchPath.c_str(); } - - /** - * Read the next entry in the directory scan. Failure indicates - * that there are no more entries. If the call is successful, - * attributes for the entry can be read with @ref IsDirectory(), - * @ref IsRegular() and @ref GetName(). - * - * @return Boolean success/failure - */ - virtual bool ReadNext() = 0; - - /** - * After a successful @ref ReadNext() this method is used to - * determine if the entry is a directory entry or not. Calling this - * method after an unsuccessful @ref ReadNext() or before - * @ref ReadNext() is called for the first time, yields undefined - * results. - * - * @return True if the entry is a directory, else false. - */ - virtual bool IsDirectory() = 0; - - - /** - * After a successful @ref ReadNext() this method is used to - * determine if the entry is a regular file entry or not. Calling - * this method after an unsuccessful @ref ReadNext() or before - * @ref ReadNext() is called for the first time, yields undefined - * results. - * - * @return True if the entry is a regular file, else false. - */ - virtual bool IsRegular() = 0; - - /** - * After a successful @ref ReadNext() this method is used to - * determine the name of the recently read directory entry. Calling - * this method after an unsuccessful @ref ReadNext() or before - * @ref ReadNext() is called for the first time, yields undefined - * results. - * - * @return A pointer to the recently read directory entry. - */ - virtual const char *GetName() = 0; - - /** - * Check whether the creation of a directory scan succeeded or - * failed (e.g. due to resource constraints). - * - * return True if the directory scan is valid. - */ - virtual bool IsValidScan() const = 0; -}; - -#ifdef __linux__ -#include -typedef FastOS_Linux_File FASTOS_PREFIX(File); -#else -#include -typedef FastOS_UNIX_File FASTOS_PREFIX(File); -#endif -typedef FastOS_UNIX_DirectoryScan FASTOS_PREFIX(DirectoryScan); diff --git a/fastos/src/vespa/fastos/file_rw_ops.cpp b/fastos/src/vespa/fastos/file_rw_ops.cpp deleted file mode 100644 index 79fe95b21f2..00000000000 --- a/fastos/src/vespa/fastos/file_rw_ops.cpp +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "file_rw_ops.h" -#include - -namespace fastos { - -File_RW_Ops::ReadFunc File_RW_Ops::_read = ::read; -File_RW_Ops::WriteFunc File_RW_Ops::_write = ::write; -File_RW_Ops::PreadFunc File_RW_Ops::_pread = ::pread; -File_RW_Ops::PwriteFunc File_RW_Ops::_pwrite = ::pwrite; - -} diff --git a/fastos/src/vespa/fastos/file_rw_ops.h b/fastos/src/vespa/fastos/file_rw_ops.h deleted file mode 100644 index 4f7aa6f082f..00000000000 --- a/fastos/src/vespa/fastos/file_rw_ops.h +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include - - -namespace fastos { - -/* - * Class handling pointers to functions used by FastOS_File for read - * and writa access. Unit tests might modify pointers to inject errors. - */ -class File_RW_Ops -{ - using ReadFunc = ssize_t (*)(int fd, void* buf, size_t count); - using WriteFunc = ssize_t (*)(int fd, const void* buf, size_t count); - using PreadFunc = ssize_t (*)(int fd, void* buf, size_t count, off_t offset); - using PwriteFunc = ssize_t (*)(int fd, const void* buf, size_t count, off_t offset); - -public: - static ReadFunc _read; - static WriteFunc _write; - static PreadFunc _pread; - static PwriteFunc _pwrite; - - static ssize_t read(int fd, void* buf, size_t count) { return _read(fd, buf, count); } - static ssize_t write(int fd, const void* buf, size_t count) { return _write(fd, buf, count); } - static ssize_t pread(int fd, void* buf, size_t count, off_t offset) { return _pread(fd, buf, count, offset); } - static ssize_t pwrite(int fd, const void* buf, size_t count, off_t offset) { return _pwrite(fd, buf, count, offset); } -}; - -} diff --git a/fastos/src/vespa/fastos/linux_file.cpp b/fastos/src/vespa/fastos/linux_file.cpp deleted file mode 100644 index 6fb29782957..00000000000 --- a/fastos/src/vespa/fastos/linux_file.cpp +++ /dev/null @@ -1,436 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** -****************************************************************************** -* @author Oivind H. Danielsen -* @date Creation date: 2000-09-21 -* @file -* Implementation of FastOS_Linux_File methods. -*****************************************************************************/ - -#ifdef __linux__ -#include "file.h" -#include "file_rw_ops.h" -#include -#include -#include -#include -#include -#include -#include - -using fastos::File_RW_Ops; - -const size_t FastOS_Linux_File::_directIOFileAlign = 4096; -const size_t FastOS_Linux_File::_directIOMemAlign = 4096; - -FastOS_Linux_File::FastOS_Linux_File(const char *filename) - : FastOS_UNIX_File(filename), - _cachedSize(-1), - _filePointer(-1) -{ -} - -FastOS_Linux_File::~FastOS_Linux_File () { - bool ok = Close(); - assert(ok); -} - -#define DIRECTIOPOSSIBLE(buf, len, off) \ - (((off) & (_directIOFileAlign - 1)) == 0 && \ - ((len) & (_directIOFileAlign - 1)) == 0 && \ - (reinterpret_cast(buf) & (_directIOMemAlign - 1)) == 0) - -ssize_t -FastOS_Linux_File::readInternal(int fh, void *buffer, size_t length, int64_t readOffset) -{ - char * data = static_cast(buffer); - ssize_t has_read(0); - while (has_read < ssize_t(length)) { - size_t lenNow = std::min(getChunkSize(), length - has_read); - ssize_t readNow = File_RW_Ops::pread(fh, data + has_read, lenNow, readOffset + has_read); - if (readNow > 0) { - has_read += readNow; - } else { - return (has_read > 0) ? has_read : readNow; - } - } - return has_read; -} - - -ssize_t -FastOS_Linux_File::readInternal(int fh, void *buffer, size_t length) -{ - char * data = static_cast(buffer); - ssize_t has_read(0); - while (has_read < ssize_t(length)) { - size_t lenNow = std::min(getChunkSize(), length - has_read); - ssize_t readNow = File_RW_Ops::read(fh, data + has_read, lenNow); - if (readNow > 0) { - has_read += readNow; - } else { - return (has_read > 0) ? has_read : readNow; - } - } - return has_read; -} - - -ssize_t -FastOS_Linux_File::writeInternal(int fh, const void *buffer, size_t length, int64_t writeOffset) -{ - return File_RW_Ops::pwrite(fh, buffer, length, writeOffset); -} - -ssize_t -FastOS_Linux_File::writeInternal(int fh, const void *buffer, size_t length) -{ - return File_RW_Ops::write(fh, buffer, length); -} - - -ssize_t FastOS_Linux_File::readUnalignedEnd(void *buffer, size_t length, int64_t readOffset) -{ - if (length == 0) { return 0; } - int fh = open(GetFileName(), O_RDONLY, 0664); - if (fh < 0) { - std::ostringstream os; - os << "Failed opening file " << GetFileName() << " for reading the unaligend end due to : " << getLastErrorString(); - throw std::runtime_error(os.str()); - } - ssize_t readResult = readInternal(fh, buffer, length, readOffset); - close(fh); - return readResult; -} - -ssize_t FastOS_Linux_File::writeUnalignedEnd(const void *buffer, size_t length, int64_t writeOffset) -{ - if (length == 0) { return 0; } - int fh = open(GetFileName(), O_WRONLY | O_SYNC, 0664); - if (fh < 0) { - std::ostringstream os; - os << "Failed opening file " << GetFileName() << " for reading the unaligend end due to : " << getLastErrorString(); - throw std::runtime_error(os.str()); - } - ssize_t writeResult = writeInternal(fh, buffer, length, writeOffset); - close(fh); - return writeResult; -} - -ssize_t -FastOS_Linux_File::ReadBufInternal(void *buffer, size_t length, int64_t readOffset) -{ - if (length == 0) { return 0; } - ssize_t readResult; - - if (_directIOEnabled) { - if (DIRECTIOPOSSIBLE(buffer, length, readOffset)) { - readResult = readInternal(_filedes, buffer, length, readOffset); - } else { - size_t alignedLength(length & ~(_directIOFileAlign - 1)); - if (DIRECTIOPOSSIBLE(buffer, alignedLength, readOffset)) { - size_t remain(length - alignedLength); - if (alignedLength > 0) { - readResult = readInternal(_filedes, buffer, alignedLength, readOffset); - } else { - readResult = 0; - } - if (static_cast(readResult) == alignedLength && (remain != 0)) { - ssize_t readResult2 = readUnalignedEnd(static_cast(buffer) + alignedLength, - remain, readOffset + alignedLength); - if (readResult == 0) { - readResult = readResult2; - } else if (readResult2 > 0) { - readResult += readResult2; - } - } - } else { - throw DirectIOException(GetFileName(), buffer, length, readOffset); - } - } - } else { - readResult = readInternal(_filedes, buffer, length, readOffset); - } - - if (readResult < 0) { - perror("pread error"); - } - - return readResult; -} - -void -FastOS_Linux_File::ReadBuf(void *buffer, size_t length, int64_t readOffset) -{ - ssize_t readResult(ReadBufInternal(buffer, length, readOffset)); - if (static_cast(readResult) != length) { - std::string errorString = (readResult != -1) - ? std::string("short read") - : getLastErrorString(); - std::ostringstream os; - os << "Fatal: Reading " << length << " bytes, got " << readResult << " from '" - << GetFileName() << "' failed: " << errorString; - throw std::runtime_error(os.str()); - } -} - - -ssize_t -FastOS_Linux_File::Read(void *buffer, size_t len) -{ - if (_directIOEnabled) { - ssize_t readResult = ReadBufInternal(buffer, len, _filePointer); - if (readResult > 0) { - _filePointer += readResult; - } - return readResult; - } else { - return readInternal(_filedes, buffer, len); - } -} - - -ssize_t -FastOS_Linux_File::Write2(const void *buffer, size_t length) -{ - const char * data = static_cast(buffer); - ssize_t written(0); - while (written < ssize_t(length)) { - size_t lenNow = std::min(getChunkSize(), length - written); - ssize_t writtenNow = internalWrite2(data + written, lenNow); - if (writtenNow > 0) { - written += writtenNow; - } else { - return (written > 0) ? written : writtenNow;; - } - } - return written; -} - -ssize_t -FastOS_Linux_File::internalWrite2(const void *buffer, size_t length) -{ - ssize_t writeRes; - if (_directIOEnabled) { - if (DIRECTIOPOSSIBLE(buffer, length, _filePointer)) { - writeRes = writeInternal(_filedes, buffer, length, _filePointer); - } else { - size_t alignedLength(length & ~(_directIOFileAlign - 1)); - if (DIRECTIOPOSSIBLE(buffer, alignedLength, _filePointer)) { - size_t remain(length - alignedLength); - if (alignedLength > 0) { - writeRes = writeInternal(_filedes, buffer, alignedLength, _filePointer); - } else { - writeRes = 0; - } - if (static_cast(writeRes) == alignedLength && remain != 0) { - ssize_t writeRes2 = writeUnalignedEnd(static_cast(buffer) + alignedLength, - remain, _filePointer + alignedLength); - if (writeRes == 0) { - writeRes = writeRes2; - } else if (writeRes2 > 0) { - writeRes += writeRes2; - } - } - } else { - throw DirectIOException(GetFileName(), buffer, length, _filePointer); - } - } - if (writeRes > 0) { - _filePointer += writeRes; - if (_filePointer > _cachedSize) { - _cachedSize = _filePointer; - } - } - } else { - writeRes = writeInternal(_filedes, buffer, length); - } - - return writeRes; -} - - -bool -FastOS_Linux_File::SetPosition(int64_t desiredPosition) -{ - bool rc = FastOS_UNIX_File::SetPosition(desiredPosition); - - if (rc && _directIOEnabled) { - _filePointer = desiredPosition; - } - - return rc; -} - - -int64_t -FastOS_Linux_File::GetPosition() -{ - return _directIOEnabled ? _filePointer : FastOS_UNIX_File::GetPosition(); -} - - -bool -FastOS_Linux_File::SetSize(int64_t newSize) -{ - bool rc = FastOS_UNIX_File::SetSize(newSize); - - if (rc) { - _cachedSize = newSize; - } - return rc; -} - - -namespace { - void * align(void * p, size_t alignment) { - const size_t alignMask(alignment-1); - return reinterpret_cast((reinterpret_cast(p) + alignMask) & ~alignMask); - } -} - -void * -FastOS_Linux_File::AllocateDirectIOBuffer (size_t byteSize, void *&realPtr) -{ - size_t dummy1, dummy2; - size_t memoryAlignment; - - GetDirectIORestrictions(memoryAlignment, dummy1, dummy2); - - realPtr = malloc(byteSize + memoryAlignment - 1); - return align(realPtr, memoryAlignment); -} - -size_t -FastOS_Linux_File::getMaxDirectIOMemAlign() -{ - return _directIOMemAlign; -} - - -bool -FastOS_Linux_File::GetDirectIORestrictions (size_t &memoryAlignment, size_t &transferGranularity, size_t &transferMaximum) -{ - if (_directIOEnabled) { - memoryAlignment = _directIOMemAlign; - transferGranularity = _directIOFileAlign; - transferMaximum = 0x7FFFFFFF; - return true; - } else { - return FastOS_UNIX_File::GetDirectIORestrictions(memoryAlignment, transferGranularity, transferMaximum); - } -} - - -bool -FastOS_Linux_File::DirectIOPadding (int64_t offset, size_t length, size_t &padBefore, size_t &padAfter) -{ - if (_directIOEnabled) { - - padBefore = offset & (_directIOFileAlign - 1); - padAfter = _directIOFileAlign - ((padBefore + length) & (_directIOFileAlign - 1)); - - if (padAfter == _directIOFileAlign) { - padAfter = 0; - } - if (int64_t(offset+length+padAfter) > _cachedSize) { - // _cachedSize is not really trustworthy, so if we suspect it is not correct, we correct it. - // The main reason is that it will not reflect the file being extended by another filedescriptor. - _cachedSize = GetSize(); - } - if ((padAfter != 0) && - (static_cast(offset + length + padAfter) > _cachedSize) && - (static_cast(offset + length) <= _cachedSize)) - { - padAfter = _cachedSize - (offset + length); - } - - if (static_cast(offset + length + padAfter) <= static_cast(_cachedSize)) { - return true; - } - } - - padAfter = 0; - padBefore = 0; - - return false; -} - - -void -FastOS_Linux_File::EnableDirectIO() -{ - if (!IsOpened()) { - _directIOEnabled = true; - } -} - - -bool -FastOS_Linux_File::Open(unsigned int openFlags, const char *filename) -{ - bool rc; - _cachedSize = -1; - _filePointer = -1; - if (_directIOEnabled && (_openFlags & FASTOS_FILE_OPEN_STDFLAGS) != 0) { - _directIOEnabled = false; - } - if (_syncWritesEnabled) { - openFlags |= FASTOS_FILE_OPEN_SYNCWRITES; - } - if (_directIOEnabled) { - rc = FastOS_UNIX_File::Open(openFlags | FASTOS_FILE_OPEN_DIRECTIO, filename); - if ( ! rc ) { //Retry without directIO. - rc = FastOS_UNIX_File::Open(openFlags | FASTOS_FILE_OPEN_SYNCWRITES, filename); - } - if (rc) { - int fadviseOptions = getFAdviseOptions(); - if (POSIX_FADV_NORMAL != fadviseOptions) { - rc = (posix_fadvise(_filedes, 0, 0, fadviseOptions) == 0); - if (!rc) { - bool close_ok = Close(); - assert(close_ok); - } - } - } - if (rc) { - bool sync_ok = Sync(); - assert(sync_ok); - _cachedSize = GetSize(); - _filePointer = 0; - } - } else { - rc = FastOS_UNIX_File::Open(openFlags, filename); - if (rc && (POSIX_FADV_NORMAL != getFAdviseOptions())) { - rc = (posix_fadvise(_filedes, 0, 0, getFAdviseOptions()) == 0); - if (!rc) { - bool close_ok = Close(); - assert(close_ok); - } - } - } - return rc; -} - -int -FastOS_Linux_File::count_open_files() -{ - static const char * const fd_dir_name = "/proc/self/fd"; - int count = 0; - DIR *dp = opendir(fd_dir_name); - if (dp != nullptr) { - struct dirent *ptr; - while ((ptr = readdir(dp)) != nullptr) { - if ((strcmp(".", ptr->d_name) != 0) && (strcmp("..", ptr->d_name) != 0)) { - ++count; - } - } - closedir(dp); - } else { - std::error_code ec(errno, std::system_category()); - fprintf(stderr, "could not scan directory %s: %s\n", fd_dir_name, ec.message().c_str()); - } - return count; -} - -#endif diff --git a/fastos/src/vespa/fastos/linux_file.h b/fastos/src/vespa/fastos/linux_file.h deleted file mode 100644 index 2481b163210..00000000000 --- a/fastos/src/vespa/fastos/linux_file.h +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** -****************************************************************************** -* @author Oivind H. Danielsen -* @date Creation date: 2000-09-21 -* @file -* Class definition for FastOS_Linux_File. -*****************************************************************************/ - -#pragma once - -#include "unix_file.h" - -/** - * This is the Linux implementation of @ref FastOS_File. Most - * methods are inherited from @ref FastOS_UNIX_File. - */ -class FastOS_Linux_File : public FastOS_UNIX_File -{ -public: - using FastOS_UNIX_File::ReadBuf; -protected: - int64_t _cachedSize; - int64_t _filePointer; // Only maintained/used in directio mode - -public: - FastOS_Linux_File (const char *filename = nullptr); - ~FastOS_Linux_File () override; - bool GetDirectIORestrictions(size_t &memoryAlignment, size_t &transferGranularity, size_t &transferMaximum) override; - bool DirectIOPadding(int64_t offset, size_t length, size_t &padBefore, size_t &padAfter) override; - void EnableDirectIO() override; - bool SetPosition(int64_t desiredPosition) override; - int64_t GetPosition() override; - bool SetSize(int64_t newSize) override; - void ReadBuf(void *buffer, size_t length, int64_t readOffset) override; - void *AllocateDirectIOBuffer(size_t byteSize, void *&realPtr) override; - - - [[nodiscard]] ssize_t Read(void *buffer, size_t len) override; - [[nodiscard]] ssize_t Write2(const void *buffer, size_t len) override; - bool Open(unsigned int openFlags, const char *filename) override; - - static size_t getMaxDirectIOMemAlign(); - static int count_open_files(); -private: - ssize_t internalWrite2(const void *buffer, size_t len); - ssize_t readUnalignedEnd(void *buffer, size_t length, int64_t readOffset); - ssize_t writeUnalignedEnd(const void *buffer, size_t length, int64_t readOffset); - ssize_t ReadBufInternal(void *buffer, size_t length, int64_t readOffset); - ssize_t readInternal(int fh, void *buffer, size_t length, int64_t readOffset); - ssize_t readInternal(int fh, void *buffer, size_t length); - static ssize_t writeInternal(int fh, const void *buffer, size_t length, int64_t writeOffset); - static ssize_t writeInternal(int fh, const void *buffer, size_t length); - - static const size_t _directIOFileAlign; - static const size_t _directIOMemAlign; -}; diff --git a/fastos/src/vespa/fastos/types.h b/fastos/src/vespa/fastos/types.h deleted file mode 100644 index 69dd3e5231c..00000000000 --- a/fastos/src/vespa/fastos/types.h +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#define FASTOS_PREFIX(a) FastOS_##a - -// New macros to support the new gcc visibility features. -#define VESPA_DLL_EXPORT __attribute__ ((visibility("default"))) -#define VESPA_DLL_LOCAL __attribute__ ((visibility("hidden"))) diff --git a/fastos/src/vespa/fastos/unix_file.cpp b/fastos/src/vespa/fastos/unix_file.cpp deleted file mode 100644 index 7c4cde19125..00000000000 --- a/fastos/src/vespa/fastos/unix_file.cpp +++ /dev/null @@ -1,590 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** -****************************************************************************** -* @author Oivind H. Danielsen -* @date Creation date: 2000-01-18 -* @file -* Implementation of FastOS_UNIX_File methods. -*****************************************************************************/ - -#include "file.h" -#include -#include -#include -#include -#include -#include -#include -#include -#ifdef __linux__ -#include -#else -#include -#endif -#ifdef __APPLE__ -#include -#include -#endif -#include "file_rw_ops.h" - -using fastos::File_RW_Ops; - -int FastOS_UNIX_File::GetLastOSError() { - return errno; -} - -ssize_t -FastOS_UNIX_File::Read(void *buffer, size_t len) -{ - return File_RW_Ops::read(_filedes, buffer, len); -} - - -ssize_t -FastOS_UNIX_File::Write2(const void *buffer, size_t len) -{ - return File_RW_Ops::write(_filedes, buffer, len); -} - -bool -FastOS_UNIX_File::SetPosition(int64_t desiredPosition) -{ - int64_t position = lseek(_filedes, desiredPosition, SEEK_SET); - - return (position == desiredPosition); -} - - -int64_t -FastOS_UNIX_File::GetPosition() -{ - return lseek(_filedes, 0, SEEK_CUR); -} - -void FastOS_UNIX_File::ReadBuf(void *buffer, size_t length, - int64_t readOffset) -{ - ssize_t readResult; - - readResult = File_RW_Ops::pread(_filedes, buffer, length, readOffset); - if (static_cast(readResult) != length) { - std::string errorString = readResult != -1 ? - std::string("short read") : - FastOS_FileInterface::getLastErrorString(); - std::ostringstream os; - os << "Fatal: Reading " << length << " bytes, got " << readResult << " from '" - << GetFileName() << "' failed: " << errorString; - throw std::runtime_error(os.str()); - } -} - -bool -FastOS_UNIX_File::Stat(const char *filename, FastOS_StatInfo *statInfo) -{ - bool rc = false; - - struct stat stbuf{}; - int lstatres; - - do { - lstatres = lstat(filename, &stbuf); - } while (lstatres == -1 && errno == EINTR); - if (lstatres == 0) { - statInfo->_error = FastOS_StatInfo::Ok; - statInfo->_isRegular = S_ISREG(stbuf.st_mode); - statInfo->_isDirectory = S_ISDIR(stbuf.st_mode); - statInfo->_size = static_cast(stbuf.st_size); - statInfo->_modifiedTime = stbuf.st_mtime; -#ifdef __linux__ - statInfo->_modifiedTimeNS = stbuf.st_mtim.tv_sec; - statInfo->_modifiedTimeNS *= 1000000000; - statInfo->_modifiedTimeNS += stbuf.st_mtim.tv_nsec; -#elif defined(__APPLE__) - statInfo->_modifiedTimeNS = stbuf.st_mtimespec.tv_sec; - statInfo->_modifiedTimeNS *= 1000000000; - statInfo->_modifiedTimeNS += stbuf.st_mtimespec.tv_nsec; -#else - statInfo->_modifiedTimeNS = stbuf.st_mtime; - statInfo->_modifiedTimeNS *= 1000000000; -#endif - rc = true; - } else { - if (errno == ENOENT) { - statInfo->_error = FastOS_StatInfo::FileNotFound; - } else { - statInfo->_error = FastOS_StatInfo::Unknown; - } - } - - return rc; -} - -bool FastOS_UNIX_File::SetCurrentDirectory (const char *pathName) { return (chdir(pathName) == 0); } - - -int FastOS_UNIX_File::GetMaximumFilenameLength (const char *pathName) -{ - return pathconf(pathName, _PC_NAME_MAX); -} - -int FastOS_UNIX_File::GetMaximumPathLength(const char *pathName) -{ - return pathconf(pathName, _PC_PATH_MAX); -} - -std::string -FastOS_UNIX_File::getCurrentDirectory() -{ - std::string res; - int maxPathLen = FastOS_File::GetMaximumPathLength("."); - if (maxPathLen == -1) { - maxPathLen = 16384; - } else if (maxPathLen < 512) { - maxPathLen = 512; - } - - char *currentDir = new char [maxPathLen + 1]; - - if (getcwd(currentDir, maxPathLen) != nullptr) { - res = currentDir; - } - delete [] currentDir; - - return res; -} - - -unsigned int -FastOS_UNIX_File::CalcAccessFlags(unsigned int openFlags) -{ - unsigned int accessFlags=0; - - if ((openFlags & (FASTOS_FILE_OPEN_READ | FASTOS_FILE_OPEN_DIRECTIO)) != 0) { - if ((openFlags & FASTOS_FILE_OPEN_WRITE) != 0) { - // Open for reading and writing - accessFlags = O_RDWR; - } else { - // Open for reading only - accessFlags = O_RDONLY; - } - } else { - // Open for writing only - accessFlags = O_WRONLY; - } - - if (((openFlags & FASTOS_FILE_OPEN_EXISTING) == 0) && ((openFlags & FASTOS_FILE_OPEN_WRITE) != 0)) { - // Create file if it does not exist - accessFlags |= O_CREAT; - } - -#if defined(O_SYNC) - if ((openFlags & FASTOS_FILE_OPEN_SYNCWRITES) != 0) - accessFlags |= O_SYNC; -#elif defined(O_FSYNC) - if ((openFlags & FASTOS_FILE_OPEN_SYNCWRITES) != 0) - accessFlags |= O_FSYNC; -#endif - -#ifdef __linux__ - if ((openFlags & FASTOS_FILE_OPEN_DIRECTIO) != 0) { - accessFlags |= O_DIRECT; - } -#endif - - if ((openFlags & FASTOS_FILE_OPEN_TRUNCATE) != 0) { - // Truncate file on open - accessFlags |= O_TRUNC; - } - return accessFlags; -} - -#ifdef __linux__ -constexpr int ALWAYS_SUPPORTED_MMAP_FLAGS = ~MAP_HUGETLB; -#else -constexpr int ALWAYS_SUPPORTED_MMAP_FLAGS = ~0; -#endif - -bool -FastOS_UNIX_File::Open(unsigned int openFlags, const char *filename) -{ - bool rc = false; - assert(_filedes == -1); - - if ((openFlags & FASTOS_FILE_OPEN_STDFLAGS) != 0) { - FILE *file; - - switch(openFlags & FASTOS_FILE_OPEN_STDFLAGS) { - - case FASTOS_FILE_OPEN_STDOUT: - file = stdout; - SetFileName("stdout"); - break; - - case FASTOS_FILE_OPEN_STDERR: - file = stderr; - SetFileName("stderr"); - break; - - default: - fprintf(stderr, "Invalid open-flags %08X\n", openFlags); - abort(); - } - -#ifdef __linux__ - _filedes = file->_fileno; -#else - _filedes = fileno(file); -#endif - _openFlags = openFlags; - rc = true; - } else { - if (filename != nullptr) { - SetFileName(filename); - } - unsigned int accessFlags = CalcAccessFlags(openFlags); - - _filedes = open(_filename.c_str(), accessFlags, 0664); - - rc = (_filedes != -1); - - if (rc) { - _openFlags = openFlags; - if (_mmapEnabled) { - int64_t filesize = GetSize(); - auto mlen = static_cast(filesize); - if ((static_cast(mlen) == filesize) && (mlen > 0)) { - void *mbase = mmap(nullptr, mlen, PROT_READ, MAP_SHARED | _mmapFlags, _filedes, 0); - if (mbase == MAP_FAILED) { - mbase = mmap(nullptr, mlen, PROT_READ, MAP_SHARED | (_mmapFlags & ALWAYS_SUPPORTED_MMAP_FLAGS), _filedes, 0); - } - if (mbase != MAP_FAILED) { -#ifdef __linux__ - int fadviseOptions = getFAdviseOptions(); - int eCode(0); - if (POSIX_FADV_RANDOM == fadviseOptions) { - eCode = posix_madvise(mbase, mlen, POSIX_MADV_RANDOM); - } else if (POSIX_FADV_SEQUENTIAL == fadviseOptions) { - eCode = posix_madvise(mbase, mlen, POSIX_MADV_SEQUENTIAL); - } - if (eCode != 0) { - fprintf(stderr, "Failed: posix_madvise(%p, %ld, %d) = %d\n", mbase, mlen, fadviseOptions, eCode); - } -#endif - _mmapbase = mbase; - _mmaplen = mlen; - } else { - close(_filedes); - _filedes = -1; - std::ostringstream os; - os << "mmap of file '" << GetFileName() << "' with flags '" << std::hex << (MAP_SHARED | _mmapFlags) << std::dec - << "' failed with error :'" << getErrorString(GetLastOSError()) << "'"; - throw std::runtime_error(os.str()); - } - } - } - } - - } - - return rc; -} - -void FastOS_UNIX_File::dropFromCache() const -{ -#ifdef __linux__ - posix_fadvise(_filedes, 0, 0, POSIX_FADV_DONTNEED); -#endif -} - - -bool -FastOS_UNIX_File::Close() -{ - bool ok = true; - - if (_filedes >= 0) { - if ((_openFlags & FASTOS_FILE_OPEN_STDFLAGS) != 0) { - ok = true; - } else { - do { - ok = (close(_filedes) == 0); - } while (!ok && errno == EINTR); - } - - if (_mmapbase != nullptr) { - madvise(_mmapbase, _mmaplen, MADV_DONTNEED); - munmap(static_cast(_mmapbase), _mmaplen); - _mmapbase = nullptr; - _mmaplen = 0; - } - - _filedes = -1; - } - - _openFlags = 0; - - return ok; -} - - -int64_t -FastOS_UNIX_File::GetSize() -{ - int64_t fileSize=-1; - struct stat stbuf{}; - - assert(IsOpened()); - - int res = fstat(_filedes, &stbuf); - - if (res == 0) { - fileSize = stbuf.st_size; - } - - return fileSize; -} - - -time_t -FastOS_UNIX_File::GetModificationTime() -{ - struct stat stbuf{}; - assert(IsOpened()); - - int res = fstat(_filedes, &stbuf); - assert(res == 0); - (void) res; - - return stbuf.st_mtime; -} - - -bool -FastOS_UNIX_File::Delete(const char *name) -{ - return (unlink(name) == 0); -} - - -bool -FastOS_UNIX_File::Delete() -{ - assert( ! IsOpened()); - - return (unlink(_filename.c_str()) == 0); -} - -bool FastOS_UNIX_File::Rename (const char *currentFileName, const char *newFileName) -{ - bool rc = false; - - // Enforce documentation. If the destination file exists, - // fail Rename. - FastOS_StatInfo statInfo{}; - if (!FastOS_File::Stat(newFileName, &statInfo)) { - rc = (rename(currentFileName, newFileName) == 0); - } else { - errno = EEXIST; - } - return rc; -} - -bool -FastOS_UNIX_File::Sync() -{ - assert(IsOpened()); - - return (fsync(_filedes) == 0); -} - - -bool -FastOS_UNIX_File::SetSize(int64_t newSize) -{ - bool rc = false; - - if (ftruncate(_filedes, static_cast(newSize)) == 0) { - rc = SetPosition(newSize); - } - - return rc; -} - - -FastOS_File::Error -FastOS_UNIX_File::TranslateError (const int osError) -{ - switch(osError) { - case ENOENT: return ERR_NOENT; // No such file or directory - case ENOMEM: return ERR_NOMEM; // Not enough memory - case EACCES: return ERR_ACCES; // Permission denied - case EEXIST: return ERR_EXIST; // File exists - case EINVAL: return ERR_INVAL; // Invalid argument - case ENOSPC: return ERR_NOSPC; // No space left on device - case EINTR: return ERR_INTR; // interrupt - case EAGAIN: return ERR_AGAIN; // Resource unavailable, try again - case EBUSY: return ERR_BUSY; // Device or resource busy - case EIO: return ERR_IO; // I/O error - case EPERM: return ERR_PERM; // Not owner - case ENODEV: return ERR_NODEV; // No such device - case ENXIO: return ERR_NXIO; // Device not configured - default: break; - } - - if (osError == ENFILE) - return ERR_NFILE; - - if (osError == EMFILE) - return ERR_MFILE; - - return ERR_UNKNOWN; -} - - -std::string -FastOS_UNIX_File::getErrorString(const int osError) -{ - std::error_code ec(osError, std::system_category()); - return ec.message(); -} - - -int64_t FastOS_UNIX_File::GetFreeDiskSpace (const char *path) -{ - struct statfs statBuf{}; - int statVal = statfs(path, &statBuf); - if (statVal == 0) { - return int64_t(statBuf.f_bavail) * int64_t(statBuf.f_bsize); - } - - return -1; -} - -int -FastOS_UNIX_File::count_open_files() -{ -#ifdef __APPLE__ - int buffer_size = proc_pidinfo(getpid(), PROC_PIDLISTFDS, 0, nullptr, 0); - return buffer_size / sizeof(proc_fdinfo); -#else - return 0; -#endif -} - -FastOS_UNIX_DirectoryScan::FastOS_UNIX_DirectoryScan(const char *searchPath) - : FastOS_DirectoryScanInterface(searchPath), - _statRun(false), - _isDirectory(false), - _isRegular(false), - _statName(nullptr), - _statFilenameP(nullptr), - _dir(nullptr), - _dp(nullptr) -{ - _dir = opendir(searchPath); - - const int minimumLength = 512 + 1; - const int defaultLength = 16384; - - int maxNameLength = FastOS_File::GetMaximumFilenameLength(searchPath); - int maxPathLength = FastOS_File::GetMaximumPathLength(searchPath); - int nameLength = maxNameLength + 1 + maxPathLength; - - if ((maxNameLength == -1) || - (maxPathLength == -1) || - (nameLength < minimumLength)) - { - nameLength = defaultLength; - } - - _statName = new char [nameLength + 1]; // Include null - - strcpy(_statName, searchPath); - strcat(_statName, "/"); - - _statFilenameP = &_statName[strlen(_statName)]; -} - - -FastOS_UNIX_DirectoryScan::~FastOS_UNIX_DirectoryScan() -{ - if (_dir != nullptr) { - closedir(_dir); - _dir = nullptr; - } - delete [] _statName; -} - - -bool -FastOS_UNIX_DirectoryScan::ReadNext() -{ - _statRun = false; - - if (_dir != nullptr) { - _dp = readdir(_dir); - return (_dp != nullptr); - } - - return false; -} - - -void -FastOS_UNIX_DirectoryScan::DoStat() -{ - struct stat stbuf{}; - - assert(_dp != nullptr); - - strcpy(_statFilenameP, _dp->d_name); - - if (lstat(_statName, &stbuf) == 0) { - _isRegular = S_ISREG(stbuf.st_mode); - _isDirectory = S_ISDIR(stbuf.st_mode); - } else { - printf("lstat failed for [%s]\n", _dp->d_name); - _isRegular = false; - _isDirectory = false; - } - - _statRun = true; -} - - -bool -FastOS_UNIX_DirectoryScan::IsDirectory() -{ - if (!_statRun) { - DoStat(); - } - - return _isDirectory; -} - - -bool -FastOS_UNIX_DirectoryScan::IsRegular() -{ - if (!_statRun) { - DoStat(); - } - - return _isRegular; -} - - -const char * -FastOS_UNIX_DirectoryScan::GetName() -{ - assert(_dp != nullptr); - - return static_cast(_dp->d_name); -} - - -bool -FastOS_UNIX_DirectoryScan::IsValidScan() const -{ - return _dir != nullptr; -} diff --git a/fastos/src/vespa/fastos/unix_file.h b/fastos/src/vespa/fastos/unix_file.h deleted file mode 100644 index 31e45f8d2fa..00000000000 --- a/fastos/src/vespa/fastos/unix_file.h +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** -****************************************************************************** -* @author Oivind H. Danielsen -* @date Creation date: 2000-01-18 -* @file -* Class definitions for FastOS_UNIX_File and FastOS_UNIX_DirectoryScan. -*****************************************************************************/ - -#pragma once - -#include - -/** - * This is the generic UNIX implementation of @ref FastOS_FileInterface. - */ -class FastOS_UNIX_File : public FastOS_FileInterface -{ -public: - using FastOS_FileInterface::ReadBuf; -private: - FastOS_UNIX_File(const FastOS_UNIX_File&); - FastOS_UNIX_File& operator=(const FastOS_UNIX_File&); - -protected: - void *_mmapbase; - size_t _mmaplen; - int _filedes; - int _mmapFlags; - bool _mmapEnabled; - - static unsigned int CalcAccessFlags(unsigned int openFlags); - -public: - static bool Rename (const char *currentFileName, const char *newFileName); - bool Rename (const char *newFileName) override { - return FastOS_FileInterface::Rename(newFileName); - } - - static bool Stat(const char *filename, FastOS_StatInfo *statInfo); - - static std::string getCurrentDirectory(); - - static bool SetCurrentDirectory (const char *pathName); - static int GetMaximumFilenameLength (const char *pathName); - static int GetMaximumPathLength (const char *pathName); - - FastOS_UNIX_File(const char *filename=nullptr) - : FastOS_FileInterface(filename), - _mmapbase(nullptr), - _mmaplen(0), - _filedes(-1), - _mmapFlags(0), - _mmapEnabled(false) - { } - - void ReadBuf(void *buffer, size_t length, int64_t readOffset) override; - [[nodiscard]] ssize_t Read(void *buffer, size_t len) override; - [[nodiscard]] ssize_t Write2(const void *buffer, size_t len) override; - bool Open(unsigned int openFlags, const char *filename) override; - [[nodiscard]] bool Close() override; - bool IsOpened() const override { return _filedes >= 0; } - - void enableMemoryMap(int flags) override { - _mmapEnabled = true; - _mmapFlags = flags; - } - - void *MemoryMapPtr(int64_t position) const override { - if (_mmapbase != nullptr) { - if (position < int64_t(_mmaplen)) { - return static_cast(static_cast(_mmapbase) + position); - } else { // This is an indication that the file size has changed and a remap/reopen must be done. - return nullptr; - } - } else { - return nullptr; - } - } - - bool IsMemoryMapped() const override { return _mmapbase != nullptr; } - bool SetPosition(int64_t desiredPosition) override; - int64_t GetPosition() override; - int64_t GetSize() override; - time_t GetModificationTime() override; - bool Delete() override; - [[nodiscard]] bool Sync() override; - bool SetSize(int64_t newSize) override; - void dropFromCache() const override; - - static bool Delete(const char *filename); - static int GetLastOSError(); - static Error TranslateError(const int osError); - static std::string getErrorString(const int osError); - static int64_t GetFreeDiskSpace (const char *path); - static int count_open_files(); -}; - -#include -/** - * This is the generic UNIX implementation of @ref FastOS_DirectoryScan. - */ -class FastOS_UNIX_DirectoryScan : public FastOS_DirectoryScanInterface -{ -private: - FastOS_UNIX_DirectoryScan(const FastOS_UNIX_DirectoryScan&); - FastOS_UNIX_DirectoryScan& operator=(const FastOS_UNIX_DirectoryScan&); - - bool _statRun; - bool _isDirectory; - bool _isRegular; - char *_statName; - char *_statFilenameP; - - void DoStat(); - -protected: - DIR *_dir; - struct dirent *_dp; - -public: - FastOS_UNIX_DirectoryScan(const char *searchPath); - ~FastOS_UNIX_DirectoryScan(); - - bool ReadNext() override; - bool IsDirectory() override; - bool IsRegular() override; - const char *GetName() override; - bool IsValidScan() const override; -}; diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 2720d8786cb..8f1b687449a 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -26,6 +26,8 @@ vespa_define_module( src/apps/vespa-validate-hostname TESTS + ${VESPALIB_DIRECTIO_TESTDIR} + ${VESPALIB_PROCESS_MEMORY_STATS_TESTDIR} src/tests/alloc src/tests/approx src/tests/array @@ -38,9 +40,9 @@ vespa_define_module( src/tests/bits src/tests/box src/tests/btree - src/tests/btree/btree_store src/tests/btree/btree-scan-speed src/tests/btree/btree-stress + src/tests/btree/btree_store src/tests/clock src/tests/component src/tests/compress @@ -73,7 +75,6 @@ vespa_define_module( src/tests/datastore/unique_store src/tests/datastore/unique_store_dictionary src/tests/datastore/unique_store_string_allocator - ${VESPALIB_DIRECTIO_TESTDIR} src/tests/detect_type_benchmark src/tests/dotproduct src/tests/drop-file-from-cache @@ -86,6 +87,9 @@ vespa_define_module( src/tests/executor_idle_tracking src/tests/explore_modern_cpp src/tests/false + src/tests/fastlib/io + src/tests/fastlib/text + src/tests/fastos src/tests/fiddle src/tests/fileheader src/tests/floatingpointtype @@ -95,6 +99,7 @@ vespa_define_module( src/tests/guard src/tests/host_name src/tests/hwaccelrated + src/tests/invokeservice src/tests/io/fileutil src/tests/io/mapped_file_input src/tests/issue @@ -134,7 +139,6 @@ vespa_define_module( src/tests/printable src/tests/priority_queue src/tests/process - ${VESPALIB_PROCESS_MEMORY_STATS_TESTDIR} src/tests/programoptions src/tests/random src/tests/referencecounter @@ -144,14 +148,14 @@ vespa_define_module( src/tests/runnable_pair src/tests/rusage src/tests/sequencedtaskexecutor - src/tests/shutdownguard - src/tests/singleexecutor src/tests/sha1 src/tests/shared_operation_throttler src/tests/shared_string_repo src/tests/sharedptr + src/tests/shutdownguard src/tests/signalhandler src/tests/simple_thread_bundle + src/tests/singleexecutor src/tests/slime src/tests/slime/external_data_value src/tests/slime/summary-feature-benchmark @@ -204,17 +208,15 @@ vespa_define_module( src/tests/util/string_escape src/tests/valgrind src/tests/visit_ranges - src/tests/invokeservice src/tests/wakeup src/tests/xmlserializable src/tests/zcurve - src/tests/fastlib/io - src/tests/fastlib/text LIBS src/vespa/fastlib/io src/vespa/fastlib/text src/vespa/fastlib/text/apps + src/vespa/fastos src/vespa/vespalib src/vespa/vespalib/btree src/vespa/vespalib/component diff --git a/vespalib/src/tests/fastos/CMakeLists.txt b/vespalib/src/tests/fastos/CMakeLists.txt new file mode 100644 index 00000000000..6ea986ac0b0 --- /dev/null +++ b/vespalib/src/tests/fastos/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(fastos_file_test_app TEST + SOURCES + file_test.cpp + DEPENDS + vespalib + GTest::GTest +) +vespa_add_test(NAME fastos_file_test_app COMMAND fastos_file_test_app) diff --git a/vespalib/src/tests/fastos/file_test.cpp b/vespalib/src/tests/fastos/file_test.cpp new file mode 100644 index 00000000000..d0f8bbfd98b --- /dev/null +++ b/vespalib/src/tests/fastos/file_test.cpp @@ -0,0 +1,555 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "tests.h" +#include +#include +#include +#include +#include + +class FileTest : public BaseTest +{ +public: + const std::string srcDir = getenv("SOURCE_DIRECTORY") ? getenv("SOURCE_DIRECTORY") : "."; + const std::string roFilename = srcDir + "/hello.txt"; + const std::string woFilename = "generated/writeonlytest.txt"; + const std::string rwFilename = "generated/readwritetest.txt"; + + void GetCurrentDirTest () + { + TestHeader ("Get Current Directory Test"); + + std::string currentDir = FastOS_File::getCurrentDirectory(); + + Progress(!currentDir.empty(), + "Current dir: %s", !currentDir.empty() ? + currentDir.c_str() : ""); + + bool dirrc = FastOS_File::SetCurrentDirectory(".."); + + std::string parentDir; + + if (dirrc) { + parentDir = FastOS_File::getCurrentDirectory(); + } + + Progress(dirrc && strcmp(currentDir.c_str(), parentDir.c_str()) != 0, + "Parent dir: %s", !parentDir.empty() ? + parentDir.c_str() : ""); + + dirrc = FastOS_File::SetCurrentDirectory(currentDir.c_str()); + + Progress(dirrc, "Changed back to working directory."); + + PrintSeparator(); + } + + void MemoryMapTest (int mmap_flags) + { + TestHeader ("Memory Map Test"); + + int i; + const int bufSize = 1000; + + std::filesystem::create_directory(std::filesystem::path("generated")); + FastOS_File file("generated/memorymaptest"); + + bool rc = file.OpenReadWrite(); + Progress(rc, "Opening file 'generated/memorymaptest'"); + + if (rc) { + char *buffer = new char [bufSize]; + for (i = 0; i < bufSize; i++) { + buffer[i] = i % 256; + } + ssize_t wroteB = file.Write2(buffer, bufSize); + Progress(wroteB == bufSize, "Writing %d bytes to file", bufSize); + + bool close_ok = file.Close(); + assert(close_ok); + file.enableMemoryMap(mmap_flags); + + rc = file.OpenReadOnly(); + + Progress(rc, "Opening file 'generated/memorymaptest' read-only"); + if (rc) { + bool mmapEnabled; + char *mmapBuffer = nullptr; + + mmapEnabled = file.IsMemoryMapped(); + mmapBuffer = static_cast(file.MemoryMapPtr(0)); + + Progress(rc, "Memory mapping %s", + mmapEnabled ? "enabled" : "disabled"); + Progress(rc, "Map address: 0x%p", mmapBuffer); + + if (mmapEnabled) { + rc = 0; + for (i = 0; i < bufSize; i++) { + rc |= (mmapBuffer[i] == i % 256); + } + Progress(rc, "Reading %d bytes from memory map", bufSize); + } + } + delete [] buffer; + } + std::filesystem::remove_all(std::filesystem::path("generated")); + PrintSeparator(); + } + + void DirectIOTest () + { + TestHeader ("Direct Disk IO Test"); + + int i; + const int bufSize = 40000; + + std::filesystem::create_directory(std::filesystem::path("generated")); + FastOS_File file("generated/diotest"); + + bool rc = file.OpenWriteOnly(); + Progress(rc, "Opening file 'generated/diotest' write-only"); + + if (rc) { + char *buffer = new char [bufSize]; + + for (i=0; i + (reinterpret_cast(alignPtr) & + (memoryAlignment - 1)); + if (align != 0) { + alignPtr = &alignPtr[memoryAlignment - align]; + } + int residue = bufSize; + int pos=0; + while (residue > 0) { + int readThisTime = eachRead * transferGranularity; + if (readThisTime > residue) { + readThisTime = residue; + } + file.ReadBuf(alignPtr, readThisTime, pos); + + for (i=0; iOpenReadOnly()) { + int64_t filesize; + filesize = myFile->GetSize(); + ProgressI64((filesize == 27), "File size: %ld", filesize); + + char dummyData[6] = "Dummy"; + bool writeResult = myFile->CheckedWrite(dummyData, 6); + + if (writeResult) { + Progress(false, "Should not be able to write a file opened for read-only access."); + } else { + char dummyData2[28]; + Progress(true, "Write failed with read-only access."); + + bool rc = myFile->SetPosition(1); + Progress(rc, "Setting position to 1"); + + if (rc) { + ssize_t readBytes; + int64_t filePosition; + readBytes = myFile->Read(dummyData2, 28); + + Progress(readBytes == 26, "Attempting to read 28 bytes, should get 26. Got: %d", readBytes); + + filePosition = myFile->GetPosition(); + Progress(filePosition == 27, "File position should now be 27. Was: %d", int(filePosition)); + + readBytes = myFile->Read(dummyData2, 6); + Progress(readBytes == 0, "We should now get 0 bytes. Read: %d bytes", readBytes); + + filePosition = myFile->GetPosition(); + Progress(filePosition == 27, "File position should now be 27. Was: %d", int(filePosition)); + } + } + } else { + Progress(false, "Unable to open file '%s'.", roFilename.c_str()); + } + delete(myFile); + PrintSeparator(); + } + + void WriteOnlyTest () + { + TestHeader("Write-Only Test"); + std::filesystem::create_directory(std::filesystem::path("generated")); + + FastOS_File *myFile = new FastOS_File(woFilename.c_str()); + + if (myFile->OpenWriteOnly()) { + int64_t filesize; + filesize = myFile->GetSize(); + + ProgressI64((filesize == 0), "File size: %ld", filesize); + + char dummyData[6] = "Dummy"; + bool writeResult = myFile->CheckedWrite(dummyData, 6); + + if (!writeResult) { + Progress(false, "Should be able to write to file opened for write-only access."); + } else { + Progress(true, "Write 6 bytes ok."); + + int64_t filePosition = myFile->GetPosition(); + if (filePosition == 6) { + Progress(true, "Fileposition is now 6."); + + if (myFile->SetPosition(0)) { + Progress(true, "SetPosition(0) success."); + filePosition = myFile->GetPosition(); + + if (filePosition == 0) { + Progress(true, "Fileposition is now 0."); + + int readBytes = myFile->Read(dummyData, 6); + + if (readBytes != 6) { + Progress(true, "Trying to read a write-only file should fail and it did."); + Progress(true, "Return code was: %d.", readBytes); + } else { + Progress(false, "Read on a file with write-only access should fail, but it didn't."); + } + } else { + ProgressI64(false, "Fileposition should be 6, but was %ld.", filePosition); + } + } else { + Progress(false, "SetPosition(0) failed"); + } + } else { + ProgressI64(false, "Fileposition should be 6, but was %ld.", filePosition); + } + } + bool closeResult = myFile->Close(); + Progress(closeResult, "Close file."); + } else { + Progress(false, "Unable to open file '%s'.", woFilename.c_str()); + } + + + bool deleteResult = myFile->Delete(); + Progress(deleteResult, "Delete file '%s'.", woFilename.c_str()); + + delete(myFile); + std::filesystem::remove_all(std::filesystem::path("generated")); + PrintSeparator(); + } + + void ReadWriteTest () + { + TestHeader("Read/Write Test"); + std::filesystem::create_directory(std::filesystem::path("generated")); + + FastOS_File *myFile = new FastOS_File(rwFilename.c_str()); + + if (myFile->OpenExisting()) { + Progress(false, "OpenExisting() should not work when '%s' does not exist.", rwFilename.c_str()); + bool close_ok = myFile->Close(); + assert(close_ok); + } else { + Progress(true, "OpenExisting() should fail when '%s' does not exist, and it did.", rwFilename.c_str()); + } + + if (myFile->OpenReadWrite()) { + int64_t filesize; + + filesize = myFile->GetSize(); + + ProgressI64((filesize == 0), "File size: %ld", filesize); + + char dummyData[6] = "Dummy"; + + bool writeResult = myFile->CheckedWrite(dummyData, 6); + + if (!writeResult) { + Progress(false, "Should be able to write to file opened for read/write access."); + } else { + Progress(true, "Write 6 bytes ok."); + + int64_t filePosition = myFile->GetPosition(); + + if (filePosition == 6) { + Progress(true, "Fileposition is now 6."); + + if (myFile->SetPosition(0)) { + Progress(true, "SetPosition(0) success."); + filePosition = myFile->GetPosition(); + + if (filePosition == 0) { + Progress(true, "Fileposition is now 0."); + + char dummyData2[7]; + int readBytes = myFile->Read(dummyData2, 6); + + if (readBytes == 6) { + Progress(true, "Reading 6 bytes worked."); + + int cmpResult = memcmp(dummyData, dummyData2, 6); + Progress((cmpResult == 0), "Comparing the written and read result.\n"); + + bool rc = myFile->SetPosition(1); + Progress(rc, "Setting position to 1"); + + if (rc) { + readBytes = myFile->Read(dummyData2, 7); + + Progress(readBytes == 5, "Attempting to read 7 bytes, should get 5. Got: %d", readBytes); + + filePosition = myFile->GetPosition(); + Progress(filePosition == 6, "File position should now be 6. Was: %d", int(filePosition)); + + readBytes = myFile->Read(dummyData2, 6); + Progress(readBytes == 0, "We should not be able to read any more. Read: %d bytes", readBytes); + + filePosition = myFile->GetPosition(); + Progress(filePosition == 6, "File position should now be 6. Was: %d", int(filePosition)); + } + } else { + Progress(false, "Reading 6 bytes failed."); + } + } else { + ProgressI64(false, "Fileposition should be 6, but was %ld.", filePosition); + } + } else { + Progress(false, "SetPosition(0) failed"); + } + } else { + ProgressI64(false, "Fileposition should be 6, but was %ld.", filePosition); + } + } + + bool closeResult = myFile->Close(); + Progress(closeResult, "Close file."); + } else { + Progress(false, "Unable to open file '%s'.", rwFilename.c_str()); + } + bool deleteResult = myFile->Delete(); + Progress(deleteResult, "Delete file '%s'.", rwFilename.c_str()); + + delete(myFile); + std::filesystem::remove_all(std::filesystem::path("generated")); + PrintSeparator(); + } + + void ScanDirectoryTest() + { + TestHeader("DirectoryScan Test"); + + FastOS_DirectoryScan *scanDir = new FastOS_DirectoryScan("."); + + while (scanDir->ReadNext()) { + const char *name = scanDir->GetName(); + bool isDirectory = scanDir->IsDirectory(); + bool isRegular = scanDir->IsRegular(); + + printf("%-30s %s\n", name, + isDirectory ? "DIR" : (isRegular ? "FILE" : "UNKN")); + } + + delete(scanDir); + PrintSeparator(); + } + + void ReadBufTest () + { + TestHeader("ReadBuf Test"); + + FastOS_File file(roFilename.c_str()); + + char buffer[20]; + + if (file.OpenReadOnly()) { + int64_t position = file.GetPosition(); + Progress(position == 0, "File pointer should be 0 after opening file"); + + ssize_t has_read = file.Read(buffer, 4); + Progress(has_read == 4, "Must read 4 bytes"); + buffer[4] = '\0'; + position = file.GetPosition(); + Progress(position == 4, "File pointer should be 4 after reading 4 bytes"); + Progress(strcmp(buffer, "This") == 0, "[This]=[%s]", buffer); + + file.ReadBuf(buffer, 6, 8); + buffer[6] = '\0'; + position = file.GetPosition(); + Progress(position == 4, "File pointer should still be 4 after ReadBuf"); + Progress(strcmp(buffer, "a test") == 0, "[a test]=[%s]", buffer); + } + + PrintSeparator(); + } + + void DiskFreeSpaceTest () + { + TestHeader("DiskFreeSpace Test"); + + int64_t freeSpace = FastOS_File::GetFreeDiskSpace(roFilename.c_str()); + ProgressI64(freeSpace != -1, "DiskFreeSpace using file ('hello.txt'): %ld MB.", freeSpace == -1 ? -1 : freeSpace/(1024*1024)); + freeSpace = FastOS_File::GetFreeDiskSpace("."); + ProgressI64(freeSpace != -1, "DiskFreeSpace using dir (.): %ld MB.", freeSpace == -1 ? -1 : freeSpace/(1024*1024)); + PrintSeparator(); + } + + void MaxLengthTest () + { + TestHeader ("Max Lengths Test"); + + int maxval = FastOS_File::GetMaximumFilenameLength("."); + Progress(maxval > 5 && maxval < (512*1024), + "Maximum filename length = %d", maxval); + + maxval = FastOS_File::GetMaximumPathLength("."); + Progress(maxval > 5 && maxval < (512*1024), + "Maximum path length = %d", maxval); + + PrintSeparator(); + } + + int Main () override + { + printf("This test should be run in the 'tests' directory.\n\n"); + printf("grep for the string '%s' to detect failures.\n\n", failString); + + GetCurrentDirTest(); + DirectIOTest(); + MaxLengthTest(); + DiskFreeSpaceTest(); + ReadOnlyTest(); + WriteOnlyTest(); + ReadWriteTest(); + ScanDirectoryTest(); + ReadBufTest(); + MemoryMapTest(0); +#ifdef __linux__ + MemoryMapTest(MAP_HUGETLB); +#endif + + PrintSeparator(); + printf("END OF TEST (%s)\n", _argv[0]); + + return allWasOk() ? 0 : 1; + } + FileTest(); + ~FileTest(); +}; + +FileTest::FileTest() { } +FileTest::~FileTest() { } + + +int main (int argc, char **argv) +{ + FileTest app; + + setvbuf(stdout, nullptr, _IOLBF, 8192); + return app.Entry(argc, argv); +} diff --git a/vespalib/src/tests/fastos/hello.txt b/vespalib/src/tests/fastos/hello.txt new file mode 100644 index 00000000000..62a405393a6 --- /dev/null +++ b/vespalib/src/tests/fastos/hello.txt @@ -0,0 +1 @@ +This is a test input file. diff --git a/vespalib/src/tests/fastos/tests.h b/vespalib/src/tests/fastos/tests.h new file mode 100644 index 00000000000..9cd7a10ab48 --- /dev/null +++ b/vespalib/src/tests/fastos/tests.h @@ -0,0 +1,149 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include +#include +#include +#include + +class BaseTest +{ +private: + BaseTest(const BaseTest&); + BaseTest &operator=(const BaseTest&); + + int totallen; + bool _allOkFlag; +public: + int _argc; + char **_argv; + + const char *okString; + const char *failString; + + BaseTest () + : totallen(70), + _allOkFlag(true), + _argc(0), + _argv(nullptr), + okString("SUCCESS"), + failString("FAILURE") + { + } + + virtual int Main() = 0; + + int Entry(int argc, char **argv) { + _argc = argc; + _argv = argv; + return Main(); + } + + virtual ~BaseTest() {}; + + bool allWasOk() const { return _allOkFlag; } + + void PrintSeparator () + { + for(int i=0; i +#include +#include +#include +#include + +DirectIOException::DirectIOException(const char * fileName, const void * buffer, size_t length, int64_t offset) : + std::exception(), + _what(), + _fileName(fileName), + _buffer(buffer), + _length(length), + _offset(offset) +{ + std::ostringstream os; + os << "DirectIO failed for file '" << fileName << "' buffer=0x" << std::hex << reinterpret_cast(buffer); + os << " length=0x" << length << " offset=0x" << offset; + _what = os.str(); +} + +DirectIOException::~DirectIOException() {} + +#ifdef __linux__ +int FastOS_FileInterface::_defaultFAdviseOptions = POSIX_FADV_NORMAL; +#else +int FastOS_FileInterface::_defaultFAdviseOptions = 0; +#endif + +static const size_t MAX_CHUNK_SIZE = 0x4000000; // 64 MB + +FastOS_FileInterface::FastOS_FileInterface(const char *filename) + : _fAdviseOptions(_defaultFAdviseOptions), + _chunkSize(MAX_CHUNK_SIZE), + _filename(), + _openFlags(0), + _directIOEnabled(false), + _syncWritesEnabled(false) +{ + if (filename != nullptr) + SetFileName(filename); +} + + +FastOS_FileInterface::~FastOS_FileInterface() = default; + +void +FastOS_FileInterface::ReadBuf(void *buffer, size_t length) +{ + ssize_t readResult = Read(buffer, length); + + if ((readResult == -1) || (static_cast(readResult) != length)) { + std::string errorString = readResult != -1 ? + std::string("short read") : + FastOS_FileInterface::getLastErrorString(); + std::ostringstream os; + os << "Fatal: Reading " << length << " bytes from '" << GetFileName() << "' failed: " << errorString; + throw std::runtime_error(os.str()); + } +} + +void +FastOS_FileInterface::WriteBuf(const void *buffer, size_t length) +{ + WriteBufInternal(buffer, length); +} + +void +FastOS_FileInterface::WriteBufInternal(const void *buffer, size_t length) +{ + ssize_t writeResult = Write2(buffer, length); + if (length - writeResult != 0) { + std::string errorString = writeResult != -1 ? + std::string("short write") : + FastOS_FileInterface::getLastErrorString(); + std::ostringstream os; + os << "Fatal: Writing " << length << " bytes to '" << GetFileName() << "' failed (wrote " << writeResult << "): " << errorString; + throw std::runtime_error(os.str()); + } +} + +bool +FastOS_FileInterface::CheckedWrite(const void *buffer, size_t len) +{ + ssize_t writeResult = Write2(buffer, len); + if (writeResult < 0) { + std::string errorString = FastOS_FileInterface::getLastErrorString(); + fprintf(stderr, "Writing %lu bytes to '%s' failed: %s\n", + static_cast(len), + GetFileName(), + errorString.c_str()); + return false; + } + if (writeResult != (ssize_t)len) { + fprintf(stderr, "Short write, tried to write %lu bytes to '%s', only wrote %lu bytes\n", + static_cast(len), + GetFileName(), + static_cast(writeResult)); + return false; + } + return true; +} + + +void +FastOS_FileInterface::ReadBuf(void *buffer, size_t length, int64_t readOffset) +{ + if (!SetPosition(readOffset)) { + std::string errorString = FastOS_FileInterface::getLastErrorString(); + std::ostringstream os; + os << "Fatal: Setting fileoffset to " << readOffset << " in '" << GetFileName() << "' : " << errorString; + throw std::runtime_error(os.str()); + } + ReadBuf(buffer, length); +} + + +void +FastOS_FileInterface::EnableDirectIO() +{ + // Only subclasses with support for DirectIO do something here. +} + + +void +FastOS_FileInterface::EnableSyncWrites() +{ + if (!IsOpened()) + _syncWritesEnabled = true; +} + + +bool +FastOS_FileInterface:: +GetDirectIORestrictions(size_t &memoryAlignment, + size_t &transferGranularity, + size_t &transferMaximum) +{ + memoryAlignment = 1; + transferGranularity = 1; + transferMaximum = 0x7FFFFFFF; + return false; +} + +bool +FastOS_FileInterface::DirectIOPadding(int64_t offset, + size_t buflen, + size_t &padBefore, + size_t &padAfter) +{ + (void)offset; + (void)buflen; + padBefore = 0; + padAfter = 0; + return false; +} + + +void * +FastOS_FileInterface::allocateGenericDirectIOBuffer(size_t byteSize, void *&realPtr) +{ + realPtr = malloc(byteSize); // Default - use malloc allignment + return realPtr; +} + +size_t +FastOS_FileInterface::getMaxDirectIOMemAlign() +{ + return 1u; +} + +void * +FastOS_FileInterface::AllocateDirectIOBuffer(size_t byteSize, void *&realPtr) +{ + return allocateGenericDirectIOBuffer(byteSize, realPtr); +} + +void +FastOS_FileInterface::enableMemoryMap(int mmapFlags) +{ + // Only subclases with support for memory mapping do something here. + (void) mmapFlags; +} + + +void * +FastOS_FileInterface::MemoryMapPtr(int64_t position) const +{ + // Only subclases with support for memory mapping do something here. + (void) position; + return nullptr; +} + + +bool +FastOS_FileInterface::IsMemoryMapped() const +{ + // Only subclases with support for memory mapping do something here. + return false; +} + +void +FastOS_FileInterface::SetFileName(const char *filename) +{ + _filename = filename; +} + + +const char * +FastOS_FileInterface::GetFileName() const +{ + return _filename.c_str(); +} + + +bool +FastOS_FileInterface::OpenReadWrite(const char *filename) +{ + return Open(FASTOS_FILE_OPEN_READ | + FASTOS_FILE_OPEN_WRITE, filename); +} + + +bool +FastOS_FileInterface::OpenExisting(bool abortIfNotExist, + const char *filename) +{ + bool rc = Open(FASTOS_FILE_OPEN_READ | + FASTOS_FILE_OPEN_WRITE | + FASTOS_FILE_OPEN_EXISTING, + filename); + + if (abortIfNotExist && (!rc)) { + std::string errorString = + FastOS_FileInterface::getLastErrorString(); + fprintf(stderr, + "Cannot open %s: %s\n", + filename, + errorString.c_str()); + abort(); + } + + return rc; +} + + +bool +FastOS_FileInterface::OpenReadOnlyExisting(bool abortIfNotExist, + const char *filename) +{ + bool rc = Open(FASTOS_FILE_OPEN_READ | + FASTOS_FILE_OPEN_EXISTING, + filename); + + if (abortIfNotExist && (!rc)) { + std::string errorString = + FastOS_FileInterface::getLastErrorString(); + fprintf(stderr, + "Cannot open %s: %s\n", + filename, + errorString.c_str()); + abort(); + } + + return rc; +} + + +bool +FastOS_FileInterface::OpenWriteOnlyTruncate(const char *filename) +{ + // printf("********* OpenWriteOnlyTruncate %s\n", filename); + return Open(FASTOS_FILE_OPEN_WRITE | + FASTOS_FILE_OPEN_CREATE | + FASTOS_FILE_OPEN_TRUNCATE, + filename); +} + + +bool +FastOS_FileInterface::OpenWriteOnlyExisting(bool abortIfNotExist, + const char *filename) +{ + bool rc = Open(FASTOS_FILE_OPEN_WRITE | + FASTOS_FILE_OPEN_EXISTING, + filename); + + if (abortIfNotExist && (!rc)) { + std::string errorString = + FastOS_FileInterface::getLastErrorString(); + fprintf(stderr, + "Cannot open %s: %s\n", + filename, + errorString.c_str()); + abort(); + } + + return rc; +} + +bool +FastOS_FileInterface::OpenReadOnly(const char *filename) +{ + return Open(FASTOS_FILE_OPEN_READ | + FASTOS_FILE_OPEN_EXISTING, + filename); +} + + +bool +FastOS_FileInterface::OpenWriteOnly(const char *filename) +{ + return Open(FASTOS_FILE_OPEN_WRITE, filename); +} + +FastOS_File::Error +FastOS_FileInterface::GetLastError() +{ + return FastOS_File::TranslateError(FastOS_File::GetLastOSError()); +} + + +std::string +FastOS_FileInterface::getLastErrorString() +{ + int err = FastOS_File::GetLastOSError(); + return FastOS_File::getErrorString(err); +} + +bool FastOS_FileInterface::Rename (const char *newFileName) +{ + bool rc=false; + if (FastOS_File::Rename(GetFileName(), newFileName)) { + SetFileName(newFileName); + rc = true; + } + return rc; +} + +void FastOS_FileInterface::dropFromCache() const +{ +} + +FastOS_DirectoryScanInterface::FastOS_DirectoryScanInterface(const char *path) + : _searchPath(path) +{ +} + +FastOS_DirectoryScanInterface::~FastOS_DirectoryScanInterface() = default; diff --git a/vespalib/src/vespa/fastos/file.h b/vespalib/src/vespa/fastos/file.h new file mode 100644 index 00000000000..1cf6fee71dd --- /dev/null +++ b/vespalib/src/vespa/fastos/file.h @@ -0,0 +1,745 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +//************************************************************************ +/** + * @file + * Class definitions for FastOS_File, FastOS_DirectoryScan and + * FastOS_StatInfo. + * + * @author Div, Oivind H. Danielsen + */ + +#pragma once + +#include "types.h" +#include +#include + +constexpr int FASTOS_FILE_OPEN_READ = (1<<0); +constexpr int FASTOS_FILE_OPEN_WRITE = (1<<1); +constexpr int FASTOS_FILE_OPEN_EXISTING = (1<<2); +constexpr int FASTOS_FILE_OPEN_CREATE = (1<<3); +constexpr int FASTOS_FILE_OPEN_TRUNCATE = (1<<4); +constexpr int FASTOS_FILE_OPEN_STDOUT = (2<<5); +constexpr int FASTOS_FILE_OPEN_STDERR = (3<<5); +constexpr int FASTOS_FILE_OPEN_STDFLAGS = (3<<5); +constexpr int FASTOS_FILE_OPEN_DIRECTIO = (1<<7); +constexpr int FASTOS_FILE_OPEN_SYNCWRITES = (1<<9); // synchronous writes + +/** + * This class contains regular file-access functionality. + * + * Example (how to read 10 bytes from the file 'hello.txt'): + * + * @code + * void Foo::Bar () + * { + * FastOS_File file("hello.txt"); + * + * if(file.OpenReadOnly()) + * { + * char buffer[10]; + * + * if(file.Read(buffer, 10) == 10) + * { + * // Read success + * } + * else + * { + * // Read failure + * } + * } + * else + * { + * // Unable to open file 'hello.txt' + * } + * } + * @endcode + */ + +class DirectIOException : public std::exception +{ +public: + DirectIOException(const char * fileName, const void * buffer, size_t length, int64_t offset); + ~DirectIOException(); + const char* what() const noexcept override { return _what.c_str(); } + const void * getBuffer() const { return _buffer; } + size_t getLength() const { return _length; } + int64_t getOffset() const { return _offset; } + const std::string & getFileName() const { return _fileName; } +private: + std::string _what; + std::string _fileName; + const void * _buffer; + size_t _length; + int64_t _offset; +}; + +class FastOS_FileInterface +{ +private: + FastOS_FileInterface (const FastOS_FileInterface&); + FastOS_FileInterface& operator=(const FastOS_FileInterface&); + + // Default options for madvise used on every file opening. Default is FADV_NORMAL + // Set with setDefaultFAdviseOptions() application wide. + // And setFAdviseOptions() per file. + static int _defaultFAdviseOptions; + int _fAdviseOptions; + size_t _chunkSize; + void WriteBufInternal(const void *buffer, size_t length); + +protected: + std::string _filename; + unsigned int _openFlags; + bool _directIOEnabled; + bool _syncWritesEnabled; + +public: + static void setDefaultFAdviseOptions(int options) { _defaultFAdviseOptions = options; } + int getFAdviseOptions() const { return _fAdviseOptions; } + void setFAdviseOptions(int options) { _fAdviseOptions = options; } + + /** + * Return path separator string. This will yield "/" on UNIX systems. + * @return pointer to path separator character string + */ + static const char *GetPathSeparator() { return "/"; } + + /** + * Constructor. A filename could be supplied at this point, or specified + * later using @ref SetFileName() or @ref Open(). + * @param filename a filename (optional) + */ + FastOS_FileInterface(const char *filename=nullptr); + + /** + * Destructor. If the current file is open, the destructor will close + * it for you. + */ + virtual ~FastOS_FileInterface(); + + /** + * Associate a new filename with this object. This filename will be + * used when performing @ref Open() (unless @ref Open() specifies a + * different filename). + * @param filename filename character string (will be copied internally) + */ + virtual void SetFileName(const char *filename); + + /** + * Return the filename associated with the File object. If no filename + * has been set, an empty string is returned instead. + * @return The filename associated with the File object + */ + virtual const char *GetFileName() const; + + /** + * Open a file. + * @param openFlags A combination of the flags: FASTOS_FILE_OPEN_READ, + * (Is read-access desired?), FASTOS_FILE_OPEN_WRITE + * (Is write-access desired?), and + * FASTOS_FILE_OPEN_EXISTING (The file to be opened + * should already exist. If the file does not exist, + * the call will fail.). + * @param filename You may optionally specify a filename here. This + * will replace the currently associated filename + * (if any) set using either the constructor, + * @ref SetFileName() or a previous call to + * @ref Open(). + * @return Boolean success/failure + */ + virtual bool Open(unsigned int openFlags, const char *filename=nullptr) = 0; + + /** + * Open a file for read/write access. The file will be created if it does + * not already exist. + * @param filename You may optionally specify a filename here. This + * will replace the currently associated filename + * (if any) set using either the constructor, + * @ref SetFileName() or a previous call to @ref Open(). + * @return Boolean success/failure + */ + bool OpenReadWrite(const char *filename=nullptr); + + /** + * Open a file for read/write access. This method fails if the file does + * not already exist. + * @param abortIfNotExist Abort the program if the file does not exist. + * @param filename You may optionally specify a filename here. This + * will replace the currently associated filename + * (if any) set using either the constructor, + * @ref SetFileName() or a previous call to + * @ref Open(). + * @return Boolean success/failure + */ + bool OpenExisting(bool abortIfNotExist=false, const char *filename=nullptr); + + /** + * Open a file for read access. This method fails if the file does + * not already exist. + * @param abortIfNotExist Abort the program if the file does not exist. + * @param filename You may optionally specify a filename here. This + * will replace the currently associated filename + * (if any) set using either the constructor, + * @ref SetFileName() or a previous call to + * @ref Open(). + * @return Boolean success/failure + */ + bool OpenReadOnlyExisting (bool abortIfNotExist=false, const char *filename=nullptr); + + /** + * Open a file for write access. If the file does not exist, it is created. + * If the file exists, it is truncated to 0 bytes. + * @param filename You may optionally specify a filename here. This + * will replace the currently associated filename + * (if any) set using either the constructor, + * @ref SetFileName() or a previous call to + * @ref Open(). + * @return Boolean success/failure + */ + bool OpenWriteOnlyTruncate(const char *filename=nullptr); + + /** + * Open a file for write access. This method fails if the file does + * not already exist. + * @param abortIfNotExist Abort the program if the file does not exist. + * @param filename You may optionally specify a filename here. This + * will replace the currently associated filename + * (if any) set using either the constructor, + * @ref SetFileName() or a previous call to + * @ref Open(). + * @return Boolean success/failure + */ + bool OpenWriteOnlyExisting (bool abortIfNotExist=false, const char *filename=nullptr); + + /** + * Open a file for read-access only. This method fails if the file does + * not already exist. + * @param filename You may optionally specify a filename here. This + * will replace the currently associated filename + * (if any) set using either the constructor, + * @ref SetFileName() or a previous call to @ref Open(). + * @return Boolean success/failure + */ + bool OpenReadOnly(const char *filename=nullptr); + + /** + * Open a file for write-access only. The file will be created if it does + * not already exist. + * @param filename You may optionally specify a filename here. This + * will replace the currently associated filename + * (if any) set using either the constructor, + * @ref SetFileName() or a previous call to @ref Open(). + * @return Boolean success/failure + */ + bool OpenWriteOnly(const char *filename=nullptr); + + /** + * Close the file. The call will successfully do nothing if the file + * already is closed. + * @return Boolean success/failure + */ + [[nodiscard]] virtual bool Close() = 0; + + /** + * Is the file currently opened? + * @return true if the file is opened, else false + */ + virtual bool IsOpened() const = 0; + + /** + * Read [length] bytes into [buffer]. + * @param buffer buffer pointer + * @param length number of bytes to read + * @return The number of bytes which was actually read, + * or -1 on error. + */ + [[nodiscard]] virtual ssize_t Read(void *buffer, size_t length) = 0; + + /** + * Write [len] bytes from [buffer]. This is just a wrapper for + * Write2, which does the same thing but returns the number of + * bytes written instead of just a bool. + * @param buffer buffer pointer + * @param len number of bytes to write + * @return Boolean success/failure + */ + [[nodiscard]] bool CheckedWrite(const void *buffer, size_t len); + + /** + * Write [len] bytes from [buffer]. + * @param buffer buffer pointer + * @param len number of bytes to write + * @return The number of bytes actually written, or -1 on error + */ + [[nodiscard]] virtual ssize_t Write2(const void *buffer, size_t len) = 0; + + /** + * Read [length] bytes into [buffer]. Caution! If the actual number + * of bytes read != [length], an error message is printed to stderr + * and the program aborts. + * + * The method is included for backwards compatibility reasons. + * @param buffer buffer pointer + * @param length number of bytes to read + */ + virtual void ReadBuf(void *buffer, size_t length); + + /** + * Write [length] bytes from [buffer] in chunks. Caution! If the write fails, + * an error message is printed to stderr and the program aborts. + * + * The method is included for backwards compatibility reasons. + * @param buffer buffer pointer + * @param length number of bytes to write + */ + virtual void WriteBuf(const void *buffer, size_t length); + + + /** + * Read [length] bytes at file offset [readOffset] into [buffer]. + * Only thread-safe if an OS-specific implementation exists. + * + * Caution! If the actual number of bytes read != [length], an + * error message is printed to stderr and the program aborts. + * + * @param buffer buffer pointer + * @param length number of bytes to read + * @param readOffset file offset where the read operation starts + */ + virtual void ReadBuf(void *buffer, size_t length, int64_t readOffset); + + /** + * Set the filepointer. The next @ref Read() or @ref Write() will + * continue from this position. + * @param position position of the new file pointer (in bytes) + * @return Boolean success/failure + */ + virtual bool SetPosition(int64_t position) = 0; + + /** + * Get the filepointer. -1 is returned if the operation fails. + * @return current position of file pointer (in bytes) + */ + virtual int64_t GetPosition() = 0; + + /** + * const version of @link GetPosition + */ + int64_t getPosition() const { return const_cast(this)->GetPosition(); } + + /** + * Return the file size. This method requires that the file is + * currently opened. If you wish to determine the file size + * without opening the file, use @ref Stat(). + * If an error occurs, the returned value is -1. + * @return file size (in bytes) + */ + virtual int64_t GetSize() = 0; + + /** + * const version of @link GetSize + */ + int64_t getSize() const { return const_cast(this)->GetSize(); } + + /** + * Return the time when file was last modified. + * @return time of last modification + */ + virtual time_t GetModificationTime() = 0; + + /** + * Delete the file. This method requires that the file is + * currently not opened. + * @return Boolean success/failure + */ + virtual bool Delete() = 0; + + /** + * Rename/move a file or directory. This method requires that + * the file is currently not opened. A move operation is + * supported as long as the source and destination reside + * on the same volume/device. + * The method fails if the destination already exists. + * @param newFileName New file name + * @return Boolean success/failure + */ + virtual bool Rename (const char *newFileName); + + /** + * Force completion of pending disk writes (flush cache). + */ + [[nodiscard]] virtual bool Sync() = 0; + + /** + * Are we in some kind of file read mode? + */ + bool IsReadMode() { + return ((_openFlags & FASTOS_FILE_OPEN_READ) != 0); + } + + /** + * Are we in some kind of file write mode? + */ + bool IsWriteMode() { + return ((_openFlags & FASTOS_FILE_OPEN_WRITE) != 0); + } + + /** + * Truncate or extend the file to the new size. + * The file pointer is also set to [newSize]. + * @ref SetSize() requires write access to the file. + * @param newSize New file size + * @return Boolean success/failure. + */ + virtual bool SetSize(int64_t newSize) = 0; + + /** + * Enable direct disk I/O (disable OS buffering & cache). Reads + * and writes will be performed directly to or from the user + * program buffer, provided appropriate size and alignment + * restrictions are met. If the restrictions are not met, a + * normal read or write is performed as a fallback. + * Call this before opening a file, and query + * @ref GetDirectIORestrictions() after a file is opened to get the + * neccessary alignment restrictions. It is possible that direct + * disk I/O could not be enabled. In that case + * @ref GetDirectIORestrictions will return false. + */ + virtual void EnableDirectIO(); + + virtual void EnableSyncWrites(); + + bool useSyncWrites() const { return _syncWritesEnabled; } + + /** + * Set the write chunk size used in WriteBuf. + */ + void setChunkSize(size_t chunkSize) { _chunkSize = chunkSize; } + size_t getChunkSize() const { return _chunkSize; } + + /** + * Get restrictions for direct disk I/O. The file should be opened + * before this method is called. + * Even though direct disk I/O is enabled through @ref EnableDirectIO(), + * this method could still return false, indicating that direct disk I/O + * is not being used for this file. This could be caused by either: no + * OS support for direct disk I/O, direct disk I/O might only be implemented + * for certain access-modes, the file is on a network drive or other + * partition where direct disk I/O is disallowed. + * + * The restriction-arguments are always filled with valid data, independant + * of the return code and direct disk I/O availability. + * + * @param memoryAlignment Buffer alignment restriction + * @param transferGranularity All transfers must be a multiple of + * [transferGranularity] bytes. All + * file offsets for these transfers must + * also be a multiple of [transferGranularity] + * bytes. + * @param transferMaximum All transfers must be <= [transferMaximum] + * bytes. + * @return True if direct disk I/O is being used for this file, else false. + */ + virtual bool + GetDirectIORestrictions(size_t &memoryAlignment, + size_t &transferGranularity, + size_t &transferMaximum); + + /** + * Retrieve the required padding for direct I/O to be used. + * + * @param offset File offset + * @param buflen Buffer length + * @param padBefore Number of pad bytes needed in front of the buffer + * @param padAfter Number of pad bytes needed after the buffer + * + * @return True if the file access can be accomplished with + * direct I/O, else false. + */ + virtual bool DirectIOPadding(int64_t offset, + size_t buflen, + size_t &padBefore, + size_t &padAfter); + + /** + * Allocate a buffer properly alligned with regards to direct io + * access restrictions. + * @param byteSize Number of bytes to be allocated + * @param realPtr Reference where the actual pointer returned + * from malloc will be saved. Use free() with + * this pointer to deallocate the buffer. + * This value is always set. + * @return Alligned pointer value or nullptr if out of memory + */ + static void *allocateGenericDirectIOBuffer(size_t byteSize, void *&realPtr); + + /** + * Get maximum memory alignment for directio buffers. + * @return maximum memory alignment for directio buffers. + */ + static size_t getMaxDirectIOMemAlign(); + + /** + * Allocate a buffer properly alligned with regards to direct io + * access restrictions. + * @param byteSize Number of bytes to be allocated + * @param realPtr Reference where the actual pointer returned + * from malloc will be saved. Use free() with + * this pointer to deallocate the buffer. + * This value is always set. + * @return Alligned pointer value or nullptr if out of memory + */ + virtual void *AllocateDirectIOBuffer(size_t byteSize, void *&realPtr); + + /** + * Enable mapping of complete file contents into the address space of the + * running process. This only works for small files. This operation + * will be ignored on some file types. + */ + virtual void enableMemoryMap(int mmapFlags); + + /** + * Inquiry about where in memory file data is located. + * @return location of file data in memory. If the file is not mapped, + * nullptr is returned. + */ + virtual void *MemoryMapPtr(int64_t position) const; + + /** + * Inquiry if file content is mapped into memory. + * @return true if file is mapped in memory, false otherwise. + */ + virtual bool IsMemoryMapped() const; + + /** + * Will drop whatever is in the FS cache when called. Does not have effect in the future. + **/ + virtual void dropFromCache() const; + + enum Error + { + ERR_ZERO = 1, // No error New style + ERR_NOENT, // No such file or directory + ERR_NOMEM, // Not enough memory + ERR_ACCES, // Permission denied + ERR_EXIST, // File exists + ERR_INVAL, // Invalid argument + ERR_NFILE, // File table overflow + ERR_MFILE, // Too many open files + ERR_NOSPC, // No space left on device + ERR_INTR, // interrupt + ERR_AGAIN, // Resource unavailable, try again + ERR_BUSY, // Device or resource busy + ERR_IO, // I/O error + ERR_PERM, // Not owner + ERR_NODEV, // No such device + ERR_NXIO, // Device not configured + ERR_UNKNOWN, // Unknown + + ERR_EZERO = 1, // No error Old style + ERR_ENOENT, // No such file or directory + ERR_ENOMEM, // Not enough memory + ERR_EACCES, // Permission denied + ERR_EEXIST, // File exists + ERR_EINVAL, // Invalid argument + ERR_ENFILE, // File table overflow + ERR_EMFILE, // Too many open files + ERR_ENOSPC, // No space left on device + ERR_EINTR, // interrupt + ERR_EAGAIN, // Resource unavailable, try again + ERR_EBUSY, // Device or resource busy + ERR_EIO, // I/O error + ERR_EPERM, // Not owner + ERR_ENODEV, // No such device + ERR_ENXIO // Device not configured + }; + + + /** + * If a file operation fails, the error code can be retrieved + * via this method. See @ref Error for possible error codes. + * @return Error code + */ + static Error GetLastError(); + + /** + * Similar to @ref GetLastError(), but this method returns a string + * instead of an error code. + * @return String describing the last error + */ + static std::string getLastErrorString(); +}; + + +/** + * The class serves as a container for information returned by + * @ref FastOS_File::Stat(). + */ +class FastOS_StatInfo +{ +public: + /** + * Possible error codes. + */ + enum StatError + { + Ok, //!< ok + Unknown, //!< unknown error + FileNotFound //!< file not found error + }; + + StatError _error; + + /** + * Is it a regular file? This field is only valid if @ref _error is + * @ref Ok. + */ + bool _isRegular; + + /** + * Is it a directory? This field is only valid if @ref _error is + * @ref Ok. + */ + bool _isDirectory; + + /** + * File size. This field is only valid if @ref _error is + * @ref Ok. + */ + int64_t _size; + + /** + * Time of last modification in seconds. + */ + time_t _modifiedTime; + + /** + * Time of last modification in seconds. + */ + uint64_t _modifiedTimeNS; +}; + + +/** + * This class enumerates the contents of a given directory. + * + * Example: + * @code + * void Foo::Bar() + * { + * // Scan and print the contents of the directory '/usr/src/include' + * + * FastOS_DirectoryScan dirScan("/usr/src/include"); + * int numEntries = 0; + * + * while(dirScan.ReadNext()) + * { + * const char *name = dirScan.GetName(); + * bool isDirectory = dirScan.IsDirectory(); + * bool isRegular = dirScan.IsRegular(); + * + * printf("%-30s %s\n", name, + * isDirectory ? "DIR" : (isRegular ? "FILE" : "UNKN")); + * + * numEntries++; + * } + * + * printf("The directory contained %d entries.\n", numEntries); + * } + * @endcode + */ +class FastOS_DirectoryScanInterface +{ +private: + FastOS_DirectoryScanInterface(const FastOS_DirectoryScanInterface&); + FastOS_DirectoryScanInterface& operator= (const FastOS_DirectoryScanInterface&); + +protected: + std::string _searchPath; + +public: + + /** + * Constructor. + * + * @param path Path of the directory to be scanned. The path string + * is copied internally. + */ + FastOS_DirectoryScanInterface(const char *path); + + /** + * Destructor. + * + * Frees operating system resources related to the directory scan. + */ + virtual ~FastOS_DirectoryScanInterface(); + + /** + * Get search path. + * This is an internal copy of the path specified in the constructor. + * @return Search path string. + */ + const char *GetSearchPath () { return _searchPath.c_str(); } + + /** + * Read the next entry in the directory scan. Failure indicates + * that there are no more entries. If the call is successful, + * attributes for the entry can be read with @ref IsDirectory(), + * @ref IsRegular() and @ref GetName(). + * + * @return Boolean success/failure + */ + virtual bool ReadNext() = 0; + + /** + * After a successful @ref ReadNext() this method is used to + * determine if the entry is a directory entry or not. Calling this + * method after an unsuccessful @ref ReadNext() or before + * @ref ReadNext() is called for the first time, yields undefined + * results. + * + * @return True if the entry is a directory, else false. + */ + virtual bool IsDirectory() = 0; + + + /** + * After a successful @ref ReadNext() this method is used to + * determine if the entry is a regular file entry or not. Calling + * this method after an unsuccessful @ref ReadNext() or before + * @ref ReadNext() is called for the first time, yields undefined + * results. + * + * @return True if the entry is a regular file, else false. + */ + virtual bool IsRegular() = 0; + + /** + * After a successful @ref ReadNext() this method is used to + * determine the name of the recently read directory entry. Calling + * this method after an unsuccessful @ref ReadNext() or before + * @ref ReadNext() is called for the first time, yields undefined + * results. + * + * @return A pointer to the recently read directory entry. + */ + virtual const char *GetName() = 0; + + /** + * Check whether the creation of a directory scan succeeded or + * failed (e.g. due to resource constraints). + * + * return True if the directory scan is valid. + */ + virtual bool IsValidScan() const = 0; +}; + +#ifdef __linux__ +#include +typedef FastOS_Linux_File FASTOS_PREFIX(File); +#else +#include +typedef FastOS_UNIX_File FASTOS_PREFIX(File); +#endif +typedef FastOS_UNIX_DirectoryScan FASTOS_PREFIX(DirectoryScan); diff --git a/vespalib/src/vespa/fastos/file_rw_ops.cpp b/vespalib/src/vespa/fastos/file_rw_ops.cpp new file mode 100644 index 00000000000..79fe95b21f2 --- /dev/null +++ b/vespalib/src/vespa/fastos/file_rw_ops.cpp @@ -0,0 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "file_rw_ops.h" +#include + +namespace fastos { + +File_RW_Ops::ReadFunc File_RW_Ops::_read = ::read; +File_RW_Ops::WriteFunc File_RW_Ops::_write = ::write; +File_RW_Ops::PreadFunc File_RW_Ops::_pread = ::pread; +File_RW_Ops::PwriteFunc File_RW_Ops::_pwrite = ::pwrite; + +} diff --git a/vespalib/src/vespa/fastos/file_rw_ops.h b/vespalib/src/vespa/fastos/file_rw_ops.h new file mode 100644 index 00000000000..4f7aa6f082f --- /dev/null +++ b/vespalib/src/vespa/fastos/file_rw_ops.h @@ -0,0 +1,33 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include + + +namespace fastos { + +/* + * Class handling pointers to functions used by FastOS_File for read + * and writa access. Unit tests might modify pointers to inject errors. + */ +class File_RW_Ops +{ + using ReadFunc = ssize_t (*)(int fd, void* buf, size_t count); + using WriteFunc = ssize_t (*)(int fd, const void* buf, size_t count); + using PreadFunc = ssize_t (*)(int fd, void* buf, size_t count, off_t offset); + using PwriteFunc = ssize_t (*)(int fd, const void* buf, size_t count, off_t offset); + +public: + static ReadFunc _read; + static WriteFunc _write; + static PreadFunc _pread; + static PwriteFunc _pwrite; + + static ssize_t read(int fd, void* buf, size_t count) { return _read(fd, buf, count); } + static ssize_t write(int fd, const void* buf, size_t count) { return _write(fd, buf, count); } + static ssize_t pread(int fd, void* buf, size_t count, off_t offset) { return _pread(fd, buf, count, offset); } + static ssize_t pwrite(int fd, const void* buf, size_t count, off_t offset) { return _pwrite(fd, buf, count, offset); } +}; + +} diff --git a/vespalib/src/vespa/fastos/linux_file.cpp b/vespalib/src/vespa/fastos/linux_file.cpp new file mode 100644 index 00000000000..6fb29782957 --- /dev/null +++ b/vespalib/src/vespa/fastos/linux_file.cpp @@ -0,0 +1,436 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** +****************************************************************************** +* @author Oivind H. Danielsen +* @date Creation date: 2000-09-21 +* @file +* Implementation of FastOS_Linux_File methods. +*****************************************************************************/ + +#ifdef __linux__ +#include "file.h" +#include "file_rw_ops.h" +#include +#include +#include +#include +#include +#include +#include + +using fastos::File_RW_Ops; + +const size_t FastOS_Linux_File::_directIOFileAlign = 4096; +const size_t FastOS_Linux_File::_directIOMemAlign = 4096; + +FastOS_Linux_File::FastOS_Linux_File(const char *filename) + : FastOS_UNIX_File(filename), + _cachedSize(-1), + _filePointer(-1) +{ +} + +FastOS_Linux_File::~FastOS_Linux_File () { + bool ok = Close(); + assert(ok); +} + +#define DIRECTIOPOSSIBLE(buf, len, off) \ + (((off) & (_directIOFileAlign - 1)) == 0 && \ + ((len) & (_directIOFileAlign - 1)) == 0 && \ + (reinterpret_cast(buf) & (_directIOMemAlign - 1)) == 0) + +ssize_t +FastOS_Linux_File::readInternal(int fh, void *buffer, size_t length, int64_t readOffset) +{ + char * data = static_cast(buffer); + ssize_t has_read(0); + while (has_read < ssize_t(length)) { + size_t lenNow = std::min(getChunkSize(), length - has_read); + ssize_t readNow = File_RW_Ops::pread(fh, data + has_read, lenNow, readOffset + has_read); + if (readNow > 0) { + has_read += readNow; + } else { + return (has_read > 0) ? has_read : readNow; + } + } + return has_read; +} + + +ssize_t +FastOS_Linux_File::readInternal(int fh, void *buffer, size_t length) +{ + char * data = static_cast(buffer); + ssize_t has_read(0); + while (has_read < ssize_t(length)) { + size_t lenNow = std::min(getChunkSize(), length - has_read); + ssize_t readNow = File_RW_Ops::read(fh, data + has_read, lenNow); + if (readNow > 0) { + has_read += readNow; + } else { + return (has_read > 0) ? has_read : readNow; + } + } + return has_read; +} + + +ssize_t +FastOS_Linux_File::writeInternal(int fh, const void *buffer, size_t length, int64_t writeOffset) +{ + return File_RW_Ops::pwrite(fh, buffer, length, writeOffset); +} + +ssize_t +FastOS_Linux_File::writeInternal(int fh, const void *buffer, size_t length) +{ + return File_RW_Ops::write(fh, buffer, length); +} + + +ssize_t FastOS_Linux_File::readUnalignedEnd(void *buffer, size_t length, int64_t readOffset) +{ + if (length == 0) { return 0; } + int fh = open(GetFileName(), O_RDONLY, 0664); + if (fh < 0) { + std::ostringstream os; + os << "Failed opening file " << GetFileName() << " for reading the unaligend end due to : " << getLastErrorString(); + throw std::runtime_error(os.str()); + } + ssize_t readResult = readInternal(fh, buffer, length, readOffset); + close(fh); + return readResult; +} + +ssize_t FastOS_Linux_File::writeUnalignedEnd(const void *buffer, size_t length, int64_t writeOffset) +{ + if (length == 0) { return 0; } + int fh = open(GetFileName(), O_WRONLY | O_SYNC, 0664); + if (fh < 0) { + std::ostringstream os; + os << "Failed opening file " << GetFileName() << " for reading the unaligend end due to : " << getLastErrorString(); + throw std::runtime_error(os.str()); + } + ssize_t writeResult = writeInternal(fh, buffer, length, writeOffset); + close(fh); + return writeResult; +} + +ssize_t +FastOS_Linux_File::ReadBufInternal(void *buffer, size_t length, int64_t readOffset) +{ + if (length == 0) { return 0; } + ssize_t readResult; + + if (_directIOEnabled) { + if (DIRECTIOPOSSIBLE(buffer, length, readOffset)) { + readResult = readInternal(_filedes, buffer, length, readOffset); + } else { + size_t alignedLength(length & ~(_directIOFileAlign - 1)); + if (DIRECTIOPOSSIBLE(buffer, alignedLength, readOffset)) { + size_t remain(length - alignedLength); + if (alignedLength > 0) { + readResult = readInternal(_filedes, buffer, alignedLength, readOffset); + } else { + readResult = 0; + } + if (static_cast(readResult) == alignedLength && (remain != 0)) { + ssize_t readResult2 = readUnalignedEnd(static_cast(buffer) + alignedLength, + remain, readOffset + alignedLength); + if (readResult == 0) { + readResult = readResult2; + } else if (readResult2 > 0) { + readResult += readResult2; + } + } + } else { + throw DirectIOException(GetFileName(), buffer, length, readOffset); + } + } + } else { + readResult = readInternal(_filedes, buffer, length, readOffset); + } + + if (readResult < 0) { + perror("pread error"); + } + + return readResult; +} + +void +FastOS_Linux_File::ReadBuf(void *buffer, size_t length, int64_t readOffset) +{ + ssize_t readResult(ReadBufInternal(buffer, length, readOffset)); + if (static_cast(readResult) != length) { + std::string errorString = (readResult != -1) + ? std::string("short read") + : getLastErrorString(); + std::ostringstream os; + os << "Fatal: Reading " << length << " bytes, got " << readResult << " from '" + << GetFileName() << "' failed: " << errorString; + throw std::runtime_error(os.str()); + } +} + + +ssize_t +FastOS_Linux_File::Read(void *buffer, size_t len) +{ + if (_directIOEnabled) { + ssize_t readResult = ReadBufInternal(buffer, len, _filePointer); + if (readResult > 0) { + _filePointer += readResult; + } + return readResult; + } else { + return readInternal(_filedes, buffer, len); + } +} + + +ssize_t +FastOS_Linux_File::Write2(const void *buffer, size_t length) +{ + const char * data = static_cast(buffer); + ssize_t written(0); + while (written < ssize_t(length)) { + size_t lenNow = std::min(getChunkSize(), length - written); + ssize_t writtenNow = internalWrite2(data + written, lenNow); + if (writtenNow > 0) { + written += writtenNow; + } else { + return (written > 0) ? written : writtenNow;; + } + } + return written; +} + +ssize_t +FastOS_Linux_File::internalWrite2(const void *buffer, size_t length) +{ + ssize_t writeRes; + if (_directIOEnabled) { + if (DIRECTIOPOSSIBLE(buffer, length, _filePointer)) { + writeRes = writeInternal(_filedes, buffer, length, _filePointer); + } else { + size_t alignedLength(length & ~(_directIOFileAlign - 1)); + if (DIRECTIOPOSSIBLE(buffer, alignedLength, _filePointer)) { + size_t remain(length - alignedLength); + if (alignedLength > 0) { + writeRes = writeInternal(_filedes, buffer, alignedLength, _filePointer); + } else { + writeRes = 0; + } + if (static_cast(writeRes) == alignedLength && remain != 0) { + ssize_t writeRes2 = writeUnalignedEnd(static_cast(buffer) + alignedLength, + remain, _filePointer + alignedLength); + if (writeRes == 0) { + writeRes = writeRes2; + } else if (writeRes2 > 0) { + writeRes += writeRes2; + } + } + } else { + throw DirectIOException(GetFileName(), buffer, length, _filePointer); + } + } + if (writeRes > 0) { + _filePointer += writeRes; + if (_filePointer > _cachedSize) { + _cachedSize = _filePointer; + } + } + } else { + writeRes = writeInternal(_filedes, buffer, length); + } + + return writeRes; +} + + +bool +FastOS_Linux_File::SetPosition(int64_t desiredPosition) +{ + bool rc = FastOS_UNIX_File::SetPosition(desiredPosition); + + if (rc && _directIOEnabled) { + _filePointer = desiredPosition; + } + + return rc; +} + + +int64_t +FastOS_Linux_File::GetPosition() +{ + return _directIOEnabled ? _filePointer : FastOS_UNIX_File::GetPosition(); +} + + +bool +FastOS_Linux_File::SetSize(int64_t newSize) +{ + bool rc = FastOS_UNIX_File::SetSize(newSize); + + if (rc) { + _cachedSize = newSize; + } + return rc; +} + + +namespace { + void * align(void * p, size_t alignment) { + const size_t alignMask(alignment-1); + return reinterpret_cast((reinterpret_cast(p) + alignMask) & ~alignMask); + } +} + +void * +FastOS_Linux_File::AllocateDirectIOBuffer (size_t byteSize, void *&realPtr) +{ + size_t dummy1, dummy2; + size_t memoryAlignment; + + GetDirectIORestrictions(memoryAlignment, dummy1, dummy2); + + realPtr = malloc(byteSize + memoryAlignment - 1); + return align(realPtr, memoryAlignment); +} + +size_t +FastOS_Linux_File::getMaxDirectIOMemAlign() +{ + return _directIOMemAlign; +} + + +bool +FastOS_Linux_File::GetDirectIORestrictions (size_t &memoryAlignment, size_t &transferGranularity, size_t &transferMaximum) +{ + if (_directIOEnabled) { + memoryAlignment = _directIOMemAlign; + transferGranularity = _directIOFileAlign; + transferMaximum = 0x7FFFFFFF; + return true; + } else { + return FastOS_UNIX_File::GetDirectIORestrictions(memoryAlignment, transferGranularity, transferMaximum); + } +} + + +bool +FastOS_Linux_File::DirectIOPadding (int64_t offset, size_t length, size_t &padBefore, size_t &padAfter) +{ + if (_directIOEnabled) { + + padBefore = offset & (_directIOFileAlign - 1); + padAfter = _directIOFileAlign - ((padBefore + length) & (_directIOFileAlign - 1)); + + if (padAfter == _directIOFileAlign) { + padAfter = 0; + } + if (int64_t(offset+length+padAfter) > _cachedSize) { + // _cachedSize is not really trustworthy, so if we suspect it is not correct, we correct it. + // The main reason is that it will not reflect the file being extended by another filedescriptor. + _cachedSize = GetSize(); + } + if ((padAfter != 0) && + (static_cast(offset + length + padAfter) > _cachedSize) && + (static_cast(offset + length) <= _cachedSize)) + { + padAfter = _cachedSize - (offset + length); + } + + if (static_cast(offset + length + padAfter) <= static_cast(_cachedSize)) { + return true; + } + } + + padAfter = 0; + padBefore = 0; + + return false; +} + + +void +FastOS_Linux_File::EnableDirectIO() +{ + if (!IsOpened()) { + _directIOEnabled = true; + } +} + + +bool +FastOS_Linux_File::Open(unsigned int openFlags, const char *filename) +{ + bool rc; + _cachedSize = -1; + _filePointer = -1; + if (_directIOEnabled && (_openFlags & FASTOS_FILE_OPEN_STDFLAGS) != 0) { + _directIOEnabled = false; + } + if (_syncWritesEnabled) { + openFlags |= FASTOS_FILE_OPEN_SYNCWRITES; + } + if (_directIOEnabled) { + rc = FastOS_UNIX_File::Open(openFlags | FASTOS_FILE_OPEN_DIRECTIO, filename); + if ( ! rc ) { //Retry without directIO. + rc = FastOS_UNIX_File::Open(openFlags | FASTOS_FILE_OPEN_SYNCWRITES, filename); + } + if (rc) { + int fadviseOptions = getFAdviseOptions(); + if (POSIX_FADV_NORMAL != fadviseOptions) { + rc = (posix_fadvise(_filedes, 0, 0, fadviseOptions) == 0); + if (!rc) { + bool close_ok = Close(); + assert(close_ok); + } + } + } + if (rc) { + bool sync_ok = Sync(); + assert(sync_ok); + _cachedSize = GetSize(); + _filePointer = 0; + } + } else { + rc = FastOS_UNIX_File::Open(openFlags, filename); + if (rc && (POSIX_FADV_NORMAL != getFAdviseOptions())) { + rc = (posix_fadvise(_filedes, 0, 0, getFAdviseOptions()) == 0); + if (!rc) { + bool close_ok = Close(); + assert(close_ok); + } + } + } + return rc; +} + +int +FastOS_Linux_File::count_open_files() +{ + static const char * const fd_dir_name = "/proc/self/fd"; + int count = 0; + DIR *dp = opendir(fd_dir_name); + if (dp != nullptr) { + struct dirent *ptr; + while ((ptr = readdir(dp)) != nullptr) { + if ((strcmp(".", ptr->d_name) != 0) && (strcmp("..", ptr->d_name) != 0)) { + ++count; + } + } + closedir(dp); + } else { + std::error_code ec(errno, std::system_category()); + fprintf(stderr, "could not scan directory %s: %s\n", fd_dir_name, ec.message().c_str()); + } + return count; +} + +#endif diff --git a/vespalib/src/vespa/fastos/linux_file.h b/vespalib/src/vespa/fastos/linux_file.h new file mode 100644 index 00000000000..2481b163210 --- /dev/null +++ b/vespalib/src/vespa/fastos/linux_file.h @@ -0,0 +1,57 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** +****************************************************************************** +* @author Oivind H. Danielsen +* @date Creation date: 2000-09-21 +* @file +* Class definition for FastOS_Linux_File. +*****************************************************************************/ + +#pragma once + +#include "unix_file.h" + +/** + * This is the Linux implementation of @ref FastOS_File. Most + * methods are inherited from @ref FastOS_UNIX_File. + */ +class FastOS_Linux_File : public FastOS_UNIX_File +{ +public: + using FastOS_UNIX_File::ReadBuf; +protected: + int64_t _cachedSize; + int64_t _filePointer; // Only maintained/used in directio mode + +public: + FastOS_Linux_File (const char *filename = nullptr); + ~FastOS_Linux_File () override; + bool GetDirectIORestrictions(size_t &memoryAlignment, size_t &transferGranularity, size_t &transferMaximum) override; + bool DirectIOPadding(int64_t offset, size_t length, size_t &padBefore, size_t &padAfter) override; + void EnableDirectIO() override; + bool SetPosition(int64_t desiredPosition) override; + int64_t GetPosition() override; + bool SetSize(int64_t newSize) override; + void ReadBuf(void *buffer, size_t length, int64_t readOffset) override; + void *AllocateDirectIOBuffer(size_t byteSize, void *&realPtr) override; + + + [[nodiscard]] ssize_t Read(void *buffer, size_t len) override; + [[nodiscard]] ssize_t Write2(const void *buffer, size_t len) override; + bool Open(unsigned int openFlags, const char *filename) override; + + static size_t getMaxDirectIOMemAlign(); + static int count_open_files(); +private: + ssize_t internalWrite2(const void *buffer, size_t len); + ssize_t readUnalignedEnd(void *buffer, size_t length, int64_t readOffset); + ssize_t writeUnalignedEnd(const void *buffer, size_t length, int64_t readOffset); + ssize_t ReadBufInternal(void *buffer, size_t length, int64_t readOffset); + ssize_t readInternal(int fh, void *buffer, size_t length, int64_t readOffset); + ssize_t readInternal(int fh, void *buffer, size_t length); + static ssize_t writeInternal(int fh, const void *buffer, size_t length, int64_t writeOffset); + static ssize_t writeInternal(int fh, const void *buffer, size_t length); + + static const size_t _directIOFileAlign; + static const size_t _directIOMemAlign; +}; diff --git a/vespalib/src/vespa/fastos/types.h b/vespalib/src/vespa/fastos/types.h new file mode 100644 index 00000000000..69dd3e5231c --- /dev/null +++ b/vespalib/src/vespa/fastos/types.h @@ -0,0 +1,9 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#define FASTOS_PREFIX(a) FastOS_##a + +// New macros to support the new gcc visibility features. +#define VESPA_DLL_EXPORT __attribute__ ((visibility("default"))) +#define VESPA_DLL_LOCAL __attribute__ ((visibility("hidden"))) diff --git a/vespalib/src/vespa/fastos/unix_file.cpp b/vespalib/src/vespa/fastos/unix_file.cpp new file mode 100644 index 00000000000..7c4cde19125 --- /dev/null +++ b/vespalib/src/vespa/fastos/unix_file.cpp @@ -0,0 +1,590 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** +****************************************************************************** +* @author Oivind H. Danielsen +* @date Creation date: 2000-01-18 +* @file +* Implementation of FastOS_UNIX_File methods. +*****************************************************************************/ + +#include "file.h" +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef __linux__ +#include +#else +#include +#endif +#ifdef __APPLE__ +#include +#include +#endif +#include "file_rw_ops.h" + +using fastos::File_RW_Ops; + +int FastOS_UNIX_File::GetLastOSError() { + return errno; +} + +ssize_t +FastOS_UNIX_File::Read(void *buffer, size_t len) +{ + return File_RW_Ops::read(_filedes, buffer, len); +} + + +ssize_t +FastOS_UNIX_File::Write2(const void *buffer, size_t len) +{ + return File_RW_Ops::write(_filedes, buffer, len); +} + +bool +FastOS_UNIX_File::SetPosition(int64_t desiredPosition) +{ + int64_t position = lseek(_filedes, desiredPosition, SEEK_SET); + + return (position == desiredPosition); +} + + +int64_t +FastOS_UNIX_File::GetPosition() +{ + return lseek(_filedes, 0, SEEK_CUR); +} + +void FastOS_UNIX_File::ReadBuf(void *buffer, size_t length, + int64_t readOffset) +{ + ssize_t readResult; + + readResult = File_RW_Ops::pread(_filedes, buffer, length, readOffset); + if (static_cast(readResult) != length) { + std::string errorString = readResult != -1 ? + std::string("short read") : + FastOS_FileInterface::getLastErrorString(); + std::ostringstream os; + os << "Fatal: Reading " << length << " bytes, got " << readResult << " from '" + << GetFileName() << "' failed: " << errorString; + throw std::runtime_error(os.str()); + } +} + +bool +FastOS_UNIX_File::Stat(const char *filename, FastOS_StatInfo *statInfo) +{ + bool rc = false; + + struct stat stbuf{}; + int lstatres; + + do { + lstatres = lstat(filename, &stbuf); + } while (lstatres == -1 && errno == EINTR); + if (lstatres == 0) { + statInfo->_error = FastOS_StatInfo::Ok; + statInfo->_isRegular = S_ISREG(stbuf.st_mode); + statInfo->_isDirectory = S_ISDIR(stbuf.st_mode); + statInfo->_size = static_cast(stbuf.st_size); + statInfo->_modifiedTime = stbuf.st_mtime; +#ifdef __linux__ + statInfo->_modifiedTimeNS = stbuf.st_mtim.tv_sec; + statInfo->_modifiedTimeNS *= 1000000000; + statInfo->_modifiedTimeNS += stbuf.st_mtim.tv_nsec; +#elif defined(__APPLE__) + statInfo->_modifiedTimeNS = stbuf.st_mtimespec.tv_sec; + statInfo->_modifiedTimeNS *= 1000000000; + statInfo->_modifiedTimeNS += stbuf.st_mtimespec.tv_nsec; +#else + statInfo->_modifiedTimeNS = stbuf.st_mtime; + statInfo->_modifiedTimeNS *= 1000000000; +#endif + rc = true; + } else { + if (errno == ENOENT) { + statInfo->_error = FastOS_StatInfo::FileNotFound; + } else { + statInfo->_error = FastOS_StatInfo::Unknown; + } + } + + return rc; +} + +bool FastOS_UNIX_File::SetCurrentDirectory (const char *pathName) { return (chdir(pathName) == 0); } + + +int FastOS_UNIX_File::GetMaximumFilenameLength (const char *pathName) +{ + return pathconf(pathName, _PC_NAME_MAX); +} + +int FastOS_UNIX_File::GetMaximumPathLength(const char *pathName) +{ + return pathconf(pathName, _PC_PATH_MAX); +} + +std::string +FastOS_UNIX_File::getCurrentDirectory() +{ + std::string res; + int maxPathLen = FastOS_File::GetMaximumPathLength("."); + if (maxPathLen == -1) { + maxPathLen = 16384; + } else if (maxPathLen < 512) { + maxPathLen = 512; + } + + char *currentDir = new char [maxPathLen + 1]; + + if (getcwd(currentDir, maxPathLen) != nullptr) { + res = currentDir; + } + delete [] currentDir; + + return res; +} + + +unsigned int +FastOS_UNIX_File::CalcAccessFlags(unsigned int openFlags) +{ + unsigned int accessFlags=0; + + if ((openFlags & (FASTOS_FILE_OPEN_READ | FASTOS_FILE_OPEN_DIRECTIO)) != 0) { + if ((openFlags & FASTOS_FILE_OPEN_WRITE) != 0) { + // Open for reading and writing + accessFlags = O_RDWR; + } else { + // Open for reading only + accessFlags = O_RDONLY; + } + } else { + // Open for writing only + accessFlags = O_WRONLY; + } + + if (((openFlags & FASTOS_FILE_OPEN_EXISTING) == 0) && ((openFlags & FASTOS_FILE_OPEN_WRITE) != 0)) { + // Create file if it does not exist + accessFlags |= O_CREAT; + } + +#if defined(O_SYNC) + if ((openFlags & FASTOS_FILE_OPEN_SYNCWRITES) != 0) + accessFlags |= O_SYNC; +#elif defined(O_FSYNC) + if ((openFlags & FASTOS_FILE_OPEN_SYNCWRITES) != 0) + accessFlags |= O_FSYNC; +#endif + +#ifdef __linux__ + if ((openFlags & FASTOS_FILE_OPEN_DIRECTIO) != 0) { + accessFlags |= O_DIRECT; + } +#endif + + if ((openFlags & FASTOS_FILE_OPEN_TRUNCATE) != 0) { + // Truncate file on open + accessFlags |= O_TRUNC; + } + return accessFlags; +} + +#ifdef __linux__ +constexpr int ALWAYS_SUPPORTED_MMAP_FLAGS = ~MAP_HUGETLB; +#else +constexpr int ALWAYS_SUPPORTED_MMAP_FLAGS = ~0; +#endif + +bool +FastOS_UNIX_File::Open(unsigned int openFlags, const char *filename) +{ + bool rc = false; + assert(_filedes == -1); + + if ((openFlags & FASTOS_FILE_OPEN_STDFLAGS) != 0) { + FILE *file; + + switch(openFlags & FASTOS_FILE_OPEN_STDFLAGS) { + + case FASTOS_FILE_OPEN_STDOUT: + file = stdout; + SetFileName("stdout"); + break; + + case FASTOS_FILE_OPEN_STDERR: + file = stderr; + SetFileName("stderr"); + break; + + default: + fprintf(stderr, "Invalid open-flags %08X\n", openFlags); + abort(); + } + +#ifdef __linux__ + _filedes = file->_fileno; +#else + _filedes = fileno(file); +#endif + _openFlags = openFlags; + rc = true; + } else { + if (filename != nullptr) { + SetFileName(filename); + } + unsigned int accessFlags = CalcAccessFlags(openFlags); + + _filedes = open(_filename.c_str(), accessFlags, 0664); + + rc = (_filedes != -1); + + if (rc) { + _openFlags = openFlags; + if (_mmapEnabled) { + int64_t filesize = GetSize(); + auto mlen = static_cast(filesize); + if ((static_cast(mlen) == filesize) && (mlen > 0)) { + void *mbase = mmap(nullptr, mlen, PROT_READ, MAP_SHARED | _mmapFlags, _filedes, 0); + if (mbase == MAP_FAILED) { + mbase = mmap(nullptr, mlen, PROT_READ, MAP_SHARED | (_mmapFlags & ALWAYS_SUPPORTED_MMAP_FLAGS), _filedes, 0); + } + if (mbase != MAP_FAILED) { +#ifdef __linux__ + int fadviseOptions = getFAdviseOptions(); + int eCode(0); + if (POSIX_FADV_RANDOM == fadviseOptions) { + eCode = posix_madvise(mbase, mlen, POSIX_MADV_RANDOM); + } else if (POSIX_FADV_SEQUENTIAL == fadviseOptions) { + eCode = posix_madvise(mbase, mlen, POSIX_MADV_SEQUENTIAL); + } + if (eCode != 0) { + fprintf(stderr, "Failed: posix_madvise(%p, %ld, %d) = %d\n", mbase, mlen, fadviseOptions, eCode); + } +#endif + _mmapbase = mbase; + _mmaplen = mlen; + } else { + close(_filedes); + _filedes = -1; + std::ostringstream os; + os << "mmap of file '" << GetFileName() << "' with flags '" << std::hex << (MAP_SHARED | _mmapFlags) << std::dec + << "' failed with error :'" << getErrorString(GetLastOSError()) << "'"; + throw std::runtime_error(os.str()); + } + } + } + } + + } + + return rc; +} + +void FastOS_UNIX_File::dropFromCache() const +{ +#ifdef __linux__ + posix_fadvise(_filedes, 0, 0, POSIX_FADV_DONTNEED); +#endif +} + + +bool +FastOS_UNIX_File::Close() +{ + bool ok = true; + + if (_filedes >= 0) { + if ((_openFlags & FASTOS_FILE_OPEN_STDFLAGS) != 0) { + ok = true; + } else { + do { + ok = (close(_filedes) == 0); + } while (!ok && errno == EINTR); + } + + if (_mmapbase != nullptr) { + madvise(_mmapbase, _mmaplen, MADV_DONTNEED); + munmap(static_cast(_mmapbase), _mmaplen); + _mmapbase = nullptr; + _mmaplen = 0; + } + + _filedes = -1; + } + + _openFlags = 0; + + return ok; +} + + +int64_t +FastOS_UNIX_File::GetSize() +{ + int64_t fileSize=-1; + struct stat stbuf{}; + + assert(IsOpened()); + + int res = fstat(_filedes, &stbuf); + + if (res == 0) { + fileSize = stbuf.st_size; + } + + return fileSize; +} + + +time_t +FastOS_UNIX_File::GetModificationTime() +{ + struct stat stbuf{}; + assert(IsOpened()); + + int res = fstat(_filedes, &stbuf); + assert(res == 0); + (void) res; + + return stbuf.st_mtime; +} + + +bool +FastOS_UNIX_File::Delete(const char *name) +{ + return (unlink(name) == 0); +} + + +bool +FastOS_UNIX_File::Delete() +{ + assert( ! IsOpened()); + + return (unlink(_filename.c_str()) == 0); +} + +bool FastOS_UNIX_File::Rename (const char *currentFileName, const char *newFileName) +{ + bool rc = false; + + // Enforce documentation. If the destination file exists, + // fail Rename. + FastOS_StatInfo statInfo{}; + if (!FastOS_File::Stat(newFileName, &statInfo)) { + rc = (rename(currentFileName, newFileName) == 0); + } else { + errno = EEXIST; + } + return rc; +} + +bool +FastOS_UNIX_File::Sync() +{ + assert(IsOpened()); + + return (fsync(_filedes) == 0); +} + + +bool +FastOS_UNIX_File::SetSize(int64_t newSize) +{ + bool rc = false; + + if (ftruncate(_filedes, static_cast(newSize)) == 0) { + rc = SetPosition(newSize); + } + + return rc; +} + + +FastOS_File::Error +FastOS_UNIX_File::TranslateError (const int osError) +{ + switch(osError) { + case ENOENT: return ERR_NOENT; // No such file or directory + case ENOMEM: return ERR_NOMEM; // Not enough memory + case EACCES: return ERR_ACCES; // Permission denied + case EEXIST: return ERR_EXIST; // File exists + case EINVAL: return ERR_INVAL; // Invalid argument + case ENOSPC: return ERR_NOSPC; // No space left on device + case EINTR: return ERR_INTR; // interrupt + case EAGAIN: return ERR_AGAIN; // Resource unavailable, try again + case EBUSY: return ERR_BUSY; // Device or resource busy + case EIO: return ERR_IO; // I/O error + case EPERM: return ERR_PERM; // Not owner + case ENODEV: return ERR_NODEV; // No such device + case ENXIO: return ERR_NXIO; // Device not configured + default: break; + } + + if (osError == ENFILE) + return ERR_NFILE; + + if (osError == EMFILE) + return ERR_MFILE; + + return ERR_UNKNOWN; +} + + +std::string +FastOS_UNIX_File::getErrorString(const int osError) +{ + std::error_code ec(osError, std::system_category()); + return ec.message(); +} + + +int64_t FastOS_UNIX_File::GetFreeDiskSpace (const char *path) +{ + struct statfs statBuf{}; + int statVal = statfs(path, &statBuf); + if (statVal == 0) { + return int64_t(statBuf.f_bavail) * int64_t(statBuf.f_bsize); + } + + return -1; +} + +int +FastOS_UNIX_File::count_open_files() +{ +#ifdef __APPLE__ + int buffer_size = proc_pidinfo(getpid(), PROC_PIDLISTFDS, 0, nullptr, 0); + return buffer_size / sizeof(proc_fdinfo); +#else + return 0; +#endif +} + +FastOS_UNIX_DirectoryScan::FastOS_UNIX_DirectoryScan(const char *searchPath) + : FastOS_DirectoryScanInterface(searchPath), + _statRun(false), + _isDirectory(false), + _isRegular(false), + _statName(nullptr), + _statFilenameP(nullptr), + _dir(nullptr), + _dp(nullptr) +{ + _dir = opendir(searchPath); + + const int minimumLength = 512 + 1; + const int defaultLength = 16384; + + int maxNameLength = FastOS_File::GetMaximumFilenameLength(searchPath); + int maxPathLength = FastOS_File::GetMaximumPathLength(searchPath); + int nameLength = maxNameLength + 1 + maxPathLength; + + if ((maxNameLength == -1) || + (maxPathLength == -1) || + (nameLength < minimumLength)) + { + nameLength = defaultLength; + } + + _statName = new char [nameLength + 1]; // Include null + + strcpy(_statName, searchPath); + strcat(_statName, "/"); + + _statFilenameP = &_statName[strlen(_statName)]; +} + + +FastOS_UNIX_DirectoryScan::~FastOS_UNIX_DirectoryScan() +{ + if (_dir != nullptr) { + closedir(_dir); + _dir = nullptr; + } + delete [] _statName; +} + + +bool +FastOS_UNIX_DirectoryScan::ReadNext() +{ + _statRun = false; + + if (_dir != nullptr) { + _dp = readdir(_dir); + return (_dp != nullptr); + } + + return false; +} + + +void +FastOS_UNIX_DirectoryScan::DoStat() +{ + struct stat stbuf{}; + + assert(_dp != nullptr); + + strcpy(_statFilenameP, _dp->d_name); + + if (lstat(_statName, &stbuf) == 0) { + _isRegular = S_ISREG(stbuf.st_mode); + _isDirectory = S_ISDIR(stbuf.st_mode); + } else { + printf("lstat failed for [%s]\n", _dp->d_name); + _isRegular = false; + _isDirectory = false; + } + + _statRun = true; +} + + +bool +FastOS_UNIX_DirectoryScan::IsDirectory() +{ + if (!_statRun) { + DoStat(); + } + + return _isDirectory; +} + + +bool +FastOS_UNIX_DirectoryScan::IsRegular() +{ + if (!_statRun) { + DoStat(); + } + + return _isRegular; +} + + +const char * +FastOS_UNIX_DirectoryScan::GetName() +{ + assert(_dp != nullptr); + + return static_cast(_dp->d_name); +} + + +bool +FastOS_UNIX_DirectoryScan::IsValidScan() const +{ + return _dir != nullptr; +} diff --git a/vespalib/src/vespa/fastos/unix_file.h b/vespalib/src/vespa/fastos/unix_file.h new file mode 100644 index 00000000000..31e45f8d2fa --- /dev/null +++ b/vespalib/src/vespa/fastos/unix_file.h @@ -0,0 +1,130 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** +****************************************************************************** +* @author Oivind H. Danielsen +* @date Creation date: 2000-01-18 +* @file +* Class definitions for FastOS_UNIX_File and FastOS_UNIX_DirectoryScan. +*****************************************************************************/ + +#pragma once + +#include + +/** + * This is the generic UNIX implementation of @ref FastOS_FileInterface. + */ +class FastOS_UNIX_File : public FastOS_FileInterface +{ +public: + using FastOS_FileInterface::ReadBuf; +private: + FastOS_UNIX_File(const FastOS_UNIX_File&); + FastOS_UNIX_File& operator=(const FastOS_UNIX_File&); + +protected: + void *_mmapbase; + size_t _mmaplen; + int _filedes; + int _mmapFlags; + bool _mmapEnabled; + + static unsigned int CalcAccessFlags(unsigned int openFlags); + +public: + static bool Rename (const char *currentFileName, const char *newFileName); + bool Rename (const char *newFileName) override { + return FastOS_FileInterface::Rename(newFileName); + } + + static bool Stat(const char *filename, FastOS_StatInfo *statInfo); + + static std::string getCurrentDirectory(); + + static bool SetCurrentDirectory (const char *pathName); + static int GetMaximumFilenameLength (const char *pathName); + static int GetMaximumPathLength (const char *pathName); + + FastOS_UNIX_File(const char *filename=nullptr) + : FastOS_FileInterface(filename), + _mmapbase(nullptr), + _mmaplen(0), + _filedes(-1), + _mmapFlags(0), + _mmapEnabled(false) + { } + + void ReadBuf(void *buffer, size_t length, int64_t readOffset) override; + [[nodiscard]] ssize_t Read(void *buffer, size_t len) override; + [[nodiscard]] ssize_t Write2(const void *buffer, size_t len) override; + bool Open(unsigned int openFlags, const char *filename) override; + [[nodiscard]] bool Close() override; + bool IsOpened() const override { return _filedes >= 0; } + + void enableMemoryMap(int flags) override { + _mmapEnabled = true; + _mmapFlags = flags; + } + + void *MemoryMapPtr(int64_t position) const override { + if (_mmapbase != nullptr) { + if (position < int64_t(_mmaplen)) { + return static_cast(static_cast(_mmapbase) + position); + } else { // This is an indication that the file size has changed and a remap/reopen must be done. + return nullptr; + } + } else { + return nullptr; + } + } + + bool IsMemoryMapped() const override { return _mmapbase != nullptr; } + bool SetPosition(int64_t desiredPosition) override; + int64_t GetPosition() override; + int64_t GetSize() override; + time_t GetModificationTime() override; + bool Delete() override; + [[nodiscard]] bool Sync() override; + bool SetSize(int64_t newSize) override; + void dropFromCache() const override; + + static bool Delete(const char *filename); + static int GetLastOSError(); + static Error TranslateError(const int osError); + static std::string getErrorString(const int osError); + static int64_t GetFreeDiskSpace (const char *path); + static int count_open_files(); +}; + +#include +/** + * This is the generic UNIX implementation of @ref FastOS_DirectoryScan. + */ +class FastOS_UNIX_DirectoryScan : public FastOS_DirectoryScanInterface +{ +private: + FastOS_UNIX_DirectoryScan(const FastOS_UNIX_DirectoryScan&); + FastOS_UNIX_DirectoryScan& operator=(const FastOS_UNIX_DirectoryScan&); + + bool _statRun; + bool _isDirectory; + bool _isRegular; + char *_statName; + char *_statFilenameP; + + void DoStat(); + +protected: + DIR *_dir; + struct dirent *_dp; + +public: + FastOS_UNIX_DirectoryScan(const char *searchPath); + ~FastOS_UNIX_DirectoryScan(); + + bool ReadNext() override; + bool IsDirectory() override; + bool IsRegular() override; + const char *GetName() override; + bool IsValidScan() const override; +}; diff --git a/vespalib/src/vespa/vespalib/CMakeLists.txt b/vespalib/src/vespa/vespalib/CMakeLists.txt index 9e31084c067..63876d442ef 100644 --- a/vespalib/src/vespa/vespalib/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/CMakeLists.txt @@ -30,8 +30,9 @@ vespa_add_library(vespalib $ $ $ - $ - $ + $ + $ + $ INSTALL lib64 DEPENDS ${VESPA_GCC_LIB} -- cgit v1.2.3 From da683245d8535073490653d0058db5e473253e97 Mon Sep 17 00:00:00 2001 From: Ola Aunronning Date: Tue, 28 Feb 2023 13:51:50 +0100 Subject: Count both nodes and hosts by state --- .../hosted/provision/maintenance/MetricsReporter.java | 15 +++++++++++---- .../provision/maintenance/MetricsReporterTest.java | 16 +++++++++++++--- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index 67c1c7359f7..5af74214648 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -267,13 +267,20 @@ public class MetricsReporter extends NodeRepositoryMaintainer { } private void updateNodeCountMetrics(NodeList nodes) { - Map> nodesByState = nodes.nodeType(NodeType.tenant).asList().stream() - .collect(Collectors.groupingBy(Node::state)); + var nodesByState = nodes.nodeType(NodeType.tenant) + .asList().stream() + .collect(Collectors.groupingBy(Node::state)); + + var hostsByState = nodes.nodeType(NodeType.host) + .asList().stream() + .collect(Collectors.groupingBy(Node::state)); // Count per state for (State state : State.values()) { - List nodesInState = nodesByState.getOrDefault(state, List.of()); - metric.set("hostedVespa." + state.name() + "Hosts", nodesInState.size(), null); + var nodesInState = nodesByState.getOrDefault(state, List.of()); + var hostsInState = hostsByState.getOrDefault(state, List.of()); + metric.set("hostedVespa." + state.name() + "Nodes", nodesInState.size(), null); + metric.set("hostedVespa." + state.name() + "Hosts", hostsInState.size(), null); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index 44050fa747c..d7ffda542ff 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -87,7 +87,7 @@ public class MetricsReporterTest { Map expectedMetrics = new TreeMap<>(); expectedMetrics.put("zone.working", 1); - expectedMetrics.put("hostedVespa.provisionedHosts", 1); + expectedMetrics.put("hostedVespa.provisionedHosts", 0); expectedMetrics.put("hostedVespa.parkedHosts", 0); expectedMetrics.put("hostedVespa.readyHosts", 0); expectedMetrics.put("hostedVespa.reservedHosts", 0); @@ -97,6 +97,16 @@ public class MetricsReporterTest { expectedMetrics.put("hostedVespa.failedHosts", 0); expectedMetrics.put("hostedVespa.deprovisionedHosts", 0); expectedMetrics.put("hostedVespa.breakfixedHosts", 0); + expectedMetrics.put("hostedVespa.provisionedNodes", 1); + expectedMetrics.put("hostedVespa.parkedNodes", 0); + expectedMetrics.put("hostedVespa.readyNodes", 0); + expectedMetrics.put("hostedVespa.reservedNodes", 0); + expectedMetrics.put("hostedVespa.activeNodes", 0); + expectedMetrics.put("hostedVespa.inactiveNodes", 0); + expectedMetrics.put("hostedVespa.dirtyNodes", 0); + expectedMetrics.put("hostedVespa.failedNodes", 0); + expectedMetrics.put("hostedVespa.deprovisionedNodes", 0); + expectedMetrics.put("hostedVespa.breakfixedNodes", 0); expectedMetrics.put("hostedVespa.pendingRedeployments", 42); expectedMetrics.put("hostedVespa.docker.totalCapacityDisk", 0.0); expectedMetrics.put("hostedVespa.docker.totalCapacityMem", 0.0); @@ -207,8 +217,8 @@ public class MetricsReporterTest { MetricsReporter metricsReporter = metricsReporter(metric, tester); metricsReporter.maintain(); - assertEquals(0, metric.values.get("hostedVespa.readyHosts")); // Only tenants counts - assertEquals(2, metric.values.get("hostedVespa.reservedHosts")); + assertEquals(0, metric.values.get("hostedVespa.readyNodes")); // Only tenants counts + assertEquals(2, metric.values.get("hostedVespa.reservedNodes")); assertEquals(120.0, metric.values.get("hostedVespa.docker.totalCapacityDisk")); assertEquals(100.0, metric.values.get("hostedVespa.docker.totalCapacityMem")); -- cgit v1.2.3 From 57481f854f39e794c593d09c808fe6336c8e9bca Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Tue, 28 Feb 2023 12:57:10 +0100 Subject: Export refcount package --- jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java | 1 + jdisc_core/src/test/resources/exportPackages.properties | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java index 53acd8cbb1b..e6c273fafde 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java @@ -53,6 +53,7 @@ public class ExportPackages { .append("com.yahoo.jdisc.handler, ") .append("com.yahoo.jdisc.service, ") .append("com.yahoo.jdisc.statistics, ") + .append("com.yahoo.jdisc.refcount, ") .append("javax.inject;version=1.0.0, ") // TODO Vespa 9: remove. Included in guice, but not exported. Needed by container-jersey. .append("org.aopalliance.intercept, ") diff --git a/jdisc_core/src/test/resources/exportPackages.properties b/jdisc_core/src/test/resources/exportPackages.properties index 82242b1644b..b49f91842e3 100644 --- a/jdisc_core/src/test/resources/exportPackages.properties +++ b/jdisc_core/src/test/resources/exportPackages.properties @@ -1,3 +1,3 @@ #generated by com.yahoo.jdisc.core.ExportPackages #Wed Jul 20 02:55:26 CEST 2022 -exportPackages=org.osgi.framework; version\="1.10.0", org.osgi.framework.connect; version\="1.0.0", org.osgi.framework.dto; uses\:\="org.osgi.dto"; version\="1.8.0", org.osgi.framework.hooks.bundle; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.resolver; uses\:\="org.osgi.framework.wiring"; version\="1.0.0", org.osgi.framework.hooks.service; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.weaving; uses\:\="org.osgi.framework.wiring"; version\="1.1.0", org.osgi.framework.launch; uses\:\="org.osgi.framework"; version\="1.2.0", org.osgi.framework.namespace; uses\:\="org.osgi.resource"; version\="1.2.0", org.osgi.framework.startlevel; uses\:\="org.osgi.framework"; version\="1.0.0", org.osgi.framework.startlevel.dto; uses\:\="org.osgi.dto"; version\="1.0.0", org.osgi.framework.wiring; uses\:\="org.osgi.framework,org.osgi.resource"; version\="1.2.0", org.osgi.framework.wiring.dto; uses\:\="org.osgi.dto,org.osgi.resource.dto"; version\="1.3.0", org.osgi.resource; version\="1.0.1", org.osgi.resource.dto; uses\:\="org.osgi.dto"; version\="1.0.1", org.osgi.service.packageadmin; uses\:\="org.osgi.framework"; version\="1.2.1", org.osgi.service.startlevel; uses\:\="org.osgi.framework"; version\="1.1.1", org.osgi.service.url; version\="1.0.1", org.osgi.service.resolver; uses\:\="org.osgi.resource"; version\="1.1.1", org.osgi.util.tracker; uses\:\="org.osgi.framework"; version\="1.5.3", org.osgi.dto; version\="1.1.1", org.osgi.service.condition; version\="1.0.0", javax.security.auth.callback; version\="0.0.0.JavaSE_017", javax.security.auth.login; version\="0.0.0.JavaSE_017", javax.net.ssl; version\="0.0.0.JavaSE_017", java.lang.constant; version\="0.0.0.JavaSE_017", java.security.interfaces; version\="0.0.0.JavaSE_017", java.text.spi; version\="0.0.0.JavaSE_017", java.nio.channels.spi; version\="0.0.0.JavaSE_017", java.math; version\="0.0.0.JavaSE_017", java.nio.file; version\="0.0.0.JavaSE_017", java.util.concurrent.atomic; version\="0.0.0.JavaSE_017", java.security.cert; version\="0.0.0.JavaSE_017", java.security.spec; version\="0.0.0.JavaSE_017", java.nio.channels; version\="0.0.0.JavaSE_017", java.time.chrono; version\="0.0.0.JavaSE_017", javax.crypto; version\="0.0.0.JavaSE_017", java.time.zone; version\="0.0.0.JavaSE_017", java.nio.charset; version\="0.0.0.JavaSE_017", java.io; version\="0.0.0.JavaSE_017", java.util.spi; version\="0.0.0.JavaSE_017", java.net; version\="0.0.0.JavaSE_017", javax.security.cert; version\="0.0.0.JavaSE_017", java.lang.annotation; version\="0.0.0.JavaSE_017", javax.security.auth.spi; version\="0.0.0.JavaSE_017", java.util.concurrent; version\="0.0.0.JavaSE_017", java.nio.charset.spi; version\="0.0.0.JavaSE_017", javax.crypto.interfaces; version\="0.0.0.JavaSE_017", java.util; version\="0.0.0.JavaSE_017", java.security; version\="0.0.0.JavaSE_017", java.nio.file.spi; version\="0.0.0.JavaSE_017", java.nio; version\="0.0.0.JavaSE_017", java.util.jar; version\="0.0.0.JavaSE_017", javax.security.auth; version\="0.0.0.JavaSE_017", java.lang.ref; version\="0.0.0.JavaSE_017", java.util.regex; version\="0.0.0.JavaSE_017", java.net.spi; version\="0.0.0.JavaSE_017", java.lang.module; version\="0.0.0.JavaSE_017", java.lang.invoke; version\="0.0.0.JavaSE_017", java.time.format; version\="0.0.0.JavaSE_017", java.util.concurrent.locks; version\="0.0.0.JavaSE_017", java.time.temporal; version\="0.0.0.JavaSE_017", java.util.zip; version\="0.0.0.JavaSE_017", java.nio.file.attribute; version\="0.0.0.JavaSE_017", java.util.random; version\="0.0.0.JavaSE_017", java.text; version\="0.0.0.JavaSE_017", javax.crypto.spec; version\="0.0.0.JavaSE_017", java.util.stream; version\="0.0.0.JavaSE_017", java.time; version\="0.0.0.JavaSE_017", java.lang; version\="0.0.0.JavaSE_017", java.lang.runtime; version\="0.0.0.JavaSE_017", java.util.function; version\="0.0.0.JavaSE_017", javax.net; version\="0.0.0.JavaSE_017", javax.security.auth.x500; version\="0.0.0.JavaSE_017", java.lang.reflect; version\="0.0.0.JavaSE_017", javax.lang.model; version\="0.0.0.JavaSE_017", javax.annotation.processing; version\="0.0.0.JavaSE_017", javax.lang.model.element; version\="0.0.0.JavaSE_017", javax.tools; version\="0.0.0.JavaSE_017", javax.lang.model.type; version\="0.0.0.JavaSE_017", javax.lang.model.util; version\="0.0.0.JavaSE_017", java.awt.datatransfer; version\="0.0.0.JavaSE_017", java.awt.desktop; version\="0.0.0.JavaSE_017", javax.swing.plaf.synth; version\="0.0.0.JavaSE_017", java.beans; version\="0.0.0.JavaSE_017", javax.swing.text.html.parser; version\="0.0.0.JavaSE_017", javax.swing.text.rtf; version\="0.0.0.JavaSE_017", java.awt.font; version\="0.0.0.JavaSE_017", javax.imageio; version\="0.0.0.JavaSE_017", java.awt.im.spi; version\="0.0.0.JavaSE_017", java.applet; version\="0.0.0.JavaSE_017", javax.sound.midi; version\="0.0.0.JavaSE_017", java.awt.dnd; version\="0.0.0.JavaSE_017", javax.swing.text; version\="0.0.0.JavaSE_017", javax.swing.plaf.basic; version\="0.0.0.JavaSE_017", javax.swing.undo; version\="0.0.0.JavaSE_017", javax.swing.plaf; version\="0.0.0.JavaSE_017", javax.swing.filechooser; version\="0.0.0.JavaSE_017", javax.imageio.event; version\="0.0.0.JavaSE_017", javax.sound.sampled; version\="0.0.0.JavaSE_017", javax.print.attribute; version\="0.0.0.JavaSE_017", javax.print; version\="0.0.0.JavaSE_017", javax.swing.plaf.nimbus; version\="0.0.0.JavaSE_017", javax.accessibility; version\="0.0.0.JavaSE_017", java.awt.event; version\="0.0.0.JavaSE_017", javax.swing.text.html; version\="0.0.0.JavaSE_017", javax.imageio.spi; version\="0.0.0.JavaSE_017", javax.swing.border; version\="0.0.0.JavaSE_017", javax.sound.sampled.spi; version\="0.0.0.JavaSE_017", javax.imageio.plugins.bmp; version\="0.0.0.JavaSE_017", java.awt.color; version\="0.0.0.JavaSE_017", java.awt.geom; version\="0.0.0.JavaSE_017", javax.imageio.plugins.jpeg; version\="0.0.0.JavaSE_017", javax.swing.tree; version\="0.0.0.JavaSE_017", javax.imageio.plugins.tiff; version\="0.0.0.JavaSE_017", java.awt.print; version\="0.0.0.JavaSE_017", java.awt.image; version\="0.0.0.JavaSE_017", javax.imageio.metadata; version\="0.0.0.JavaSE_017", javax.swing.table; version\="0.0.0.JavaSE_017", javax.sound.midi.spi; version\="0.0.0.JavaSE_017", javax.print.attribute.standard; version\="0.0.0.JavaSE_017", javax.swing.colorchooser; version\="0.0.0.JavaSE_017", javax.swing; version\="0.0.0.JavaSE_017", java.awt.image.renderable; version\="0.0.0.JavaSE_017", javax.swing.plaf.multi; version\="0.0.0.JavaSE_017", java.awt.im; version\="0.0.0.JavaSE_017", javax.print.event; version\="0.0.0.JavaSE_017", javax.swing.plaf.metal; version\="0.0.0.JavaSE_017", java.beans.beancontext; version\="0.0.0.JavaSE_017", java.awt; version\="0.0.0.JavaSE_017", javax.imageio.stream; version\="0.0.0.JavaSE_017", javax.swing.event; version\="0.0.0.JavaSE_017", java.lang.instrument; version\="0.0.0.JavaSE_017", java.util.logging; version\="0.0.0.JavaSE_017", javax.management.timer; version\="0.0.0.JavaSE_017", javax.management; version\="0.0.0.JavaSE_017", javax.management.relation; version\="0.0.0.JavaSE_017", javax.management.loading; version\="0.0.0.JavaSE_017", javax.management.openmbean; version\="0.0.0.JavaSE_017", java.lang.management; version\="0.0.0.JavaSE_017", javax.management.remote; version\="0.0.0.JavaSE_017", javax.management.monitor; version\="0.0.0.JavaSE_017", javax.management.modelmbean; version\="0.0.0.JavaSE_017", javax.management.remote.rmi; version\="0.0.0.JavaSE_017", javax.naming.event; version\="0.0.0.JavaSE_017", javax.naming.ldap.spi; version\="0.0.0.JavaSE_017", javax.naming; version\="0.0.0.JavaSE_017", javax.naming.spi; version\="0.0.0.JavaSE_017", javax.naming.ldap; version\="0.0.0.JavaSE_017", javax.naming.directory; version\="0.0.0.JavaSE_017", java.net.http; version\="0.0.0.JavaSE_017", java.util.prefs; version\="0.0.0.JavaSE_017", java.rmi.registry; version\="0.0.0.JavaSE_017", javax.rmi.ssl; version\="0.0.0.JavaSE_017", java.rmi.dgc; version\="0.0.0.JavaSE_017", java.rmi; version\="0.0.0.JavaSE_017", java.rmi.server; version\="0.0.0.JavaSE_017", javax.script; version\="0.0.0.JavaSE_017", org.ietf.jgss; version\="0.0.0.JavaSE_017", javax.security.auth.kerberos; version\="0.0.0.JavaSE_017", javax.security.sasl; version\="0.0.0.JavaSE_017", javax.smartcardio; version\="0.0.0.JavaSE_017", java.sql; version\="0.0.0.JavaSE_017", javax.sql; version\="0.0.0.JavaSE_017", javax.sql.rowset.spi; version\="0.0.0.JavaSE_017", javax.sql.rowset.serial; version\="0.0.0.JavaSE_017", javax.sql.rowset; version\="0.0.0.JavaSE_017", javax.transaction.xa; version\="0.0.0.JavaSE_017", javax.xml; version\="0.0.0.JavaSE_017", javax.xml.transform.sax; version\="0.0.0.JavaSE_017", javax.xml.datatype; version\="0.0.0.JavaSE_017", javax.xml.catalog; version\="0.0.0.JavaSE_017", org.w3c.dom.traversal; version\="0.0.0.JavaSE_017", javax.xml.stream.events; version\="0.0.0.JavaSE_017", javax.xml.stream; version\="0.0.0.JavaSE_017", org.xml.sax; version\="0.0.0.JavaSE_017", javax.xml.transform; version\="0.0.0.JavaSE_017", javax.xml.xpath; version\="0.0.0.JavaSE_017", org.w3c.dom.events; version\="0.0.0.JavaSE_017", org.w3c.dom.ranges; version\="0.0.0.JavaSE_017", org.w3c.dom.ls; version\="0.0.0.JavaSE_017", javax.xml.stream.util; version\="0.0.0.JavaSE_017", javax.xml.namespace; version\="0.0.0.JavaSE_017", javax.xml.transform.stax; version\="0.0.0.JavaSE_017", org.xml.sax.helpers; version\="0.0.0.JavaSE_017", org.w3c.dom.views; version\="0.0.0.JavaSE_017", org.w3c.dom.bootstrap; version\="0.0.0.JavaSE_017", org.w3c.dom; version\="0.0.0.JavaSE_017", javax.xml.transform.stream; version\="0.0.0.JavaSE_017", javax.xml.transform.dom; version\="0.0.0.JavaSE_017", javax.xml.validation; version\="0.0.0.JavaSE_017", javax.xml.parsers; version\="0.0.0.JavaSE_017", org.xml.sax.ext; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.spec; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.keyinfo; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig; version\="0.0.0.JavaSE_017", com.sun.java.accessibility.util; version\="0.0.0.JavaSE_017", com.sun.tools.attach.spi; version\="0.0.0.JavaSE_017", com.sun.tools.attach; version\="0.0.0.JavaSE_017", com.sun.source.doctree; version\="0.0.0.JavaSE_017", com.sun.source.tree; version\="0.0.0.JavaSE_017", com.sun.source.util; version\="0.0.0.JavaSE_017", com.sun.tools.javac; version\="0.0.0.JavaSE_017", jdk.dynalink.beans; version\="0.0.0.JavaSE_017", jdk.dynalink.linker.support; version\="0.0.0.JavaSE_017", jdk.dynalink.support; version\="0.0.0.JavaSE_017", jdk.dynalink; version\="0.0.0.JavaSE_017", jdk.dynalink.linker; version\="0.0.0.JavaSE_017", com.sun.net.httpserver; version\="0.0.0.JavaSE_017", com.sun.net.httpserver.spi; version\="0.0.0.JavaSE_017", com.sun.jarsigner; version\="0.0.0.JavaSE_017", jdk.security.jarsigner; version\="0.0.0.JavaSE_017", jdk.javadoc.doclet; version\="0.0.0.JavaSE_017", com.sun.tools.jconsole; version\="0.0.0.JavaSE_017", com.sun.jdi.connect; version\="0.0.0.JavaSE_017", com.sun.jdi.event; version\="0.0.0.JavaSE_017", com.sun.jdi.connect.spi; version\="0.0.0.JavaSE_017", com.sun.jdi; version\="0.0.0.JavaSE_017", com.sun.jdi.request; version\="0.0.0.JavaSE_017", jdk.jfr.consumer; version\="0.0.0.JavaSE_017", jdk.jfr; version\="0.0.0.JavaSE_017", jdk.jshell; version\="0.0.0.JavaSE_017", jdk.jshell.execution; version\="0.0.0.JavaSE_017", jdk.jshell.spi; version\="0.0.0.JavaSE_017", jdk.jshell.tool; version\="0.0.0.JavaSE_017", netscape.javascript; version\="0.0.0.JavaSE_017", com.sun.management; version\="0.0.0.JavaSE_017", jdk.management.jfr; version\="0.0.0.JavaSE_017", jdk.net; version\="0.0.0.JavaSE_017", jdk.nio; version\="0.0.0.JavaSE_017", jdk.nio.mapmode; version\="0.0.0.JavaSE_017", com.sun.nio.sctp; version\="0.0.0.JavaSE_017", com.sun.security.auth.module; version\="0.0.0.JavaSE_017", com.sun.security.auth.login; version\="0.0.0.JavaSE_017", com.sun.security.auth; version\="0.0.0.JavaSE_017", com.sun.security.auth.callback; version\="0.0.0.JavaSE_017", com.sun.security.jgss; version\="0.0.0.JavaSE_017", sun.reflect; version\="0.0.0.JavaSE_017", sun.misc; version\="0.0.0.JavaSE_017", com.sun.nio.file; version\="0.0.0.JavaSE_017", jdk.swing.interop; version\="0.0.0.JavaSE_017", org.w3c.dom.stylesheets; version\="0.0.0.JavaSE_017", org.w3c.dom.html; version\="0.0.0.JavaSE_017", org.w3c.dom.xpath; version\="0.0.0.JavaSE_017", org.w3c.dom.css; version\="0.0.0.JavaSE_017", com.yahoo.jdisc, com.yahoo.jdisc.application, com.yahoo.jdisc.handler, com.yahoo.jdisc.service, com.yahoo.jdisc.statistics, javax.inject;version\=1.0.0, org.aopalliance.intercept, org.aopalliance.aop, com.google.common.annotations;version\="27.1.0",com.google.common.base;version\="27.1.0",com.google.common.cache;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent",com.google.common.collect;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.escape;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.eventbus;version\="27.1.0",com.google.common.graph;version\="27.1.0";uses\:\="com.google.common.collect",com.google.common.hash;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.html;version\="27.1.0";uses\:\="com.google.common.escape",com.google.common.io;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.graph,com.google.common.hash",com.google.common.math;version\="27.1.0",com.google.common.net;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.escape",com.google.common.primitives;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.reflect;version\="27.1.0";uses\:\="com.google.common.collect,com.google.common.io",com.google.common.util.concurrent;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent.internal",com.google.common.xml;version\="27.1.0";uses\:\="com.google.common.escape", com.google.inject;version\="1.4",com.google.inject.binder;version\="1.4",com.google.inject.matcher;version\="1.4",com.google.inject.multibindings;version\="1.4",com.google.inject.name;version\="1.4",com.google.inject.spi;version\="1.4",com.google.inject.util;version\="1.4", org.slf4j;version\=1.7.32, org.slf4j.spi;version\=1.7.32, org.slf4j.helpers;version\=1.7.32, org.slf4j.event;version\=1.7.32, org.slf4j.impl;version\=1.7.32, org.apache.commons.logging;version\=1.2, org.apache.commons.logging.impl;version\=1.2, org.apache.log4j;version\=1.2.17,org.apache.log4j.helpers;version\=1.2.17,org.apache.log4j.spi;version\=1.2.17,org.apache.log4j.xml;version\=1.2.17, com.yahoo.component.annotation;version\="1.0.0", com.yahoo.config;version\=1.0.0, com.yahoo.vespa.defaults;version\=1.0.0, ai.vespa.http;version\=1.0.0,ai.vespa.validation;version\=1.0.0,com.yahoo.binaryprefix;version\=1.0.0,com.yahoo.collections;version\=1.0.0,com.yahoo.compress;version\=1.0.0,com.yahoo.concurrent.classlock;version\=1.0.0,com.yahoo.concurrent.maintenance;version\=1.0.0,com.yahoo.concurrent;version\=1.0.0,com.yahoo.data.access.simple;version\=1.0.0,com.yahoo.data.access.slime;version\=1.0.0,com.yahoo.data.access;version\=1.0.0,com.yahoo.errorhandling;version\=1.0.0,com.yahoo.exception;version\=1.0.0,com.yahoo.geo;version\=1.0.0,com.yahoo.io.reader;version\=1.0.0,com.yahoo.io;version\=1.0.0,com.yahoo.javacc;version\=1.0.0,com.yahoo.lang;version\=1.0.0,com.yahoo.nativec;version\=1.0.0,com.yahoo.net;version\=1.0.0,com.yahoo.path;version\=1.0.0,com.yahoo.protect;version\=1.0.0,com.yahoo.reflection;version\=1.0.0,com.yahoo.slime;version\=1.0.0,com.yahoo.stream;version\=1.0.0,com.yahoo.system.execution;version\=1.0.0,com.yahoo.system;version\=1.0.0,com.yahoo.tensor.evaluation;version\=1.0.0,com.yahoo.tensor.functions;version\=1.0.0,com.yahoo.tensor.serialization;version\=1.0.0,com.yahoo.tensor;version\=1.0.0,com.yahoo.text.internal;version\=1.0.0,com.yahoo.text;version\=1.0.0,com.yahoo.time;version\=1.0.0,com.yahoo.transaction;version\=1.0.0,com.yahoo.vespa.objects;version\=1.0.0,com.yahoo.yolean.chain;version\=1.0.0,com.yahoo.yolean.concurrent;version\=1.0.0,com.yahoo.yolean.function;version\=1.0.0,com.yahoo.yolean.system;version\=1.0.0,com.yahoo.yolean.trace;version\=1.0.0,com.yahoo.yolean;version\=1.0.0, com.yahoo.log.event;version\=1.0.0,com.yahoo.log.impl;version\=1.0.0,com.yahoo.log;version\=1.0.0, javax.xml.bind;version\="2.3";uses\:\="javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.annotation;version\="2.3";uses\:\="javax.xml.bind,javax.xml.parsers,javax.xml.transform,javax.xml.transform.dom,org.w3c.dom",javax.xml.bind.annotation.adapters;version\="2.3",javax.xml.bind.attachment;version\="2.3";uses\:\="javax.activation",javax.xml.bind.helpers;version\="2.3";uses\:\="javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.util;version\="2.3";uses\:\="javax.xml.bind,javax.xml.transform.sax", com.sun.istack;version\="3.0.5";uses\:\="javax.activation,javax.xml.stream,org.xml.sax,org.xml.sax.helpers",com.sun.istack.localization;version\="3.0.5",com.sun.istack.logging;version\="3.0.5",com.sun.xml.bind;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.annotation;version\="2.3.0",com.sun.xml.bind.api;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.api.impl;version\="2.3.0",com.sun.xml.bind.marshaller;uses\:\="javax.xml.parsers,org.w3c.dom,org.xml.sax,org.xml.sax.helpers";version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;version\="2.3.0",com.sun.xml.bind.v2;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.core;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.impl,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.namespace,javax.xml.transform";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.nav";version\="2.3.0",com.sun.xml.bind.v2.model.nav;uses\:\="com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.util;uses\:\="javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.xml.bind.v2.model.annotation,javax.activation,javax.xml.bind,javax.xml.bind.annotation.adapters";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen.episode;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="javax.xml.parsers,javax.xml.transform,javax.xml.validation,javax.xml.xpath";version\="2.3.0",com.sun.xml.txw2;uses\:\="com.sun.xml.txw2.output,javax.xml.namespace";version\="2.3.0",com.sun.xml.txw2.annotation;version\="2.3.0",com.sun.xml.txw2.output;uses\:\="com.sun.xml.txw2,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.dom,javax.xml.transform.sax,javax.xml.transform.stream,org.w3c.dom,org.xml.sax,org.xml.sax.ext,org.xml.sax.helpers";version\="2.3.0", com.sun.xml.bind;uses\:\="com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.datatype,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.api;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.xml.bind,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.marshaller;version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;uses\:\="com.sun.xml.bind,javax.xml.bind.helpers,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,javax.xml.bind";version\="2.3.0",com.sun.xml.bind.v2.bytecode;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.model.runtime;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.namespace,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.istack,com.sun.xml.bind.api,com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.property,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.output;uses\:\="com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.runtime,com.sun.xml.fastinfoset.stax,javax.xml.stream,org.jvnet.staxex,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.property;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,com.sun.xml.bind.v2.runtime.unmarshaller,com.sun.xml.bind.v2.util,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect.opt;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="com.sun.xml.bind,com.sun.xml.bind.api,com.sun.xml.bind.unmarshaller,com.sun.xml.bind.util,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.reflect,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.txw2.output,javax.xml.bind,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.schemagen.xmlschema;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.namespace,javax.xml.transform.stream,org.xml.sax";version\="2.3.0", javax.activation;uses\:\="com.sun.activation.registries";version\="1.2",com.sun.activation.viewers;uses\:\="javax.activation";version\="1.2.0",com.sun.activation.registries;version\="1.2.0" +exportPackages=org.osgi.framework; version\="1.10.0", org.osgi.framework.connect; version\="1.0.0", org.osgi.framework.dto; uses\:\="org.osgi.dto"; version\="1.8.0", org.osgi.framework.hooks.bundle; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.resolver; uses\:\="org.osgi.framework.wiring"; version\="1.0.0", org.osgi.framework.hooks.service; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.weaving; uses\:\="org.osgi.framework.wiring"; version\="1.1.0", org.osgi.framework.launch; uses\:\="org.osgi.framework"; version\="1.2.0", org.osgi.framework.namespace; uses\:\="org.osgi.resource"; version\="1.2.0", org.osgi.framework.startlevel; uses\:\="org.osgi.framework"; version\="1.0.0", org.osgi.framework.startlevel.dto; uses\:\="org.osgi.dto"; version\="1.0.0", org.osgi.framework.wiring; uses\:\="org.osgi.framework,org.osgi.resource"; version\="1.2.0", org.osgi.framework.wiring.dto; uses\:\="org.osgi.dto,org.osgi.resource.dto"; version\="1.3.0", org.osgi.resource; version\="1.0.1", org.osgi.resource.dto; uses\:\="org.osgi.dto"; version\="1.0.1", org.osgi.service.packageadmin; uses\:\="org.osgi.framework"; version\="1.2.1", org.osgi.service.startlevel; uses\:\="org.osgi.framework"; version\="1.1.1", org.osgi.service.url; version\="1.0.1", org.osgi.service.resolver; uses\:\="org.osgi.resource"; version\="1.1.1", org.osgi.util.tracker; uses\:\="org.osgi.framework"; version\="1.5.3", org.osgi.dto; version\="1.1.1", org.osgi.service.condition; version\="1.0.0", javax.security.auth.callback; version\="0.0.0.JavaSE_017", javax.security.auth.login; version\="0.0.0.JavaSE_017", javax.net.ssl; version\="0.0.0.JavaSE_017", java.lang.constant; version\="0.0.0.JavaSE_017", java.security.interfaces; version\="0.0.0.JavaSE_017", java.text.spi; version\="0.0.0.JavaSE_017", java.nio.channels.spi; version\="0.0.0.JavaSE_017", java.math; version\="0.0.0.JavaSE_017", java.nio.file; version\="0.0.0.JavaSE_017", java.util.concurrent.atomic; version\="0.0.0.JavaSE_017", java.security.cert; version\="0.0.0.JavaSE_017", java.security.spec; version\="0.0.0.JavaSE_017", java.nio.channels; version\="0.0.0.JavaSE_017", java.time.chrono; version\="0.0.0.JavaSE_017", javax.crypto; version\="0.0.0.JavaSE_017", java.time.zone; version\="0.0.0.JavaSE_017", java.nio.charset; version\="0.0.0.JavaSE_017", java.io; version\="0.0.0.JavaSE_017", java.util.spi; version\="0.0.0.JavaSE_017", java.net; version\="0.0.0.JavaSE_017", javax.security.cert; version\="0.0.0.JavaSE_017", java.lang.annotation; version\="0.0.0.JavaSE_017", javax.security.auth.spi; version\="0.0.0.JavaSE_017", java.util.concurrent; version\="0.0.0.JavaSE_017", java.nio.charset.spi; version\="0.0.0.JavaSE_017", javax.crypto.interfaces; version\="0.0.0.JavaSE_017", java.util; version\="0.0.0.JavaSE_017", java.security; version\="0.0.0.JavaSE_017", java.nio.file.spi; version\="0.0.0.JavaSE_017", java.nio; version\="0.0.0.JavaSE_017", java.util.jar; version\="0.0.0.JavaSE_017", javax.security.auth; version\="0.0.0.JavaSE_017", java.lang.ref; version\="0.0.0.JavaSE_017", java.util.regex; version\="0.0.0.JavaSE_017", java.net.spi; version\="0.0.0.JavaSE_017", java.lang.module; version\="0.0.0.JavaSE_017", java.lang.invoke; version\="0.0.0.JavaSE_017", java.time.format; version\="0.0.0.JavaSE_017", java.util.concurrent.locks; version\="0.0.0.JavaSE_017", java.time.temporal; version\="0.0.0.JavaSE_017", java.util.zip; version\="0.0.0.JavaSE_017", java.nio.file.attribute; version\="0.0.0.JavaSE_017", java.util.random; version\="0.0.0.JavaSE_017", java.text; version\="0.0.0.JavaSE_017", javax.crypto.spec; version\="0.0.0.JavaSE_017", java.util.stream; version\="0.0.0.JavaSE_017", java.time; version\="0.0.0.JavaSE_017", java.lang; version\="0.0.0.JavaSE_017", java.lang.runtime; version\="0.0.0.JavaSE_017", java.util.function; version\="0.0.0.JavaSE_017", javax.net; version\="0.0.0.JavaSE_017", javax.security.auth.x500; version\="0.0.0.JavaSE_017", java.lang.reflect; version\="0.0.0.JavaSE_017", javax.lang.model; version\="0.0.0.JavaSE_017", javax.annotation.processing; version\="0.0.0.JavaSE_017", javax.lang.model.element; version\="0.0.0.JavaSE_017", javax.tools; version\="0.0.0.JavaSE_017", javax.lang.model.type; version\="0.0.0.JavaSE_017", javax.lang.model.util; version\="0.0.0.JavaSE_017", java.awt.datatransfer; version\="0.0.0.JavaSE_017", java.awt.desktop; version\="0.0.0.JavaSE_017", javax.swing.plaf.synth; version\="0.0.0.JavaSE_017", java.beans; version\="0.0.0.JavaSE_017", javax.swing.text.html.parser; version\="0.0.0.JavaSE_017", javax.swing.text.rtf; version\="0.0.0.JavaSE_017", java.awt.font; version\="0.0.0.JavaSE_017", javax.imageio; version\="0.0.0.JavaSE_017", java.awt.im.spi; version\="0.0.0.JavaSE_017", java.applet; version\="0.0.0.JavaSE_017", javax.sound.midi; version\="0.0.0.JavaSE_017", java.awt.dnd; version\="0.0.0.JavaSE_017", javax.swing.text; version\="0.0.0.JavaSE_017", javax.swing.plaf.basic; version\="0.0.0.JavaSE_017", javax.swing.undo; version\="0.0.0.JavaSE_017", javax.swing.plaf; version\="0.0.0.JavaSE_017", javax.swing.filechooser; version\="0.0.0.JavaSE_017", javax.imageio.event; version\="0.0.0.JavaSE_017", javax.sound.sampled; version\="0.0.0.JavaSE_017", javax.print.attribute; version\="0.0.0.JavaSE_017", javax.print; version\="0.0.0.JavaSE_017", javax.swing.plaf.nimbus; version\="0.0.0.JavaSE_017", javax.accessibility; version\="0.0.0.JavaSE_017", java.awt.event; version\="0.0.0.JavaSE_017", javax.swing.text.html; version\="0.0.0.JavaSE_017", javax.imageio.spi; version\="0.0.0.JavaSE_017", javax.swing.border; version\="0.0.0.JavaSE_017", javax.sound.sampled.spi; version\="0.0.0.JavaSE_017", javax.imageio.plugins.bmp; version\="0.0.0.JavaSE_017", java.awt.color; version\="0.0.0.JavaSE_017", java.awt.geom; version\="0.0.0.JavaSE_017", javax.imageio.plugins.jpeg; version\="0.0.0.JavaSE_017", javax.swing.tree; version\="0.0.0.JavaSE_017", javax.imageio.plugins.tiff; version\="0.0.0.JavaSE_017", java.awt.print; version\="0.0.0.JavaSE_017", java.awt.image; version\="0.0.0.JavaSE_017", javax.imageio.metadata; version\="0.0.0.JavaSE_017", javax.swing.table; version\="0.0.0.JavaSE_017", javax.sound.midi.spi; version\="0.0.0.JavaSE_017", javax.print.attribute.standard; version\="0.0.0.JavaSE_017", javax.swing.colorchooser; version\="0.0.0.JavaSE_017", javax.swing; version\="0.0.0.JavaSE_017", java.awt.image.renderable; version\="0.0.0.JavaSE_017", javax.swing.plaf.multi; version\="0.0.0.JavaSE_017", java.awt.im; version\="0.0.0.JavaSE_017", javax.print.event; version\="0.0.0.JavaSE_017", javax.swing.plaf.metal; version\="0.0.0.JavaSE_017", java.beans.beancontext; version\="0.0.0.JavaSE_017", java.awt; version\="0.0.0.JavaSE_017", javax.imageio.stream; version\="0.0.0.JavaSE_017", javax.swing.event; version\="0.0.0.JavaSE_017", java.lang.instrument; version\="0.0.0.JavaSE_017", java.util.logging; version\="0.0.0.JavaSE_017", javax.management.timer; version\="0.0.0.JavaSE_017", javax.management; version\="0.0.0.JavaSE_017", javax.management.relation; version\="0.0.0.JavaSE_017", javax.management.loading; version\="0.0.0.JavaSE_017", javax.management.openmbean; version\="0.0.0.JavaSE_017", java.lang.management; version\="0.0.0.JavaSE_017", javax.management.remote; version\="0.0.0.JavaSE_017", javax.management.monitor; version\="0.0.0.JavaSE_017", javax.management.modelmbean; version\="0.0.0.JavaSE_017", javax.management.remote.rmi; version\="0.0.0.JavaSE_017", javax.naming.event; version\="0.0.0.JavaSE_017", javax.naming.ldap.spi; version\="0.0.0.JavaSE_017", javax.naming; version\="0.0.0.JavaSE_017", javax.naming.spi; version\="0.0.0.JavaSE_017", javax.naming.ldap; version\="0.0.0.JavaSE_017", javax.naming.directory; version\="0.0.0.JavaSE_017", java.net.http; version\="0.0.0.JavaSE_017", java.util.prefs; version\="0.0.0.JavaSE_017", java.rmi.registry; version\="0.0.0.JavaSE_017", javax.rmi.ssl; version\="0.0.0.JavaSE_017", java.rmi.dgc; version\="0.0.0.JavaSE_017", java.rmi; version\="0.0.0.JavaSE_017", java.rmi.server; version\="0.0.0.JavaSE_017", javax.script; version\="0.0.0.JavaSE_017", org.ietf.jgss; version\="0.0.0.JavaSE_017", javax.security.auth.kerberos; version\="0.0.0.JavaSE_017", javax.security.sasl; version\="0.0.0.JavaSE_017", javax.smartcardio; version\="0.0.0.JavaSE_017", java.sql; version\="0.0.0.JavaSE_017", javax.sql; version\="0.0.0.JavaSE_017", javax.sql.rowset.spi; version\="0.0.0.JavaSE_017", javax.sql.rowset.serial; version\="0.0.0.JavaSE_017", javax.sql.rowset; version\="0.0.0.JavaSE_017", javax.transaction.xa; version\="0.0.0.JavaSE_017", javax.xml; version\="0.0.0.JavaSE_017", javax.xml.transform.sax; version\="0.0.0.JavaSE_017", javax.xml.datatype; version\="0.0.0.JavaSE_017", javax.xml.catalog; version\="0.0.0.JavaSE_017", org.w3c.dom.traversal; version\="0.0.0.JavaSE_017", javax.xml.stream.events; version\="0.0.0.JavaSE_017", javax.xml.stream; version\="0.0.0.JavaSE_017", org.xml.sax; version\="0.0.0.JavaSE_017", javax.xml.transform; version\="0.0.0.JavaSE_017", javax.xml.xpath; version\="0.0.0.JavaSE_017", org.w3c.dom.events; version\="0.0.0.JavaSE_017", org.w3c.dom.ranges; version\="0.0.0.JavaSE_017", org.w3c.dom.ls; version\="0.0.0.JavaSE_017", javax.xml.stream.util; version\="0.0.0.JavaSE_017", javax.xml.namespace; version\="0.0.0.JavaSE_017", javax.xml.transform.stax; version\="0.0.0.JavaSE_017", org.xml.sax.helpers; version\="0.0.0.JavaSE_017", org.w3c.dom.views; version\="0.0.0.JavaSE_017", org.w3c.dom.bootstrap; version\="0.0.0.JavaSE_017", org.w3c.dom; version\="0.0.0.JavaSE_017", javax.xml.transform.stream; version\="0.0.0.JavaSE_017", javax.xml.transform.dom; version\="0.0.0.JavaSE_017", javax.xml.validation; version\="0.0.0.JavaSE_017", javax.xml.parsers; version\="0.0.0.JavaSE_017", org.xml.sax.ext; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.spec; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.keyinfo; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig; version\="0.0.0.JavaSE_017", com.sun.java.accessibility.util; version\="0.0.0.JavaSE_017", com.sun.tools.attach.spi; version\="0.0.0.JavaSE_017", com.sun.tools.attach; version\="0.0.0.JavaSE_017", com.sun.source.doctree; version\="0.0.0.JavaSE_017", com.sun.source.tree; version\="0.0.0.JavaSE_017", com.sun.source.util; version\="0.0.0.JavaSE_017", com.sun.tools.javac; version\="0.0.0.JavaSE_017", jdk.dynalink.beans; version\="0.0.0.JavaSE_017", jdk.dynalink.linker.support; version\="0.0.0.JavaSE_017", jdk.dynalink.support; version\="0.0.0.JavaSE_017", jdk.dynalink; version\="0.0.0.JavaSE_017", jdk.dynalink.linker; version\="0.0.0.JavaSE_017", com.sun.net.httpserver; version\="0.0.0.JavaSE_017", com.sun.net.httpserver.spi; version\="0.0.0.JavaSE_017", com.sun.jarsigner; version\="0.0.0.JavaSE_017", jdk.security.jarsigner; version\="0.0.0.JavaSE_017", jdk.javadoc.doclet; version\="0.0.0.JavaSE_017", com.sun.tools.jconsole; version\="0.0.0.JavaSE_017", com.sun.jdi.connect; version\="0.0.0.JavaSE_017", com.sun.jdi.event; version\="0.0.0.JavaSE_017", com.sun.jdi.connect.spi; version\="0.0.0.JavaSE_017", com.sun.jdi; version\="0.0.0.JavaSE_017", com.sun.jdi.request; version\="0.0.0.JavaSE_017", jdk.jfr.consumer; version\="0.0.0.JavaSE_017", jdk.jfr; version\="0.0.0.JavaSE_017", jdk.jshell; version\="0.0.0.JavaSE_017", jdk.jshell.execution; version\="0.0.0.JavaSE_017", jdk.jshell.spi; version\="0.0.0.JavaSE_017", jdk.jshell.tool; version\="0.0.0.JavaSE_017", netscape.javascript; version\="0.0.0.JavaSE_017", com.sun.management; version\="0.0.0.JavaSE_017", jdk.management.jfr; version\="0.0.0.JavaSE_017", jdk.net; version\="0.0.0.JavaSE_017", jdk.nio; version\="0.0.0.JavaSE_017", jdk.nio.mapmode; version\="0.0.0.JavaSE_017", com.sun.nio.sctp; version\="0.0.0.JavaSE_017", com.sun.security.auth.module; version\="0.0.0.JavaSE_017", com.sun.security.auth.login; version\="0.0.0.JavaSE_017", com.sun.security.auth; version\="0.0.0.JavaSE_017", com.sun.security.auth.callback; version\="0.0.0.JavaSE_017", com.sun.security.jgss; version\="0.0.0.JavaSE_017", sun.reflect; version\="0.0.0.JavaSE_017", sun.misc; version\="0.0.0.JavaSE_017", com.sun.nio.file; version\="0.0.0.JavaSE_017", jdk.swing.interop; version\="0.0.0.JavaSE_017", org.w3c.dom.stylesheets; version\="0.0.0.JavaSE_017", org.w3c.dom.html; version\="0.0.0.JavaSE_017", org.w3c.dom.xpath; version\="0.0.0.JavaSE_017", org.w3c.dom.css; version\="0.0.0.JavaSE_017", com.yahoo.jdisc, com.yahoo.jdisc.application, com.yahoo.jdisc.handler, com.yahoo.jdisc.service, com.yahoo.jdisc.statistics, com.yahoo.jdisc.refcount, javax.inject;version\=1.0.0, org.aopalliance.intercept, org.aopalliance.aop, com.google.common.annotations;version\="27.1.0",com.google.common.base;version\="27.1.0",com.google.common.cache;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent",com.google.common.collect;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.escape;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.eventbus;version\="27.1.0",com.google.common.graph;version\="27.1.0";uses\:\="com.google.common.collect",com.google.common.hash;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.html;version\="27.1.0";uses\:\="com.google.common.escape",com.google.common.io;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.graph,com.google.common.hash",com.google.common.math;version\="27.1.0",com.google.common.net;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.escape",com.google.common.primitives;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.reflect;version\="27.1.0";uses\:\="com.google.common.collect,com.google.common.io",com.google.common.util.concurrent;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent.internal",com.google.common.xml;version\="27.1.0";uses\:\="com.google.common.escape", com.google.inject;version\="1.4",com.google.inject.binder;version\="1.4",com.google.inject.matcher;version\="1.4",com.google.inject.multibindings;version\="1.4",com.google.inject.name;version\="1.4",com.google.inject.spi;version\="1.4",com.google.inject.util;version\="1.4", org.slf4j;version\=1.7.32, org.slf4j.spi;version\=1.7.32, org.slf4j.helpers;version\=1.7.32, org.slf4j.event;version\=1.7.32, org.slf4j.impl;version\=1.7.32, org.apache.commons.logging;version\=1.2, org.apache.commons.logging.impl;version\=1.2, org.apache.log4j;version\=1.2.17,org.apache.log4j.helpers;version\=1.2.17,org.apache.log4j.spi;version\=1.2.17,org.apache.log4j.xml;version\=1.2.17, com.yahoo.component.annotation;version\="1.0.0", com.yahoo.config;version\=1.0.0, com.yahoo.vespa.defaults;version\=1.0.0, ai.vespa.http;version\=1.0.0,ai.vespa.validation;version\=1.0.0,com.yahoo.binaryprefix;version\=1.0.0,com.yahoo.collections;version\=1.0.0,com.yahoo.compress;version\=1.0.0,com.yahoo.concurrent.classlock;version\=1.0.0,com.yahoo.concurrent.maintenance;version\=1.0.0,com.yahoo.concurrent;version\=1.0.0,com.yahoo.data.access.simple;version\=1.0.0,com.yahoo.data.access.slime;version\=1.0.0,com.yahoo.data.access;version\=1.0.0,com.yahoo.errorhandling;version\=1.0.0,com.yahoo.exception;version\=1.0.0,com.yahoo.geo;version\=1.0.0,com.yahoo.io.reader;version\=1.0.0,com.yahoo.io;version\=1.0.0,com.yahoo.javacc;version\=1.0.0,com.yahoo.lang;version\=1.0.0,com.yahoo.nativec;version\=1.0.0,com.yahoo.net;version\=1.0.0,com.yahoo.path;version\=1.0.0,com.yahoo.protect;version\=1.0.0,com.yahoo.reflection;version\=1.0.0,com.yahoo.slime;version\=1.0.0,com.yahoo.stream;version\=1.0.0,com.yahoo.system.execution;version\=1.0.0,com.yahoo.system;version\=1.0.0,com.yahoo.tensor.evaluation;version\=1.0.0,com.yahoo.tensor.functions;version\=1.0.0,com.yahoo.tensor.serialization;version\=1.0.0,com.yahoo.tensor;version\=1.0.0,com.yahoo.text.internal;version\=1.0.0,com.yahoo.text;version\=1.0.0,com.yahoo.time;version\=1.0.0,com.yahoo.transaction;version\=1.0.0,com.yahoo.vespa.objects;version\=1.0.0,com.yahoo.yolean.chain;version\=1.0.0,com.yahoo.yolean.concurrent;version\=1.0.0,com.yahoo.yolean.function;version\=1.0.0,com.yahoo.yolean.system;version\=1.0.0,com.yahoo.yolean.trace;version\=1.0.0,com.yahoo.yolean;version\=1.0.0, com.yahoo.log.event;version\=1.0.0,com.yahoo.log.impl;version\=1.0.0,com.yahoo.log;version\=1.0.0, javax.xml.bind;version\="2.3";uses\:\="javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.annotation;version\="2.3";uses\:\="javax.xml.bind,javax.xml.parsers,javax.xml.transform,javax.xml.transform.dom,org.w3c.dom",javax.xml.bind.annotation.adapters;version\="2.3",javax.xml.bind.attachment;version\="2.3";uses\:\="javax.activation",javax.xml.bind.helpers;version\="2.3";uses\:\="javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.util;version\="2.3";uses\:\="javax.xml.bind,javax.xml.transform.sax", com.sun.istack;version\="3.0.5";uses\:\="javax.activation,javax.xml.stream,org.xml.sax,org.xml.sax.helpers",com.sun.istack.localization;version\="3.0.5",com.sun.istack.logging;version\="3.0.5",com.sun.xml.bind;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.annotation;version\="2.3.0",com.sun.xml.bind.api;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.api.impl;version\="2.3.0",com.sun.xml.bind.marshaller;uses\:\="javax.xml.parsers,org.w3c.dom,org.xml.sax,org.xml.sax.helpers";version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;version\="2.3.0",com.sun.xml.bind.v2;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.core;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.impl,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.namespace,javax.xml.transform";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.nav";version\="2.3.0",com.sun.xml.bind.v2.model.nav;uses\:\="com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.util;uses\:\="javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.xml.bind.v2.model.annotation,javax.activation,javax.xml.bind,javax.xml.bind.annotation.adapters";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen.episode;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="javax.xml.parsers,javax.xml.transform,javax.xml.validation,javax.xml.xpath";version\="2.3.0",com.sun.xml.txw2;uses\:\="com.sun.xml.txw2.output,javax.xml.namespace";version\="2.3.0",com.sun.xml.txw2.annotation;version\="2.3.0",com.sun.xml.txw2.output;uses\:\="com.sun.xml.txw2,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.dom,javax.xml.transform.sax,javax.xml.transform.stream,org.w3c.dom,org.xml.sax,org.xml.sax.ext,org.xml.sax.helpers";version\="2.3.0", com.sun.xml.bind;uses\:\="com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.datatype,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.api;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.xml.bind,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.marshaller;version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;uses\:\="com.sun.xml.bind,javax.xml.bind.helpers,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,javax.xml.bind";version\="2.3.0",com.sun.xml.bind.v2.bytecode;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.model.runtime;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.namespace,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.istack,com.sun.xml.bind.api,com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.property,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.output;uses\:\="com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.runtime,com.sun.xml.fastinfoset.stax,javax.xml.stream,org.jvnet.staxex,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.property;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,com.sun.xml.bind.v2.runtime.unmarshaller,com.sun.xml.bind.v2.util,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect.opt;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="com.sun.xml.bind,com.sun.xml.bind.api,com.sun.xml.bind.unmarshaller,com.sun.xml.bind.util,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.reflect,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.txw2.output,javax.xml.bind,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.schemagen.xmlschema;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.namespace,javax.xml.transform.stream,org.xml.sax";version\="2.3.0", javax.activation;uses\:\="com.sun.activation.registries";version\="1.2",com.sun.activation.viewers;uses\:\="javax.activation";version\="1.2.0",com.sun.activation.registries;version\="1.2.0" -- cgit v1.2.3 From d1517e7985ad819e43f00e4ed1977d813c8b19a3 Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Tue, 28 Feb 2023 14:25:41 +0100 Subject: Add comment about updating `ExportPackagesIT` --- jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java | 1 + 1 file changed, 1 insertion(+) diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java index e6c273fafde..1ae25aa0cfa 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java @@ -45,6 +45,7 @@ public class ExportPackages { } } + // Make sure to update the junit integration test `ExportPackagesIT.java` if the set of exported packages is modified. private static String getExportPackages(String[] jars) throws IOException { StringBuilder out = new StringBuilder(); out.append(getSystemPackages()).append(", ") -- cgit v1.2.3 From 226bfb610c94fc795c95882ee3b48e2556019bcb Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Tue, 28 Feb 2023 14:26:33 +0100 Subject: Set last modified timestamp for file reference to be used Avoid file references being deleted by maintainer after having checked that file reference already exists --- .../com/yahoo/vespa/config/server/filedistribution/FileDirectory.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index 85b30f4d303..454ab9e490d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -23,6 +23,7 @@ import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; +import java.time.Clock; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.logging.Level; @@ -153,6 +154,8 @@ public class FileDirectory extends AbstractComponent { return true; } + // update last modified time so that maintainer deleting unused file references considers this as recently used + existingFile.setLastModified(Clock.systemUTC().instant().toEpochMilli()); log.log(Level.FINE, "Directory for file reference '" + fileReference.value() + "' already exists and has all content"); return false; } -- cgit v1.2.3 From aa8c6839a9f9c50eeaee6a5c31c380abcc753f3a Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Tue, 28 Feb 2023 13:31:10 +0000 Subject: Unify on steady time --- metrics/src/tests/metricmanagertest.cpp | 53 ++++++++++------------ metrics/src/tests/snapshottest.cpp | 14 +++--- metrics/src/vespa/metrics/metricmanager.cpp | 21 +++++---- metrics/src/vespa/metrics/metricmanager.h | 4 +- storage/src/tests/common/metricstest.cpp | 2 +- .../src/tests/storageserver/statereportertest.cpp | 2 +- 6 files changed, 46 insertions(+), 50 deletions(-) diff --git a/metrics/src/tests/metricmanagertest.cpp b/metrics/src/tests/metricmanagertest.cpp index 1ba1cad9463..6d6f21ea7b0 100644 --- a/metrics/src/tests/metricmanagertest.cpp +++ b/metrics/src/tests/metricmanagertest.cpp @@ -364,10 +364,10 @@ class FakeTimer : public MetricManager::Timer { std::atomic _time; public: FakeTimer(time_t startTime = 0) : _time(startTime) {} - time_t getTime() const override { return load_relaxed(_time); } + time_point getTime() const override { return time_point(vespalib::from_s(load_relaxed(_time))); } void set_time(time_t t) noexcept { store_relaxed(_time, t); } // Not safe for multiple writers, only expected to be called by test. - void add_time(time_t t) noexcept { set_time(getTime() + t); } + void add_time(time_t t) noexcept { set_time(load_relaxed(_time) + t); } }; struct BriefValuePrinter : public MetricVisitor { @@ -556,8 +556,8 @@ TEST_F(MetricManagerTest, test_snapshots) TEST_F(MetricManagerTest, test_json_output) { - FakeTimer* timer = new FakeTimer(1000); - std::unique_ptr timerImpl(timer); + auto timerImpl = std::make_unique(1000); + FakeTimer & timer = *timerImpl; MetricManager mm(std::move(timerImpl)); TestMetricSet mySet; { @@ -582,7 +582,7 @@ TEST_F(MetricManagerTest, test_json_output) mySet.val10.a.val2.addValue(2); mySet.val10.b.val1.addValue(1); - timer->set_time(1300); + timer.set_time(1300); takeSnapshots(mm, 1300); // Create json output @@ -879,9 +879,7 @@ TEST_F(MetricManagerTest, json_output_can_have_multiple_sets_with_same_name) TEST_F(MetricManagerTest, test_text_output) { - FakeTimer* timer = new FakeTimer(1000); - std::unique_ptr timerImpl(timer); - MetricManager mm(std::move(timerImpl)); + MetricManager mm(std::make_unique(1000)); TestMetricSet mySet; { MetricLockGuard lockGuard(mm.getMetricLock()); @@ -954,10 +952,7 @@ namespace { std::mutex& _output_mutex; FakeTimer& _timer; - MyUpdateHook(std::ostringstream& output, - std::mutex& output_mutex, - const char* name, - FakeTimer& timer) + MyUpdateHook(std::ostringstream& output, std::mutex& output_mutex, const char* name, FakeTimer& timer) : UpdateHook(name), _output(output), _output_mutex(output_mutex), @@ -967,7 +962,7 @@ namespace { void updateMetrics(const MetricLockGuard & ) override { std::lock_guard lock(_output_mutex); // updateMetrics() called from metric manager thread - _output << _timer.getTime() << ": " << getName() << " called\n"; + _output << vespalib::count_s(_timer.getTime().time_since_epoch()) << ": " << getName() << " called\n"; } }; } @@ -976,8 +971,8 @@ TEST_F(MetricManagerTest, test_update_hooks) { std::mutex output_mutex; std::ostringstream output; - FakeTimer* timer = new FakeTimer(1000); - std::unique_ptr timerImpl(timer); + auto timerImpl = std::make_unique(1000); + FakeTimer & timer = *timerImpl; // Add a metric set just so one exist TestMetricSet mySet; MetricManager mm(std::move(timerImpl)); @@ -986,9 +981,9 @@ TEST_F(MetricManagerTest, test_update_hooks) mm.registerMetric(lockGuard, mySet.set); } - MyUpdateHook preInitShort(output, output_mutex, "BIS", *timer); - MyUpdateHook preInitLong(output, output_mutex, "BIL", *timer); - MyUpdateHook preInitInfinite(output, output_mutex, "BII", *timer); + MyUpdateHook preInitShort(output, output_mutex, "BIS", timer); + MyUpdateHook preInitLong(output, output_mutex, "BIL", timer); + MyUpdateHook preInitInfinite(output, output_mutex, "BII", timer); mm.addMetricUpdateHook(preInitShort, 5); mm.addMetricUpdateHook(preInitLong, 300); mm.addMetricUpdateHook(preInitInfinite, 0); @@ -1007,55 +1002,55 @@ TEST_F(MetricManagerTest, test_update_hooks) "consumer[1].tags[0] snaptest\n")); output << "Init done\n"; - MyUpdateHook postInitShort(output, output_mutex, "AIS", *timer); - MyUpdateHook postInitLong(output, output_mutex, "AIL", *timer); - MyUpdateHook postInitInfinite(output, output_mutex, "AII", *timer); + MyUpdateHook postInitShort(output, output_mutex, "AIS", timer); + MyUpdateHook postInitLong(output, output_mutex, "AIL", timer); + MyUpdateHook postInitInfinite(output, output_mutex, "AII", timer); mm.addMetricUpdateHook(postInitShort, 5); mm.addMetricUpdateHook(postInitLong, 400); mm.addMetricUpdateHook(postInitInfinite, 0); // After 5 seconds the short ones should get another. - timer->set_time(1006); + timer.set_time(1006); waitForTimeProcessed(mm, 1006); // After 4 more seconds the short ones should get another // since last update was a second late. (Stable periods, process time // should not affect how often they are updated) - timer->set_time(1010); + timer.set_time(1010); waitForTimeProcessed(mm, 1010); // Bumping considerably ahead, such that next update is in the past, // we should only get one update called in this period. - timer->set_time(1200); + timer.set_time(1200); waitForTimeProcessed(mm, 1200); // No updates at this time. - timer->set_time(1204); + timer.set_time(1204); waitForTimeProcessed(mm, 1204); // Give all hooks an update mm.updateMetrics(true); // Last update should not have interfered with periods - timer->set_time(1205); + timer.set_time(1205); waitForTimeProcessed(mm, 1205); // Time is just ahead of a snapshot. - timer->set_time(1299); + timer.set_time(1299); waitForTimeProcessed(mm, 1299); // At time 1300 we are at a 5 minute snapshot bump // All hooks should thus get an update. The one with matching period // should only get one - timer->set_time(1300); + timer.set_time(1300); waitForTimeProcessed(mm, 1300); // The snapshot time currently doesn't count for the metric at period // 400. It will get an event at this time. - timer->set_time(1450); + timer.set_time(1450); waitForTimeProcessed(mm, 1450); std::string expected( diff --git a/metrics/src/tests/snapshottest.cpp b/metrics/src/tests/snapshottest.cpp index b4eb4a1353c..4d2ea96c36d 100644 --- a/metrics/src/tests/snapshottest.cpp +++ b/metrics/src/tests/snapshottest.cpp @@ -122,15 +122,15 @@ struct TestMetricSet : public MetricSet { SubMetricSet set2; SumMetric setSum; - TestMetricSet(vespalib::stringref name, MetricSet* owner = 0); + TestMetricSet(vespalib::stringref name); ~TestMetricSet(); void incValues(); }; -TestMetricSet::TestMetricSet(vespalib::stringref name, MetricSet* owner) - : MetricSet(name, {}, "", owner), +TestMetricSet::TestMetricSet(vespalib::stringref name) + : MetricSet(name, {}, "", nullptr), set1("set1", this), set2("set2", this), setSum("setSum", {}, "", this) @@ -149,7 +149,7 @@ TestMetricSet::incValues() { struct FakeTimer : public MetricManager::Timer { uint32_t _timeInSecs; FakeTimer() : _timeInSecs(1) {} - time_t getTime() const override { return _timeInSecs; } + time_point getTime() const override { return time_point(vespalib::from_s(_timeInSecs)); } }; void ASSERT_VALUE(int32_t value, const MetricSnapshot & snapshot, const char *name) __attribute__((noinline)); @@ -176,8 +176,7 @@ TEST_F(SnapshotTest, test_snapshot_two_days) TestMetricSet set("test"); FakeTimer* timer; - MetricManager mm( - std::unique_ptr(timer = new FakeTimer)); + MetricManager mm(std::unique_ptr(timer = new FakeTimer)); { MetricLockGuard lockGuard(mm.getMetricLock()); mm.registerMetric(lockGuard, set); @@ -215,10 +214,9 @@ TEST_F(SnapshotTest, test_snapshot_two_days) << "\n"; */ - const MetricSnapshot* snap = 0; // active snapshot MetricLockGuard lockGuard(mm.getMetricLock()); - snap = &mm.getActiveMetrics(lockGuard); + const MetricSnapshot* snap = &mm.getActiveMetrics(lockGuard); ASSERT_VALUE(0, *snap, "test.set1.set1.count1"); ASSERT_VALUE(0, *snap, "test.set1.set1.countSum"); diff --git a/metrics/src/vespa/metrics/metricmanager.cpp b/metrics/src/vespa/metrics/metricmanager.cpp index 9135dd86a16..edf909131a0 100644 --- a/metrics/src/vespa/metrics/metricmanager.cpp +++ b/metrics/src/vespa/metrics/metricmanager.cpp @@ -26,13 +26,15 @@ using vespalib::IllegalStateException; using vespalib::IllegalArgumentException; using vespalib::make_string_short::fmt; using vespalib::count_ms; +using vespalib::count_s; +using vespalib::from_s; MetricManager::ConsumerSpec::ConsumerSpec() = default; MetricManager::ConsumerSpec::~ConsumerSpec() = default; -time_t +time_point MetricManager::Timer::getTime() const { - return vespalib::count_s(vespalib::system_clock::now().time_since_epoch()); + return vespalib::steady_clock::now(); } void @@ -119,7 +121,7 @@ MetricManager::addMetricUpdateHook(UpdateHook& hook, uint32_t period) std::lock_guard sync(_waiter); // If we've already initialized manager, log period has been set. // In this case. Call first time after period - hook._nextCall = _timer->getTime() + period; + hook._nextCall = count_s(_timer->getTime().time_since_epoch()) + period; if (period == 0) { for (UpdateHook * sHook : _snapshotUpdateHooks) { if (sHook == &hook) { @@ -464,7 +466,7 @@ MetricManager::configure(const MetricLockGuard & , std::unique_ptr confi // Set up snapshots only first time. We don't allow live reconfig // of snapshot periods. - time_t currentTime(_timer->getTime()); + time_t currentTime = count_s(_timer->getTime().time_since_epoch()); _activeMetrics.setFromTime(currentTime); uint32_t count = 1; for (uint32_t i = 0; i< snapshotPeriods.size(); ++i) { @@ -731,7 +733,7 @@ MetricManager::run() // we constantly add next time to do something from the last timer. // For that to work, we need to initialize timers on first iteration // to set them to current time. - time_t currentTime = _timer->getTime(); + time_t currentTime = count_s(_timer->getTime().time_since_epoch()); for (auto & snapshot : _snapshots) { snapshot->setFromTime(currentTime); } @@ -741,12 +743,13 @@ MetricManager::run() // Ensure correct time for first snapshot _snapshots[0]->getSnapshot().setToTime(currentTime); while (!stop_requested()) { - currentTime = _timer->getTime(); + time_point now = _timer->getTime(); + currentTime = count_s(now.time_since_epoch()); time_t next = tick(sync, currentTime); if (currentTime < next) { - size_t ms = (next - currentTime) * 1000; - _cond.wait_for(sync, std::chrono::milliseconds(ms)); - _sleepTimes.addValue(ms); + vespalib::duration wait_time = from_s(next - currentTime); + _cond.wait_for(sync, wait_time); + _sleepTimes.addValue(count_ms(wait_time)); } else { _sleepTimes.addValue(0); } diff --git a/metrics/src/vespa/metrics/metricmanager.h b/metrics/src/vespa/metrics/metricmanager.h index 3c81d9abd6a..c3ce37b451f 100644 --- a/metrics/src/vespa/metrics/metricmanager.h +++ b/metrics/src/vespa/metrics/metricmanager.h @@ -67,8 +67,8 @@ public: struct Timer { virtual ~Timer() = default; - virtual time_t getTime() const; - time_point getTimeInMilliSecs() const { return time_point(vespalib::from_s(getTime())); } + virtual time_point getTime() const; + time_point getTimeInMilliSecs() const { return getTime(); } }; /** diff --git a/storage/src/tests/common/metricstest.cpp b/storage/src/tests/common/metricstest.cpp index 76c6a9cf991..60b2fa0de6c 100644 --- a/storage/src/tests/common/metricstest.cpp +++ b/storage/src/tests/common/metricstest.cpp @@ -52,7 +52,7 @@ namespace { { framework::Clock& _clock; explicit MetricClock(framework::Clock& c) : _clock(c) {} - [[nodiscard]] time_t getTime() const override { return vespalib::count_s(_clock.getMonotonicTime().time_since_epoch()); } + [[nodiscard]] metrics::time_point getTime() const override { return _clock.getMonotonicTime(); } }; } diff --git a/storage/src/tests/storageserver/statereportertest.cpp b/storage/src/tests/storageserver/statereportertest.cpp index 2570b5232eb..afa1b4a2c14 100644 --- a/storage/src/tests/storageserver/statereportertest.cpp +++ b/storage/src/tests/storageserver/statereportertest.cpp @@ -53,7 +53,7 @@ struct MetricClock : public metrics::MetricManager::Timer { framework::Clock& _clock; explicit MetricClock(framework::Clock& c) : _clock(c) {} - [[nodiscard]] time_t getTime() const override { return vespalib::count_s(_clock.getMonotonicTime().time_since_epoch()); } + [[nodiscard]] metrics::time_point getTime() const override { return _clock.getMonotonicTime(); } }; } -- cgit v1.2.3 From 19f42845eb32e76adc7a73edddd9bb6254309998 Mon Sep 17 00:00:00 2001 From: HÃ¥vard Pettersen Date: Tue, 28 Feb 2023 13:18:22 +0000 Subject: remove fastos --- CMakeLists.txt | 3 ++- config/CMakeLists.txt | 1 - configdefinitions/CMakeLists.txt | 1 - configutil/CMakeLists.txt | 1 - document/CMakeLists.txt | 1 - documentapi/CMakeLists.txt | 1 - fastos/.gitignore | 11 ----------- fastos/CMakeLists.txt | 7 ------- fastos/OWNERS | 2 -- fastos/README | 4 ---- fastos/src/.gitignore | 3 --- fastos/src/vespa/fastos/.gitignore | 28 ---------------------------- fastos/src/vespa/fastos/CMakeLists.txt | 15 --------------- fbench/CMakeLists.txt | 1 - fbench/src/httpclient/CMakeLists.txt | 1 - fbench/src/test/CMakeLists.txt | 1 - fileacquirer/CMakeLists.txt | 1 - jrt_test/CMakeLists.txt | 1 - logd/CMakeLists.txt | 1 - lowercasing_test/CMakeLists.txt | 1 - metrics/CMakeLists.txt | 1 - persistence/CMakeLists.txt | 1 - searchcore/CMakeLists.txt | 1 - searchlib/CMakeLists.txt | 1 - storage/CMakeLists.txt | 1 - storageserver/CMakeLists.txt | 1 - streamingvisitors/CMakeLists.txt | 1 - vbench/CMakeLists.txt | 1 - vdslib/CMakeLists.txt | 1 - vdstestlib/CMakeLists.txt | 1 - vespaclient/CMakeLists.txt | 1 - vespalib/CMakeLists.txt | 1 - vespalog/src/test/threads/CMakeLists.txt | 1 - vespalog/src/vespa/log/CMakeLists.txt | 1 + vespamalloc/src/tests/test1/CMakeLists.txt | 3 +++ 35 files changed, 6 insertions(+), 96 deletions(-) delete mode 100644 fastos/.gitignore delete mode 100644 fastos/CMakeLists.txt delete mode 100644 fastos/OWNERS delete mode 100644 fastos/README delete mode 100644 fastos/src/.gitignore delete mode 100644 fastos/src/vespa/fastos/.gitignore delete mode 100644 fastos/src/vespa/fastos/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 5463c4215a0..ce11196725f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,8 @@ vespa_use_default_cmake_prefix_path() SET(CMAKE_FIND_PACKAGE_SORT_ORDER NATURAL) SET(CMAKE_FIND_PACKAGE_SORT_DIRECTION DEC) +find_package(Threads REQUIRED) + find_package(LLVM REQUIRED CONFIG) message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") @@ -99,7 +101,6 @@ add_subdirectory(docprocs) add_subdirectory(document) add_subdirectory(documentapi) add_subdirectory(eval) -add_subdirectory(fastos) add_subdirectory(fbench) add_subdirectory(fileacquirer) add_subdirectory(filedistribution) diff --git a/config/CMakeLists.txt b/config/CMakeLists.txt index 1c04aa3eaa8..561ddbc078c 100644 --- a/config/CMakeLists.txt +++ b/config/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalib vespalog fnet diff --git a/configdefinitions/CMakeLists.txt b/configdefinitions/CMakeLists.txt index c374e93904e..80b53d1dc0a 100644 --- a/configdefinitions/CMakeLists.txt +++ b/configdefinitions/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalib config_cloudconfig diff --git a/configutil/CMakeLists.txt b/configutil/CMakeLists.txt index 4ec9ad13648..a1d699ae009 100644 --- a/configutil/CMakeLists.txt +++ b/configutil/CMakeLists.txt @@ -2,7 +2,6 @@ vespa_define_module( DEPENDS vespadefaults - fastos config_cloudconfig vbench vespalib diff --git a/document/CMakeLists.txt b/document/CMakeLists.txt index 88dbe2816d9..e1e4d8ff5cc 100644 --- a/document/CMakeLists.txt +++ b/document/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib config_cloudconfig diff --git a/documentapi/CMakeLists.txt b/documentapi/CMakeLists.txt index 9261bcf9114..beeda4afeb4 100644 --- a/documentapi/CMakeLists.txt +++ b/documentapi/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog config_cloudconfig vespalib diff --git a/fastos/.gitignore b/fastos/.gitignore deleted file mode 100644 index 54e2680a6d8..00000000000 --- a/fastos/.gitignore +++ /dev/null @@ -1,11 +0,0 @@ -*.ilk -*.pdb -.Build_completed -.Dist_completed -.Install_completed -.PreBuild_completed -bin -include -lib -update.log -Makefile diff --git a/fastos/CMakeLists.txt b/fastos/CMakeLists.txt deleted file mode 100644 index 60813c569e4..00000000000 --- a/fastos/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_define_module( - LIBS - src/vespa/fastos - - TESTS -) diff --git a/fastos/OWNERS b/fastos/OWNERS deleted file mode 100644 index 912f61c59b8..00000000000 --- a/fastos/OWNERS +++ /dev/null @@ -1,2 +0,0 @@ -baldersheim -arnej27959 diff --git a/fastos/README b/fastos/README deleted file mode 100644 index ed9afabffb8..00000000000 --- a/fastos/README +++ /dev/null @@ -1,4 +0,0 @@ -Old OS abstraction layer - -obsolete, to be replaced with implementations using -standard C++14 threads and newer unix networking APIs diff --git a/fastos/src/.gitignore b/fastos/src/.gitignore deleted file mode 100644 index 2e8e6fd906a..00000000000 --- a/fastos/src/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/Makefile.ini -/config_command.sh -/project.dsw diff --git a/fastos/src/vespa/fastos/.gitignore b/fastos/src/vespa/fastos/.gitignore deleted file mode 100644 index 004799df5b4..00000000000 --- a/fastos/src/vespa/fastos/.gitignore +++ /dev/null @@ -1,28 +0,0 @@ -*.So -*.core -*.exe -*.ilk -*.pdb -.depend -.depend.NEW -Debug -Makefile -Makefile.factory -Makefile.overrides -Makefile.pre -Release -autoconf.h -config_command.bat -config_command.sh -fastconfig -fastconfig.exe -fastos.lib -fastosconfig.h -libfastos.a -makefeatures -makemake -processtest.log -test.txt -vc60.idb -vc60.pdb -/libfastos.so.5.1 diff --git a/fastos/src/vespa/fastos/CMakeLists.txt b/fastos/src/vespa/fastos/CMakeLists.txt deleted file mode 100644 index 29810a4f296..00000000000 --- a/fastos/src/vespa/fastos/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(fastos_objects OBJECT - SOURCES -) - -vespa_add_library(fastos - SOURCES - $ - INSTALL lib64 - DEPENDS - ${CMAKE_DL_LIBS} -) - -find_package(Threads REQUIRED) -target_link_libraries(fastos PUBLIC ${CMAKE_THREAD_LIBS_INIT}) diff --git a/fbench/CMakeLists.txt b/fbench/CMakeLists.txt index ff287d221ec..3f1d78a66a0 100644 --- a/fbench/CMakeLists.txt +++ b/fbench/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalib APPS diff --git a/fbench/src/httpclient/CMakeLists.txt b/fbench/src/httpclient/CMakeLists.txt index 163a68f9c98..2d5e3b32437 100644 --- a/fbench/src/httpclient/CMakeLists.txt +++ b/fbench/src/httpclient/CMakeLists.txt @@ -4,5 +4,4 @@ vespa_add_library(fbench_httpclient STATIC httpclient.cpp DEPENDS fbench_util - fastos ) diff --git a/fbench/src/test/CMakeLists.txt b/fbench/src/test/CMakeLists.txt index d13b6b82a81..c81d818ed06 100644 --- a/fbench/src/test/CMakeLists.txt +++ b/fbench/src/test/CMakeLists.txt @@ -26,6 +26,5 @@ vespa_add_executable(fbench_clientstatus_app TEST clientstatus.cpp DEPENDS fbench_util - fastos ) vespa_add_test(NAME fbench_clientstatus_app COMMAND fbench_clientstatus_app) diff --git a/fileacquirer/CMakeLists.txt b/fileacquirer/CMakeLists.txt index 13150f58ba3..cc18dc2bd84 100644 --- a/fileacquirer/CMakeLists.txt +++ b/fileacquirer/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib config_cloudconfig diff --git a/jrt_test/CMakeLists.txt b/jrt_test/CMakeLists.txt index ea8c8b94faa..a678cbf112a 100644 --- a/jrt_test/CMakeLists.txt +++ b/jrt_test/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib fnet diff --git a/logd/CMakeLists.txt b/logd/CMakeLists.txt index d02b99a393a..5823ebf54a7 100644 --- a/logd/CMakeLists.txt +++ b/logd/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib config_cloudconfig diff --git a/lowercasing_test/CMakeLists.txt b/lowercasing_test/CMakeLists.txt index b08a6ef350d..119209d4227 100644 --- a/lowercasing_test/CMakeLists.txt +++ b/lowercasing_test/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib searchlib diff --git a/metrics/CMakeLists.txt b/metrics/CMakeLists.txt index d75c17d4a00..5b95d8635d4 100644 --- a/metrics/CMakeLists.txt +++ b/metrics/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib config_cloudconfig diff --git a/persistence/CMakeLists.txt b/persistence/CMakeLists.txt index d87f172bfb5..072e273338b 100644 --- a/persistence/CMakeLists.txt +++ b/persistence/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib document diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index d6c353e46c9..131460b0384 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos fnet vespalog vespalib diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index 4ccab1a9380..44051a96578 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib vespaeval diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt index 4b7c12b0f31..1062a89f055 100644 --- a/storage/CMakeLists.txt +++ b/storage/CMakeLists.txt @@ -2,7 +2,6 @@ vespa_define_module( DEPENDS vespadefaults - fastos metrics config_cloudconfig configdefinitions diff --git a/storageserver/CMakeLists.txt b/storageserver/CMakeLists.txt index ee3335c4921..a2f9d0b776e 100644 --- a/storageserver/CMakeLists.txt +++ b/storageserver/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos storage streamingvisitors diff --git a/streamingvisitors/CMakeLists.txt b/streamingvisitors/CMakeLists.txt index 2c7f01ddf37..fede7087d8d 100644 --- a/streamingvisitors/CMakeLists.txt +++ b/streamingvisitors/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog storage config_cloudconfig diff --git a/vbench/CMakeLists.txt b/vbench/CMakeLists.txt index 3fb5df8cd20..e78913262be 100644 --- a/vbench/CMakeLists.txt +++ b/vbench/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalib LIBS diff --git a/vdslib/CMakeLists.txt b/vdslib/CMakeLists.txt index 0f8144b99e9..1276323f83b 100644 --- a/vdslib/CMakeLists.txt +++ b/vdslib/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib config_cloudconfig diff --git a/vdstestlib/CMakeLists.txt b/vdstestlib/CMakeLists.txt index d0a921672c2..7f478989332 100644 --- a/vdstestlib/CMakeLists.txt +++ b/vdstestlib/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalib TESTS diff --git a/vespaclient/CMakeLists.txt b/vespaclient/CMakeLists.txt index 912b35fa763..6e82b83517e 100644 --- a/vespaclient/CMakeLists.txt +++ b/vespaclient/CMakeLists.txt @@ -2,7 +2,6 @@ vespa_define_module( DEPENDS vespadefaults - fastos configdefinitions config_cloudconfig vespalog diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 8f1b687449a..8175e75875a 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -6,7 +6,6 @@ endif() vespa_define_module( DEPENDS - fastos vespalog EXTERNAL_DEPENDS diff --git a/vespalog/src/test/threads/CMakeLists.txt b/vespalog/src/test/threads/CMakeLists.txt index 00de16ff005..19fe2511025 100644 --- a/vespalog/src/test/threads/CMakeLists.txt +++ b/vespalog/src/test/threads/CMakeLists.txt @@ -4,6 +4,5 @@ vespa_add_executable(vespalog_threads_test_app TEST testthreads.cpp DEPENDS vespalog - fastos ) vespa_add_test(NAME vespalog_threads_test_app COMMAND vespalog_threads_test_app vespa.log ENVIRONMENT "VESPA_LOG_TARGET=file:vespa.log") diff --git a/vespalog/src/vespa/log/CMakeLists.txt b/vespalog/src/vespa/log/CMakeLists.txt index dbeba17334b..82536b1f4e9 100644 --- a/vespalog/src/vespa/log/CMakeLists.txt +++ b/vespalog/src/vespa/log/CMakeLists.txt @@ -18,3 +18,4 @@ vespa_add_library(vespalog reject-filter.cpp INSTALL lib64 ) +target_link_libraries(vespalog PUBLIC ${CMAKE_THREAD_LIBS_INIT}) diff --git a/vespamalloc/src/tests/test1/CMakeLists.txt b/vespamalloc/src/tests/test1/CMakeLists.txt index dd7c92a4dac..f3ce9f70f45 100644 --- a/vespamalloc/src/tests/test1/CMakeLists.txt +++ b/vespamalloc/src/tests/test1/CMakeLists.txt @@ -8,12 +8,15 @@ vespa_add_executable(vespamalloc_testatomic_app TEST vespamalloc_util EXTERNAL_DEPENDS ${VESPA_ATOMIC_LIB} + dl ) vespa_add_test(NAME vespamalloc_testatomic_app NO_VALGRIND COMMAND vespamalloc_testatomic_app) vespa_add_executable(vespamalloc_new_test_app TEST SOURCES new_test.cpp + EXTERNAL_DEPENDS + dl ) vespa_add_test(NAME vespamalloc_new_test_app NO_VALGRIND COMMAND vespamalloc_new_test_app) -- cgit v1.2.3 From ddd8f0e9a70aa1099133bc6893ca7c488653bb5b Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Tue, 28 Feb 2023 14:06:18 +0000 Subject: Remove fastos/types. --- .../proton/matching/docid_range_scheduler.h | 3 +- searchlib/src/vespa/searchlib/common/bitvector.h | 2 -- .../searchlib/predicate/predicate_posting_list.h | 5 ++-- .../src/vespa/searchlib/queryeval/hitcollector.h | 1 - .../storageapi/mbusprot/serializationhelper.h | 34 ---------------------- vespalib/src/vespa/fastos/file.h | 3 +- vespalib/src/vespa/fastos/types.h | 9 ------ .../src/vespa/vespalib/hwaccelrated/avxprivate.hpp | 1 - 8 files changed, 7 insertions(+), 51 deletions(-) delete mode 100644 vespalib/src/vespa/fastos/types.h diff --git a/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h b/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h index 177d19b06f9..cd4d1686eeb 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h +++ b/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h @@ -3,13 +3,14 @@ #pragma once #include -#include #include #include #include #include #include +#define VESPA_DLL_LOCAL __attribute__ ((visibility("hidden"))) + namespace proton::matching { /** diff --git a/searchlib/src/vespa/searchlib/common/bitvector.h b/searchlib/src/vespa/searchlib/common/bitvector.h index 67f3e2ad502..149e57390af 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.h +++ b/searchlib/src/vespa/searchlib/common/bitvector.h @@ -3,10 +3,8 @@ #pragma once #include "bitword.h" -#include #include #include -#include #include #include diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_posting_list.h b/searchlib/src/vespa/searchlib/predicate/predicate_posting_list.h index 0bf33f1d0e5..0de9be332fa 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_posting_list.h +++ b/searchlib/src/vespa/searchlib/predicate/predicate_posting_list.h @@ -3,7 +3,8 @@ #include #include -#include + +#define VESPA_DLL_LOCAL __attribute__ ((visibility("hidden"))) /** * Interface for posting lists used by PredicateSearch. @@ -25,7 +26,7 @@ protected: public: using UP = std::unique_ptr; - virtual ~PredicatePostingList() {} + virtual ~PredicatePostingList() = default; /* * Moves to next document after the one supplied. diff --git a/searchlib/src/vespa/searchlib/queryeval/hitcollector.h b/searchlib/src/vespa/searchlib/queryeval/hitcollector.h index fe686f3e0bf..e244c856c4a 100644 --- a/searchlib/src/vespa/searchlib/queryeval/hitcollector.h +++ b/searchlib/src/vespa/searchlib/queryeval/hitcollector.h @@ -9,7 +9,6 @@ #include #include #include -#include namespace search::queryeval { diff --git a/storage/src/vespa/storageapi/mbusprot/serializationhelper.h b/storage/src/vespa/storageapi/mbusprot/serializationhelper.h index 457a6178704..671ffbddd6f 100644 --- a/storage/src/vespa/storageapi/mbusprot/serializationhelper.h +++ b/storage/src/vespa/storageapi/mbusprot/serializationhelper.h @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include #include #include #include @@ -13,12 +12,6 @@ namespace storage::mbusprot { class SerializationHelper { public: - static int64_t getLong(document::ByteBuffer& buf) { - int64_t tmp; - buf.getLongNetwork(tmp); - return tmp; - } - static int32_t getInt(document::ByteBuffer& buf) { int32_t tmp; buf.getIntNetwork(tmp); @@ -46,26 +39,6 @@ public: return s; } - static bool getBoolean(document::ByteBuffer& buf) { - uint8_t tmp; - buf.getByte(tmp); - return (tmp == 1); - } - - static api::ReturnCode getReturnCode(document::ByteBuffer& buf) { - api::ReturnCode::Result result = (api::ReturnCode::Result) getInt(buf); - vespalib::stringref message = getString(buf); - return api::ReturnCode(result, message); - } - - static void putReturnCode(const api::ReturnCode& code, vespalib::GrowableByteBuffer& buf) - { - buf.putInt(code.getResult()); - buf.putString(code.getMessage()); - } - - static const uint32_t BUCKET_INFO_SERIALIZED_SIZE = sizeof(uint32_t) * 3; - static document::GlobalId getGlobalId(document::ByteBuffer& buf) { std::vector buffer(getShort(buf)); for (uint32_t i=0; i #include +#define FASTOS_PREFIX(a) FastOS_##a + constexpr int FASTOS_FILE_OPEN_READ = (1<<0); constexpr int FASTOS_FILE_OPEN_WRITE = (1<<1); constexpr int FASTOS_FILE_OPEN_EXISTING = (1<<2); diff --git a/vespalib/src/vespa/fastos/types.h b/vespalib/src/vespa/fastos/types.h deleted file mode 100644 index 69dd3e5231c..00000000000 --- a/vespalib/src/vespa/fastos/types.h +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#define FASTOS_PREFIX(a) FastOS_##a - -// New macros to support the new gcc visibility features. -#define VESPA_DLL_EXPORT __attribute__ ((visibility("default"))) -#define VESPA_DLL_LOCAL __attribute__ ((visibility("hidden"))) diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp b/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp index f5d87e14802..3bdbb7a81ff 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp +++ b/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp @@ -3,7 +3,6 @@ #pragma once #include "private_helpers.hpp" -#include namespace vespalib::hwaccelrated::avx { -- cgit v1.2.3 From fd9d20d62003b677f172ca07603c657f89873088 Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Tue, 28 Feb 2023 15:09:35 +0100 Subject: Add temporary test artifacts to gitignore --- config-model/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/config-model/.gitignore b/config-model/.gitignore index 6edd041cbe8..7e7e597675d 100644 --- a/config-model/.gitignore +++ b/config-model/.gitignore @@ -4,5 +4,6 @@ /target /src/test/integration/*/copy/ /src/test/integration/*/models.generated/ +/src/test/derived/*/models.generated/ *.cfg.actual /var/ -- cgit v1.2.3 From a79e364692a3470b489ade4d12368df451601d1c Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Tue, 28 Feb 2023 15:09:48 +0100 Subject: Introduce feature flag to enable global phase --- .../src/main/java/com/yahoo/config/model/api/ModelContext.java | 1 + .../main/java/com/yahoo/config/model/deploy/TestProperties.java | 1 + .../com/yahoo/vespa/model/container/search/ContainerSearch.java | 6 +++++- .../com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java | 2 +- .../java/com/yahoo/vespa/model/container/ContainerClusterTest.java | 2 +- .../com/yahoo/vespa/config/server/deploy/ModelContextImpl.java | 3 +++ .../main/resources/configdefinitions/container.qr-searchers.def | 3 +++ .../src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java | 5 ++++- flags/src/main/java/com/yahoo/vespa/flags/Flags.java | 7 +++++++ 9 files changed, 26 insertions(+), 4 deletions(-) diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 04de4d2e277..d5c074a191d 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -111,6 +111,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"tokle"}) default boolean useRestrictedDataPlaneBindings() { return false; } @ModelFeatureFlag(owners = {"arnej","baldersheim"}, removeAfter = "8.110") default boolean useOldJdiscContainerStartup() { return false; } @ModelFeatureFlag(owners = {"tokle, bjorncs"}, removeAfter = "8.108") default boolean enableDataPlaneFilter() { return true; } + @ModelFeatureFlag(owners = {"arnej, bjorncs"}) default boolean enableGlobalPhase() { return false; } //Below are all flags that must be kept until 7 is out of the door @ModelFeatureFlag(owners = {"arnej"}, removeAfter="7.last") default boolean ignoreThreadStackSizes() { return false; } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 49194a5d1bb..ecbb990f096 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -137,6 +137,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean useRestrictedDataPlaneBindings() { return useRestrictedDataPlaneBindings; } @Override public Optional cloudAccount() { return cloudAccount; } @Override public boolean allowUserFilters() { return allowUserFilters; } + @Override public boolean enableGlobalPhase() { return true; } // Enable global-phase by default for unit tests only public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) { this.sharedStringRepoNoReclaim = sharedStringRepoNoReclaim; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index 14c25ee7452..08aae9cb0a3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.search; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.container.QrSearchersConfig; import com.yahoo.prelude.semantics.SemanticRulesConfig; import com.yahoo.search.config.IndexInfoConfig; @@ -45,13 +46,15 @@ public class ContainerSearch extends ContainerSubsystem private final ApplicationContainerCluster owningCluster; private final List searchClusters = new LinkedList<>(); + private final boolean globalPhase; private QueryProfiles queryProfiles; private SemanticRules semanticRules; private PageTemplates pageTemplates; - public ContainerSearch(ApplicationContainerCluster cluster, SearchChains chains) { + public ContainerSearch(DeployState deployState, ApplicationContainerCluster cluster, SearchChains chains) { super(chains); + this.globalPhase = deployState.featureFlags().enableGlobalPhase(); this.owningCluster = cluster; owningCluster.addComponent(Component.fromClassAndBundle(CompiledQueryProfileRegistry.class, SEARCH_AND_DOCPROC_BUNDLE)); @@ -153,6 +156,7 @@ public class ContainerSearch extends ContainerSubsystem if ( ! (sys instanceof IndexedSearchCluster)) { scB.storagecluster(new QrSearchersConfig.Searchcluster.Storagecluster.Builder(). routespec(((StreamingSearchCluster)sys).getStorageRouteSpec())); + scB.globalphase(globalPhase); } builder.searchcluster(scB); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index ceba3864296..a639c158d62 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -732,7 +732,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder { SearchChains searchChains = new DomSearchChainsBuilder() .build(deployState, containerCluster, producerSpec); - ContainerSearch containerSearch = new ContainerSearch(containerCluster, searchChains); + ContainerSearch containerSearch = new ContainerSearch(deployState, containerCluster, searchChains); applyApplicationPackageDirectoryConfigs(deployState.getApplicationPackage(), containerSearch); containerSearch.setQueryProfiles(deployState.getQueryProfiles()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index ce2d5ca2da5..5973ef56962 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -485,7 +485,7 @@ public class ContainerClusterTest { if (isCombinedCluster) cluster.setHostClusterId("test-content-cluster"); cluster.setMemoryPercentage(memoryPercentage); - cluster.setSearch(new ContainerSearch(cluster, new SearchChains(cluster, "search-chain"))); + cluster.setSearch(new ContainerSearch(root.getDeployState(), cluster, new SearchChains(cluster, "search-chain"))); return cluster; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 1d80740ffe0..d61590c2e84 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -202,6 +202,7 @@ public class ModelContextImpl implements ModelContext { private final int rpc_events_before_wakeup; private final boolean useRestrictedDataPlaneBindings; private final int heapPercentage; + private final boolean enableGlobalPhase; public FeatureFlags(FlagSource source, ApplicationId appId, Version version) { this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -246,6 +247,7 @@ public class ModelContextImpl implements ModelContext { this.queryDispatchWarmup = flagValue(source, appId, version, PermanentFlags.QUERY_DISPATCH_WARMUP); this.useRestrictedDataPlaneBindings = flagValue(source, appId, version, Flags.RESTRICT_DATA_PLANE_BINDINGS); this.heapPercentage = flagValue(source, appId, version, PermanentFlags.HEAP_SIZE_PERCENTAGE); + this.enableGlobalPhase = flagValue(source, appId, version, PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW); } @Override public int heapSizePercentage() { return heapPercentage; } @@ -298,6 +300,7 @@ public class ModelContextImpl implements ModelContext { return defVal; } @Override public boolean useRestrictedDataPlaneBindings() { return useRestrictedDataPlaneBindings; } + @Override public boolean enableGlobalPhase() { return enableGlobalPhase; } private static V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag flag) { return flag.bindTo(source) diff --git a/container-core/src/main/resources/configdefinitions/container.qr-searchers.def b/container-core/src/main/resources/configdefinitions/container.qr-searchers.def index e89ee40d792..034e7fc442b 100644 --- a/container-core/src/main/resources/configdefinitions/container.qr-searchers.def +++ b/container-core/src/main/resources/configdefinitions/container.qr-searchers.def @@ -71,3 +71,6 @@ searchcluster[].indexingmode enum { REALTIME, STREAMING } default=REALTIME ## Storage cluster route to use for search cluster if indexingmode is streaming. searchcluster[].storagecluster.routespec string default="" + +## Enable global phase ranking +searchcluster[].globalphase bool default=false diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 7787d7d7702..08d8d54bc53 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -66,6 +66,7 @@ public class ClusterSearcher extends Searcher { private final VespaBackEndSearcher server; private final Executor executor; private final GlobalPhaseRanker globalPhaseHelper; + private final boolean enableGlobalPhase; @Inject public ClusterSearcher(ComponentId id, @@ -105,6 +106,7 @@ public class ClusterSearcher extends Searcher { server = searchDispatch(searchClusterIndex, searchClusterName, uniqueServerId, docSumParams, documentDbConfig, schemaInfo, dispatchers); } + enableGlobalPhase = searchClusterConfig.globalphase(); } private static QrSearchersConfig.Searchcluster getSearchClusterConfigFromClusterName(QrSearchersConfig config, String name) { @@ -164,6 +166,7 @@ public class ClusterSearcher extends Searcher { server = searcher; this.executor = executor; this.globalPhaseHelper = null; + this.enableGlobalPhase = false; } /** Do not use, for internal testing purposes only. **/ @@ -245,7 +248,7 @@ public class ClusterSearcher extends Searcher { } String schema = restrict.iterator().next(); Result result = searcher.search(query, execution); - if (globalPhaseHelper != null) { + if (globalPhaseHelper != null && enableGlobalPhase) { globalPhaseHelper.process(query, result, schema); } return result; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 9b4b04a3d62..dcb69d5f77b 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -352,6 +352,13 @@ public class Flags { "Takes effect on next config server container start", ZONE_ID); + public static final UnboundBooleanFlag ENABLE_GLOBAL_PHASE = defineFeatureFlag( + "enable-global-phase", false, + List.of("arnej", "bjorncs"), "2023-02-28", "2024-01-10", + "Enable global phase ranking", + "Takes effect at redeployment", + APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List owners, String createdAt, String expiresAt, String description, -- cgit v1.2.3 From 613e8879466573ad5a0ce3e594bfa51828110fc6 Mon Sep 17 00:00:00 2001 From: Tor Brede Vekterli Date: Tue, 28 Feb 2023 13:54:43 +0000 Subject: Pass `stringref`s to Portal response rendering Don't need to potentially instantiate temporaries since the data is always immediately committed to another buffer. --- vespalib/src/vespa/vespalib/portal/http_connection.cpp | 14 +++++++++----- vespalib/src/vespa/vespalib/portal/http_connection.h | 6 +++--- vespalib/src/vespa/vespalib/portal/portal.cpp | 6 +++--- vespalib/src/vespa/vespalib/portal/portal.h | 6 +++--- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/vespalib/src/vespa/vespalib/portal/http_connection.cpp b/vespalib/src/vespa/vespalib/portal/http_connection.cpp index 6ea56e2659c..3d8edf2fc2e 100644 --- a/vespalib/src/vespa/vespalib/portal/http_connection.cpp +++ b/vespalib/src/vespa/vespalib/portal/http_connection.cpp @@ -245,14 +245,16 @@ HttpConnection::handle_event(bool, bool) } void -HttpConnection::respond_with_content(const vespalib::string &content_type, - const vespalib::string &content) +HttpConnection::respond_with_content(vespalib::stringref content_type, + vespalib::stringref content) { { OutputWriter dst(_output, CHUNK_SIZE); dst.printf("HTTP/1.1 200 OK\r\n"); dst.printf("Connection: close\r\n"); - dst.printf("Content-Type: %s\r\n", content_type.c_str()); + dst.printf("Content-Type: "); + dst.write(content_type.data(), content_type.size()); + dst.printf("\r\n"); dst.printf("Content-Length: %zu\r\n", content.size()); emit_http_security_headers(dst); dst.printf("\r\n"); @@ -263,11 +265,13 @@ HttpConnection::respond_with_content(const vespalib::string &content_type, } void -HttpConnection::respond_with_error(int code, const vespalib::string &msg) +HttpConnection::respond_with_error(int code, vespalib::stringref msg) { { OutputWriter dst(_output, CHUNK_SIZE); - dst.printf("HTTP/1.1 %d %s\r\n", code, msg.c_str()); + dst.printf("HTTP/1.1 %d ", code); + dst.write(msg.data(), msg.size()); + dst.printf("\r\n"); dst.printf("Connection: close\r\n"); dst.printf("\r\n"); } diff --git a/vespalib/src/vespa/vespalib/portal/http_connection.h b/vespalib/src/vespa/vespalib/portal/http_connection.h index 03d23351e7d..8540cb87e1d 100644 --- a/vespalib/src/vespa/vespalib/portal/http_connection.h +++ b/vespalib/src/vespa/vespalib/portal/http_connection.h @@ -53,9 +53,9 @@ public: // Precondition: handshake must have been completed const net::ConnectionAuthContext &auth_context() const noexcept { return *_auth_ctx; } - void respond_with_content(const vespalib::string &content_type, - const vespalib::string &content); - void respond_with_error(int code, const vespalib::string &msg); + void respond_with_content(vespalib::stringref content_type, + vespalib::stringref content); + void respond_with_error(int code, const vespalib::stringref msg); }; } // namespace vespalib::portal diff --git a/vespalib/src/vespa/vespalib/portal/portal.cpp b/vespalib/src/vespa/vespalib/portal/portal.cpp index a98562f6504..691ddaef495 100644 --- a/vespalib/src/vespa/vespalib/portal/portal.cpp +++ b/vespalib/src/vespa/vespalib/portal/portal.cpp @@ -77,8 +77,8 @@ Portal::GetRequest::export_params() const } void -Portal::GetRequest::respond_with_content(const vespalib::string &content_type, - const vespalib::string &content) +Portal::GetRequest::respond_with_content(vespalib::stringref content_type, + vespalib::stringref content) { assert(active()); _conn->respond_with_content(content_type, content); @@ -86,7 +86,7 @@ Portal::GetRequest::respond_with_content(const vespalib::string &content_type, } void -Portal::GetRequest::respond_with_error(int code, const vespalib::string &msg) +Portal::GetRequest::respond_with_error(int code, vespalib::stringref msg) { assert(active()); _conn->respond_with_error(code, msg); diff --git a/vespalib/src/vespa/vespalib/portal/portal.h b/vespalib/src/vespa/vespalib/portal/portal.h index 314dd6e7de9..6954d800c91 100644 --- a/vespalib/src/vespa/vespalib/portal/portal.h +++ b/vespalib/src/vespa/vespalib/portal/portal.h @@ -65,9 +65,9 @@ public: bool has_param(const vespalib::string &name) const; const vespalib::string &get_param(const vespalib::string &name) const; std::map export_params() const; - void respond_with_content(const vespalib::string &content_type, - const vespalib::string &content); - void respond_with_error(int code, const vespalib::string &msg); + void respond_with_content(vespalib::stringref content_type, + vespalib::stringref content); + void respond_with_error(int code, vespalib::stringref msg); const net::ConnectionAuthContext &auth_context() const noexcept; ~GetRequest(); }; -- cgit v1.2.3 From 59714d2b2afb952e1b402948b52e42862cf26c8f Mon Sep 17 00:00:00 2001 From: HÃ¥vard Pettersen Date: Tue, 28 Feb 2023 14:16:54 +0000 Subject: re-try on signals (EINTR) and check return values --- vespalib/src/vespa/vespalib/net/wakeup_pipe.cpp | 24 +++++++++++++++--------- vespalib/src/vespa/vespalib/net/wakeup_pipe.h | 7 +++++-- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/vespalib/src/vespa/vespalib/net/wakeup_pipe.cpp b/vespalib/src/vespa/vespalib/net/wakeup_pipe.cpp index 60900289e79..b8025bfcf9f 100644 --- a/vespalib/src/vespa/vespalib/net/wakeup_pipe.cpp +++ b/vespalib/src/vespa/vespalib/net/wakeup_pipe.cpp @@ -2,34 +2,40 @@ #include "wakeup_pipe.h" #include "socket_utils.h" +#include #include namespace vespalib { WakeupPipe::WakeupPipe() - : _pipe() + : _reader(), + _writer() { - socketutils::nonblocking_pipe(_pipe); + int pipe[2]; + socketutils::nonblocking_pipe(pipe); + _reader.reset(pipe[0]); + _writer.reset(pipe[1]); } -WakeupPipe::~WakeupPipe() -{ - close(_pipe[0]); - close(_pipe[1]); -} +WakeupPipe::~WakeupPipe() = default; void WakeupPipe::write_token() { char token = 'T'; - [[maybe_unused]] ssize_t res = write(_pipe[1], &token, 1); + ssize_t res = _writer.write(&token, 1); + if (res < 0) { + res = -errno; + } + REQUIRE(res > 0 || res == -EAGAIN || res == -EWOULDBLOCK); } void WakeupPipe::read_tokens() { char token_trash[128]; - [[maybe_unused]] ssize_t res = read(_pipe[0], token_trash, sizeof(token_trash)); + ssize_t res = _reader.read(token_trash, sizeof(token_trash)); + REQUIRE(res > 0); } } diff --git a/vespalib/src/vespa/vespalib/net/wakeup_pipe.h b/vespalib/src/vespa/vespalib/net/wakeup_pipe.h index b52f7f9e32d..36c88b205c0 100644 --- a/vespalib/src/vespa/vespalib/net/wakeup_pipe.h +++ b/vespalib/src/vespa/vespalib/net/wakeup_pipe.h @@ -2,6 +2,8 @@ #pragma once +#include "socket_handle.h" + namespace vespalib { //----------------------------------------------------------------------------- @@ -15,11 +17,12 @@ namespace vespalib { **/ class WakeupPipe { private: - int _pipe[2]; + SocketHandle _reader; + SocketHandle _writer; public: WakeupPipe(); ~WakeupPipe(); - int get_read_fd() const { return _pipe[0]; } + int get_read_fd() const { return _reader.get(); } void write_token(); void read_tokens(); }; -- cgit v1.2.3 From 439e7046af4594b28af440f52c9d8e90be6c6432 Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Tue, 28 Feb 2023 15:42:18 +0100 Subject: Upgrade Maven Surefire Plugin --- parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parent/pom.xml b/parent/pom.xml index 2fcf5527302..273e9b0ef8f 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -1170,7 +1170,7 @@ 0.6.0 3.21.7 1.3.5 - 2.22.2 + 3.0.0-M9 2.35.0 0.16 3.8.0 -- cgit v1.2.3 From f08a1a0269fe4df06acc3d88088092d251668fac Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Tue, 28 Feb 2023 14:44:34 +0000 Subject: Ensure we stay with system time until limitation has been lifted in a type safe way. --- metrics/src/vespa/metrics/metricmanager.cpp | 2 +- metrics/src/vespa/metrics/updatehook.h | 2 +- storage/src/tests/common/metricstest.cpp | 2 +- storage/src/tests/storageserver/statereportertest.cpp | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/metrics/src/vespa/metrics/metricmanager.cpp b/metrics/src/vespa/metrics/metricmanager.cpp index edf909131a0..df83001a4e2 100644 --- a/metrics/src/vespa/metrics/metricmanager.cpp +++ b/metrics/src/vespa/metrics/metricmanager.cpp @@ -34,7 +34,7 @@ MetricManager::ConsumerSpec::~ConsumerSpec() = default; time_point MetricManager::Timer::getTime() const { - return vespalib::steady_clock::now(); + return vespalib::system_clock::now(); } void diff --git a/metrics/src/vespa/metrics/updatehook.h b/metrics/src/vespa/metrics/updatehook.h index f355197bbb7..58d9ef0d743 100644 --- a/metrics/src/vespa/metrics/updatehook.h +++ b/metrics/src/vespa/metrics/updatehook.h @@ -6,7 +6,7 @@ namespace metrics { -using time_point = vespalib::steady_time; +using time_point = vespalib::system_time; class MetricLockGuard { public: diff --git a/storage/src/tests/common/metricstest.cpp b/storage/src/tests/common/metricstest.cpp index 60b2fa0de6c..7231a071319 100644 --- a/storage/src/tests/common/metricstest.cpp +++ b/storage/src/tests/common/metricstest.cpp @@ -52,7 +52,7 @@ namespace { { framework::Clock& _clock; explicit MetricClock(framework::Clock& c) : _clock(c) {} - [[nodiscard]] metrics::time_point getTime() const override { return _clock.getMonotonicTime(); } + [[nodiscard]] metrics::time_point getTime() const override { return _clock.getSystemTime(); } }; } diff --git a/storage/src/tests/storageserver/statereportertest.cpp b/storage/src/tests/storageserver/statereportertest.cpp index afa1b4a2c14..3a772c1ddde 100644 --- a/storage/src/tests/storageserver/statereportertest.cpp +++ b/storage/src/tests/storageserver/statereportertest.cpp @@ -53,7 +53,7 @@ struct MetricClock : public metrics::MetricManager::Timer { framework::Clock& _clock; explicit MetricClock(framework::Clock& c) : _clock(c) {} - [[nodiscard]] metrics::time_point getTime() const override { return _clock.getMonotonicTime(); } + [[nodiscard]] metrics::time_point getTime() const override { return _clock.getSystemTime(); } }; } -- cgit v1.2.3 From d4944343ef4993b46b5ef641fc4e6d2765344ddd Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Tue, 28 Feb 2023 16:34:30 +0100 Subject: Only setup RankProfilesEvaluator components if feature flag is enabled --- .../yahoo/vespa/model/container/search/ContainerSearch.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index 08aae9cb0a3..5d4ec598250 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -77,10 +77,12 @@ public class ContainerSearch extends ContainerSubsystem if (searchCluster instanceof IndexedSearchCluster indexed) { var dispatcher = new DispatcherComponent(indexed); owningCluster.addComponent(dispatcher); - for (var documentDb : indexed.getDocumentDbs()) { - var factory = new RankProfilesEvaluatorComponent(documentDb); - if (! owningCluster.getComponentsMap().containsKey(factory.getComponentId())) { - owningCluster.addComponent(factory); + if (globalPhase) { + for (var documentDb : indexed.getDocumentDbs()) { + var factory = new RankProfilesEvaluatorComponent(documentDb); + if (! owningCluster.getComponentsMap().containsKey(factory.getComponentId())) { + owningCluster.addComponent(factory); + } } } } -- cgit v1.2.3 From 003f019d7579e49f4ec7609ef8eac26ada6ae753 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Tue, 28 Feb 2023 16:55:23 +0100 Subject: Revert "remove fastos" --- CMakeLists.txt | 3 +-- config/CMakeLists.txt | 1 + configdefinitions/CMakeLists.txt | 1 + configutil/CMakeLists.txt | 1 + document/CMakeLists.txt | 1 + documentapi/CMakeLists.txt | 1 + fastos/.gitignore | 11 +++++++++++ fastos/CMakeLists.txt | 7 +++++++ fastos/OWNERS | 2 ++ fastos/README | 4 ++++ fastos/src/.gitignore | 3 +++ fastos/src/vespa/fastos/.gitignore | 28 ++++++++++++++++++++++++++++ fastos/src/vespa/fastos/CMakeLists.txt | 15 +++++++++++++++ fbench/CMakeLists.txt | 1 + fbench/src/httpclient/CMakeLists.txt | 1 + fbench/src/test/CMakeLists.txt | 1 + fileacquirer/CMakeLists.txt | 1 + jrt_test/CMakeLists.txt | 1 + logd/CMakeLists.txt | 1 + lowercasing_test/CMakeLists.txt | 1 + metrics/CMakeLists.txt | 1 + persistence/CMakeLists.txt | 1 + searchcore/CMakeLists.txt | 1 + searchlib/CMakeLists.txt | 1 + storage/CMakeLists.txt | 1 + storageserver/CMakeLists.txt | 1 + streamingvisitors/CMakeLists.txt | 1 + vbench/CMakeLists.txt | 1 + vdslib/CMakeLists.txt | 1 + vdstestlib/CMakeLists.txt | 1 + vespaclient/CMakeLists.txt | 1 + vespalib/CMakeLists.txt | 1 + vespalog/src/test/threads/CMakeLists.txt | 1 + vespalog/src/vespa/log/CMakeLists.txt | 1 - vespamalloc/src/tests/test1/CMakeLists.txt | 3 --- 35 files changed, 96 insertions(+), 6 deletions(-) create mode 100644 fastos/.gitignore create mode 100644 fastos/CMakeLists.txt create mode 100644 fastos/OWNERS create mode 100644 fastos/README create mode 100644 fastos/src/.gitignore create mode 100644 fastos/src/vespa/fastos/.gitignore create mode 100644 fastos/src/vespa/fastos/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index ce11196725f..5463c4215a0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,8 +28,6 @@ vespa_use_default_cmake_prefix_path() SET(CMAKE_FIND_PACKAGE_SORT_ORDER NATURAL) SET(CMAKE_FIND_PACKAGE_SORT_DIRECTION DEC) -find_package(Threads REQUIRED) - find_package(LLVM REQUIRED CONFIG) message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") @@ -101,6 +99,7 @@ add_subdirectory(docprocs) add_subdirectory(document) add_subdirectory(documentapi) add_subdirectory(eval) +add_subdirectory(fastos) add_subdirectory(fbench) add_subdirectory(fileacquirer) add_subdirectory(filedistribution) diff --git a/config/CMakeLists.txt b/config/CMakeLists.txt index 561ddbc078c..1c04aa3eaa8 100644 --- a/config/CMakeLists.txt +++ b/config/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS + fastos vespalib vespalog fnet diff --git a/configdefinitions/CMakeLists.txt b/configdefinitions/CMakeLists.txt index 80b53d1dc0a..c374e93904e 100644 --- a/configdefinitions/CMakeLists.txt +++ b/configdefinitions/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS + fastos vespalib config_cloudconfig diff --git a/configutil/CMakeLists.txt b/configutil/CMakeLists.txt index a1d699ae009..4ec9ad13648 100644 --- a/configutil/CMakeLists.txt +++ b/configutil/CMakeLists.txt @@ -2,6 +2,7 @@ vespa_define_module( DEPENDS vespadefaults + fastos config_cloudconfig vbench vespalib diff --git a/document/CMakeLists.txt b/document/CMakeLists.txt index e1e4d8ff5cc..88dbe2816d9 100644 --- a/document/CMakeLists.txt +++ b/document/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS + fastos vespalog vespalib config_cloudconfig diff --git a/documentapi/CMakeLists.txt b/documentapi/CMakeLists.txt index beeda4afeb4..9261bcf9114 100644 --- a/documentapi/CMakeLists.txt +++ b/documentapi/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS + fastos vespalog config_cloudconfig vespalib diff --git a/fastos/.gitignore b/fastos/.gitignore new file mode 100644 index 00000000000..54e2680a6d8 --- /dev/null +++ b/fastos/.gitignore @@ -0,0 +1,11 @@ +*.ilk +*.pdb +.Build_completed +.Dist_completed +.Install_completed +.PreBuild_completed +bin +include +lib +update.log +Makefile diff --git a/fastos/CMakeLists.txt b/fastos/CMakeLists.txt new file mode 100644 index 00000000000..60813c569e4 --- /dev/null +++ b/fastos/CMakeLists.txt @@ -0,0 +1,7 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_define_module( + LIBS + src/vespa/fastos + + TESTS +) diff --git a/fastos/OWNERS b/fastos/OWNERS new file mode 100644 index 00000000000..912f61c59b8 --- /dev/null +++ b/fastos/OWNERS @@ -0,0 +1,2 @@ +baldersheim +arnej27959 diff --git a/fastos/README b/fastos/README new file mode 100644 index 00000000000..ed9afabffb8 --- /dev/null +++ b/fastos/README @@ -0,0 +1,4 @@ +Old OS abstraction layer + +obsolete, to be replaced with implementations using +standard C++14 threads and newer unix networking APIs diff --git a/fastos/src/.gitignore b/fastos/src/.gitignore new file mode 100644 index 00000000000..2e8e6fd906a --- /dev/null +++ b/fastos/src/.gitignore @@ -0,0 +1,3 @@ +/Makefile.ini +/config_command.sh +/project.dsw diff --git a/fastos/src/vespa/fastos/.gitignore b/fastos/src/vespa/fastos/.gitignore new file mode 100644 index 00000000000..004799df5b4 --- /dev/null +++ b/fastos/src/vespa/fastos/.gitignore @@ -0,0 +1,28 @@ +*.So +*.core +*.exe +*.ilk +*.pdb +.depend +.depend.NEW +Debug +Makefile +Makefile.factory +Makefile.overrides +Makefile.pre +Release +autoconf.h +config_command.bat +config_command.sh +fastconfig +fastconfig.exe +fastos.lib +fastosconfig.h +libfastos.a +makefeatures +makemake +processtest.log +test.txt +vc60.idb +vc60.pdb +/libfastos.so.5.1 diff --git a/fastos/src/vespa/fastos/CMakeLists.txt b/fastos/src/vespa/fastos/CMakeLists.txt new file mode 100644 index 00000000000..29810a4f296 --- /dev/null +++ b/fastos/src/vespa/fastos/CMakeLists.txt @@ -0,0 +1,15 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(fastos_objects OBJECT + SOURCES +) + +vespa_add_library(fastos + SOURCES + $ + INSTALL lib64 + DEPENDS + ${CMAKE_DL_LIBS} +) + +find_package(Threads REQUIRED) +target_link_libraries(fastos PUBLIC ${CMAKE_THREAD_LIBS_INIT}) diff --git a/fbench/CMakeLists.txt b/fbench/CMakeLists.txt index 3f1d78a66a0..ff287d221ec 100644 --- a/fbench/CMakeLists.txt +++ b/fbench/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS + fastos vespalib APPS diff --git a/fbench/src/httpclient/CMakeLists.txt b/fbench/src/httpclient/CMakeLists.txt index 2d5e3b32437..163a68f9c98 100644 --- a/fbench/src/httpclient/CMakeLists.txt +++ b/fbench/src/httpclient/CMakeLists.txt @@ -4,4 +4,5 @@ vespa_add_library(fbench_httpclient STATIC httpclient.cpp DEPENDS fbench_util + fastos ) diff --git a/fbench/src/test/CMakeLists.txt b/fbench/src/test/CMakeLists.txt index c81d818ed06..d13b6b82a81 100644 --- a/fbench/src/test/CMakeLists.txt +++ b/fbench/src/test/CMakeLists.txt @@ -26,5 +26,6 @@ vespa_add_executable(fbench_clientstatus_app TEST clientstatus.cpp DEPENDS fbench_util + fastos ) vespa_add_test(NAME fbench_clientstatus_app COMMAND fbench_clientstatus_app) diff --git a/fileacquirer/CMakeLists.txt b/fileacquirer/CMakeLists.txt index cc18dc2bd84..13150f58ba3 100644 --- a/fileacquirer/CMakeLists.txt +++ b/fileacquirer/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS + fastos vespalog vespalib config_cloudconfig diff --git a/jrt_test/CMakeLists.txt b/jrt_test/CMakeLists.txt index a678cbf112a..ea8c8b94faa 100644 --- a/jrt_test/CMakeLists.txt +++ b/jrt_test/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS + fastos vespalog vespalib fnet diff --git a/logd/CMakeLists.txt b/logd/CMakeLists.txt index 5823ebf54a7..d02b99a393a 100644 --- a/logd/CMakeLists.txt +++ b/logd/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS + fastos vespalog vespalib config_cloudconfig diff --git a/lowercasing_test/CMakeLists.txt b/lowercasing_test/CMakeLists.txt index 119209d4227..b08a6ef350d 100644 --- a/lowercasing_test/CMakeLists.txt +++ b/lowercasing_test/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS + fastos vespalog vespalib searchlib diff --git a/metrics/CMakeLists.txt b/metrics/CMakeLists.txt index 5b95d8635d4..d75c17d4a00 100644 --- a/metrics/CMakeLists.txt +++ b/metrics/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS + fastos vespalog vespalib config_cloudconfig diff --git a/persistence/CMakeLists.txt b/persistence/CMakeLists.txt index 072e273338b..d87f172bfb5 100644 --- a/persistence/CMakeLists.txt +++ b/persistence/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS + fastos vespalog vespalib document diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index 131460b0384..d6c353e46c9 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS + fastos fnet vespalog vespalib diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index 44051a96578..4ccab1a9380 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS + fastos vespalog vespalib vespaeval diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt index 1062a89f055..4b7c12b0f31 100644 --- a/storage/CMakeLists.txt +++ b/storage/CMakeLists.txt @@ -2,6 +2,7 @@ vespa_define_module( DEPENDS vespadefaults + fastos metrics config_cloudconfig configdefinitions diff --git a/storageserver/CMakeLists.txt b/storageserver/CMakeLists.txt index a2f9d0b776e..ee3335c4921 100644 --- a/storageserver/CMakeLists.txt +++ b/storageserver/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS + fastos storage streamingvisitors diff --git a/streamingvisitors/CMakeLists.txt b/streamingvisitors/CMakeLists.txt index fede7087d8d..2c7f01ddf37 100644 --- a/streamingvisitors/CMakeLists.txt +++ b/streamingvisitors/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS + fastos vespalog storage config_cloudconfig diff --git a/vbench/CMakeLists.txt b/vbench/CMakeLists.txt index e78913262be..3fb5df8cd20 100644 --- a/vbench/CMakeLists.txt +++ b/vbench/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS + fastos vespalib LIBS diff --git a/vdslib/CMakeLists.txt b/vdslib/CMakeLists.txt index 1276323f83b..0f8144b99e9 100644 --- a/vdslib/CMakeLists.txt +++ b/vdslib/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS + fastos vespalog vespalib config_cloudconfig diff --git a/vdstestlib/CMakeLists.txt b/vdstestlib/CMakeLists.txt index 7f478989332..d0a921672c2 100644 --- a/vdstestlib/CMakeLists.txt +++ b/vdstestlib/CMakeLists.txt @@ -1,6 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS + fastos vespalib TESTS diff --git a/vespaclient/CMakeLists.txt b/vespaclient/CMakeLists.txt index 6e82b83517e..912b35fa763 100644 --- a/vespaclient/CMakeLists.txt +++ b/vespaclient/CMakeLists.txt @@ -2,6 +2,7 @@ vespa_define_module( DEPENDS vespadefaults + fastos configdefinitions config_cloudconfig vespalog diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 8175e75875a..8f1b687449a 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -6,6 +6,7 @@ endif() vespa_define_module( DEPENDS + fastos vespalog EXTERNAL_DEPENDS diff --git a/vespalog/src/test/threads/CMakeLists.txt b/vespalog/src/test/threads/CMakeLists.txt index 19fe2511025..00de16ff005 100644 --- a/vespalog/src/test/threads/CMakeLists.txt +++ b/vespalog/src/test/threads/CMakeLists.txt @@ -4,5 +4,6 @@ vespa_add_executable(vespalog_threads_test_app TEST testthreads.cpp DEPENDS vespalog + fastos ) vespa_add_test(NAME vespalog_threads_test_app COMMAND vespalog_threads_test_app vespa.log ENVIRONMENT "VESPA_LOG_TARGET=file:vespa.log") diff --git a/vespalog/src/vespa/log/CMakeLists.txt b/vespalog/src/vespa/log/CMakeLists.txt index 82536b1f4e9..dbeba17334b 100644 --- a/vespalog/src/vespa/log/CMakeLists.txt +++ b/vespalog/src/vespa/log/CMakeLists.txt @@ -18,4 +18,3 @@ vespa_add_library(vespalog reject-filter.cpp INSTALL lib64 ) -target_link_libraries(vespalog PUBLIC ${CMAKE_THREAD_LIBS_INIT}) diff --git a/vespamalloc/src/tests/test1/CMakeLists.txt b/vespamalloc/src/tests/test1/CMakeLists.txt index f3ce9f70f45..dd7c92a4dac 100644 --- a/vespamalloc/src/tests/test1/CMakeLists.txt +++ b/vespamalloc/src/tests/test1/CMakeLists.txt @@ -8,15 +8,12 @@ vespa_add_executable(vespamalloc_testatomic_app TEST vespamalloc_util EXTERNAL_DEPENDS ${VESPA_ATOMIC_LIB} - dl ) vespa_add_test(NAME vespamalloc_testatomic_app NO_VALGRIND COMMAND vespamalloc_testatomic_app) vespa_add_executable(vespamalloc_new_test_app TEST SOURCES new_test.cpp - EXTERNAL_DEPENDS - dl ) vespa_add_test(NAME vespamalloc_new_test_app NO_VALGRIND COMMAND vespamalloc_new_test_app) -- cgit v1.2.3 From c0b9b3547b2ecca8472168b2a0747d8a0ba1ab76 Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Tue, 28 Feb 2023 17:34:09 +0100 Subject: Bundle 'container-search-and-docproc' requires 'model-evaluation'+'model-integration' --- .../yahoo/vespa/model/container/ContainerModelEvaluation.java | 6 +++--- .../java/com/yahoo/vespa/model/container/PlatformBundles.java | 9 ++++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java index 3d9a8441ed5..0be3c825614 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java @@ -26,9 +26,9 @@ public class ContainerModelEvaluation implements OnnxModelsConfig.Producer, RankingExpressionsConfig.Producer { - private final static String EVALUATION_BUNDLE_NAME = "model-evaluation"; - private final static String INTEGRATION_BUNDLE_NAME = "model-integration"; - private final static String ONNXRUNTIME_BUNDLE_NAME = "container-onnxruntime.jar"; + public final static String EVALUATION_BUNDLE_NAME = "model-evaluation"; + public final static String INTEGRATION_BUNDLE_NAME = "model-integration"; + public final static String ONNXRUNTIME_BUNDLE_NAME = "container-onnxruntime.jar"; private final static String EVALUATOR_NAME = ModelsEvaluator.class.getName(); private final static String REST_HANDLER_NAME = "ai.vespa.models.handler.ModelsEvaluationHandler"; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java index 29b1edc1397..19df9a4064f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java @@ -10,6 +10,10 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import static com.yahoo.vespa.model.container.ContainerModelEvaluation.EVALUATION_BUNDLE_NAME; +import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATION_BUNDLE_NAME; +import static com.yahoo.vespa.model.container.ContainerModelEvaluation.ONNXRUNTIME_BUNDLE_NAME; + /** * NOTE: Stable ordering of bundles in config is handled by {@link ContainerCluster#addPlatformBundle(Path)} * @@ -53,7 +57,10 @@ public class PlatformBundles { public static final Set SEARCH_AND_DOCPROC_BUNDLES = toBundlePaths( SEARCH_AND_DOCPROC_BUNDLE, "docprocs", - "linguistics-components" + "linguistics-components", + EVALUATION_BUNDLE_NAME, + INTEGRATION_BUNDLE_NAME, + ONNXRUNTIME_BUNDLE_NAME ); private static Set toBundlePaths(String... bundleNames) { -- cgit v1.2.3 From 8a109b406dd19c2b773938d49c7ad858e09f68df Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Tue, 28 Feb 2023 19:38:09 +0100 Subject: Replace empty autoscaling --- .../src/main/java/com/yahoo/config/provision/ClusterInfo.java | 2 ++ .../java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java | 4 ++++ .../vespa/hosted/provision/maintenance/AutoscalingMaintainer.java | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java new file mode 100644 index 00000000000..cec5fb1a1ed --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java @@ -0,0 +1,2 @@ +package com.yahoo.config.provision;public class ClusterInfo { +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java index 9506bba73e7..2cc43a1eb33 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java @@ -77,6 +77,10 @@ public class Autoscaling { return peak.equals(Load.zero()); } + public boolean isPresent() { + return ! isEmpty(); + } + @Override public boolean equals(Object o) { if ( ! (o instanceof Autoscaling other)) return false; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index 674c20e25f2..27a07fffca3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -79,7 +79,7 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { Autoscaling autoscaling = null; if (cluster.target().resources().isEmpty() || current.equals(cluster.target().resources().get())) { autoscaling = autoscaler.autoscale(application.get(), cluster, clusterNodes); - if ( ! autoscaling.isEmpty()) // Ignore empties we'll get from servers recently started + if ( autoscaling.isPresent() || cluster.target().isEmpty()) // Ignore empty from recently started servers cluster = cluster.withTarget(autoscaling); } -- cgit v1.2.3