diff options
author | Morten Tokle <mortent@yahooinc.com> | 2023-03-01 08:11:05 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-03-01 08:11:05 +0100 |
commit | 52625c4410dc46b27fab4061fc2d0f34b3faa9f2 (patch) | |
tree | 86985170b3a7ed10e8a82605720d1acb0e832b83 | |
parent | 6540b004e3a7eea35da0956910331e69bcb87f8c (diff) | |
parent | 42f5944056ed2b9d46b1b5538cbcccf0b369ca44 (diff) |
Merge branch 'master' into mortent/revert-26208
99 files changed, 1656 insertions, 940 deletions
diff --git a/client/go/internal/cli/cmd/root.go b/client/go/internal/cli/cmd/root.go index faba6bbbfd4..70e0afbcd32 100644 --- a/client/go/internal/cli/cmd/root.go +++ b/client/go/internal/cli/cmd/root.go @@ -250,6 +250,7 @@ func (c *CLI) configureCommands() { rootCmd.AddCommand(statusCmd) // status rootCmd.AddCommand(newTestCmd(c)) // test rootCmd.AddCommand(newVersionCmd(c)) // version + rootCmd.AddCommand(newVisitCmd(c)) // visit } func (c *CLI) printErr(err error, hints ...string) { @@ -263,6 +264,10 @@ func (c *CLI) printSuccess(msg ...interface{}) { fmt.Fprintln(c.Stdout, color.GreenString("Success:"), fmt.Sprint(msg...)) } +func (c *CLI) printDebug(msg ...interface{}) { + fmt.Fprintln(c.Stderr, color.CyanString("Debug:"), fmt.Sprint(msg...)) +} + func (c *CLI) printWarning(msg interface{}, hints ...string) { fmt.Fprintln(c.Stderr, color.YellowString("Warning:"), msg) for _, hint := range hints { diff --git a/client/go/internal/cli/cmd/visit.go b/client/go/internal/cli/cmd/visit.go new file mode 100644 index 00000000000..1022d74354d --- /dev/null +++ b/client/go/internal/cli/cmd/visit.go @@ -0,0 +1,348 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa visit command +// Author: arnej + +package cmd + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +type visitArgs struct { + contentCluster string + fieldSet string + selection string + makeFeed bool + jsonLines bool + pretty bool + debugMode bool + chunkCount int + cli *CLI +} + +func (v *visitArgs) writeBytes(b []byte) { + v.cli.Stdout.Write(b) +} + +func (v *visitArgs) writeString(s string) { + v.writeBytes([]byte(s)) +} + +func (v *visitArgs) debugPrint(s string) { + if v.debugMode { + v.cli.printDebug(s) + } +} + +func (v *visitArgs) dumpDocuments(documents []DocumentBlob) { + comma := false + pretty := false + if v.makeFeed { + comma = true + pretty = v.pretty + } else if !v.jsonLines { + return + } + for _, value := range documents { + if pretty { + var prettyJSON bytes.Buffer + parseError := json.Indent(&prettyJSON, value.blob, "", " ") + if parseError != nil { + v.writeBytes(value.blob) + } else { + v.writeBytes(prettyJSON.Bytes()) + } + } else { + v.writeBytes(value.blob) + } + if comma { + v.writeString(",\n") + } else { + v.writeString("\n") + } + } +} + +var totalDocCount int + +func newVisitCmd(cli *CLI) *cobra.Command { + var ( + vArgs visitArgs + ) + cmd := &cobra.Command{ + Use: "visit", + Short: "Visit and print all documents in a vespa cluster", + Long: `Run visiting to retrieve all documents. + +By default prints each document received on its own line (JSON-L format). +`, + Example: `$ vespa visit # get documents from any cluster +$ vespa visit --content-cluster search # get documents from cluster named "search" +`, + Args: cobra.MaximumNArgs(0), + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + vArgs.cli = cli + service, err := documentService(cli) + if err != nil { + return err + } + result := probeHandler(service, cli) + if result.Success { + result = visitClusters(&vArgs, service) + } + if !result.Success { + return fmt.Errorf("visit failed: %s", result.Message) + } + vArgs.debugPrint(fmt.Sprintf("sum of 'documentCount': %d", totalDocCount)) + return nil + }, + } + cmd.Flags().StringVar(&vArgs.contentCluster, "content-cluster", "*", `Which content cluster to visit documents from`) + cmd.Flags().StringVar(&vArgs.fieldSet, "field-set", "", `Which fieldset to ask for`) + cmd.Flags().StringVar(&vArgs.selection, "selection", "", `select subset of cluster`) + cmd.Flags().BoolVar(&vArgs.debugMode, "debug-mode", false, `print debugging output`) + cmd.Flags().BoolVar(&vArgs.jsonLines, "json-lines", true, `output documents as JSON lines`) + cmd.Flags().BoolVar(&vArgs.makeFeed, "make-feed", false, `output JSON array suitable for vespa-feeder`) + cmd.Flags().BoolVar(&vArgs.pretty, "pretty-json", false, `format pretty JSON`) + cmd.Flags().IntVar(&vArgs.chunkCount, "chunk-count", 1000, `chunk by count`) + return cmd +} + +type HandlersInfo struct { + Handlers []struct { + HandlerId string `json:"id"` + HandlerClass string `json:"class"` + HandlerBundle string `json:"bundle"` + ServerBindings []string `json:"serverBindings"` + } `json:"handlers"` +} + +func parseHandlersOutput(r io.Reader) (*HandlersInfo, error) { + var handlersInfo HandlersInfo + codec := json.NewDecoder(r) + err := codec.Decode(&handlersInfo) + return &handlersInfo, err +} + +func probeHandler(service *vespa.Service, cli *CLI) (res util.OperationResult) { + urlPath := service.BaseURL + "/" + url, urlParseError := url.Parse(urlPath) + if urlParseError != nil { + return util.Failure("Invalid request path: '" + urlPath + "': " + urlParseError.Error()) + } + request := &http.Request{ + URL: url, + Method: "GET", + } + timeout := time.Duration(90) * time.Second + response, err := service.Do(request, timeout) + if err != nil { + return util.Failure("Request failed: " + err.Error()) + } + defer response.Body.Close() + if response.StatusCode == 200 { + handlersInfo, err := parseHandlersOutput(response.Body) + if err != nil || len(handlersInfo.Handlers) == 0 { + cli.printWarning("Could not parse JSON response from"+urlPath, err.Error()) + return util.Failure("Bad endpoint") + } + for _, h := range handlersInfo.Handlers { + if strings.HasSuffix(h.HandlerClass, "DocumentV1ApiHandler") { + for _, binding := range h.ServerBindings { + if strings.Contains(binding, "/document/v1/") { + return util.Success("handler OK") + } + } + w := fmt.Sprintf("expected /document/v1/ binding, but got: %v", h.ServerBindings) + cli.printWarning(w) + } + } + cli.printWarning("Missing /document/v1/ API; add <document-api /> to the container cluster delcaration in services.xml") + return util.Failure("Missing /document/v1 API") + } else { + return util.FailureWithPayload(service.Description()+" at "+request.URL.Host+": "+response.Status, util.ReaderToJSON(response.Body)) + } +} + +func visitClusters(vArgs *visitArgs, service *vespa.Service) (res util.OperationResult) { + clusters := []string{ + vArgs.contentCluster, + } + if vArgs.contentCluster == "*" { + clusters = probeVisit(vArgs, service) + } + if vArgs.makeFeed { + vArgs.writeString("[\n") + } + for _, c := range clusters { + vArgs.contentCluster = c + res = runVisit(vArgs, service) + if !res.Success { + return res + } + vArgs.debugPrint("Success: " + res.Message) + } + if vArgs.makeFeed { + vArgs.writeString("{}\n]\n") + } + return res +} + +func probeVisit(vArgs *visitArgs, service *vespa.Service) []string { + clusters := make([]string, 0, 3) + vvo, _ := runOneVisit(vArgs, service, "") + if vvo != nil { + msg := vvo.ErrorMsg + if strings.Contains(msg, "no content cluster '*'") { + for idx, value := range strings.Split(msg, ",") { + if idx > 0 { + parts := strings.Split(value, "'") + if len(parts) == 3 { + clusters = append(clusters, parts[1]) + } + } + } + } + } + return clusters +} + +func runVisit(vArgs *visitArgs, service *vespa.Service) (res util.OperationResult) { + vArgs.debugPrint(fmt.Sprintf("trying to visit: '%s'", vArgs.contentCluster)) + var totalDocuments int = 0 + var continuationToken string + for { + var vvo *VespaVisitOutput + vvo, res = runOneVisit(vArgs, service, continuationToken) + if !res.Success { + if vvo != nil && vvo.ErrorMsg != "" { + vArgs.cli.printWarning(vvo.ErrorMsg) + } + return res + } + vArgs.dumpDocuments(vvo.Documents) + vArgs.debugPrint(fmt.Sprintf("got %d documents", len(vvo.Documents))) + totalDocuments += len(vvo.Documents) + continuationToken = vvo.Continuation + if continuationToken == "" { + break + } + } + res.Message = fmt.Sprintf("%s [%d documents visited]", res.Message, totalDocuments) + return +} + +func quoteArgForUrl(arg string) string { + var buf strings.Builder + buf.Grow(len(arg)) + for _, r := range arg { + switch { + case 'a' <= r && r <= 'z': + buf.WriteRune(r) + case 'A' <= r && r <= 'Z': + buf.WriteRune(r) + case '0' <= r && r <= '9': + buf.WriteRune(r) + case r <= ' ' || r > '~': + buf.WriteRune('+') + default: + s := fmt.Sprintf("%s%02X", "%", r) + buf.WriteString(s) + } + } + return buf.String() +} + +func runOneVisit(vArgs *visitArgs, service *vespa.Service, contToken string) (*VespaVisitOutput, util.OperationResult) { + urlPath := service.BaseURL + "/document/v1/?cluster=" + quoteArgForUrl(vArgs.contentCluster) + if vArgs.fieldSet != "" { + urlPath = urlPath + "&fieldSet=" + quoteArgForUrl(vArgs.fieldSet) + } + if vArgs.selection != "" { + urlPath = urlPath + "&selection=" + quoteArgForUrl(vArgs.selection) + } + if contToken != "" { + urlPath = urlPath + "&continuation=" + contToken + } + if vArgs.chunkCount > 0 { + urlPath = urlPath + fmt.Sprintf("&wantedDocumentCount=%d", vArgs.chunkCount) + } + url, urlParseError := url.Parse(urlPath) + if urlParseError != nil { + return nil, util.Failure("Invalid request path: '" + urlPath + "': " + urlParseError.Error()) + } + request := &http.Request{ + URL: url, + Method: "GET", + } + timeout := time.Duration(900) * time.Second + response, err := service.Do(request, timeout) + if err != nil { + return nil, util.Failure("Request failed: " + err.Error()) + } + defer response.Body.Close() + vvo, err := parseVisitOutput(response.Body) + if response.StatusCode == 200 { + if err == nil { + totalDocCount += vvo.DocumentCount + if vvo.DocumentCount != len(vvo.Documents) { + vArgs.cli.printWarning(fmt.Sprintf("Inconsistent contents from: %v", url)) + vArgs.cli.printWarning(fmt.Sprintf("claimed count: %d", vvo.DocumentCount)) + vArgs.cli.printWarning(fmt.Sprintf("document blobs: %d", len(vvo.Documents))) + return nil, util.Failure("Inconsistent contents from document API") + } + return vvo, util.Success("visited " + vArgs.contentCluster) + } else { + return nil, util.Failure("error reading response: " + err.Error()) + } + } else if response.StatusCode/100 == 4 { + return vvo, util.FailureWithPayload("Invalid document operation: "+response.Status, util.ReaderToJSON(response.Body)) + } else { + return vvo, util.FailureWithPayload(service.Description()+" at "+request.URL.Host+": "+response.Status, util.ReaderToJSON(response.Body)) + } +} + +type DocumentBlob struct { + blob []byte +} + +func (d *DocumentBlob) UnmarshalJSON(data []byte) error { + d.blob = make([]byte, len(data)) + copy(d.blob, data) + return nil +} + +func (d *DocumentBlob) MarshalJSON() ([]byte, error) { + return d.blob, nil +} + +type VespaVisitOutput struct { + PathId string `json:"pathId"` + Documents []DocumentBlob `json:"documents"` + DocumentCount int `json:"documentCount"` + Continuation string `json:"continuation"` + ErrorMsg string `json:"message"` +} + +func parseVisitOutput(r io.Reader) (*VespaVisitOutput, error) { + codec := json.NewDecoder(r) + var parsedJson VespaVisitOutput + err := codec.Decode(&parsedJson) + if err != nil { + return nil, fmt.Errorf("could not decode JSON, error: %s", err.Error()) + } + return &parsedJson, nil +} diff --git a/client/go/internal/cli/cmd/visit_test.go b/client/go/internal/cli/cmd/visit_test.go new file mode 100644 index 00000000000..4302680b9d9 --- /dev/null +++ b/client/go/internal/cli/cmd/visit_test.go @@ -0,0 +1,142 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package cmd + +import ( + "fmt" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/internal/mock" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +const ( + normalpre = `{"pathId":"/document/v1/","documents":[` + document1 = `{"id":"id:t:m::1","fields":{"title":"t"}}` + document2 = `{"id":"id:t:m::2","fields":{"title":"t2"}}` + document3 = `{"id":"id:t:m::3","fields":{"ar":"xyz","w":63,"title":"xyzzy","year":2000}}` + + savedresponse = `{"pathId":"/document/v1/","documents":[{"id":"id:test:music::1921492307","fields":{"title":"song","year":2010}},{"id":"id:test:music::p_try-this-clean-bonus-dvd-_music_1922003403","fields":{"artist":"xyz","weight":600000,"song":"hate","title":"xyz","year":2000}}],"documentCount":2,"continuation":"AAAACAAAAAAAAAAJAAAAAAAAAAgAAAAAAAABAAAAAAEgAAAAAAAAEAAAAAAAAAAA"}` + + saveddoc0 = `{"id":"id:test:music::1921492307","fields":{"title":"song","year":2010}}` + saveddoc1 = `{"id":"id:test:music::p_try-this-clean-bonus-dvd-_music_1922003403","fields":{"artist":"xyz","weight":600000,"song":"hate","title":"xyz","year":2000}}` + handlersResponse = `{ + "handlers" : [ { + "id" : "com.yahoo.container.usability.BindingsOverviewHandler", + "class" : "com.yahoo.container.usability.BindingsOverviewHandler", + "bundle" : "container-disc:8.0.0", + "serverBindings" : [ "http://*/" ] + }, { + "id" : "com.yahoo.document.restapi.resource.DocumentV1ApiHandler", + "class" : "com.yahoo.document.restapi.resource.DocumentV1ApiHandler", + "bundle" : "vespaclient-container-plugin:8.0.0", + "serverBindings" : [ "http://*/document/v1/*", "http://*/document/v1/*/" ] + } ] +}` + clusterStarResponse = `{"pathId":"/document/v1/","message":"Your Vespa deployment has no content cluster '*', only 'fooCC'"}` +) + +func TestQuoteFunc(t *testing.T) { + var buf []byte = make([]byte, 3) + buf[0] = 'a' + buf[2] = 'z' + for i := 0; i < 256; i++ { + buf[1] = byte(i) + s := string(buf) + res := quoteArgForUrl(s) + if i < 32 || i > 127 { + assert.Equal(t, "a+z", res) + } else { + fmt.Printf("res %3d => '%s'\n", i, res) + } + } +} + +// low-level (unit) test +func TestRunOneVisit(t *testing.T) { + withResponse := func(client *mock.HTTPClient) { + client.NextResponseString(200, savedresponse) + } + op := func(service *vespa.Service) { + vArgs := visitArgs{ + contentCluster: "fooCC", + } + vvo, res := runOneVisit(&vArgs, service, "BBBB") + assert.Equal(t, true, res.Success) + assert.Equal(t, "visited fooCC", res.Message) + assert.Equal(t, "/document/v1/", vvo.PathId) + assert.Equal(t, "", vvo.ErrorMsg) + assert.Equal(t, "AAAACAAAAAAAAAAJAAAAAAAAAAgAAAAAAAABAAAAAAEgAAAAAAAAEAAAAAAAAAAA", vvo.Continuation) + assert.Equal(t, 2, vvo.DocumentCount) + assert.Equal(t, 2, len(vvo.Documents)) + assert.Equal(t, saveddoc0, string(vvo.Documents[0].blob)) + assert.Equal(t, saveddoc1, string(vvo.Documents[1].blob)) + } + req := withMockClient(t, withResponse, op) + assert.Equal(t, "cluster=fooCC&continuation=BBBB", req.URL.RawQuery) + + op = func(service *vespa.Service) { + vArgs := visitArgs{ + contentCluster: "search", + fieldSet: "[id]", + selection: "music.year>2000", + chunkCount: 123, + } + vvo, res := runOneVisit(&vArgs, service, "asdf") + assert.Equal(t, true, res.Success) + assert.Equal(t, 2, vvo.DocumentCount) + } + req = withMockClient(t, withResponse, op) + assert.Equal(t, "cluster=search&fieldSet=%5Bid%5D&selection=music%2Eyear%3E2000&continuation=asdf&wantedDocumentCount=123", req.URL.RawQuery) +} + +func withMockClient(t *testing.T, prepCli func(*mock.HTTPClient), runOp func(*vespa.Service)) *http.Request { + client := &mock.HTTPClient{} + prepCli(client) + cli, _, _ := newTestCLI(t) + cli.httpClient = client + service, _ := documentService(cli) + runOp(service) + return client.LastRequest +} + +func TestVisitCommand(t *testing.T) { + assertVisitResults( + []string{ + "visit", + "--json-lines", + }, + t, + []string{ + normalpre + + document1 + + `],"documentCount":1,"continuation":"CAFE"}`, + normalpre + + document2 + + "," + + document3 + + `],"documentCount":2}`, + }, + "cluster=fooCC&continuation=CAFE&wantedDocumentCount=1000", + document1+"\n"+ + document2+"\n"+ + document3+"\n") +} + +func assertVisitResults(arguments []string, t *testing.T, responses []string, queryPart, output string) { + client := &mock.HTTPClient{} + client.NextResponseString(200, handlersResponse) + client.NextResponseString(400, clusterStarResponse) + for _, resp := range responses { + client.NextResponseString(200, resp) + } + cli, stdout, stderr := newTestCLI(t) + cli.httpClient = client + assert.Nil(t, cli.Run(arguments...)) + assert.Equal(t, output, stdout.String()) + assert.Equal(t, "", stderr.String()) + assert.Equal(t, queryPart, client.LastRequest.URL.RawQuery) + assert.Equal(t, "/document/v1/", client.LastRequest.URL.Path) + assert.Equal(t, "GET", client.LastRequest.Method) +} diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 04de4d2e277..d5c074a191d 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -111,6 +111,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"tokle"}) default boolean useRestrictedDataPlaneBindings() { return false; } @ModelFeatureFlag(owners = {"arnej","baldersheim"}, removeAfter = "8.110") default boolean useOldJdiscContainerStartup() { return false; } @ModelFeatureFlag(owners = {"tokle, bjorncs"}, removeAfter = "8.108") default boolean enableDataPlaneFilter() { return true; } + @ModelFeatureFlag(owners = {"arnej, bjorncs"}) default boolean enableGlobalPhase() { return false; } //Below are all flags that must be kept until 7 is out of the door @ModelFeatureFlag(owners = {"arnej"}, removeAfter="7.last") default boolean ignoreThreadStackSizes() { return false; } diff --git a/config-model/.gitignore b/config-model/.gitignore index 6edd041cbe8..7e7e597675d 100644 --- a/config-model/.gitignore +++ b/config-model/.gitignore @@ -4,5 +4,6 @@ /target /src/test/integration/*/copy/ /src/test/integration/*/models.generated/ +/src/test/derived/*/models.generated/ *.cfg.actual /var/ diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 49194a5d1bb..ecbb990f096 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -137,6 +137,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean useRestrictedDataPlaneBindings() { return useRestrictedDataPlaneBindings; } @Override public Optional<CloudAccount> cloudAccount() { return cloudAccount; } @Override public boolean allowUserFilters() { return allowUserFilters; } + @Override public boolean enableGlobalPhase() { return true; } // Enable global-phase by default for unit tests only public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) { this.sharedStringRepoNoReclaim = sharedStringRepoNoReclaim; diff --git a/config-model/src/main/java/com/yahoo/schema/RankProfile.java b/config-model/src/main/java/com/yahoo/schema/RankProfile.java index ad6eb038058..7cb0a088f5f 100644 --- a/config-model/src/main/java/com/yahoo/schema/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/RankProfile.java @@ -1019,6 +1019,9 @@ public class RankProfile implements Cloneable { var recorder = new InputRecorder(needInputs); recorder.transform(globalPhaseRanking.function().getBody(), context); for (String input : needInputs) { + if (input.startsWith("constant(") || input.startsWith("query(")) { + continue; + } try { addMatchFeatures(new FeatureList(input)); } catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) { diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java index b0f63ebb732..4e7988a2006 100644 --- a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java @@ -3,16 +3,13 @@ package com.yahoo.schema.expressiontransforms; import com.yahoo.schema.FeatureNames; import com.yahoo.schema.RankProfile; -import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.Reference; -import com.yahoo.searchlib.rankingexpression.parser.ParseException; import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; -import java.io.StringReader; import java.util.Set; /** @@ -86,13 +83,7 @@ public class InputRecorder extends ExpressionTransformer<RankProfileTransformCon throw new IllegalArgumentException("missing onnx model: " + arg); } for (String onnxInput : model.getInputMap().values()) { - var reader = new StringReader(onnxInput); - try { - var asExpression = new RankingExpression(reader); - transform(asExpression.getRoot(), context); - } catch (ParseException e) { - throw new IllegalArgumentException("illegal onnx input '" + onnxInput + "': " + e.getMessage()); - } + neededInputs.add(onnxInput); } return; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java index 3d9a8441ed5..0be3c825614 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java @@ -26,9 +26,9 @@ public class ContainerModelEvaluation implements OnnxModelsConfig.Producer, RankingExpressionsConfig.Producer { - private final static String EVALUATION_BUNDLE_NAME = "model-evaluation"; - private final static String INTEGRATION_BUNDLE_NAME = "model-integration"; - private final static String ONNXRUNTIME_BUNDLE_NAME = "container-onnxruntime.jar"; + public final static String EVALUATION_BUNDLE_NAME = "model-evaluation"; + public final static String INTEGRATION_BUNDLE_NAME = "model-integration"; + public final static String ONNXRUNTIME_BUNDLE_NAME = "container-onnxruntime.jar"; private final static String EVALUATOR_NAME = ModelsEvaluator.class.getName(); private final static String REST_HANDLER_NAME = "ai.vespa.models.handler.ModelsEvaluationHandler"; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java index 29b1edc1397..19df9a4064f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java @@ -10,6 +10,10 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import static com.yahoo.vespa.model.container.ContainerModelEvaluation.EVALUATION_BUNDLE_NAME; +import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATION_BUNDLE_NAME; +import static com.yahoo.vespa.model.container.ContainerModelEvaluation.ONNXRUNTIME_BUNDLE_NAME; + /** * NOTE: Stable ordering of bundles in config is handled by {@link ContainerCluster#addPlatformBundle(Path)} * @@ -53,7 +57,10 @@ public class PlatformBundles { public static final Set<Path> SEARCH_AND_DOCPROC_BUNDLES = toBundlePaths( SEARCH_AND_DOCPROC_BUNDLE, "docprocs", - "linguistics-components" + "linguistics-components", + EVALUATION_BUNDLE_NAME, + INTEGRATION_BUNDLE_NAME, + ONNXRUNTIME_BUNDLE_NAME ); private static Set<Path> toBundlePaths(String... bundleNames) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index 86c48407775..5d4ec598250 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -1,12 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.search; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.container.QrSearchersConfig; import com.yahoo.prelude.semantics.SemanticRulesConfig; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.search.pagetemplates.PageTemplatesConfig; import com.yahoo.search.query.profile.config.QueryProfilesConfig; +import com.yahoo.search.ranking.RankProfilesEvaluatorFactory; import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; @@ -44,18 +46,22 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> private final ApplicationContainerCluster owningCluster; private final List<SearchCluster> searchClusters = new LinkedList<>(); + private final boolean globalPhase; private QueryProfiles queryProfiles; private SemanticRules semanticRules; private PageTemplates pageTemplates; - public ContainerSearch(ApplicationContainerCluster cluster, SearchChains chains) { + public ContainerSearch(DeployState deployState, ApplicationContainerCluster cluster, SearchChains chains) { super(chains); + this.globalPhase = deployState.featureFlags().enableGlobalPhase(); this.owningCluster = cluster; owningCluster.addComponent(Component.fromClassAndBundle(CompiledQueryProfileRegistry.class, SEARCH_AND_DOCPROC_BUNDLE)); owningCluster.addComponent(Component.fromClassAndBundle(com.yahoo.search.schema.SchemaInfo.class, SEARCH_AND_DOCPROC_BUNDLE)); owningCluster.addComponent(Component.fromClassAndBundle(SearchStatusExtension.class, SEARCH_AND_DOCPROC_BUNDLE)); + owningCluster.addComponent(Component.fromClassAndBundle(RankProfilesEvaluatorFactory.class, SEARCH_AND_DOCPROC_BUNDLE)); + owningCluster.addComponent(Component.fromClassAndBundle(com.yahoo.search.ranking.GlobalPhaseRanker.class, SEARCH_AND_DOCPROC_BUNDLE)); cluster.addSearchAndDocprocBundles(); } @@ -68,9 +74,18 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> /** Adds a Dispatcher component to the owning container cluster for each search cluster */ private void initializeDispatchers(Collection<SearchCluster> searchClusters) { for (SearchCluster searchCluster : searchClusters) { - if ( ! ( searchCluster instanceof IndexedSearchCluster)) continue; - var dispatcher = new DispatcherComponent((IndexedSearchCluster)searchCluster); - owningCluster.addComponent(dispatcher); + if (searchCluster instanceof IndexedSearchCluster indexed) { + var dispatcher = new DispatcherComponent(indexed); + owningCluster.addComponent(dispatcher); + if (globalPhase) { + for (var documentDb : indexed.getDocumentDbs()) { + var factory = new RankProfilesEvaluatorComponent(documentDb); + if (! owningCluster.getComponentsMap().containsKey(factory.getComponentId())) { + owningCluster.addComponent(factory); + } + } + } + } } } @@ -143,6 +158,7 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> if ( ! (sys instanceof IndexedSearchCluster)) { scB.storagecluster(new QrSearchersConfig.Searchcluster.Storagecluster.Builder(). routespec(((StreamingSearchCluster)sys).getStorageRouteSpec())); + scB.globalphase(globalPhase); } builder.searchcluster(scB); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/RankProfilesEvaluatorComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RankProfilesEvaluatorComponent.java new file mode 100644 index 00000000000..75a2802ee53 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RankProfilesEvaluatorComponent.java @@ -0,0 +1,49 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search; + +import com.yahoo.config.model.producer.AnyConfigProducer; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.search.ranking.RankProfilesEvaluator; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; +import com.yahoo.vespa.model.container.ContainerModelEvaluation; +import com.yahoo.vespa.model.container.PlatformBundles; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.search.DocumentDatabase; + +public class RankProfilesEvaluatorComponent + extends Component<AnyConfigProducer, ComponentModel> + implements + RankProfilesConfig.Producer, + RankingConstantsConfig.Producer, + RankingExpressionsConfig.Producer, + OnnxModelsConfig.Producer +{ + private final DocumentDatabase ddb; + + public RankProfilesEvaluatorComponent(DocumentDatabase db) { + super(toComponentModel(db.getSchemaName())); + ddb = db; + } + + private static ComponentModel toComponentModel(String p) { + String myComponentId = "ranking-expression-evaluator." + p; + return new ComponentModel(myComponentId, + RankProfilesEvaluator.class.getName(), + PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE); + } + + @Override + public void getConfig(RankProfilesConfig.Builder builder) { ddb.getConfig(builder); } + + @Override + public void getConfig(RankingExpressionsConfig.Builder builder) { ddb.getConfig(builder); } + + @Override + public void getConfig(RankingConstantsConfig.Builder builder) { ddb.getConfig(builder); } + + @Override + public void getConfig(OnnxModelsConfig.Builder builder) { ddb.getConfig(builder); } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index ceba3864296..a639c158d62 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -732,7 +732,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { SearchChains searchChains = new DomSearchChainsBuilder() .build(deployState, containerCluster, producerSpec); - ContainerSearch containerSearch = new ContainerSearch(containerCluster, searchChains); + ContainerSearch containerSearch = new ContainerSearch(deployState, containerCluster, searchChains); applyApplicationPackageDirectoryConfigs(deployState.getApplicationPackage(), containerSearch); containerSearch.setQueryProfiles(deployState.getQueryProfiles()); diff --git a/config-model/src/test/derived/globalphase_onnx_inside/files/ax_plus_b.onnx b/config-model/src/test/derived/globalphase_onnx_inside/files/ax_plus_b.onnx new file mode 100644 index 00000000000..17282d13dc3 --- /dev/null +++ b/config-model/src/test/derived/globalphase_onnx_inside/files/ax_plus_b.onnx @@ -0,0 +1,23 @@ +:© + +matrix_X +vector_AXA"MatMul + +XA +vector_Bvector_Y"AddlrZ +matrix_X + + +Z +vector_A + + +Z +vector_B + + +b +vector_Y + + +B
\ No newline at end of file diff --git a/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg b/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg new file mode 100644 index 00000000000..35bb1ccc3d2 --- /dev/null +++ b/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg @@ -0,0 +1,34 @@ +rankprofile[].name "default" +rankprofile[].fef.property[].name "rankingExpression(handicap).rankingScript" +rankprofile[].fef.property[].value "query(yy)" +rankprofile[].fef.property[].name "rankingExpression(handicap).type" +rankprofile[].fef.property[].value "tensor(d0[2])" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "rankingExpression(firstphase)" +rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" +rankprofile[].fef.property[].value "reduce(attribute(aa), sum)" +rankprofile[].fef.property[].name "vespa.rank.globalphase" +rankprofile[].fef.property[].value "rankingExpression(globalphase)" +rankprofile[].fef.property[].name "rankingExpression(globalphase).rankingScript" +rankprofile[].fef.property[].value "reduce(constant(ww) * (onnx(inside).foobar - rankingExpression(handicap)), sum)" +rankprofile[].fef.property[].name "vespa.match.feature" +rankprofile[].fef.property[].value "attribute(aa)" +rankprofile[].fef.property[].name "vespa.globalphase.rerankcount" +rankprofile[].fef.property[].value "13" +rankprofile[].fef.property[].name "vespa.type.attribute.aa" +rankprofile[].fef.property[].value "tensor(d1[3])" +rankprofile[].fef.property[].name "vespa.type.query.bb" +rankprofile[].fef.property[].value "tensor(d0[2])" +rankprofile[].fef.property[].name "vespa.type.query.yy" +rankprofile[].fef.property[].value "tensor(d0[2])" +rankprofile[].name "unranked" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "value(0)" +rankprofile[].fef.property[].name "vespa.hitcollector.heapsize" +rankprofile[].fef.property[].value "0" +rankprofile[].fef.property[].name "vespa.hitcollector.arraysize" +rankprofile[].fef.property[].value "0" +rankprofile[].fef.property[].name "vespa.dump.ignoredefaultfeatures" +rankprofile[].fef.property[].value "true" +rankprofile[].fef.property[].name "vespa.type.attribute.aa" +rankprofile[].fef.property[].value "tensor(d1[3])" diff --git a/config-model/src/test/derived/globalphase_onnx_inside/test.sd b/config-model/src/test/derived/globalphase_onnx_inside/test.sd new file mode 100644 index 00000000000..c38e318ce6b --- /dev/null +++ b/config-model/src/test/derived/globalphase_onnx_inside/test.sd @@ -0,0 +1,42 @@ +schema test { + + document test { + field aa type tensor(d1[3]) { + indexing: attribute + } + } + + constant xx { + file: files/const_xx.json + type: tensor(d0[2],d1[3]) + } + constant ww { + file: files/const_ww.json + type: tensor(d0[2]) + } + + rank-profile default { + inputs { + query(bb) tensor(d0[2]) + query(yy) tensor(d0[2]) + } + onnx-model inside { + file: files/ax_plus_b.onnx + input vector_A: attribute(aa) + input matrix_X: constant(xx) + input vector_B: query(bb) + output vector_Y: foobar + } + first-phase { + expression: sum(attribute(aa)) + } + function handicap() { + expression: query(yy) + } + global-phase { + rerank-count: 13 + expression: sum(constant(ww) * (onnx(inside).foobar - handicap)) + } + } + +} diff --git a/config-model/src/test/derived/rankingexpression/rank-profiles.cfg b/config-model/src/test/derived/rankingexpression/rank-profiles.cfg index ea8bc5f77e6..e3947e9e46f 100644 --- a/config-model/src/test/derived/rankingexpression/rank-profiles.cfg +++ b/config-model/src/test/derived/rankingexpression/rank-profiles.cfg @@ -410,8 +410,6 @@ rankprofile[].fef.property[].value "rankingExpression(globalphase)" rankprofile[].fef.property[].name "rankingExpression(globalphase).rankingScript" rankprofile[].fef.property[].value "rankingExpression(myplus) + reduce(rankingExpression(mymul), sum) + firstPhase" rankprofile[].fef.property[].name "vespa.match.feature" -rankprofile[].fef.property[].value "query(fromq)" -rankprofile[].fef.property[].name "vespa.match.feature" rankprofile[].fef.property[].value "firstPhase" rankprofile[].fef.property[].name "vespa.match.feature" rankprofile[].fef.property[].value "attribute(t1)" diff --git a/config-model/src/test/java/com/yahoo/schema/derived/GlobalPhaseOnnxModelsTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/GlobalPhaseOnnxModelsTestCase.java new file mode 100644 index 00000000000..2ff33dd70d8 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/GlobalPhaseOnnxModelsTestCase.java @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.parser.ParseException; +import org.junit.jupiter.api.Test; +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Tests exporting with global-phase and ONNX models + * + * @author arnej + */ +public class GlobalPhaseOnnxModelsTestCase extends AbstractExportingTestCase { + + @Test + void testModelInRankProfile() throws IOException, ParseException { + assertCorrectDeriving("globalphase_onnx_inside"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index ce2d5ca2da5..5973ef56962 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -485,7 +485,7 @@ public class ContainerClusterTest { if (isCombinedCluster) cluster.setHostClusterId("test-content-cluster"); cluster.setMemoryPercentage(memoryPercentage); - cluster.setSearch(new ContainerSearch(cluster, new SearchChains(cluster, "search-chain"))); + cluster.setSearch(new ContainerSearch(root.getDeployState(), cluster, new SearchChains(cluster, "search-chain"))); return cluster; } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java new file mode 100644 index 00000000000..cec5fb1a1ed --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java @@ -0,0 +1,2 @@ +package com.yahoo.config.provision;public class ClusterInfo { +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java b/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java index a614e251dca..89f7755729c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java @@ -38,9 +38,15 @@ public class GetConfigContext { return new GetConfigContext(app, handler, trace); } + public static GetConfigContext empty() { + return new GetConfigContext(null, null, null); + } + public static GetConfigContext testContext(ApplicationId app) { return new GetConfigContext(app, null, null); } + + public boolean isEmpty() { return app == null && requestHandler == null && trace == null; } /** * Helper to produce a log preamble with the tenant and app id diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 1d80740ffe0..d61590c2e84 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -202,6 +202,7 @@ public class ModelContextImpl implements ModelContext { private final int rpc_events_before_wakeup; private final boolean useRestrictedDataPlaneBindings; private final int heapPercentage; + private final boolean enableGlobalPhase; public FeatureFlags(FlagSource source, ApplicationId appId, Version version) { this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -246,6 +247,7 @@ public class ModelContextImpl implements ModelContext { this.queryDispatchWarmup = flagValue(source, appId, version, PermanentFlags.QUERY_DISPATCH_WARMUP); this.useRestrictedDataPlaneBindings = flagValue(source, appId, version, Flags.RESTRICT_DATA_PLANE_BINDINGS); this.heapPercentage = flagValue(source, appId, version, PermanentFlags.HEAP_SIZE_PERCENTAGE); + this.enableGlobalPhase = flagValue(source, appId, version, PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW); } @Override public int heapSizePercentage() { return heapPercentage; } @@ -298,6 +300,7 @@ public class ModelContextImpl implements ModelContext { return defVal; } @Override public boolean useRestrictedDataPlaneBindings() { return useRestrictedDataPlaneBindings; } + @Override public boolean enableGlobalPhase() { return enableGlobalPhase; } private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index 85b30f4d303..454ab9e490d 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -23,6 +23,7 @@ import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; +import java.time.Clock; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.logging.Level; @@ -153,6 +154,8 @@ public class FileDirectory extends AbstractComponent { return true; } + // update last modified time so that maintainer deleting unused file references considers this as recently used + existingFile.setLastModified(Clock.systemUTC().instant().toEpochMilli()); log.log(Level.FINE, "Directory for file reference '" + fileReference.value() + "' already exists and has all content"); return false; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java index 1c419ce047a..5e7fff8c922 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.config.server.rpc; import com.yahoo.cloud.config.SentinelConfig; -import com.yahoo.collections.Pair; import com.yahoo.component.Version; import com.yahoo.config.provision.TenantName; import com.yahoo.container.di.config.ApplicationBundlesConfig; @@ -37,12 +36,10 @@ class GetConfigProcessor implements Runnable { private static final Logger log = Logger.getLogger(GetConfigProcessor.class.getName()); private static final String localHostName = HostName.getLocalhost(); - private static final PayloadChecksums emptyApplicationBundlesConfigChecksums = PayloadChecksums.fromPayload(Payload.from(ConfigPayload.fromInstance(new ApplicationBundlesConfig.Builder().build()))); private final JRTServerConfigRequest request; - /* True only when this request has expired its server timeout and we need to respond to the client */ private final boolean forceResponse; private final RpcServer rpcServer; @@ -75,13 +72,13 @@ class GetConfigProcessor implements Runnable { } // TODO: Increment statistics (Metrics) failed counters when requests fail - public Pair<GetConfigContext, Long> getConfig(JRTServerConfigRequest request) { + public Optional<DelayedConfig> resolveConfig(JRTServerConfigRequest request) { // Request has already been detached if ( ! request.validateParameters()) { // Error code is set in verifyParameters if parameters are not OK. log.log(Level.WARNING, "Parameters for request " + request + " did not validate: " + request.errorCode() + " : " + request.errorMessage()); respond(request); - return null; + return Optional.empty(); } Trace trace = request.getRequestTrace(); debugLog(trace, "GetConfigProcessor.run() on " + localHostName); @@ -92,13 +89,13 @@ class GetConfigProcessor implements Runnable { // fabricate an empty request to cause the sentinel to stop all running services if (rpcServer.canReturnEmptySentinelConfig() && rpcServer.allTenantsLoaded() && tenant.isEmpty() && isSentinelConfigRequest(request)) { returnEmpty(request); - return null; + return Optional.empty(); } GetConfigContext context = rpcServer.createGetConfigContext(tenant, request, trace); - if (context == null || ! context.requestHandler().hasApplication(context.applicationId(), Optional.empty())) { + if (context.isEmpty() || ! context.requestHandler().hasApplication(context.applicationId(), Optional.empty())) { handleError(request, APPLICATION_NOT_LOADED, "No application exists"); - return null; + return Optional.empty(); } logPre = TenantRepository.logPre(context.applicationId()); @@ -109,7 +106,7 @@ class GetConfigProcessor implements Runnable { if ( ! context.requestHandler().hasApplication(context.applicationId(), vespaVersion)) { handleError(request, ErrorCode.UNKNOWN_VESPA_VERSION, "Unknown Vespa version in request: " + printableVespaVersion(vespaVersion)); - return null; + return Optional.empty(); } ConfigResponse config; @@ -117,14 +114,14 @@ class GetConfigProcessor implements Runnable { config = rpcServer.resolveConfig(request, context, vespaVersion); } catch (UnknownConfigDefinitionException e) { handleError(request, ErrorCode.UNKNOWN_DEFINITION, "Unknown config definition " + request.getConfigKey()); - return null; + return Optional.empty(); } catch (UnknownConfigIdException e) { handleError(request, ErrorCode.ILLEGAL_CONFIGID, "Illegal config id " + request.getConfigKey().getConfigId()); - return null; + return Optional.empty(); } catch (Throwable e) { log.log(Level.SEVERE, "Unexpected error handling config request", e); handleError(request, ErrorCode.INTERNAL_ERROR, "Internal error " + e.getMessage()); - return null; + return Optional.empty(); } // config == null is not an error, but indicates that the config will be returned later. @@ -135,7 +132,7 @@ class GetConfigProcessor implements Runnable { && ! context.requestHandler().compatibleWith(vespaVersion, context.applicationId()) // ... with a runtime version incompatible with the deploying version ... && ! emptyApplicationBundlesConfigChecksums.matches(config.getPayloadChecksums())) { // ... and there actually are incompatible user bundles, then return no config: handleError(request, ErrorCode.INCOMPATIBLE_VESPA_VERSION, "Version " + printableVespaVersion(vespaVersion) + " is binary incompatible with the latest deployed version"); - return null; + return Optional.empty(); } // debugLog(trace, "config response before encoding:" + config.toString()); @@ -144,23 +141,24 @@ class GetConfigProcessor implements Runnable { respond(request); } else { debugLog(trace, "delaying response " + request.getShortDescription()); - return new Pair<>(context, config != null ? config.getGeneration() : 0); + return Optional.of(new DelayedConfig(context, config != null ? config.getGeneration() : 0)); } - return null; + return Optional.empty(); } @Override public void run() { - Pair<GetConfigContext, Long> delayed = getConfig(request); + Optional<DelayedConfig> delayed = resolveConfig(request); - if (delayed != null) { - rpcServer.delayResponse(request, delayed.getFirst()); - if (rpcServer.hasNewerGeneration(delayed.getFirst().applicationId(), delayed.getSecond())) { + delayed.ifPresent(d -> { + GetConfigContext context = d.context(); + rpcServer.delayResponse(request, context); + if (rpcServer.hasNewerGeneration(context.applicationId(), d.generation())) { // This will ensure that if the config activation train left the station while I was boarding, // another train will immediately be scheduled. - rpcServer.configActivated(delayed.getFirst().applicationId()); + rpcServer.configActivated(context.applicationId()); } - } + }); } private boolean isSentinelConfigRequest(JRTServerConfigRequest request) { @@ -194,4 +192,6 @@ class GetConfigProcessor implements Runnable { } } + private record DelayedConfig(GetConfigContext context, long generation) {} + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index 2d731370b70..7a1b2d2aeef 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -406,7 +406,7 @@ public class RpcServer implements Runnable, ConfigActivationListener, TenantList metrics.incUnknownHostRequests(); trace.trace(TRACELEVEL, msg); log.log(Level.WARNING, msg); - return null; + return GetConfigContext.empty(); } RequestHandler handler = requestHandler.get(); ApplicationId applicationId = handler.resolveApplicationId(request.getClientHostName()); diff --git a/container-core/src/main/resources/configdefinitions/container.qr-searchers.def b/container-core/src/main/resources/configdefinitions/container.qr-searchers.def index e89ee40d792..034e7fc442b 100644 --- a/container-core/src/main/resources/configdefinitions/container.qr-searchers.def +++ b/container-core/src/main/resources/configdefinitions/container.qr-searchers.def @@ -71,3 +71,6 @@ searchcluster[].indexingmode enum { REALTIME, STREAMING } default=REALTIME ## Storage cluster route to use for search cluster if indexingmode is streaming. searchcluster[].storagecluster.routespec string default="" + +## Enable global phase ranking +searchcluster[].globalphase bool default=false diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 729aebf2fc2..08d8d54bc53 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -20,6 +20,7 @@ import com.yahoo.search.Searcher; import com.yahoo.search.config.ClusterConfig; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.query.ParameterParser; +import com.yahoo.search.ranking.GlobalPhaseRanker; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.searchchain.Execution; @@ -64,6 +65,8 @@ public class ClusterSearcher extends Searcher { private final VespaBackEndSearcher server; private final Executor executor; + private final GlobalPhaseRanker globalPhaseHelper; + private final boolean enableGlobalPhase; @Inject public ClusterSearcher(ComponentId id, @@ -73,10 +76,12 @@ public class ClusterSearcher extends Searcher { DocumentdbInfoConfig documentDbConfig, SchemaInfo schemaInfo, ComponentRegistry<Dispatcher> dispatchers, + GlobalPhaseRanker globalPhaseHelper, VipStatus vipStatus, VespaDocumentAccess access) { super(id); this.executor = executor; + this.globalPhaseHelper = globalPhaseHelper; int searchClusterIndex = clusterConfig.clusterId(); searchClusterName = clusterConfig.clusterName(); QrSearchersConfig.Searchcluster searchClusterConfig = getSearchClusterConfigFromClusterName(qrsConfig, searchClusterName); @@ -101,6 +106,7 @@ public class ClusterSearcher extends Searcher { server = searchDispatch(searchClusterIndex, searchClusterName, uniqueServerId, docSumParams, documentDbConfig, schemaInfo, dispatchers); } + enableGlobalPhase = searchClusterConfig.globalphase(); } private static QrSearchersConfig.Searchcluster getSearchClusterConfigFromClusterName(QrSearchersConfig config, String name) { @@ -159,7 +165,10 @@ public class ClusterSearcher extends Searcher { maxQueryCacheTimeout = DEFAULT_MAX_QUERY_CACHE_TIMEOUT; server = searcher; this.executor = executor; + this.globalPhaseHelper = null; + this.enableGlobalPhase = false; } + /** Do not use, for internal testing purposes only. **/ ClusterSearcher(Set<String> schemas) { this(schemas, null, null); @@ -169,7 +178,7 @@ public class ClusterSearcher extends Searcher { public void fill(com.yahoo.search.Result result, String summaryClass, Execution execution) { Query query = result.getQuery(); - VespaBackEndSearcher searcher = server; + Searcher searcher = server; if (searcher != null) { if (query.getTimeLeft() > 0) { searcher.fill(result, summaryClass, execution); @@ -190,7 +199,7 @@ public class ClusterSearcher extends Searcher { public Result search(Query query, Execution execution) { validateQueryTimeout(query); validateQueryCache(query); - VespaBackEndSearcher searcher = server; + Searcher searcher = server; if (searcher == null) { return new Result(query, ErrorMessage.createNoBackendsInService("Could not search")); } @@ -228,8 +237,21 @@ public class ClusterSearcher extends Searcher { } else { String docType = schemas.iterator().next(); query.getModel().setRestrict(docType); - return searcher.search(query, execution); + return perSchemaSearch(searcher, query, execution); + } + } + + private Result perSchemaSearch(Searcher searcher, Query query, Execution execution) { + Set<String> restrict = query.getModel().getRestrict(); + if (restrict.size() != 1) { + throw new IllegalStateException("perSchemaSearch must always be called with 1 schema, got: " + restrict.size()); + } + String schema = restrict.iterator().next(); + Result result = searcher.search(query, execution); + if (globalPhaseHelper != null && enableGlobalPhase) { + globalPhaseHelper.process(query, result, schema); } + return result; } private static void processResult(Query query, FutureTask<Result> task, Result mergedResult) { @@ -248,12 +270,12 @@ public class ClusterSearcher extends Searcher { Set<String> schemas = resolveSchemas(query, execution.context().getIndexFacts()); List<Query> queries = createQueries(query, schemas); if (queries.size() == 1) { - return searcher.search(queries.get(0), execution); + return perSchemaSearch(searcher, queries.get(0), execution); } else { Result mergedResult = new Result(query); List<FutureTask<Result>> pending = new ArrayList<>(queries.size()); for (Query q : queries) { - FutureTask<Result> task = new FutureTask<>(() -> searcher.search(q, execution)); + FutureTask<Result> task = new FutureTask<>(() -> perSchemaSearch(searcher, q, execution)); try { executor.execute(task); pending.add(task); diff --git a/container-search/src/main/java/com/yahoo/search/ranking/Evaluator.java b/container-search/src/main/java/com/yahoo/search/ranking/Evaluator.java new file mode 100644 index 00000000000..d2edb776c92 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/Evaluator.java @@ -0,0 +1,14 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import com.yahoo.tensor.Tensor; + +import java.util.Collection; + +interface Evaluator { + Collection<String> needInputs(); + + Evaluator bind(String name, Tensor value); + + double evaluateScore(); +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java new file mode 100644 index 00000000000..87213362acd --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java @@ -0,0 +1,118 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import ai.vespa.models.evaluation.FunctionEvaluator; +import ai.vespa.models.evaluation.Model; +import com.yahoo.component.annotation.Inject; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.tensor.Tensor; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.logging.Logger; +import java.util.function.Supplier; + +public class GlobalPhaseRanker { + + private static final Logger logger = Logger.getLogger(GlobalPhaseRanker.class.getName()); + private final RankProfilesEvaluatorFactory factory; + private final Set<String> skipProcessing = new HashSet<>(); + private final Map<String, Supplier<FunctionEvaluator>> scorers = new HashMap<>(); + + @Inject + public GlobalPhaseRanker(RankProfilesEvaluatorFactory factory) { + this.factory = factory; + logger.info("using factory: " + factory); + } + + public void process(Query query, Result result, String schema) { + var functionEvaluatorSource = underlying(query, schema); + if (functionEvaluatorSource == null) { + return; + } + var prepared = findFromQuery(query, functionEvaluatorSource.get().function().arguments()); + Supplier<Evaluator> supplier = () -> { + var evaluator = functionEvaluatorSource.get(); + var simple = new SimpleEvaluator(evaluator); + for (var entry : prepared) { + simple.bind(entry.name(), entry.value()); + } + return simple; + }; + // TODO need to get rerank-count somehow + int rerank = 7; + ResultReranker.rerankHits(result, new HitRescorer(supplier), rerank); + } + + record NameAndValue(String name, Tensor value) { } + + /* do this only once per query: */ + List<NameAndValue> findFromQuery(Query query, List<String> needInputs) { + List<NameAndValue> result = new ArrayList<>(); + var ranking = query.getRanking(); + var rankFeatures = ranking.getFeatures(); + var rankProps = ranking.getProperties().asMap(); + for (String needed : needInputs) { + var optRef = com.yahoo.searchlib.rankingexpression.Reference.simple(needed); + if (optRef.isEmpty()) continue; + var ref = optRef.get(); + if (ref.name().equals("constant")) { + // XXX in theory, we should be able to avoid this + result.add(new NameAndValue(needed, null)); + continue; + } + if (ref.isSimple() && ref.name().equals("query")) { + String queryFeatureName = ref.simpleArgument().get(); + // searchers are recommended to place query features here: + var feature = rankFeatures.getTensor(queryFeatureName); + if (feature.isPresent()) { + result.add(new NameAndValue(needed, feature.get())); + } else { + // but other ways of setting query features end up in the properties: + var objList = rankProps.get(queryFeatureName); + if (objList != null && objList.size() == 1 && objList.get(0) instanceof Tensor t) { + result.add(new NameAndValue(needed, t)); + } + } + } + } + return result; + } + + private Supplier<FunctionEvaluator> underlying(Query query, String schema) { + String rankProfile = query.getRanking().getProfile(); + String key = schema + " with rank profile " + rankProfile; + if (skipProcessing.contains(key)) { + return null; + } + Supplier<FunctionEvaluator> supplier = scorers.get(key); + if (supplier != null) { + return supplier; + } + try { + var proxy = factory.proxyForSchema(schema); + var model = proxy.modelForRankProfile(rankProfile); + supplier = () -> model.evaluatorOf("globalphase"); + if (supplier.get() == null) { + supplier = null; + } + } catch (IllegalArgumentException e) { + logger.info("no global-phase for " + key + " because: " + e.getMessage()); + supplier = null; + } + if (supplier == null) { + skipProcessing.add(key); + } else { + scorers.put(key, supplier); + } + return supplier; + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/HitRescorer.java b/container-search/src/main/java/com/yahoo/search/ranking/HitRescorer.java new file mode 100644 index 00000000000..ebdbbb693f1 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/HitRescorer.java @@ -0,0 +1,56 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import com.yahoo.search.result.FeatureData; +import com.yahoo.search.result.Hit; + +import java.util.function.Supplier; +import java.util.logging.Logger; + +class HitRescorer { + + private static final Logger logger = Logger.getLogger(HitRescorer.class.getName()); + + private final Supplier<Evaluator> evaluatorSource; + + public HitRescorer(Supplier<Evaluator> evaluatorSource) { + this.evaluatorSource = evaluatorSource; + } + + boolean rescoreHit(Hit hit) { + var features = hit.getField("matchfeatures"); + if (features instanceof FeatureData matchFeatures) { + var scorer = evaluatorSource.get(); + for (String argName : scorer.needInputs()) { + var asTensor = matchFeatures.getTensor(argName); + if (asTensor == null) { + asTensor = matchFeatures.getTensor(alternate(argName)); + } + if (asTensor != null) { + scorer.bind(argName, asTensor); + } else { + logger.warning("Missing match-feature for Evaluator argument: " + argName); + return false; + } + } + double newScore = scorer.evaluateScore(); + hit.setRelevance(newScore); + return true; + } else { + logger.warning("Hit without match-features: " + hit); + return false; + } + } + + private static final String RE_PREFIX = "rankingExpression("; + private static final String RE_SUFFIX = ")"; + private static final int RE_PRE_LEN = RE_PREFIX.length(); + private static final int RE_SUF_LEN = RE_SUFFIX.length(); + + static String alternate(String argName) { + if (argName.startsWith(RE_PREFIX) && argName.endsWith(RE_SUFFIX)) { + return argName.substring(RE_PRE_LEN, argName.length() - RE_SUF_LEN); + } + return argName; + } +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluator.java b/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluator.java new file mode 100644 index 00000000000..ccb9b9837fe --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluator.java @@ -0,0 +1,53 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.search.ranking; + +import ai.vespa.models.evaluation.FunctionEvaluator; +import ai.vespa.models.evaluation.Model; +import ai.vespa.models.evaluation.ModelsEvaluator; +import com.yahoo.api.annotations.Beta; +import com.yahoo.component.AbstractComponent; +import com.yahoo.component.annotation.Inject; +import com.yahoo.filedistribution.fileacquirer.FileAcquirer; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; + +/** + * proxy for model-evaluation components + * @author arnej + */ +@Beta +public class RankProfilesEvaluator extends AbstractComponent { + + private final ModelsEvaluator evaluator; + + @Inject + public RankProfilesEvaluator( + RankProfilesConfig rankProfilesConfig, + RankingConstantsConfig constantsConfig, + RankingExpressionsConfig expressionsConfig, + OnnxModelsConfig onnxModelsConfig, + FileAcquirer fileAcquirer) + { + this.evaluator = new ModelsEvaluator( + rankProfilesConfig, + constantsConfig, + expressionsConfig, + onnxModelsConfig, + fileAcquirer); + } + + public Model modelForRankProfile(String rankProfile) { + var m = evaluator.models().get(rankProfile); + if (m == null) { + throw new IllegalArgumentException("unknown rankprofile: " + rankProfile); + } + return m; + } + + public FunctionEvaluator evaluatorForFunction(String rankProfile, String functionName) { + return modelForRankProfile(rankProfile).evaluatorOf(functionName); + } +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluatorFactory.java b/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluatorFactory.java new file mode 100644 index 00000000000..edb05ed9788 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluatorFactory.java @@ -0,0 +1,40 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.search.ranking; + +import com.yahoo.api.annotations.Beta; +import com.yahoo.component.annotation.Inject; +import com.yahoo.component.provider.ComponentRegistry; + +/** + * factory for model-evaluation proxies + * @author arnej + */ +@Beta +public class RankProfilesEvaluatorFactory { + + private final ComponentRegistry<RankProfilesEvaluator> registry; + + @Inject + public RankProfilesEvaluatorFactory(ComponentRegistry<RankProfilesEvaluator> registry) { + this.registry = registry; + } + + public RankProfilesEvaluator proxyForSchema(String schemaName) { + var component = registry.getComponent("ranking-expression-evaluator." + schemaName); + if (component == null) { + throw new IllegalArgumentException("ranking expression evaluator for schema '" + schemaName + "' not found"); + } + return component; + } + + public String toString() { + var buf = new StringBuilder(); + buf.append(this.getClass().getName()).append(" containing: ["); + for (var id : registry.allComponentsById().keySet()) { + buf.append(" ").append(id.toString()); + } + buf.append(" ]"); + return buf.toString(); + } +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/ResultReranker.java b/container-search/src/main/java/com/yahoo/search/ranking/ResultReranker.java new file mode 100644 index 00000000000..11b3fa7390a --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/ResultReranker.java @@ -0,0 +1,91 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.logging.Logger; + +class ResultReranker { + + private static final Logger logger = Logger.getLogger(ResultReranker.class.getName()); + + // scale and adjust the score according to the range + // of the original and final score values to avoid that + // a score from the backend is larger than finalScores_low + static class Ranges { + private double initialScores_high = -Double.MAX_VALUE; + private double initialScores_low = Double.MAX_VALUE; + private double finalScores_high = -Double.MAX_VALUE; + private double finalScores_low = Double.MAX_VALUE; + + boolean valid() { + return (initialScores_high >= initialScores_low + && + finalScores_high >= finalScores_low); + } + void withInitialScore(double score) { + if (score < initialScores_low) initialScores_low = score; + if (score > initialScores_high) initialScores_high = score; + } + void withFinalScore(double score) { + if (score < finalScores_low) finalScores_low = score; + if (score > finalScores_high) finalScores_high = score; + } + private double initialRange() { + double r = initialScores_high - initialScores_low; + if (r < 1.0) r = 1.0; + return r; + } + private double finalRange() { + double r = finalScores_high - finalScores_low; + if (r < 1.0) r = 1.0; + return r; + } + double scale() { return finalRange() / initialRange(); } + double bias() { return finalScores_low - initialScores_low * scale(); } + } + + static void rerankHits(Result result, HitRescorer hitRescorer, int rerankCount) { + List<Hit> hitsToRescore = new ArrayList<>(); + // consider doing recursive iteration explicitly instead of using deepIterator? + for (var iterator = result.hits().deepIterator(); iterator.hasNext();) { + Hit hit = iterator.next(); + if (hit.isMeta() || hit instanceof HitGroup) { + continue; + } + // what about hits inside grouping results? + // they are inside GroupingListHit, we won't recurse into it; so we won't see them. + hitsToRescore.add(hit); + } + // we can't be 100% certain that hits were sorted according to relevance: + hitsToRescore.sort(Comparator.naturalOrder()); + var ranges = new Ranges(); + for (var iterator = hitsToRescore.iterator(); rerankCount > 0 && iterator.hasNext(); ) { + Hit hit = iterator.next(); + double oldScore = hit.getRelevance().getScore(); + boolean didRerank = hitRescorer.rescoreHit(hit); + if (didRerank) { + ranges.withInitialScore(oldScore); + ranges.withFinalScore(hit.getRelevance().getScore()); + --rerankCount; + iterator.remove(); + } + } + // if any hits are left in the list, they need rescaling: + if (ranges.valid()) { + double scale = ranges.scale(); + double bias = ranges.bias(); + for (Hit hit : hitsToRescore) { + double oldScore = hit.getRelevance().getScore(); + hit.setRelevance(oldScore * scale + bias); + } + } + result.hits().sort(); + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java b/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java new file mode 100644 index 00000000000..f247eab1649 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java @@ -0,0 +1,51 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import ai.vespa.models.evaluation.FunctionEvaluator; +import com.yahoo.search.result.FeatureData; +import com.yahoo.search.result.Hit; +import com.yahoo.tensor.Tensor; + +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +class SimpleEvaluator implements Evaluator { + + private final FunctionEvaluator evaluator; + private final Set<String> neededInputs; + + public SimpleEvaluator(FunctionEvaluator prototype) { + this.evaluator = prototype; + this.neededInputs = new HashSet<String>(prototype.function().arguments()); + } + + @Override + public Collection<String> needInputs() { return List.copyOf(neededInputs); } + + @Override + public SimpleEvaluator bind(String name, Tensor value) { + if (value != null) evaluator.bind(name, value); + neededInputs.remove(name); + return this; + } + + @Override + public double evaluateScore() { + return evaluator.evaluate().asDouble(); + } + + @Override + public String toString() { + var buf = new StringBuilder(); + buf.append("SimpleEvaluator("); + buf.append(evaluator.function().toString()); + buf.append(")["); + for (String arg : neededInputs) { + buf.append("{").append(arg).append("}"); + } + buf.append("]"); + return buf.toString(); + } +} diff --git a/fastos/src/tests/mazeserver.cpp b/container-search/src/main/java/com/yahoo/search/ranking/package-info.java index b62e98645f2..a86a5c1e52f 100644 --- a/fastos/src/tests/mazeserver.cpp +++ b/container-search/src/main/java/com/yahoo/search/ranking/package-info.java @@ -1,4 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#define DO_MAZE_SERVER 1 -#include "sockettest.cpp" +@ExportPackage +package com.yahoo.search.ranking; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java index 5df8d2e5444..06ae9923dae 100644 --- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java @@ -464,6 +464,7 @@ public class ClusterSearcherTestCase { documentDbConfig.build(), new SchemaInfo(List.of(schema.build()), Map.of()), dispatchers, + null, vipStatus, null); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java index cf9db1517a0..f6029eade37 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java @@ -1,6 +1,5 @@ package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; @@ -9,7 +8,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeReposi import com.yahoo.yolean.Exceptions; import java.time.Duration; -import java.util.stream.Stream; +import java.util.Collection; /** * This pulls application deployment information from the node repo on all config servers, @@ -28,29 +27,35 @@ public class DeploymentInfoMaintainer extends ControllerMaintainer { @Override protected double maintain() { - controller().applications().asList().stream() - .flatMap(this::mapApplicationToInstances) - .flatMap(this::mapInstanceToDeployments) - .forEach(this::updateDeploymentInfo); - return 1.0; - } - - private Stream<Instance> mapApplicationToInstances(Application application) { - return application.instances().values().stream(); + int attempts = 0; + int failures = 0; + for (var application : controller().applications().asList()) { + for (var instance : application.instances().values()) { + for (var deployment : instanceDeployments(instance)) { + attempts++; + if ( ! updateDeploymentInfo(deployment)) + failures++; + } + } + } + return asSuccessFactor(attempts, failures); } - private Stream<DeploymentId> mapInstanceToDeployments(Instance instance) { + private Collection<DeploymentId> instanceDeployments(Instance instance) { return instance.deployments().keySet().stream() .filter(zoneId -> !zoneId.environment().isTest()) - .map(zoneId -> new DeploymentId(instance.id(), zoneId)); + .map(zoneId -> new DeploymentId(instance.id(), zoneId)) + .toList(); } - private void updateDeploymentInfo(DeploymentId id) { + private boolean updateDeploymentInfo(DeploymentId id) { try { controller().applications().deploymentInfo().put(id, nodeRepository.getApplication(id.zoneId(), id.applicationId())); + return true; } catch (ConfigServerException e) { log.info("Could not retrieve deployment info for " + id + ": " + Exceptions.toMessageString(e)); + return false; } } diff --git a/dist/vespa.spec b/dist/vespa.spec index 6bed9904124..e880734979a 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -54,8 +54,7 @@ BuildRequires: gcc-toolset-12-libatomic-devel BuildRequires: maven BuildRequires: maven-openjdk17 BuildRequires: vespa-pybind11-devel -BuildRequires: python3-pytest -BuildRequires: python36-devel +BuildRequires: python38-devel BuildRequires: glibc-langpack-en %endif %if 0%{?el9} @@ -377,7 +376,7 @@ Summary: Vespa - The open big data serving engine - ann-benchmark Requires: %{name}-base-libs = %{version}-%{release} Requires: %{name}-libs = %{version}-%{release} %if 0%{?el8} -Requires: python36 +Requires: python38 %endif %if 0%{?el9} Requires: python3 diff --git a/fastos/CMakeLists.txt b/fastos/CMakeLists.txt index c17752e234c..60813c569e4 100644 --- a/fastos/CMakeLists.txt +++ b/fastos/CMakeLists.txt @@ -4,5 +4,4 @@ vespa_define_module( src/vespa/fastos TESTS - src/tests ) diff --git a/fastos/src/tests/.gitignore b/fastos/src/tests/.gitignore deleted file mode 100644 index ccb5f0210ab..00000000000 --- a/fastos/src/tests/.gitignore +++ /dev/null @@ -1,17 +0,0 @@ -/Makefile -/backtracetest -/backtracetest.log -/filetest -/filetest.log -/processtest -/processtest.log -/sockettest -/sockettest.log -/threadtest -/threadtest.log -/timetest -/timetest.log -/typetest -/typetest.log -/usecputest -*test_app diff --git a/fastos/src/tests/CMakeLists.txt b/fastos/src/tests/CMakeLists.txt deleted file mode 100644 index 7d4b014b6b3..00000000000 --- a/fastos/src/tests/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(fastos_filetest_app TEST - SOURCES - filetest.cpp - DEPENDS - fastos -) -vespa_add_test(NAME fastos_filetest_app NO_VALGRIND COMMAND fastos_filetest_app) -vespa_add_executable(fastos_typetest_app TEST - SOURCES - typetest.cpp - DEPENDS - fastos -) -vespa_add_test(NAME fastos_typetest_app NO_VALGRIND COMMAND fastos_typetest_app) diff --git a/fastos/src/tests/coretest2.cpp b/fastos/src/tests/coretest2.cpp deleted file mode 100644 index bd93623922b..00000000000 --- a/fastos/src/tests/coretest2.cpp +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - - - - -static void -bomb(void) -{ - char *p; - - p = nullptr; - *p = 4; -} - -void *BomberRun(void *arg) -{ - (void) arg; - bomb(); - return nullptr; -}; - -static int -bombMain(void) -{ - pthread_t thread; - void *ret; - - (void) pthread_create(&thread, nullptr, BomberRun, nullptr); - ret = nullptr; - (void) pthread_join(thread, &ret); - return (0); -} - - -int -main(int argc, char **argv) -{ - (void) argc; - (void) argv; - return bombMain(); -} diff --git a/fastos/src/tests/performancetest.cpp b/fastos/src/tests/performancetest.cpp deleted file mode 100644 index f566979957a..00000000000 --- a/fastos/src/tests/performancetest.cpp +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <stdlib.h> - -#include "tests.h" - -void PerformanceTest (char *buffer); - -int main (int argc, char **argv) -{ - (void)argc; - (void)argv; - - void (*test)(char *buffer) = PerformanceTest; - - test(nullptr); - return 0; -} - -void PerformanceTest (char *buffer) -{ - // Cause exception - *static_cast<char *>(nullptr) = 'e'; - -#if 1 - FastOS_File file("test.txt"); - - if(file.OpenReadOnly()) - { - file.Read(buffer, 20); - file.Write2(buffer, 20); - file.Read(buffer, 20); - file.Write2(buffer, 20); - file.Read(buffer, 20); - file.Write2(buffer, 20); - } -#else - - int filedes = open("test.txt", O_RDONLY, 0664); - - if(filedes != -1) - { - write(filedes, buffer, 20); - read(filedes, buffer, 20); - write(filedes, buffer, 20); - read(filedes, buffer, 20); - write(filedes, buffer, 20); - read(filedes, buffer, 20); - write(filedes, buffer, 20); - - close(filedes); - } -#endif -} - diff --git a/fastos/src/tests/typetest.cpp b/fastos/src/tests/typetest.cpp deleted file mode 100644 index 7871275e17c..00000000000 --- a/fastos/src/tests/typetest.cpp +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "tests.h" -#include <vespa/fastos/file.h> - -class TypeTest : public BaseTest -{ -private: - - void ObjectSizeTest () - { - TestHeader("Object Sizes (bytes)"); - - Progress(true, "FastOS_DirectoryScan %d", sizeof(FastOS_DirectoryScan)); - Progress(true, "FastOS_File: %d", sizeof(FastOS_File)); - Progress(true, "FastOS_StatInfo %d", sizeof(FastOS_StatInfo)); - - PrintSeparator(); - } - -public: - virtual ~TypeTest() {}; - - int Main () override - { - printf("grep for the string '%s' to detect failures.\n\n", failString); - - ObjectSizeTest(); - - PrintSeparator(); - printf("END OF TEST (%s)\n", _argv[0]); - - return allWasOk() ? 0 : 1; - } -}; - - -int main (int argc, char **argv) -{ - setvbuf(stdout, nullptr, _IOLBF, 8192); - TypeTest app; - return app.Entry(argc, argv); -} - diff --git a/fastos/src/vespa/fastos/CMakeLists.txt b/fastos/src/vespa/fastos/CMakeLists.txt index c37eb43b884..29810a4f296 100644 --- a/fastos/src/vespa/fastos/CMakeLists.txt +++ b/fastos/src/vespa/fastos/CMakeLists.txt @@ -1,10 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(fastos_objects OBJECT SOURCES - file.cpp - file_rw_ops.cpp - linux_file.cpp - unix_file.cpp ) vespa_add_library(fastos diff --git a/fastos/src/vespa/fastos/types.h b/fastos/src/vespa/fastos/types.h deleted file mode 100644 index 69dd3e5231c..00000000000 --- a/fastos/src/vespa/fastos/types.h +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#define FASTOS_PREFIX(a) FastOS_##a - -// New macros to support the new gcc visibility features. -#define VESPA_DLL_EXPORT __attribute__ ((visibility("default"))) -#define VESPA_DLL_LOCAL __attribute__ ((visibility("hidden"))) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 3989b45b9ac..0aec24d4d24 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -345,6 +345,13 @@ public class Flags { "Takes effect on the next tick.", ZONE_ID, NODE_TYPE, HOSTNAME); + public static final UnboundBooleanFlag ENABLE_GLOBAL_PHASE = defineFeatureFlag( + "enable-global-phase", false, + List.of("arnej", "bjorncs"), "2023-02-28", "2024-01-10", + "Enable global phase ranking", + "Takes effect at redeployment", + APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java index 53acd8cbb1b..1ae25aa0cfa 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java @@ -45,6 +45,7 @@ public class ExportPackages { } } + // Make sure to update the junit integration test `ExportPackagesIT.java` if the set of exported packages is modified. private static String getExportPackages(String[] jars) throws IOException { StringBuilder out = new StringBuilder(); out.append(getSystemPackages()).append(", ") @@ -53,6 +54,7 @@ public class ExportPackages { .append("com.yahoo.jdisc.handler, ") .append("com.yahoo.jdisc.service, ") .append("com.yahoo.jdisc.statistics, ") + .append("com.yahoo.jdisc.refcount, ") .append("javax.inject;version=1.0.0, ") // TODO Vespa 9: remove. Included in guice, but not exported. Needed by container-jersey. .append("org.aopalliance.intercept, ") diff --git a/jdisc_core/src/test/resources/exportPackages.properties b/jdisc_core/src/test/resources/exportPackages.properties index 82242b1644b..b49f91842e3 100644 --- a/jdisc_core/src/test/resources/exportPackages.properties +++ b/jdisc_core/src/test/resources/exportPackages.properties @@ -1,3 +1,3 @@ #generated by com.yahoo.jdisc.core.ExportPackages #Wed Jul 20 02:55:26 CEST 2022 -exportPackages=org.osgi.framework; version\="1.10.0", org.osgi.framework.connect; version\="1.0.0", org.osgi.framework.dto; uses\:\="org.osgi.dto"; version\="1.8.0", org.osgi.framework.hooks.bundle; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.resolver; uses\:\="org.osgi.framework.wiring"; version\="1.0.0", org.osgi.framework.hooks.service; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.weaving; uses\:\="org.osgi.framework.wiring"; version\="1.1.0", org.osgi.framework.launch; uses\:\="org.osgi.framework"; version\="1.2.0", org.osgi.framework.namespace; uses\:\="org.osgi.resource"; version\="1.2.0", org.osgi.framework.startlevel; uses\:\="org.osgi.framework"; version\="1.0.0", org.osgi.framework.startlevel.dto; uses\:\="org.osgi.dto"; version\="1.0.0", org.osgi.framework.wiring; uses\:\="org.osgi.framework,org.osgi.resource"; version\="1.2.0", org.osgi.framework.wiring.dto; uses\:\="org.osgi.dto,org.osgi.resource.dto"; version\="1.3.0", org.osgi.resource; version\="1.0.1", org.osgi.resource.dto; uses\:\="org.osgi.dto"; version\="1.0.1", org.osgi.service.packageadmin; uses\:\="org.osgi.framework"; version\="1.2.1", org.osgi.service.startlevel; uses\:\="org.osgi.framework"; version\="1.1.1", org.osgi.service.url; version\="1.0.1", org.osgi.service.resolver; uses\:\="org.osgi.resource"; version\="1.1.1", org.osgi.util.tracker; uses\:\="org.osgi.framework"; version\="1.5.3", org.osgi.dto; version\="1.1.1", org.osgi.service.condition; version\="1.0.0", javax.security.auth.callback; version\="0.0.0.JavaSE_017", javax.security.auth.login; version\="0.0.0.JavaSE_017", javax.net.ssl; version\="0.0.0.JavaSE_017", java.lang.constant; version\="0.0.0.JavaSE_017", java.security.interfaces; version\="0.0.0.JavaSE_017", java.text.spi; version\="0.0.0.JavaSE_017", java.nio.channels.spi; version\="0.0.0.JavaSE_017", java.math; version\="0.0.0.JavaSE_017", java.nio.file; version\="0.0.0.JavaSE_017", java.util.concurrent.atomic; version\="0.0.0.JavaSE_017", java.security.cert; version\="0.0.0.JavaSE_017", java.security.spec; version\="0.0.0.JavaSE_017", java.nio.channels; version\="0.0.0.JavaSE_017", java.time.chrono; version\="0.0.0.JavaSE_017", javax.crypto; version\="0.0.0.JavaSE_017", java.time.zone; version\="0.0.0.JavaSE_017", java.nio.charset; version\="0.0.0.JavaSE_017", java.io; version\="0.0.0.JavaSE_017", java.util.spi; version\="0.0.0.JavaSE_017", java.net; version\="0.0.0.JavaSE_017", javax.security.cert; version\="0.0.0.JavaSE_017", java.lang.annotation; version\="0.0.0.JavaSE_017", javax.security.auth.spi; version\="0.0.0.JavaSE_017", java.util.concurrent; version\="0.0.0.JavaSE_017", java.nio.charset.spi; version\="0.0.0.JavaSE_017", javax.crypto.interfaces; version\="0.0.0.JavaSE_017", java.util; version\="0.0.0.JavaSE_017", java.security; version\="0.0.0.JavaSE_017", java.nio.file.spi; version\="0.0.0.JavaSE_017", java.nio; version\="0.0.0.JavaSE_017", java.util.jar; version\="0.0.0.JavaSE_017", javax.security.auth; version\="0.0.0.JavaSE_017", java.lang.ref; version\="0.0.0.JavaSE_017", java.util.regex; version\="0.0.0.JavaSE_017", java.net.spi; version\="0.0.0.JavaSE_017", java.lang.module; version\="0.0.0.JavaSE_017", java.lang.invoke; version\="0.0.0.JavaSE_017", java.time.format; version\="0.0.0.JavaSE_017", java.util.concurrent.locks; version\="0.0.0.JavaSE_017", java.time.temporal; version\="0.0.0.JavaSE_017", java.util.zip; version\="0.0.0.JavaSE_017", java.nio.file.attribute; version\="0.0.0.JavaSE_017", java.util.random; version\="0.0.0.JavaSE_017", java.text; version\="0.0.0.JavaSE_017", javax.crypto.spec; version\="0.0.0.JavaSE_017", java.util.stream; version\="0.0.0.JavaSE_017", java.time; version\="0.0.0.JavaSE_017", java.lang; version\="0.0.0.JavaSE_017", java.lang.runtime; version\="0.0.0.JavaSE_017", java.util.function; version\="0.0.0.JavaSE_017", javax.net; version\="0.0.0.JavaSE_017", javax.security.auth.x500; version\="0.0.0.JavaSE_017", java.lang.reflect; version\="0.0.0.JavaSE_017", javax.lang.model; version\="0.0.0.JavaSE_017", javax.annotation.processing; version\="0.0.0.JavaSE_017", javax.lang.model.element; version\="0.0.0.JavaSE_017", javax.tools; version\="0.0.0.JavaSE_017", javax.lang.model.type; version\="0.0.0.JavaSE_017", javax.lang.model.util; version\="0.0.0.JavaSE_017", java.awt.datatransfer; version\="0.0.0.JavaSE_017", java.awt.desktop; version\="0.0.0.JavaSE_017", javax.swing.plaf.synth; version\="0.0.0.JavaSE_017", java.beans; version\="0.0.0.JavaSE_017", javax.swing.text.html.parser; version\="0.0.0.JavaSE_017", javax.swing.text.rtf; version\="0.0.0.JavaSE_017", java.awt.font; version\="0.0.0.JavaSE_017", javax.imageio; version\="0.0.0.JavaSE_017", java.awt.im.spi; version\="0.0.0.JavaSE_017", java.applet; version\="0.0.0.JavaSE_017", javax.sound.midi; version\="0.0.0.JavaSE_017", java.awt.dnd; version\="0.0.0.JavaSE_017", javax.swing.text; version\="0.0.0.JavaSE_017", javax.swing.plaf.basic; version\="0.0.0.JavaSE_017", javax.swing.undo; version\="0.0.0.JavaSE_017", javax.swing.plaf; version\="0.0.0.JavaSE_017", javax.swing.filechooser; version\="0.0.0.JavaSE_017", javax.imageio.event; version\="0.0.0.JavaSE_017", javax.sound.sampled; version\="0.0.0.JavaSE_017", javax.print.attribute; version\="0.0.0.JavaSE_017", javax.print; version\="0.0.0.JavaSE_017", javax.swing.plaf.nimbus; version\="0.0.0.JavaSE_017", javax.accessibility; version\="0.0.0.JavaSE_017", java.awt.event; version\="0.0.0.JavaSE_017", javax.swing.text.html; version\="0.0.0.JavaSE_017", javax.imageio.spi; version\="0.0.0.JavaSE_017", javax.swing.border; version\="0.0.0.JavaSE_017", javax.sound.sampled.spi; version\="0.0.0.JavaSE_017", javax.imageio.plugins.bmp; version\="0.0.0.JavaSE_017", java.awt.color; version\="0.0.0.JavaSE_017", java.awt.geom; version\="0.0.0.JavaSE_017", javax.imageio.plugins.jpeg; version\="0.0.0.JavaSE_017", javax.swing.tree; version\="0.0.0.JavaSE_017", javax.imageio.plugins.tiff; version\="0.0.0.JavaSE_017", java.awt.print; version\="0.0.0.JavaSE_017", java.awt.image; version\="0.0.0.JavaSE_017", javax.imageio.metadata; version\="0.0.0.JavaSE_017", javax.swing.table; version\="0.0.0.JavaSE_017", javax.sound.midi.spi; version\="0.0.0.JavaSE_017", javax.print.attribute.standard; version\="0.0.0.JavaSE_017", javax.swing.colorchooser; version\="0.0.0.JavaSE_017", javax.swing; version\="0.0.0.JavaSE_017", java.awt.image.renderable; version\="0.0.0.JavaSE_017", javax.swing.plaf.multi; version\="0.0.0.JavaSE_017", java.awt.im; version\="0.0.0.JavaSE_017", javax.print.event; version\="0.0.0.JavaSE_017", javax.swing.plaf.metal; version\="0.0.0.JavaSE_017", java.beans.beancontext; version\="0.0.0.JavaSE_017", java.awt; version\="0.0.0.JavaSE_017", javax.imageio.stream; version\="0.0.0.JavaSE_017", javax.swing.event; version\="0.0.0.JavaSE_017", java.lang.instrument; version\="0.0.0.JavaSE_017", java.util.logging; version\="0.0.0.JavaSE_017", javax.management.timer; version\="0.0.0.JavaSE_017", javax.management; version\="0.0.0.JavaSE_017", javax.management.relation; version\="0.0.0.JavaSE_017", javax.management.loading; version\="0.0.0.JavaSE_017", javax.management.openmbean; version\="0.0.0.JavaSE_017", java.lang.management; version\="0.0.0.JavaSE_017", javax.management.remote; version\="0.0.0.JavaSE_017", javax.management.monitor; version\="0.0.0.JavaSE_017", javax.management.modelmbean; version\="0.0.0.JavaSE_017", javax.management.remote.rmi; version\="0.0.0.JavaSE_017", javax.naming.event; version\="0.0.0.JavaSE_017", javax.naming.ldap.spi; version\="0.0.0.JavaSE_017", javax.naming; version\="0.0.0.JavaSE_017", javax.naming.spi; version\="0.0.0.JavaSE_017", javax.naming.ldap; version\="0.0.0.JavaSE_017", javax.naming.directory; version\="0.0.0.JavaSE_017", java.net.http; version\="0.0.0.JavaSE_017", java.util.prefs; version\="0.0.0.JavaSE_017", java.rmi.registry; version\="0.0.0.JavaSE_017", javax.rmi.ssl; version\="0.0.0.JavaSE_017", java.rmi.dgc; version\="0.0.0.JavaSE_017", java.rmi; version\="0.0.0.JavaSE_017", java.rmi.server; version\="0.0.0.JavaSE_017", javax.script; version\="0.0.0.JavaSE_017", org.ietf.jgss; version\="0.0.0.JavaSE_017", javax.security.auth.kerberos; version\="0.0.0.JavaSE_017", javax.security.sasl; version\="0.0.0.JavaSE_017", javax.smartcardio; version\="0.0.0.JavaSE_017", java.sql; version\="0.0.0.JavaSE_017", javax.sql; version\="0.0.0.JavaSE_017", javax.sql.rowset.spi; version\="0.0.0.JavaSE_017", javax.sql.rowset.serial; version\="0.0.0.JavaSE_017", javax.sql.rowset; version\="0.0.0.JavaSE_017", javax.transaction.xa; version\="0.0.0.JavaSE_017", javax.xml; version\="0.0.0.JavaSE_017", javax.xml.transform.sax; version\="0.0.0.JavaSE_017", javax.xml.datatype; version\="0.0.0.JavaSE_017", javax.xml.catalog; version\="0.0.0.JavaSE_017", org.w3c.dom.traversal; version\="0.0.0.JavaSE_017", javax.xml.stream.events; version\="0.0.0.JavaSE_017", javax.xml.stream; version\="0.0.0.JavaSE_017", org.xml.sax; version\="0.0.0.JavaSE_017", javax.xml.transform; version\="0.0.0.JavaSE_017", javax.xml.xpath; version\="0.0.0.JavaSE_017", org.w3c.dom.events; version\="0.0.0.JavaSE_017", org.w3c.dom.ranges; version\="0.0.0.JavaSE_017", org.w3c.dom.ls; version\="0.0.0.JavaSE_017", javax.xml.stream.util; version\="0.0.0.JavaSE_017", javax.xml.namespace; version\="0.0.0.JavaSE_017", javax.xml.transform.stax; version\="0.0.0.JavaSE_017", org.xml.sax.helpers; version\="0.0.0.JavaSE_017", org.w3c.dom.views; version\="0.0.0.JavaSE_017", org.w3c.dom.bootstrap; version\="0.0.0.JavaSE_017", org.w3c.dom; version\="0.0.0.JavaSE_017", javax.xml.transform.stream; version\="0.0.0.JavaSE_017", javax.xml.transform.dom; version\="0.0.0.JavaSE_017", javax.xml.validation; version\="0.0.0.JavaSE_017", javax.xml.parsers; version\="0.0.0.JavaSE_017", org.xml.sax.ext; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.spec; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.keyinfo; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig; version\="0.0.0.JavaSE_017", com.sun.java.accessibility.util; version\="0.0.0.JavaSE_017", com.sun.tools.attach.spi; version\="0.0.0.JavaSE_017", com.sun.tools.attach; version\="0.0.0.JavaSE_017", com.sun.source.doctree; version\="0.0.0.JavaSE_017", com.sun.source.tree; version\="0.0.0.JavaSE_017", com.sun.source.util; version\="0.0.0.JavaSE_017", com.sun.tools.javac; version\="0.0.0.JavaSE_017", jdk.dynalink.beans; version\="0.0.0.JavaSE_017", jdk.dynalink.linker.support; version\="0.0.0.JavaSE_017", jdk.dynalink.support; version\="0.0.0.JavaSE_017", jdk.dynalink; version\="0.0.0.JavaSE_017", jdk.dynalink.linker; version\="0.0.0.JavaSE_017", com.sun.net.httpserver; version\="0.0.0.JavaSE_017", com.sun.net.httpserver.spi; version\="0.0.0.JavaSE_017", com.sun.jarsigner; version\="0.0.0.JavaSE_017", jdk.security.jarsigner; version\="0.0.0.JavaSE_017", jdk.javadoc.doclet; version\="0.0.0.JavaSE_017", com.sun.tools.jconsole; version\="0.0.0.JavaSE_017", com.sun.jdi.connect; version\="0.0.0.JavaSE_017", com.sun.jdi.event; version\="0.0.0.JavaSE_017", com.sun.jdi.connect.spi; version\="0.0.0.JavaSE_017", com.sun.jdi; version\="0.0.0.JavaSE_017", com.sun.jdi.request; version\="0.0.0.JavaSE_017", jdk.jfr.consumer; version\="0.0.0.JavaSE_017", jdk.jfr; version\="0.0.0.JavaSE_017", jdk.jshell; version\="0.0.0.JavaSE_017", jdk.jshell.execution; version\="0.0.0.JavaSE_017", jdk.jshell.spi; version\="0.0.0.JavaSE_017", jdk.jshell.tool; version\="0.0.0.JavaSE_017", netscape.javascript; version\="0.0.0.JavaSE_017", com.sun.management; version\="0.0.0.JavaSE_017", jdk.management.jfr; version\="0.0.0.JavaSE_017", jdk.net; version\="0.0.0.JavaSE_017", jdk.nio; version\="0.0.0.JavaSE_017", jdk.nio.mapmode; version\="0.0.0.JavaSE_017", com.sun.nio.sctp; version\="0.0.0.JavaSE_017", com.sun.security.auth.module; version\="0.0.0.JavaSE_017", com.sun.security.auth.login; version\="0.0.0.JavaSE_017", com.sun.security.auth; version\="0.0.0.JavaSE_017", com.sun.security.auth.callback; version\="0.0.0.JavaSE_017", com.sun.security.jgss; version\="0.0.0.JavaSE_017", sun.reflect; version\="0.0.0.JavaSE_017", sun.misc; version\="0.0.0.JavaSE_017", com.sun.nio.file; version\="0.0.0.JavaSE_017", jdk.swing.interop; version\="0.0.0.JavaSE_017", org.w3c.dom.stylesheets; version\="0.0.0.JavaSE_017", org.w3c.dom.html; version\="0.0.0.JavaSE_017", org.w3c.dom.xpath; version\="0.0.0.JavaSE_017", org.w3c.dom.css; version\="0.0.0.JavaSE_017", com.yahoo.jdisc, com.yahoo.jdisc.application, com.yahoo.jdisc.handler, com.yahoo.jdisc.service, com.yahoo.jdisc.statistics, javax.inject;version\=1.0.0, org.aopalliance.intercept, org.aopalliance.aop, com.google.common.annotations;version\="27.1.0",com.google.common.base;version\="27.1.0",com.google.common.cache;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent",com.google.common.collect;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.escape;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.eventbus;version\="27.1.0",com.google.common.graph;version\="27.1.0";uses\:\="com.google.common.collect",com.google.common.hash;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.html;version\="27.1.0";uses\:\="com.google.common.escape",com.google.common.io;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.graph,com.google.common.hash",com.google.common.math;version\="27.1.0",com.google.common.net;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.escape",com.google.common.primitives;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.reflect;version\="27.1.0";uses\:\="com.google.common.collect,com.google.common.io",com.google.common.util.concurrent;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent.internal",com.google.common.xml;version\="27.1.0";uses\:\="com.google.common.escape", com.google.inject;version\="1.4",com.google.inject.binder;version\="1.4",com.google.inject.matcher;version\="1.4",com.google.inject.multibindings;version\="1.4",com.google.inject.name;version\="1.4",com.google.inject.spi;version\="1.4",com.google.inject.util;version\="1.4", org.slf4j;version\=1.7.32, org.slf4j.spi;version\=1.7.32, org.slf4j.helpers;version\=1.7.32, org.slf4j.event;version\=1.7.32, org.slf4j.impl;version\=1.7.32, org.apache.commons.logging;version\=1.2, org.apache.commons.logging.impl;version\=1.2, org.apache.log4j;version\=1.2.17,org.apache.log4j.helpers;version\=1.2.17,org.apache.log4j.spi;version\=1.2.17,org.apache.log4j.xml;version\=1.2.17, com.yahoo.component.annotation;version\="1.0.0", com.yahoo.config;version\=1.0.0, com.yahoo.vespa.defaults;version\=1.0.0, ai.vespa.http;version\=1.0.0,ai.vespa.validation;version\=1.0.0,com.yahoo.binaryprefix;version\=1.0.0,com.yahoo.collections;version\=1.0.0,com.yahoo.compress;version\=1.0.0,com.yahoo.concurrent.classlock;version\=1.0.0,com.yahoo.concurrent.maintenance;version\=1.0.0,com.yahoo.concurrent;version\=1.0.0,com.yahoo.data.access.simple;version\=1.0.0,com.yahoo.data.access.slime;version\=1.0.0,com.yahoo.data.access;version\=1.0.0,com.yahoo.errorhandling;version\=1.0.0,com.yahoo.exception;version\=1.0.0,com.yahoo.geo;version\=1.0.0,com.yahoo.io.reader;version\=1.0.0,com.yahoo.io;version\=1.0.0,com.yahoo.javacc;version\=1.0.0,com.yahoo.lang;version\=1.0.0,com.yahoo.nativec;version\=1.0.0,com.yahoo.net;version\=1.0.0,com.yahoo.path;version\=1.0.0,com.yahoo.protect;version\=1.0.0,com.yahoo.reflection;version\=1.0.0,com.yahoo.slime;version\=1.0.0,com.yahoo.stream;version\=1.0.0,com.yahoo.system.execution;version\=1.0.0,com.yahoo.system;version\=1.0.0,com.yahoo.tensor.evaluation;version\=1.0.0,com.yahoo.tensor.functions;version\=1.0.0,com.yahoo.tensor.serialization;version\=1.0.0,com.yahoo.tensor;version\=1.0.0,com.yahoo.text.internal;version\=1.0.0,com.yahoo.text;version\=1.0.0,com.yahoo.time;version\=1.0.0,com.yahoo.transaction;version\=1.0.0,com.yahoo.vespa.objects;version\=1.0.0,com.yahoo.yolean.chain;version\=1.0.0,com.yahoo.yolean.concurrent;version\=1.0.0,com.yahoo.yolean.function;version\=1.0.0,com.yahoo.yolean.system;version\=1.0.0,com.yahoo.yolean.trace;version\=1.0.0,com.yahoo.yolean;version\=1.0.0, com.yahoo.log.event;version\=1.0.0,com.yahoo.log.impl;version\=1.0.0,com.yahoo.log;version\=1.0.0, javax.xml.bind;version\="2.3";uses\:\="javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.annotation;version\="2.3";uses\:\="javax.xml.bind,javax.xml.parsers,javax.xml.transform,javax.xml.transform.dom,org.w3c.dom",javax.xml.bind.annotation.adapters;version\="2.3",javax.xml.bind.attachment;version\="2.3";uses\:\="javax.activation",javax.xml.bind.helpers;version\="2.3";uses\:\="javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.util;version\="2.3";uses\:\="javax.xml.bind,javax.xml.transform.sax", com.sun.istack;version\="3.0.5";uses\:\="javax.activation,javax.xml.stream,org.xml.sax,org.xml.sax.helpers",com.sun.istack.localization;version\="3.0.5",com.sun.istack.logging;version\="3.0.5",com.sun.xml.bind;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.annotation;version\="2.3.0",com.sun.xml.bind.api;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.api.impl;version\="2.3.0",com.sun.xml.bind.marshaller;uses\:\="javax.xml.parsers,org.w3c.dom,org.xml.sax,org.xml.sax.helpers";version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;version\="2.3.0",com.sun.xml.bind.v2;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.core;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.impl,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.namespace,javax.xml.transform";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.nav";version\="2.3.0",com.sun.xml.bind.v2.model.nav;uses\:\="com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.util;uses\:\="javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.xml.bind.v2.model.annotation,javax.activation,javax.xml.bind,javax.xml.bind.annotation.adapters";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen.episode;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="javax.xml.parsers,javax.xml.transform,javax.xml.validation,javax.xml.xpath";version\="2.3.0",com.sun.xml.txw2;uses\:\="com.sun.xml.txw2.output,javax.xml.namespace";version\="2.3.0",com.sun.xml.txw2.annotation;version\="2.3.0",com.sun.xml.txw2.output;uses\:\="com.sun.xml.txw2,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.dom,javax.xml.transform.sax,javax.xml.transform.stream,org.w3c.dom,org.xml.sax,org.xml.sax.ext,org.xml.sax.helpers";version\="2.3.0", com.sun.xml.bind;uses\:\="com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.datatype,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.api;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.xml.bind,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.marshaller;version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;uses\:\="com.sun.xml.bind,javax.xml.bind.helpers,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,javax.xml.bind";version\="2.3.0",com.sun.xml.bind.v2.bytecode;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.model.runtime;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.namespace,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.istack,com.sun.xml.bind.api,com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.property,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.output;uses\:\="com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.runtime,com.sun.xml.fastinfoset.stax,javax.xml.stream,org.jvnet.staxex,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.property;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,com.sun.xml.bind.v2.runtime.unmarshaller,com.sun.xml.bind.v2.util,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect.opt;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="com.sun.xml.bind,com.sun.xml.bind.api,com.sun.xml.bind.unmarshaller,com.sun.xml.bind.util,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.reflect,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.txw2.output,javax.xml.bind,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.schemagen.xmlschema;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.namespace,javax.xml.transform.stream,org.xml.sax";version\="2.3.0", javax.activation;uses\:\="com.sun.activation.registries";version\="1.2",com.sun.activation.viewers;uses\:\="javax.activation";version\="1.2.0",com.sun.activation.registries;version\="1.2.0" +exportPackages=org.osgi.framework; version\="1.10.0", org.osgi.framework.connect; version\="1.0.0", org.osgi.framework.dto; uses\:\="org.osgi.dto"; version\="1.8.0", org.osgi.framework.hooks.bundle; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.resolver; uses\:\="org.osgi.framework.wiring"; version\="1.0.0", org.osgi.framework.hooks.service; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.weaving; uses\:\="org.osgi.framework.wiring"; version\="1.1.0", org.osgi.framework.launch; uses\:\="org.osgi.framework"; version\="1.2.0", org.osgi.framework.namespace; uses\:\="org.osgi.resource"; version\="1.2.0", org.osgi.framework.startlevel; uses\:\="org.osgi.framework"; version\="1.0.0", org.osgi.framework.startlevel.dto; uses\:\="org.osgi.dto"; version\="1.0.0", org.osgi.framework.wiring; uses\:\="org.osgi.framework,org.osgi.resource"; version\="1.2.0", org.osgi.framework.wiring.dto; uses\:\="org.osgi.dto,org.osgi.resource.dto"; version\="1.3.0", org.osgi.resource; version\="1.0.1", org.osgi.resource.dto; uses\:\="org.osgi.dto"; version\="1.0.1", org.osgi.service.packageadmin; uses\:\="org.osgi.framework"; version\="1.2.1", org.osgi.service.startlevel; uses\:\="org.osgi.framework"; version\="1.1.1", org.osgi.service.url; version\="1.0.1", org.osgi.service.resolver; uses\:\="org.osgi.resource"; version\="1.1.1", org.osgi.util.tracker; uses\:\="org.osgi.framework"; version\="1.5.3", org.osgi.dto; version\="1.1.1", org.osgi.service.condition; version\="1.0.0", javax.security.auth.callback; version\="0.0.0.JavaSE_017", javax.security.auth.login; version\="0.0.0.JavaSE_017", javax.net.ssl; version\="0.0.0.JavaSE_017", java.lang.constant; version\="0.0.0.JavaSE_017", java.security.interfaces; version\="0.0.0.JavaSE_017", java.text.spi; version\="0.0.0.JavaSE_017", java.nio.channels.spi; version\="0.0.0.JavaSE_017", java.math; version\="0.0.0.JavaSE_017", java.nio.file; version\="0.0.0.JavaSE_017", java.util.concurrent.atomic; version\="0.0.0.JavaSE_017", java.security.cert; version\="0.0.0.JavaSE_017", java.security.spec; version\="0.0.0.JavaSE_017", java.nio.channels; version\="0.0.0.JavaSE_017", java.time.chrono; version\="0.0.0.JavaSE_017", javax.crypto; version\="0.0.0.JavaSE_017", java.time.zone; version\="0.0.0.JavaSE_017", java.nio.charset; version\="0.0.0.JavaSE_017", java.io; version\="0.0.0.JavaSE_017", java.util.spi; version\="0.0.0.JavaSE_017", java.net; version\="0.0.0.JavaSE_017", javax.security.cert; version\="0.0.0.JavaSE_017", java.lang.annotation; version\="0.0.0.JavaSE_017", javax.security.auth.spi; version\="0.0.0.JavaSE_017", java.util.concurrent; version\="0.0.0.JavaSE_017", java.nio.charset.spi; version\="0.0.0.JavaSE_017", javax.crypto.interfaces; version\="0.0.0.JavaSE_017", java.util; version\="0.0.0.JavaSE_017", java.security; version\="0.0.0.JavaSE_017", java.nio.file.spi; version\="0.0.0.JavaSE_017", java.nio; version\="0.0.0.JavaSE_017", java.util.jar; version\="0.0.0.JavaSE_017", javax.security.auth; version\="0.0.0.JavaSE_017", java.lang.ref; version\="0.0.0.JavaSE_017", java.util.regex; version\="0.0.0.JavaSE_017", java.net.spi; version\="0.0.0.JavaSE_017", java.lang.module; version\="0.0.0.JavaSE_017", java.lang.invoke; version\="0.0.0.JavaSE_017", java.time.format; version\="0.0.0.JavaSE_017", java.util.concurrent.locks; version\="0.0.0.JavaSE_017", java.time.temporal; version\="0.0.0.JavaSE_017", java.util.zip; version\="0.0.0.JavaSE_017", java.nio.file.attribute; version\="0.0.0.JavaSE_017", java.util.random; version\="0.0.0.JavaSE_017", java.text; version\="0.0.0.JavaSE_017", javax.crypto.spec; version\="0.0.0.JavaSE_017", java.util.stream; version\="0.0.0.JavaSE_017", java.time; version\="0.0.0.JavaSE_017", java.lang; version\="0.0.0.JavaSE_017", java.lang.runtime; version\="0.0.0.JavaSE_017", java.util.function; version\="0.0.0.JavaSE_017", javax.net; version\="0.0.0.JavaSE_017", javax.security.auth.x500; version\="0.0.0.JavaSE_017", java.lang.reflect; version\="0.0.0.JavaSE_017", javax.lang.model; version\="0.0.0.JavaSE_017", javax.annotation.processing; version\="0.0.0.JavaSE_017", javax.lang.model.element; version\="0.0.0.JavaSE_017", javax.tools; version\="0.0.0.JavaSE_017", javax.lang.model.type; version\="0.0.0.JavaSE_017", javax.lang.model.util; version\="0.0.0.JavaSE_017", java.awt.datatransfer; version\="0.0.0.JavaSE_017", java.awt.desktop; version\="0.0.0.JavaSE_017", javax.swing.plaf.synth; version\="0.0.0.JavaSE_017", java.beans; version\="0.0.0.JavaSE_017", javax.swing.text.html.parser; version\="0.0.0.JavaSE_017", javax.swing.text.rtf; version\="0.0.0.JavaSE_017", java.awt.font; version\="0.0.0.JavaSE_017", javax.imageio; version\="0.0.0.JavaSE_017", java.awt.im.spi; version\="0.0.0.JavaSE_017", java.applet; version\="0.0.0.JavaSE_017", javax.sound.midi; version\="0.0.0.JavaSE_017", java.awt.dnd; version\="0.0.0.JavaSE_017", javax.swing.text; version\="0.0.0.JavaSE_017", javax.swing.plaf.basic; version\="0.0.0.JavaSE_017", javax.swing.undo; version\="0.0.0.JavaSE_017", javax.swing.plaf; version\="0.0.0.JavaSE_017", javax.swing.filechooser; version\="0.0.0.JavaSE_017", javax.imageio.event; version\="0.0.0.JavaSE_017", javax.sound.sampled; version\="0.0.0.JavaSE_017", javax.print.attribute; version\="0.0.0.JavaSE_017", javax.print; version\="0.0.0.JavaSE_017", javax.swing.plaf.nimbus; version\="0.0.0.JavaSE_017", javax.accessibility; version\="0.0.0.JavaSE_017", java.awt.event; version\="0.0.0.JavaSE_017", javax.swing.text.html; version\="0.0.0.JavaSE_017", javax.imageio.spi; version\="0.0.0.JavaSE_017", javax.swing.border; version\="0.0.0.JavaSE_017", javax.sound.sampled.spi; version\="0.0.0.JavaSE_017", javax.imageio.plugins.bmp; version\="0.0.0.JavaSE_017", java.awt.color; version\="0.0.0.JavaSE_017", java.awt.geom; version\="0.0.0.JavaSE_017", javax.imageio.plugins.jpeg; version\="0.0.0.JavaSE_017", javax.swing.tree; version\="0.0.0.JavaSE_017", javax.imageio.plugins.tiff; version\="0.0.0.JavaSE_017", java.awt.print; version\="0.0.0.JavaSE_017", java.awt.image; version\="0.0.0.JavaSE_017", javax.imageio.metadata; version\="0.0.0.JavaSE_017", javax.swing.table; version\="0.0.0.JavaSE_017", javax.sound.midi.spi; version\="0.0.0.JavaSE_017", javax.print.attribute.standard; version\="0.0.0.JavaSE_017", javax.swing.colorchooser; version\="0.0.0.JavaSE_017", javax.swing; version\="0.0.0.JavaSE_017", java.awt.image.renderable; version\="0.0.0.JavaSE_017", javax.swing.plaf.multi; version\="0.0.0.JavaSE_017", java.awt.im; version\="0.0.0.JavaSE_017", javax.print.event; version\="0.0.0.JavaSE_017", javax.swing.plaf.metal; version\="0.0.0.JavaSE_017", java.beans.beancontext; version\="0.0.0.JavaSE_017", java.awt; version\="0.0.0.JavaSE_017", javax.imageio.stream; version\="0.0.0.JavaSE_017", javax.swing.event; version\="0.0.0.JavaSE_017", java.lang.instrument; version\="0.0.0.JavaSE_017", java.util.logging; version\="0.0.0.JavaSE_017", javax.management.timer; version\="0.0.0.JavaSE_017", javax.management; version\="0.0.0.JavaSE_017", javax.management.relation; version\="0.0.0.JavaSE_017", javax.management.loading; version\="0.0.0.JavaSE_017", javax.management.openmbean; version\="0.0.0.JavaSE_017", java.lang.management; version\="0.0.0.JavaSE_017", javax.management.remote; version\="0.0.0.JavaSE_017", javax.management.monitor; version\="0.0.0.JavaSE_017", javax.management.modelmbean; version\="0.0.0.JavaSE_017", javax.management.remote.rmi; version\="0.0.0.JavaSE_017", javax.naming.event; version\="0.0.0.JavaSE_017", javax.naming.ldap.spi; version\="0.0.0.JavaSE_017", javax.naming; version\="0.0.0.JavaSE_017", javax.naming.spi; version\="0.0.0.JavaSE_017", javax.naming.ldap; version\="0.0.0.JavaSE_017", javax.naming.directory; version\="0.0.0.JavaSE_017", java.net.http; version\="0.0.0.JavaSE_017", java.util.prefs; version\="0.0.0.JavaSE_017", java.rmi.registry; version\="0.0.0.JavaSE_017", javax.rmi.ssl; version\="0.0.0.JavaSE_017", java.rmi.dgc; version\="0.0.0.JavaSE_017", java.rmi; version\="0.0.0.JavaSE_017", java.rmi.server; version\="0.0.0.JavaSE_017", javax.script; version\="0.0.0.JavaSE_017", org.ietf.jgss; version\="0.0.0.JavaSE_017", javax.security.auth.kerberos; version\="0.0.0.JavaSE_017", javax.security.sasl; version\="0.0.0.JavaSE_017", javax.smartcardio; version\="0.0.0.JavaSE_017", java.sql; version\="0.0.0.JavaSE_017", javax.sql; version\="0.0.0.JavaSE_017", javax.sql.rowset.spi; version\="0.0.0.JavaSE_017", javax.sql.rowset.serial; version\="0.0.0.JavaSE_017", javax.sql.rowset; version\="0.0.0.JavaSE_017", javax.transaction.xa; version\="0.0.0.JavaSE_017", javax.xml; version\="0.0.0.JavaSE_017", javax.xml.transform.sax; version\="0.0.0.JavaSE_017", javax.xml.datatype; version\="0.0.0.JavaSE_017", javax.xml.catalog; version\="0.0.0.JavaSE_017", org.w3c.dom.traversal; version\="0.0.0.JavaSE_017", javax.xml.stream.events; version\="0.0.0.JavaSE_017", javax.xml.stream; version\="0.0.0.JavaSE_017", org.xml.sax; version\="0.0.0.JavaSE_017", javax.xml.transform; version\="0.0.0.JavaSE_017", javax.xml.xpath; version\="0.0.0.JavaSE_017", org.w3c.dom.events; version\="0.0.0.JavaSE_017", org.w3c.dom.ranges; version\="0.0.0.JavaSE_017", org.w3c.dom.ls; version\="0.0.0.JavaSE_017", javax.xml.stream.util; version\="0.0.0.JavaSE_017", javax.xml.namespace; version\="0.0.0.JavaSE_017", javax.xml.transform.stax; version\="0.0.0.JavaSE_017", org.xml.sax.helpers; version\="0.0.0.JavaSE_017", org.w3c.dom.views; version\="0.0.0.JavaSE_017", org.w3c.dom.bootstrap; version\="0.0.0.JavaSE_017", org.w3c.dom; version\="0.0.0.JavaSE_017", javax.xml.transform.stream; version\="0.0.0.JavaSE_017", javax.xml.transform.dom; version\="0.0.0.JavaSE_017", javax.xml.validation; version\="0.0.0.JavaSE_017", javax.xml.parsers; version\="0.0.0.JavaSE_017", org.xml.sax.ext; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.spec; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.keyinfo; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig; version\="0.0.0.JavaSE_017", com.sun.java.accessibility.util; version\="0.0.0.JavaSE_017", com.sun.tools.attach.spi; version\="0.0.0.JavaSE_017", com.sun.tools.attach; version\="0.0.0.JavaSE_017", com.sun.source.doctree; version\="0.0.0.JavaSE_017", com.sun.source.tree; version\="0.0.0.JavaSE_017", com.sun.source.util; version\="0.0.0.JavaSE_017", com.sun.tools.javac; version\="0.0.0.JavaSE_017", jdk.dynalink.beans; version\="0.0.0.JavaSE_017", jdk.dynalink.linker.support; version\="0.0.0.JavaSE_017", jdk.dynalink.support; version\="0.0.0.JavaSE_017", jdk.dynalink; version\="0.0.0.JavaSE_017", jdk.dynalink.linker; version\="0.0.0.JavaSE_017", com.sun.net.httpserver; version\="0.0.0.JavaSE_017", com.sun.net.httpserver.spi; version\="0.0.0.JavaSE_017", com.sun.jarsigner; version\="0.0.0.JavaSE_017", jdk.security.jarsigner; version\="0.0.0.JavaSE_017", jdk.javadoc.doclet; version\="0.0.0.JavaSE_017", com.sun.tools.jconsole; version\="0.0.0.JavaSE_017", com.sun.jdi.connect; version\="0.0.0.JavaSE_017", com.sun.jdi.event; version\="0.0.0.JavaSE_017", com.sun.jdi.connect.spi; version\="0.0.0.JavaSE_017", com.sun.jdi; version\="0.0.0.JavaSE_017", com.sun.jdi.request; version\="0.0.0.JavaSE_017", jdk.jfr.consumer; version\="0.0.0.JavaSE_017", jdk.jfr; version\="0.0.0.JavaSE_017", jdk.jshell; version\="0.0.0.JavaSE_017", jdk.jshell.execution; version\="0.0.0.JavaSE_017", jdk.jshell.spi; version\="0.0.0.JavaSE_017", jdk.jshell.tool; version\="0.0.0.JavaSE_017", netscape.javascript; version\="0.0.0.JavaSE_017", com.sun.management; version\="0.0.0.JavaSE_017", jdk.management.jfr; version\="0.0.0.JavaSE_017", jdk.net; version\="0.0.0.JavaSE_017", jdk.nio; version\="0.0.0.JavaSE_017", jdk.nio.mapmode; version\="0.0.0.JavaSE_017", com.sun.nio.sctp; version\="0.0.0.JavaSE_017", com.sun.security.auth.module; version\="0.0.0.JavaSE_017", com.sun.security.auth.login; version\="0.0.0.JavaSE_017", com.sun.security.auth; version\="0.0.0.JavaSE_017", com.sun.security.auth.callback; version\="0.0.0.JavaSE_017", com.sun.security.jgss; version\="0.0.0.JavaSE_017", sun.reflect; version\="0.0.0.JavaSE_017", sun.misc; version\="0.0.0.JavaSE_017", com.sun.nio.file; version\="0.0.0.JavaSE_017", jdk.swing.interop; version\="0.0.0.JavaSE_017", org.w3c.dom.stylesheets; version\="0.0.0.JavaSE_017", org.w3c.dom.html; version\="0.0.0.JavaSE_017", org.w3c.dom.xpath; version\="0.0.0.JavaSE_017", org.w3c.dom.css; version\="0.0.0.JavaSE_017", com.yahoo.jdisc, com.yahoo.jdisc.application, com.yahoo.jdisc.handler, com.yahoo.jdisc.service, com.yahoo.jdisc.statistics, com.yahoo.jdisc.refcount, javax.inject;version\=1.0.0, org.aopalliance.intercept, org.aopalliance.aop, com.google.common.annotations;version\="27.1.0",com.google.common.base;version\="27.1.0",com.google.common.cache;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent",com.google.common.collect;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.escape;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.eventbus;version\="27.1.0",com.google.common.graph;version\="27.1.0";uses\:\="com.google.common.collect",com.google.common.hash;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.html;version\="27.1.0";uses\:\="com.google.common.escape",com.google.common.io;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.graph,com.google.common.hash",com.google.common.math;version\="27.1.0",com.google.common.net;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.escape",com.google.common.primitives;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.reflect;version\="27.1.0";uses\:\="com.google.common.collect,com.google.common.io",com.google.common.util.concurrent;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent.internal",com.google.common.xml;version\="27.1.0";uses\:\="com.google.common.escape", com.google.inject;version\="1.4",com.google.inject.binder;version\="1.4",com.google.inject.matcher;version\="1.4",com.google.inject.multibindings;version\="1.4",com.google.inject.name;version\="1.4",com.google.inject.spi;version\="1.4",com.google.inject.util;version\="1.4", org.slf4j;version\=1.7.32, org.slf4j.spi;version\=1.7.32, org.slf4j.helpers;version\=1.7.32, org.slf4j.event;version\=1.7.32, org.slf4j.impl;version\=1.7.32, org.apache.commons.logging;version\=1.2, org.apache.commons.logging.impl;version\=1.2, org.apache.log4j;version\=1.2.17,org.apache.log4j.helpers;version\=1.2.17,org.apache.log4j.spi;version\=1.2.17,org.apache.log4j.xml;version\=1.2.17, com.yahoo.component.annotation;version\="1.0.0", com.yahoo.config;version\=1.0.0, com.yahoo.vespa.defaults;version\=1.0.0, ai.vespa.http;version\=1.0.0,ai.vespa.validation;version\=1.0.0,com.yahoo.binaryprefix;version\=1.0.0,com.yahoo.collections;version\=1.0.0,com.yahoo.compress;version\=1.0.0,com.yahoo.concurrent.classlock;version\=1.0.0,com.yahoo.concurrent.maintenance;version\=1.0.0,com.yahoo.concurrent;version\=1.0.0,com.yahoo.data.access.simple;version\=1.0.0,com.yahoo.data.access.slime;version\=1.0.0,com.yahoo.data.access;version\=1.0.0,com.yahoo.errorhandling;version\=1.0.0,com.yahoo.exception;version\=1.0.0,com.yahoo.geo;version\=1.0.0,com.yahoo.io.reader;version\=1.0.0,com.yahoo.io;version\=1.0.0,com.yahoo.javacc;version\=1.0.0,com.yahoo.lang;version\=1.0.0,com.yahoo.nativec;version\=1.0.0,com.yahoo.net;version\=1.0.0,com.yahoo.path;version\=1.0.0,com.yahoo.protect;version\=1.0.0,com.yahoo.reflection;version\=1.0.0,com.yahoo.slime;version\=1.0.0,com.yahoo.stream;version\=1.0.0,com.yahoo.system.execution;version\=1.0.0,com.yahoo.system;version\=1.0.0,com.yahoo.tensor.evaluation;version\=1.0.0,com.yahoo.tensor.functions;version\=1.0.0,com.yahoo.tensor.serialization;version\=1.0.0,com.yahoo.tensor;version\=1.0.0,com.yahoo.text.internal;version\=1.0.0,com.yahoo.text;version\=1.0.0,com.yahoo.time;version\=1.0.0,com.yahoo.transaction;version\=1.0.0,com.yahoo.vespa.objects;version\=1.0.0,com.yahoo.yolean.chain;version\=1.0.0,com.yahoo.yolean.concurrent;version\=1.0.0,com.yahoo.yolean.function;version\=1.0.0,com.yahoo.yolean.system;version\=1.0.0,com.yahoo.yolean.trace;version\=1.0.0,com.yahoo.yolean;version\=1.0.0, com.yahoo.log.event;version\=1.0.0,com.yahoo.log.impl;version\=1.0.0,com.yahoo.log;version\=1.0.0, javax.xml.bind;version\="2.3";uses\:\="javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.annotation;version\="2.3";uses\:\="javax.xml.bind,javax.xml.parsers,javax.xml.transform,javax.xml.transform.dom,org.w3c.dom",javax.xml.bind.annotation.adapters;version\="2.3",javax.xml.bind.attachment;version\="2.3";uses\:\="javax.activation",javax.xml.bind.helpers;version\="2.3";uses\:\="javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.util;version\="2.3";uses\:\="javax.xml.bind,javax.xml.transform.sax", com.sun.istack;version\="3.0.5";uses\:\="javax.activation,javax.xml.stream,org.xml.sax,org.xml.sax.helpers",com.sun.istack.localization;version\="3.0.5",com.sun.istack.logging;version\="3.0.5",com.sun.xml.bind;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.annotation;version\="2.3.0",com.sun.xml.bind.api;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.api.impl;version\="2.3.0",com.sun.xml.bind.marshaller;uses\:\="javax.xml.parsers,org.w3c.dom,org.xml.sax,org.xml.sax.helpers";version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;version\="2.3.0",com.sun.xml.bind.v2;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.core;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.impl,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.namespace,javax.xml.transform";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.nav";version\="2.3.0",com.sun.xml.bind.v2.model.nav;uses\:\="com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.util;uses\:\="javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.xml.bind.v2.model.annotation,javax.activation,javax.xml.bind,javax.xml.bind.annotation.adapters";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen.episode;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="javax.xml.parsers,javax.xml.transform,javax.xml.validation,javax.xml.xpath";version\="2.3.0",com.sun.xml.txw2;uses\:\="com.sun.xml.txw2.output,javax.xml.namespace";version\="2.3.0",com.sun.xml.txw2.annotation;version\="2.3.0",com.sun.xml.txw2.output;uses\:\="com.sun.xml.txw2,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.dom,javax.xml.transform.sax,javax.xml.transform.stream,org.w3c.dom,org.xml.sax,org.xml.sax.ext,org.xml.sax.helpers";version\="2.3.0", com.sun.xml.bind;uses\:\="com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.datatype,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.api;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.xml.bind,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.marshaller;version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;uses\:\="com.sun.xml.bind,javax.xml.bind.helpers,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,javax.xml.bind";version\="2.3.0",com.sun.xml.bind.v2.bytecode;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.model.runtime;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.namespace,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.istack,com.sun.xml.bind.api,com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.property,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.output;uses\:\="com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.runtime,com.sun.xml.fastinfoset.stax,javax.xml.stream,org.jvnet.staxex,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.property;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,com.sun.xml.bind.v2.runtime.unmarshaller,com.sun.xml.bind.v2.util,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect.opt;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="com.sun.xml.bind,com.sun.xml.bind.api,com.sun.xml.bind.unmarshaller,com.sun.xml.bind.util,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.reflect,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.txw2.output,javax.xml.bind,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.schemagen.xmlschema;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.namespace,javax.xml.transform.stream,org.xml.sax";version\="2.3.0", javax.activation;uses\:\="com.sun.activation.registries";version\="1.2",com.sun.activation.viewers;uses\:\="javax.activation";version\="1.2.0",com.sun.activation.registries;version\="1.2.0" diff --git a/metrics/src/tests/metricmanagertest.cpp b/metrics/src/tests/metricmanagertest.cpp index 98d03514de0..6d6f21ea7b0 100644 --- a/metrics/src/tests/metricmanagertest.cpp +++ b/metrics/src/tests/metricmanagertest.cpp @@ -5,12 +5,10 @@ #include <vespa/metrics/metricmanager.h> #include <vespa/metrics/state_api_adapter.h> #include <vespa/metrics/textwriter.h> -#include <vespa/metrics/xmlwriter.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/size_literals.h> -#include <vespa/vespalib/util/xmlstream.h> #include <vespa/vespalib/util/time.h> #include <vespa/vespalib/data/simple_buffer.h> #include <vespa/vespalib/util/atomic.h> @@ -57,7 +55,7 @@ SubMetricSet::SubMetricSet(const Metric::String & name, MetricSet* owner) valsum.addMetricToSum(val1); valsum.addMetricToSum(val2); } -SubMetricSet::~SubMetricSet() { } +SubMetricSet::~SubMetricSet() = default; struct MultiSubMetricSet { @@ -167,20 +165,15 @@ getMatchedMetrics(const vespalib::string& config) MetricLockGuard g(mm.getMetricLock()); mm.visit(g, mm.getActiveMetrics(g), visitor, "consumer"); - MetricManager::ConsumerSpec::SP consumerSpec( - mm.getConsumerSpec(g, "consumer")); - return std::pair<std::string, std::string>( - visitor.toString(), - consumerSpec.get() ? consumerSpec->toString() - : "Non-existing consumer"); + MetricManager::ConsumerSpec::SP consumerSpec(mm.getConsumerSpec(g, "consumer")); + return { visitor.toString(), consumerSpec ? consumerSpec->toString() : "Non-existing consumer" }; } } #define ASSERT_CONSUMER_MATCH(name, expected, config) \ { \ - std::pair<std::string, std::string> consumerMatch( \ - getMatchedMetrics(config)); \ + std::pair<std::string, std::string> consumerMatch(getMatchedMetrics(config)); \ EXPECT_EQ("\n" + expected, "\n" + consumerMatch.first) << (name + std::string(": ") + consumerMatch.second); \ } @@ -371,10 +364,10 @@ class FakeTimer : public MetricManager::Timer { std::atomic<time_t> _time; public: FakeTimer(time_t startTime = 0) : _time(startTime) {} - time_t getTime() const override { return load_relaxed(_time); } + time_point getTime() const override { return time_point(vespalib::from_s(load_relaxed(_time))); } void set_time(time_t t) noexcept { store_relaxed(_time, t); } // Not safe for multiple writers, only expected to be called by test. - void add_time(time_t t) noexcept { set_time(getTime() + t); } + void add_time(time_t t) noexcept { set_time(load_relaxed(_time) + t); } }; struct BriefValuePrinter : public MetricVisitor { @@ -391,8 +384,7 @@ struct BriefValuePrinter : public MetricVisitor { } }; -bool waitForTimeProcessed(const MetricManager& mm, - time_t processtime, uint32_t timeout = 120) +bool waitForTimeProcessed(const MetricManager& mm, time_t processtime, uint32_t timeout = 120) { uint32_t lastchance = time(0) + timeout; while (time(0) < lastchance) { @@ -403,23 +395,20 @@ bool waitForTimeProcessed(const MetricManager& mm, return false; } -std::string dumpAllSnapshots(const MetricManager& mm, - const std::string& consumer) +std::string dumpAllSnapshots(const MetricManager& mm, const std::string& consumer) { std::ostringstream ost; ost << "\n"; { MetricLockGuard metricLock(mm.getMetricLock()); BriefValuePrinter briefValuePrinter; - mm.visit(metricLock, mm.getActiveMetrics(metricLock), - briefValuePrinter, consumer); + mm.visit(metricLock, mm.getActiveMetrics(metricLock), briefValuePrinter, consumer); ost << "Current: " << briefValuePrinter.ost.str() << "\n"; } { MetricLockGuard metricLock(mm.getMetricLock()); BriefValuePrinter briefValuePrinter; - mm.visit(metricLock, mm.getTotalMetricSnapshot(metricLock), - briefValuePrinter, consumer); + mm.visit(metricLock, mm.getTotalMetricSnapshot(metricLock), briefValuePrinter, consumer); ost << "Total: " << briefValuePrinter.ost.str() << "\n"; } std::vector<uint32_t> periods; @@ -429,17 +418,15 @@ std::string dumpAllSnapshots(const MetricManager& mm, } for (uint32_t i=0; i<periods.size(); ++i) { MetricLockGuard metricLock(mm.getMetricLock()); - const MetricSnapshotSet& set(mm.getMetricSnapshotSet( - metricLock, periods[i])); + const MetricSnapshotSet& set(mm.getMetricSnapshotSet(metricLock, periods[i])); ost << set.getName() << "\n"; - uint32_t count = 0; - for (uint32_t j=0; j<2; ++j) { + for (uint32_t count=0,j=0; j<2; ++j) { if (set.getCount() == 1 && j == 1) continue; const MetricSnapshot& snap(set.getSnapshot(j == 1)); BriefValuePrinter briefValuePrinter; mm.visit(metricLock, snap, briefValuePrinter, consumer); ost << " " << count++ << " " << &snap.getMetrics() << ": " - << briefValuePrinter.ost.str() << "\n"; + << briefValuePrinter.ost.str() << "\n"; } } return ost.str(); @@ -452,14 +439,11 @@ std::string dumpAllSnapshots(const MetricManager& mm, MetricLockGuard lockGuard(mm.getMetricLock()); \ BriefValuePrinter briefValuePrinter; \ if (period == -1) { \ - mm.visit(lockGuard, mm.getActiveMetrics(lockGuard), \ - briefValuePrinter, "snapper"); \ + mm.visit(lockGuard, mm.getActiveMetrics(lockGuard), briefValuePrinter, "snapper"); \ } else if (period == 0) { \ - mm.visit(lockGuard, mm.getTotalMetricSnapshot(lockGuard), \ - briefValuePrinter, "snapper"); \ + mm.visit(lockGuard, mm.getTotalMetricSnapshot(lockGuard), briefValuePrinter, "snapper"); \ } else { \ - mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, period), \ - briefValuePrinter, "snapper"); \ + mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, period), briefValuePrinter, "snapper"); \ } \ EXPECT_EQ(std::string(expected), briefValuePrinter.ost.str()) << dumpAllSnapshots(mm, "snapper"); \ } @@ -474,8 +458,8 @@ std::string dumpAllSnapshots(const MetricManager& mm, TEST_F(MetricManagerTest, test_snapshots) { - FakeTimer* timer = new FakeTimer(1000); - std::unique_ptr<MetricManager::Timer> timerImpl(timer); + auto timerImpl = std::make_unique<FakeTimer>(1000); + FakeTimer & timer = *timerImpl; TestMetricSet mySet; MetricManager mm(std::move(timerImpl)); { @@ -522,7 +506,7 @@ TEST_F(MetricManagerTest, test_snapshots) mySet.val10.a.val1.addValue(7); mySet.val10.a.val2.addValue(2); mySet.val10.b.val1.addValue(1); - timer->add_time(5 * 60); + timer.add_time(5 * 60); ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60); ASSERT_VALUES(mm, 5 * 60, "2,4,4,1,7,9,1,1,8,2,10"); ASSERT_VALUES(mm, 60 * 60, ""); @@ -536,7 +520,7 @@ TEST_F(MetricManagerTest, test_snapshots) mySet.val10.a.val1.addValue(8); mySet.val10.a.val2.addValue(3); mySet.val10.b.val1.addValue(2); - timer->add_time(5 * 60); + timer.add_time(5 * 60); ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60 * 2); ASSERT_VALUES(mm, 5 * 60, "4,5,5,1,8,11,2,2,10,3,13"); ASSERT_VALUES(mm, 60 * 60, ""); @@ -544,7 +528,7 @@ TEST_F(MetricManagerTest, test_snapshots) // Adding another five minute period where nothing have happened. // Metric for last 5 minutes should be 0. - timer->add_time(5 * 60); + timer.add_time(5 * 60); ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60 * 3); ASSERT_VALUES(mm, 5 * 60, "0,0,0,0,0,0,0,0,0,0,0"); ASSERT_VALUES(mm, 60 * 60, ""); @@ -555,7 +539,7 @@ TEST_F(MetricManagerTest, test_snapshots) mySet.val6.addValue(6); for (uint32_t i=0; i<9; ++i) { // 9 x 5 minutes. Avoid snapshot bumping // due to taking snapshots in the past - timer->add_time(5 * 60); + timer.add_time(5 * 60); ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60 * (4 + i)); } ASSERT_VALUES(mm, 5 * 60, "0,0,0,0,0,0,0,0,0,0,0"); @@ -570,86 +554,10 @@ TEST_F(MetricManagerTest, test_snapshots) ASSERT_VALUES(mm, 0 * 60, "0,0,0,0,0,0,0,0,0,0,0"); } -TEST_F(MetricManagerTest, test_xml_output) -{ - FakeTimer* timer = new FakeTimer(1000); - std::unique_ptr<MetricManager::Timer> timerImpl(timer); - MetricManager mm(std::move(timerImpl)); - TestMetricSet mySet; - { - MetricLockGuard lockGuard(mm.getMetricLock()); - mm.registerMetric(lockGuard, mySet.set); - } - - // Initialize metric manager to get snapshots created. - mm.init(ConfigUri("raw:" - "consumer[2]\n" - "consumer[0].name snapper\n" - "consumer[0].tags[1]\n" - "consumer[0].tags[0] snaptest\n" - "consumer[1].name log\n" - "consumer[1].tags[1]\n" - "consumer[1].tags[0] snaptest\n")); - - takeSnapshots(mm, 1000); - - // Adding metrics to have some values in them - mySet.val6.addValue(2); - mySet.val9.val1.addValue(4); - mySet.val10.count.inc(); - mySet.val10.a.val1.addValue(7); - mySet.val10.a.val2.addValue(2); - mySet.val10.b.val1.addValue(1); - - timer->set_time(1300); - takeSnapshots(mm, 1300); - - std::string expected( - "'<snapshot name=\"5 minute\" from=\"1000\" to=\"1300\" period=\"300\">\n" - " <temp>\n" - " <val6 average=\"2\" last=\"2\" min=\"2\" max=\"2\" count=\"1\"/>\n" - " <sub>\n" - " <val1 average=\"4\" last=\"4\" min=\"4\" max=\"4\" count=\"1\"/>\n" - " <valsum average=\"4\" last=\"4\" min=\"4\" max=\"4\" count=\"1\"/>\n" - " </sub>\n" - " <multisub>\n" - " <count count=\"1\"/>\n" - " <a>\n" - " <val1 average=\"7\" last=\"7\" min=\"7\" max=\"7\" count=\"1\"/>\n" - " <valsum average=\"9\" last=\"9\"/>\n" - " </a>\n" - " <b>\n" - " <val1 average=\"1\" last=\"1\" min=\"1\" max=\"1\" count=\"1\"/>\n" - " <valsum average=\"1\" last=\"1\" min=\"1\" max=\"1\" count=\"1\"/>\n" - " </b>\n" - " <sum>\n" - " <val1 average=\"8\" last=\"8\"/>\n" - " <val2 average=\"2\" last=\"2\" min=\"2\" max=\"2\" count=\"1\"/>\n" - " <valsum average=\"10\" last=\"10\"/>\n" - " </sum>\n" - " </multisub>\n" - " </temp>\n" - "</snapshot>'"); - - std::ostringstream ost; - vespalib::XmlOutputStream xos(ost, " "); - XmlWriter writer(xos, 300, 0); - { - MetricLockGuard lockGuard(mm.getMetricLock()); - mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, 300, false), - writer, "snapper"); - } - std::string actual(ost.str()); - // Not bothering to match all the nitty gritty details as it will test - // more than it needs to. Just be here in order to check on XML output - // easily if needed. - EXPECT_EQ(expected, "'" + actual + "'"); -} - TEST_F(MetricManagerTest, test_json_output) { - FakeTimer* timer = new FakeTimer(1000); - std::unique_ptr<MetricManager::Timer> timerImpl(timer); + auto timerImpl = std::make_unique<FakeTimer>(1000); + FakeTimer & timer = *timerImpl; MetricManager mm(std::move(timerImpl)); TestMetricSet mySet; { @@ -674,7 +582,7 @@ TEST_F(MetricManagerTest, test_json_output) mySet.val10.a.val2.addValue(2); mySet.val10.b.val1.addValue(1); - timer->set_time(1300); + timer.set_time(1300); takeSnapshots(mm, 1300); // Create json output @@ -683,8 +591,7 @@ TEST_F(MetricManagerTest, test_json_output) JsonWriter writer(jsonStream); { MetricLockGuard lockGuard(mm.getMetricLock()); - mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, 300, false), - writer, "snapper"); + mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, 300, false), writer, "snapper"); } jsonStream.finalize(); std::string jsonData = as.str(); @@ -773,22 +680,19 @@ struct MetricSnapshotTestFixture JsonWriter writer(jsonStream); { MetricLockGuard lockGuard(manager.getMetricLock()); - manager.visit(lockGuard, manager.getMetricSnapshot(lockGuard, 300, false), - writer, "snapper"); + manager.visit(lockGuard, manager.getMetricSnapshot(lockGuard, 300, false), writer, "snapper"); } jsonStream.finalize(); return as.str(); } - std::string renderLastSnapshotAsText( - const std::string& matchPattern = ".*") const + std::string renderLastSnapshotAsText(const std::string& matchPattern = ".*") const { std::ostringstream ss; TextWriter writer(ss, 300, matchPattern, true); { MetricLockGuard lockGuard(manager.getMetricLock()); - manager.visit(lockGuard, manager.getMetricSnapshot(lockGuard, 300, false), - writer, "snapper"); + manager.visit(lockGuard, manager.getMetricSnapshot(lockGuard, 300, false), writer, "snapper"); } return ss.str(); } @@ -819,8 +723,7 @@ public: } std::string nthMetricDimension(size_t metricIndex, const std::string& key) { - return nthMetric(metricIndex)["dimensions"][key] - .asString().make_string(); + return nthMetric(metricIndex)["dimensions"][key].asString().make_string(); } // Verify that the nth metric has the given name and the given set of @@ -843,7 +746,7 @@ JsonMetricWrapper::JsonMetricWrapper(const std::string& jsonText) { vespalib::slime::JsonFormat::decode(vespalib::Memory(jsonText), _tree); } -JsonMetricWrapper::~JsonMetricWrapper() { } +JsonMetricWrapper::~JsonMetricWrapper() = default; struct DimensionTestMetricSet : MetricSet { @@ -851,7 +754,7 @@ struct DimensionTestMetricSet : MetricSet LongCountMetric val2; DimensionTestMetricSet(MetricSet* owner = nullptr); - ~DimensionTestMetricSet(); + ~DimensionTestMetricSet() override; }; DimensionTestMetricSet::DimensionTestMetricSet(MetricSet* owner) @@ -859,7 +762,7 @@ DimensionTestMetricSet::DimensionTestMetricSet(MetricSet* owner) val1("val1", {{"tag1"}}, "val1 desc", this), val2("val2", {{"baz", "superbaz"}}, "val2 desc", this) { } -DimensionTestMetricSet::~DimensionTestMetricSet() { } +DimensionTestMetricSet::~DimensionTestMetricSet() = default; } @@ -976,9 +879,7 @@ TEST_F(MetricManagerTest, json_output_can_have_multiple_sets_with_same_name) TEST_F(MetricManagerTest, test_text_output) { - FakeTimer* timer = new FakeTimer(1000); - std::unique_ptr<MetricManager::Timer> timerImpl(timer); - MetricManager mm(std::move(timerImpl)); + MetricManager mm(std::make_unique<FakeTimer>(1000)); TestMetricSet mySet; { MetricLockGuard lockGuard(mm.getMetricLock()); @@ -1051,10 +952,7 @@ namespace { std::mutex& _output_mutex; FakeTimer& _timer; - MyUpdateHook(std::ostringstream& output, - std::mutex& output_mutex, - const char* name, - FakeTimer& timer) + MyUpdateHook(std::ostringstream& output, std::mutex& output_mutex, const char* name, FakeTimer& timer) : UpdateHook(name), _output(output), _output_mutex(output_mutex), @@ -1064,7 +962,7 @@ namespace { void updateMetrics(const MetricLockGuard & ) override { std::lock_guard lock(_output_mutex); // updateMetrics() called from metric manager thread - _output << _timer.getTime() << ": " << getName() << " called\n"; + _output << vespalib::count_s(_timer.getTime().time_since_epoch()) << ": " << getName() << " called\n"; } }; } @@ -1073,8 +971,8 @@ TEST_F(MetricManagerTest, test_update_hooks) { std::mutex output_mutex; std::ostringstream output; - FakeTimer* timer = new FakeTimer(1000); - std::unique_ptr<MetricManager::Timer> timerImpl(timer); + auto timerImpl = std::make_unique<FakeTimer>(1000); + FakeTimer & timer = *timerImpl; // Add a metric set just so one exist TestMetricSet mySet; MetricManager mm(std::move(timerImpl)); @@ -1083,9 +981,9 @@ TEST_F(MetricManagerTest, test_update_hooks) mm.registerMetric(lockGuard, mySet.set); } - MyUpdateHook preInitShort(output, output_mutex, "BIS", *timer); - MyUpdateHook preInitLong(output, output_mutex, "BIL", *timer); - MyUpdateHook preInitInfinite(output, output_mutex, "BII", *timer); + MyUpdateHook preInitShort(output, output_mutex, "BIS", timer); + MyUpdateHook preInitLong(output, output_mutex, "BIL", timer); + MyUpdateHook preInitInfinite(output, output_mutex, "BII", timer); mm.addMetricUpdateHook(preInitShort, 5); mm.addMetricUpdateHook(preInitLong, 300); mm.addMetricUpdateHook(preInitInfinite, 0); @@ -1104,55 +1002,55 @@ TEST_F(MetricManagerTest, test_update_hooks) "consumer[1].tags[0] snaptest\n")); output << "Init done\n"; - MyUpdateHook postInitShort(output, output_mutex, "AIS", *timer); - MyUpdateHook postInitLong(output, output_mutex, "AIL", *timer); - MyUpdateHook postInitInfinite(output, output_mutex, "AII", *timer); + MyUpdateHook postInitShort(output, output_mutex, "AIS", timer); + MyUpdateHook postInitLong(output, output_mutex, "AIL", timer); + MyUpdateHook postInitInfinite(output, output_mutex, "AII", timer); mm.addMetricUpdateHook(postInitShort, 5); mm.addMetricUpdateHook(postInitLong, 400); mm.addMetricUpdateHook(postInitInfinite, 0); // After 5 seconds the short ones should get another. - timer->set_time(1006); + timer.set_time(1006); waitForTimeProcessed(mm, 1006); // After 4 more seconds the short ones should get another // since last update was a second late. (Stable periods, process time // should not affect how often they are updated) - timer->set_time(1010); + timer.set_time(1010); waitForTimeProcessed(mm, 1010); // Bumping considerably ahead, such that next update is in the past, // we should only get one update called in this period. - timer->set_time(1200); + timer.set_time(1200); waitForTimeProcessed(mm, 1200); // No updates at this time. - timer->set_time(1204); + timer.set_time(1204); waitForTimeProcessed(mm, 1204); // Give all hooks an update mm.updateMetrics(true); // Last update should not have interfered with periods - timer->set_time(1205); + timer.set_time(1205); waitForTimeProcessed(mm, 1205); // Time is just ahead of a snapshot. - timer->set_time(1299); + timer.set_time(1299); waitForTimeProcessed(mm, 1299); // At time 1300 we are at a 5 minute snapshot bump // All hooks should thus get an update. The one with matching period // should only get one - timer->set_time(1300); + timer.set_time(1300); waitForTimeProcessed(mm, 1300); // The snapshot time currently doesn't count for the metric at period // 400. It will get an event at this time. - timer->set_time(1450); + timer.set_time(1450); waitForTimeProcessed(mm, 1450); std::string expected( diff --git a/metrics/src/tests/snapshottest.cpp b/metrics/src/tests/snapshottest.cpp index b4eb4a1353c..4d2ea96c36d 100644 --- a/metrics/src/tests/snapshottest.cpp +++ b/metrics/src/tests/snapshottest.cpp @@ -122,15 +122,15 @@ struct TestMetricSet : public MetricSet { SubMetricSet set2; SumMetric<SubMetricSet> setSum; - TestMetricSet(vespalib::stringref name, MetricSet* owner = 0); + TestMetricSet(vespalib::stringref name); ~TestMetricSet(); void incValues(); }; -TestMetricSet::TestMetricSet(vespalib::stringref name, MetricSet* owner) - : MetricSet(name, {}, "", owner), +TestMetricSet::TestMetricSet(vespalib::stringref name) + : MetricSet(name, {}, "", nullptr), set1("set1", this), set2("set2", this), setSum("setSum", {}, "", this) @@ -149,7 +149,7 @@ TestMetricSet::incValues() { struct FakeTimer : public MetricManager::Timer { uint32_t _timeInSecs; FakeTimer() : _timeInSecs(1) {} - time_t getTime() const override { return _timeInSecs; } + time_point getTime() const override { return time_point(vespalib::from_s(_timeInSecs)); } }; void ASSERT_VALUE(int32_t value, const MetricSnapshot & snapshot, const char *name) __attribute__((noinline)); @@ -176,8 +176,7 @@ TEST_F(SnapshotTest, test_snapshot_two_days) TestMetricSet set("test"); FakeTimer* timer; - MetricManager mm( - std::unique_ptr<MetricManager::Timer>(timer = new FakeTimer)); + MetricManager mm(std::unique_ptr<MetricManager::Timer>(timer = new FakeTimer)); { MetricLockGuard lockGuard(mm.getMetricLock()); mm.registerMetric(lockGuard, set); @@ -215,10 +214,9 @@ TEST_F(SnapshotTest, test_snapshot_two_days) << "\n"; */ - const MetricSnapshot* snap = 0; // active snapshot MetricLockGuard lockGuard(mm.getMetricLock()); - snap = &mm.getActiveMetrics(lockGuard); + const MetricSnapshot* snap = &mm.getActiveMetrics(lockGuard); ASSERT_VALUE(0, *snap, "test.set1.set1.count1"); ASSERT_VALUE(0, *snap, "test.set1.set1.countSum"); diff --git a/metrics/src/vespa/metrics/CMakeLists.txt b/metrics/src/vespa/metrics/CMakeLists.txt index 2441bf95c0b..62254a8d620 100644 --- a/metrics/src/vespa/metrics/CMakeLists.txt +++ b/metrics/src/vespa/metrics/CMakeLists.txt @@ -18,7 +18,6 @@ vespa_add_library(metrics updatehook.cpp valuemetric.cpp valuemetricvalues.cpp - xmlwriter.cpp $<TARGET_OBJECTS:metrics_common> INSTALL lib64 diff --git a/metrics/src/vespa/metrics/metricmanager.cpp b/metrics/src/vespa/metrics/metricmanager.cpp index a0e44ddbeac..df83001a4e2 100644 --- a/metrics/src/vespa/metrics/metricmanager.cpp +++ b/metrics/src/vespa/metrics/metricmanager.cpp @@ -22,25 +22,30 @@ LOG_SETUP(".metrics.manager"); namespace metrics { using Config = MetricsmanagerConfig; +using vespalib::IllegalStateException; +using vespalib::IllegalArgumentException; +using vespalib::make_string_short::fmt; +using vespalib::count_ms; +using vespalib::count_s; +using vespalib::from_s; MetricManager::ConsumerSpec::ConsumerSpec() = default; MetricManager::ConsumerSpec::~ConsumerSpec() = default; -time_t +time_point MetricManager::Timer::getTime() const { - return vespalib::count_s(vespalib::system_clock::now().time_since_epoch()); + return vespalib::system_clock::now(); } void MetricManager::assertMetricLockLocked(const MetricLockGuard& g) const { if ( ! g.owns(_waiter)) { - throw vespalib::IllegalArgumentException("Given lock does not lock the metric lock.", VESPA_STRLOC); + throw IllegalArgumentException("Given lock does not lock the metric lock.", VESPA_STRLOC); } } void -MetricManager::ConsumerSpec::print(std::ostream& out, bool verbose, - const std::string& indent) const +MetricManager::ConsumerSpec::print(std::ostream& out, bool verbose, const std::string& indent) const { (void) verbose; out << "ConsumerSpec("; @@ -63,6 +68,10 @@ MetricManager::ConsumerSpec::addMemoryUsage(MemoryConsumption& mc) const } } +MetricManager::MetricManager() + : MetricManager(std::make_unique<Timer>()) +{ } + MetricManager::MetricManager(std::unique_ptr<Timer> timer) : _activeMetrics("Active metrics showing updates since last snapshot"), _configSubscriber(), @@ -70,9 +79,7 @@ MetricManager::MetricManager(std::unique_ptr<Timer> timer) _config(), _consumerConfig(), _snapshots(), - _totalMetrics(std::make_shared<MetricSnapshot>( - "Empty metrics before init", 0, _activeMetrics.getMetrics(), - false)), + _totalMetrics(std::make_shared<MetricSnapshot>("Empty metrics before init", 0, _activeMetrics.getMetrics(), false)), _timer(std::move(timer)), _lastProcessedTime(0), _snapshotUnsetMetrics(false), @@ -114,7 +121,7 @@ MetricManager::addMetricUpdateHook(UpdateHook& hook, uint32_t period) std::lock_guard sync(_waiter); // If we've already initialized manager, log period has been set. // In this case. Call first time after period - hook._nextCall = _timer->getTime() + period; + hook._nextCall = count_s(_timer->getTime().time_since_epoch()) + period; if (period == 0) { for (UpdateHook * sHook : _snapshotUpdateHooks) { if (sHook == &hook) { @@ -165,9 +172,8 @@ void MetricManager::init(const config::ConfigUri & uri, bool startThread) { if (isInitialized()) { - throw vespalib::IllegalStateException( - "The metric manager have already been initialized. " - "It can only be initialized once.", VESPA_STRLOC); + throw IllegalStateException("The metric manager have already been initialized. " + "It can only be initialized once.", VESPA_STRLOC); } LOG(debug, "Initializing metric manager."); _configSubscriber = std::make_unique<config::ConfigSubscriber>(uri.getContext()); @@ -233,8 +239,10 @@ struct ConsumerMetricBuilder : public MetricVisitor { bool nameRemoved; uint32_t metricCount; - Result() : tagAdded(false), tagRemoved(false), - nameAdded(false), nameRemoved(false), metricCount(0) {} + Result() + : tagAdded(false), tagRemoved(false), + nameAdded(false), nameRemoved(false), metricCount(0) + {} }; std::list<Result> result; @@ -353,9 +361,7 @@ ConsumerMetricBuilder::~ConsumerMetricBuilder() = default; void MetricManager::checkMetricsAltered(const MetricLockGuard & guard) { - if (_activeMetrics.getMetrics().isRegistrationAltered() - || _consumerConfigChanged) - { + if (_activeMetrics.getMetrics().isRegistrationAltered() || _consumerConfigChanged) { handleMetricsAltered(guard); } } @@ -385,8 +391,8 @@ MetricManager::handleMetricsAltered(const MetricLockGuard & guard) } LOG(debug, "Recreating snapshots to include altered metrics"); _totalMetrics->recreateSnapshot(_activeMetrics.getMetrics(), _snapshotUnsetMetrics); - for (uint32_t i=0; i<_snapshots.size(); ++i) { - _snapshots[i]->recreateSnapshot(_activeMetrics.getMetrics(), _snapshotUnsetMetrics); + for (const auto & snapshot: _snapshots) { + snapshot->recreateSnapshot(_activeMetrics.getMetrics(), _snapshotUnsetMetrics); } LOG(debug, "Setting new consumer config. Clearing dirty flag"); _consumerConfig.swap(configMap); @@ -394,24 +400,25 @@ MetricManager::handleMetricsAltered(const MetricLockGuard & guard) } namespace { - bool setSnapshotName(std::ostream& out, const char* name, uint32_t length, uint32_t period) - { - if (length % period != 0) return false; - out << (length / period) << ' ' << name; - if (length / period != 1) out << "s"; - return true; - } + +bool +setSnapshotName(std::ostream& out, const char* name, uint32_t length, uint32_t period) { + if (length % period != 0) return false; + out << (length / period) << ' ' << name; + if (length / period != 1) out << "s"; + return true; +} + } std::vector<MetricManager::SnapSpec> MetricManager::createSnapshotPeriods(const Config& config) { std::vector<SnapSpec> result; - try{ - for (uint32_t i=0; i<config.snapshot.periods.size(); ++i) { - uint32_t length = config.snapshot.periods[i]; + try { + for (auto length : config.snapshot.periods) { if (length < 1) - throw vespalib::IllegalStateException("Snapshot periods must be positive numbers", VESPA_STRLOC); + throw IllegalStateException("Snapshot periods must be positive numbers", VESPA_STRLOC); std::ostringstream name; if (setSnapshotName(name, "week", length, 60 * 60 * 24 * 7)) { } else if (setSnapshotName(name, "day", length, 60 * 60 * 24)) { @@ -425,16 +432,13 @@ MetricManager::createSnapshotPeriods(const Config& config) for (uint32_t i=1; i<result.size(); ++i) { if (result[i].first % result[i-1].first != 0) { std::ostringstream ost; - ost << "Period " << result[i].first - << " is not a multiplum of period " + ost << "Period " << result[i].first << " is not a multiplum of period " << result[i-1].first << " which is needs to be."; - throw vespalib::IllegalStateException( - ost.str(), VESPA_STRLOC); + throw IllegalStateException(ost.str(), VESPA_STRLOC); } } } catch (vespalib::Exception& e) { - LOG(warning, "Invalid snapshot periods specified. Using defaults: %s", - e.getMessage().c_str()); + LOG(warning, "Invalid snapshot periods specified. Using defaults: %s", e.getMessage().c_str()); result.clear(); } if (result.empty()) { @@ -454,8 +458,7 @@ MetricManager::configure(const MetricLockGuard & , std::unique_ptr<Config> confi std::ostringstream ost; config::OstreamConfigWriter w(ost); w.write(*config); - LOG(debug, "Received new config for metric manager: %s", - ost.str().c_str()); + LOG(debug, "Received new config for metric manager: %s", ost.str().c_str()); } if (_snapshots.empty()) { LOG(debug, "Initializing snapshots as this is first configure call"); @@ -463,21 +466,15 @@ MetricManager::configure(const MetricLockGuard & , std::unique_ptr<Config> confi // Set up snapshots only first time. We don't allow live reconfig // of snapshot periods. - time_t currentTime(_timer->getTime()); + time_t currentTime = count_s(_timer->getTime().time_since_epoch()); _activeMetrics.setFromTime(currentTime); uint32_t count = 1; - for (uint32_t i = 0; i< snapshotPeriods.size(); ++i) - { + for (uint32_t i = 0; i< snapshotPeriods.size(); ++i) { uint32_t nextCount = 1; if (i + 1 < snapshotPeriods.size()) { - nextCount = snapshotPeriods[i + 1].first - / snapshotPeriods[i].first; - if (snapshotPeriods[i + 1].first - % snapshotPeriods[i].first != 0) - { - throw vespalib::IllegalStateException( - "Snapshot periods must be multiplum of each other", - VESPA_STRLOC); + nextCount = snapshotPeriods[i + 1].first / snapshotPeriods[i].first; + if ((snapshotPeriods[i + 1].first % snapshotPeriods[i].first) != 0) { + throw IllegalStateException("Snapshot periods must be multiplum of each other",VESPA_STRLOC); } } _snapshots.push_back(std::make_shared<MetricSnapshotSet>( @@ -489,9 +486,7 @@ MetricManager::configure(const MetricLockGuard & , std::unique_ptr<Config> confi _totalMetrics = std::make_shared<MetricSnapshot>("All time snapshot", 0, _activeMetrics.getMetrics(), _snapshotUnsetMetrics); _totalMetrics->reset(currentTime); } - if (_config.get() == 0 - || _config->consumer.size() != config->consumer.size()) - { + if (_config.get() == 0 || (_config->consumer.size() != config->consumer.size())) { _consumerConfigChanged = true; } else { for (uint32_t i=0; i<_config->consumer.size(); ++i) { @@ -519,61 +514,55 @@ MetricManager::getConsumerSpec(const MetricLockGuard &, const Metric::String& co namespace { - struct ConsumerMetricVisitor : public MetricVisitor { - const MetricManager::ConsumerSpec& _metricsToMatch; - MetricVisitor& _client; +struct ConsumerMetricVisitor : public MetricVisitor { + const MetricManager::ConsumerSpec& _metricsToMatch; + MetricVisitor& _client; #ifdef VERIFY_ALL_METRICS_VISITED - std::set<Metric::String> _visitedMetrics; + std::set<Metric::String> _visitedMetrics; #endif - ConsumerMetricVisitor(const MetricManager::ConsumerSpec& spec, - MetricVisitor& clientVisitor) - : _metricsToMatch(spec), _client(clientVisitor) {} + ConsumerMetricVisitor(const MetricManager::ConsumerSpec& spec, MetricVisitor& clientVisitor) + : _metricsToMatch(spec), _client(clientVisitor) + {} - bool visitMetricSet(const MetricSet& metricSet, - bool autoGenerated) override - { - if (metricSet.isTopSet()) return true; - return (_metricsToMatch.contains(metricSet) - && _client.visitMetricSet(metricSet, autoGenerated)); - } - void doneVisitingMetricSet(const MetricSet& metricSet) override { - if (!metricSet.isTopSet()) { + bool visitMetricSet(const MetricSet& metricSet, bool autoGenerated) override { + if (metricSet.isTopSet()) return true; + return (_metricsToMatch.contains(metricSet) + && _client.visitMetricSet(metricSet, autoGenerated)); + } + void doneVisitingMetricSet(const MetricSet& metricSet) override { + if (!metricSet.isTopSet()) { #ifdef VERIFY_ALL_METRICS_VISITED - _visitedMetrics.insert(metricSet.getPath()); + _visitedMetrics.insert(metricSet.getPath()); #endif - _client.doneVisitingMetricSet(metricSet); - } + _client.doneVisitingMetricSet(metricSet); } - bool visitCountMetric(const AbstractCountMetric& metric, - bool autoGenerated) override - { - if (_metricsToMatch.contains(metric)) { + } + bool visitCountMetric(const AbstractCountMetric& metric, bool autoGenerated) override { + if (_metricsToMatch.contains(metric)) { #ifdef VERIFY_ALL_METRICS_VISITED - _visitedMetrics.insert(metric.getPath()); + _visitedMetrics.insert(metric.getPath()); #endif - return _client.visitCountMetric(metric, autoGenerated); - } - return true; + return _client.visitCountMetric(metric, autoGenerated); } - bool visitValueMetric(const AbstractValueMetric& metric, - bool autoGenerated) override - { - if (_metricsToMatch.contains(metric)) { + return true; + } + bool visitValueMetric(const AbstractValueMetric& metric, bool autoGenerated) override { + if (_metricsToMatch.contains(metric)) { #ifdef VERIFY_ALL_METRICS_VISITED - _visitedMetrics.insert(metric.getPath()); + _visitedMetrics.insert(metric.getPath()); #endif - return _client.visitValueMetric(metric, autoGenerated); - } - return true; + return _client.visitValueMetric(metric, autoGenerated); } - }; + return true; + } +}; } void -MetricManager::visit(const MetricLockGuard & guard, const MetricSnapshot& snapshot, MetricVisitor& visitor, - const std::string& consumer) const +MetricManager::visit(const MetricLockGuard & guard, const MetricSnapshot& snapshot, + MetricVisitor& visitor, const std::string& consumer) const { if (visitor.visitSnapshot(snapshot)) { if (consumer == "") { @@ -593,9 +582,7 @@ MetricManager::visit(const MetricLockGuard & guard, const MetricSnapshot& snapsh } #endif } else { - LOGBP(debug, - "Requested metrics for non-defined consumer '%s'.", - consumer.c_str()); + LOGBP(debug, "Requested metrics for non-defined consumer '%s'.", consumer.c_str()); } } visitor.doneVisitingSnapshot(snapshot); @@ -616,39 +603,31 @@ MetricManager::getSnapshotPeriods(const MetricLockGuard& l) const // Client should have grabbed metrics lock before doing this const MetricSnapshot& -MetricManager::getMetricSnapshot(const MetricLockGuard& l, - uint32_t period, bool getInProgressSet) const +MetricManager::getMetricSnapshot(const MetricLockGuard& l, uint32_t period, bool getInProgressSet) const { assertMetricLockLocked(l); - for (uint32_t i=0; i<_snapshots.size(); ++i) { - if (_snapshots[i]->getPeriod() == period) { - if (_snapshots[i]->getCount() == 1 && getInProgressSet) { - throw vespalib::IllegalStateException( - "No temporary snapshot for set " - + _snapshots[i]->getName(), VESPA_STRLOC); + for (const auto & snapshot : _snapshots) { + if (snapshot->getPeriod() == period) { + if (snapshot->getCount() == 1 && getInProgressSet) { + throw IllegalStateException("No temporary snapshot for set " + snapshot->getName(), VESPA_STRLOC); } - return _snapshots[i]->getSnapshot(getInProgressSet); + return snapshot->getSnapshot(getInProgressSet); } } - std::ostringstream ost; - ost << "No snapshot for period of length " << period << " exist."; - throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC); + throw IllegalArgumentException(fmt("No snapshot for period of length %u exist.", period), VESPA_STRLOC); } // Client should have grabbed metrics lock before doing this const MetricSnapshotSet& -MetricManager::getMetricSnapshotSet(const MetricLockGuard& l, - uint32_t period) const +MetricManager::getMetricSnapshotSet(const MetricLockGuard& l, uint32_t period) const { assertMetricLockLocked(l); - for (uint32_t i=0; i<_snapshots.size(); ++i) { - if (_snapshots[i]->getPeriod() == period) { - return *_snapshots[i]; + for (const auto & snapshot : _snapshots) { + if (snapshot->getPeriod() == period) { + return *snapshot; } } - std::ostringstream ost; - ost << "No snapshot set for period of length " << period << " exist."; - throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC); + throw IllegalArgumentException(fmt("No snapshot set for period of length %u exist.", period), VESPA_STRLOC); } void @@ -661,9 +640,8 @@ MetricManager::timeChangedNotification() const void MetricManager::updateMetrics(bool includeSnapshotOnlyHooks) { - LOG(debug, "Calling metric update hooks%s.", - includeSnapshotOnlyHooks ? ", including snapshot hooks" : ""); - // Ensure we're not in the way of the background thread + LOG(debug, "Calling metric update hooks%s.", includeSnapshotOnlyHooks ? ", including snapshot hooks" : ""); + // Ensure we're not in the way of the background thread MetricLockGuard sync(_waiter); LOG(debug, "Giving %zu periodic update hooks.", _periodicUpdateHooks.size()); updatePeriodicMetrics(sync, 0, true); @@ -677,30 +655,29 @@ MetricManager::updateMetrics(bool includeSnapshotOnlyHooks) time_t MetricManager::updatePeriodicMetrics(const MetricLockGuard & guard, time_t updateTime, bool outOfSchedule) { + assertMetricLockLocked(guard); time_t nextUpdateTime = std::numeric_limits<time_t>::max(); - time_t preTime = _timer->getTimeInMilliSecs(); + time_point preTime = _timer->getTimeInMilliSecs(); for (auto hook : _periodicUpdateHooks) { if (hook->_nextCall <= updateTime) { hook->updateMetrics(guard); if (hook->_nextCall + hook->_period < updateTime) { if (hook->_nextCall != 0) { - LOG(debug, "Updated hook %s at time %" PRIu64 ", but next " - "run in %u seconds have already passed as time" - " is %" PRIu64 ". Bumping next call to current " - "time + period.", + LOG(debug, "Updated hook %s at time %" PRIu64 ", but next run in %u seconds have already passed as " + "time is %" PRIu64 ". Bumping next call to current time + period.", hook->_name, static_cast<uint64_t>(hook->_nextCall), hook->_period, static_cast<uint64_t>(updateTime)); } hook->_nextCall = updateTime + hook->_period; } else { hook->_nextCall += hook->_period; } - time_t postTime = _timer->getTimeInMilliSecs(); - _periodicHookLatency.addValue(postTime - preTime); + time_point postTime = _timer->getTimeInMilliSecs(); + _periodicHookLatency.addValue(count_ms(postTime - preTime)); preTime = postTime; } else if (outOfSchedule) { hook->updateMetrics(guard); - time_t postTime = _timer->getTimeInMilliSecs(); - _periodicHookLatency.addValue(postTime - preTime); + time_point postTime = _timer->getTimeInMilliSecs(); + _periodicHookLatency.addValue(count_ms(postTime - preTime)); preTime = postTime; } nextUpdateTime = std::min(nextUpdateTime, hook->_nextCall); @@ -712,11 +689,12 @@ MetricManager::updatePeriodicMetrics(const MetricLockGuard & guard, time_t updat void MetricManager::updateSnapshotMetrics(const MetricLockGuard & guard) { - time_t preTime = _timer->getTimeInMilliSecs(); - for (auto it = _snapshotUpdateHooks.begin(); it != _snapshotUpdateHooks.end(); ++it) { - (**it).updateMetrics(guard); - time_t postTime = _timer->getTimeInMilliSecs(); - _snapshotHookLatency.addValue(postTime - preTime); + assertMetricLockLocked(guard); + time_point preTime = _timer->getTimeInMilliSecs(); + for (const auto & hook : _snapshotUpdateHooks) { + hook->updateMetrics(guard); + time_point postTime = _timer->getTimeInMilliSecs(); + _snapshotHookLatency.addValue(count_ms(postTime - preTime)); preTime = postTime; } } @@ -733,17 +711,17 @@ MetricManager::forceEventLogging() void MetricManager::reset(time_t currentTime) { - time_t preTime = _timer->getTimeInMilliSecs(); + time_point preTime = _timer->getTimeInMilliSecs(); // Resetting implies visiting metrics, which needs to grab metric lock // to avoid conflict with adding/removal of metrics std::lock_guard waiterLock(_waiter); _activeMetrics.reset(currentTime); - for (uint32_t i=0; i<_snapshots.size(); ++i) { - _snapshots[i]->reset(currentTime); + for (const auto & snapshot : _snapshots) { + snapshot->reset(currentTime); } _totalMetrics->reset(currentTime); - time_t postTime = _timer->getTimeInMilliSecs(); - _resetLatency.addValue(postTime - preTime); + time_point postTime = _timer->getTimeInMilliSecs(); + _resetLatency.addValue(count_ms(postTime - preTime)); } void @@ -755,7 +733,7 @@ MetricManager::run() // we constantly add next time to do something from the last timer. // For that to work, we need to initialize timers on first iteration // to set them to current time. - time_t currentTime = _timer->getTime(); + time_t currentTime = count_s(_timer->getTime().time_since_epoch()); for (auto & snapshot : _snapshots) { snapshot->setFromTime(currentTime); } @@ -765,12 +743,13 @@ MetricManager::run() // Ensure correct time for first snapshot _snapshots[0]->getSnapshot().setToTime(currentTime); while (!stop_requested()) { - currentTime = _timer->getTime(); + time_point now = _timer->getTime(); + currentTime = count_s(now.time_since_epoch()); time_t next = tick(sync, currentTime); if (currentTime < next) { - size_t ms = (next - currentTime) * 1000; - _cond.wait_for(sync, std::chrono::milliseconds(ms)); - _sleepTimes.addValue(ms); + vespalib::duration wait_time = from_s(next - currentTime); + _cond.wait_for(sync, wait_time); + _sleepTimes.addValue(count_ms(wait_time)); } else { _sleepTimes.addValue(0); } @@ -780,11 +759,10 @@ MetricManager::run() time_t MetricManager::tick(const MetricLockGuard & guard, time_t currentTime) { - LOG(spam, "Worker thread starting to process for time %" PRIu64 ".", - static_cast<uint64_t>(currentTime)); + LOG(spam, "Worker thread starting to process for time %" PRIu64 ".", static_cast<uint64_t>(currentTime)); // Check for new config and reconfigure - if (_configSubscriber.get() && _configSubscriber->nextConfigNow()) { + if (_configSubscriber && _configSubscriber->nextConfigNow()) { configure(guard, _configHandle->getConfig()); } @@ -811,8 +789,7 @@ MetricManager::tick(const MetricLockGuard & guard, time_t currentTime) // Do snapshotting if it is time if (nextWorkTime <= currentTime) takeSnapshots(guard, nextWorkTime); - _lastProcessedTime.store(nextWorkTime <= currentTime ? nextWorkTime : currentTime, - std::memory_order_relaxed); + _lastProcessedTime.store(nextWorkTime <= currentTime ? nextWorkTime : currentTime, std::memory_order_relaxed); LOG(spam, "Worker thread done with processing for time %" PRIu64 ".", static_cast<uint64_t>(_lastProcessedTime.load(std::memory_order_relaxed))); time_t next = _snapshots[0]->getPeriod() + _snapshots[0]->getToTime(); @@ -821,8 +798,9 @@ MetricManager::tick(const MetricLockGuard & guard, time_t currentTime) } void -MetricManager::takeSnapshots(const MetricLockGuard &, time_t timeToProcess) +MetricManager::takeSnapshots(const MetricLockGuard & guard, time_t timeToProcess) { + assertMetricLockLocked(guard); // If not time to do dump data from active snapshot yet, nothing to do if (!_snapshots[0]->timeForAnotherSnapshot(timeToProcess)) { LOG(spam, "Not time to process snapshot %s at time %" PRIu64 ". Current " @@ -832,7 +810,7 @@ MetricManager::takeSnapshots(const MetricLockGuard &, time_t timeToProcess) static_cast<uint64_t>(_snapshots[0]->getToTime())); return; } - time_t preTime = _timer->getTimeInMilliSecs(); + time_point preTime = _timer->getTimeInMilliSecs(); LOG(debug, "Updating %s snapshot and total metrics at time %" PRIu64 ".", _snapshots[0]->getName().c_str(), static_cast<uint64_t>(timeToProcess)); MetricSnapshot& firstTarget(_snapshots[0]->getNextTarget()); @@ -840,67 +818,54 @@ MetricManager::takeSnapshots(const MetricLockGuard &, time_t timeToProcess) _activeMetrics.addToSnapshot(firstTarget, false, timeToProcess); _activeMetrics.addToSnapshot(*_totalMetrics, false, timeToProcess); _activeMetrics.reset(timeToProcess); - LOG(debug, "After snapshotting, " - "active metrics goes from %" PRIu64 " to %" PRIu64", " + LOG(debug, "After snapshotting, active metrics goes from %" PRIu64 " to %" PRIu64", " "and 5 minute metrics goes from %" PRIu64 " to %" PRIu64".", static_cast<uint64_t>(_activeMetrics.getFromTime()), static_cast<uint64_t>(_activeMetrics.getToTime()), static_cast<uint64_t>(firstTarget.getFromTime()), static_cast<uint64_t>(firstTarget.getToTime())); // Update later snapshots if it is time for it for (uint32_t i=1; i<_snapshots.size(); ++i) { - LOG(debug, "Adding data from last snapshot to building snapshot of " - "next period snapshot %s.", + LOG(debug, "Adding data from last snapshot to building snapshot of next period snapshot %s.", _snapshots[i]->getName().c_str()); MetricSnapshot& target(_snapshots[i]->getNextTarget()); - _snapshots[i-1]->getSnapshot().addToSnapshot( - target, false, timeToProcess); + _snapshots[i-1]->getSnapshot().addToSnapshot(target, false, timeToProcess); target.setToTime(timeToProcess); if (!_snapshots[i]->haveCompletedNewPeriod(timeToProcess)) { - LOG(debug, "Not time to roll snapshot %s yet. %u of %u snapshot " - "taken at time %" PRIu64 ", and period of %u is not up " - "yet as we're currently processing for time %" PRIu64 ".", - _snapshots[i]->getName().c_str(), - _snapshots[i]->getBuilderCount(), - _snapshots[i]->getCount(), - static_cast<uint64_t> - (_snapshots[i]->getBuilderCount() * _snapshots[i]->getPeriod() - + _snapshots[i]->getFromTime()), - _snapshots[i]->getPeriod(), - static_cast<uint64_t>(timeToProcess)); + LOG(debug, "Not time to roll snapshot %s yet. %u of %u snapshot taken at time %" PRIu64 ", and period of %u " + "is not up yet as we're currently processing for time %" PRIu64 ".", + _snapshots[i]->getName().c_str(), _snapshots[i]->getBuilderCount(), _snapshots[i]->getCount(), + static_cast<uint64_t>(_snapshots[i]->getBuilderCount() * _snapshots[i]->getPeriod() + _snapshots[i]->getFromTime()), + _snapshots[i]->getPeriod(), static_cast<uint64_t>(timeToProcess)); break; } else { LOG(debug, "Rolled snapshot %s at time %" PRIu64 ".", - _snapshots[i]->getName().c_str(), - static_cast<uint64_t>(timeToProcess)); + _snapshots[i]->getName().c_str(), static_cast<uint64_t>(timeToProcess)); } } - time_t postTime = _timer->getTimeInMilliSecs(); - _snapshotLatency.addValue(postTime - preTime); + time_point postTime = _timer->getTimeInMilliSecs(); + _snapshotLatency.addValue(count_ms(postTime - preTime)); } MemoryConsumption::UP MetricManager::getMemoryConsumption(const MetricLockGuard & guard) const { - (void) guard; - MemoryConsumption::UP mc(new MemoryConsumption); + assertMetricLockLocked(guard); + auto mc = std::make_unique<MemoryConsumption>(); mc->_consumerCount += _consumerConfig.size(); - mc->_consumerMeta += (sizeof(ConsumerSpec::SP) + sizeof(ConsumerSpec)) - * _consumerConfig.size(); - for (auto it = _consumerConfig.begin(); it != _consumerConfig.end(); ++it) { - mc->_consumerId += mc->getStringMemoryUsage( - it->first, mc->_consumerIdUnique) - + sizeof(Metric::String); - it->second->addMemoryUsage(*mc); + mc->_consumerMeta += (sizeof(ConsumerSpec::SP) + sizeof(ConsumerSpec)) * _consumerConfig.size(); + for (const auto & consumer : _consumerConfig) { + mc->_consumerId += mc->getStringMemoryUsage(consumer.first, mc->_consumerIdUnique) + sizeof(Metric::String); + consumer.second->addMemoryUsage(*mc); } uint32_t preTotal = mc->getTotalMemoryUsage(); _activeMetrics.addMemoryUsage(*mc); uint32_t postTotal = mc->getTotalMemoryUsage(); mc->addSnapShotUsage("active", postTotal - preTotal); preTotal = postTotal; - for (uint32_t i=0; i<_snapshots.size(); ++i) { - _snapshots[i]->addMemoryUsage(*mc); + for (const auto & snapshot : _snapshots) { + snapshot->addMemoryUsage(*mc); postTotal = mc->getTotalMemoryUsage(); - mc->addSnapShotUsage(_snapshots[i]->getName(), postTotal - preTotal); + mc->addSnapShotUsage(snapshot->getName(), postTotal - preTotal); preTotal = postTotal; } _totalMetrics->addMemoryUsage(*mc); diff --git a/metrics/src/vespa/metrics/metricmanager.h b/metrics/src/vespa/metrics/metricmanager.h index 7a34f894282..c3ce37b451f 100644 --- a/metrics/src/vespa/metrics/metricmanager.h +++ b/metrics/src/vespa/metrics/metricmanager.h @@ -67,8 +67,8 @@ public: struct Timer { virtual ~Timer() = default; - virtual time_t getTime() const; - virtual time_t getTimeInMilliSecs() const { return getTime() * 1000; } + virtual time_point getTime() const; + time_point getTimeInMilliSecs() const { return getTime(); } }; /** @@ -88,8 +88,7 @@ public: return (includedMetrics.find(m.getPath()) != includedMetrics.end()); } - void print(std::ostream& out, bool verbose, - const std::string& indent) const override; + void print(std::ostream& out, bool verbose, const std::string& indent) const override; void addMemoryUsage(MemoryConsumption&) const; }; @@ -126,7 +125,8 @@ private: bool stop_requested() const { return _stop_requested.load(std::memory_order_relaxed); } public: - MetricManager(std::unique_ptr<Timer> timer = std::make_unique<Timer>()); + MetricManager(); + MetricManager(std::unique_ptr<Timer> timer); ~MetricManager(); void stop(); @@ -232,19 +232,16 @@ public: } /** While accessing the total metrics you should have the metric lock. */ - const MetricSnapshot& getTotalMetricSnapshot(const MetricLockGuard& l) const - { + const MetricSnapshot& getTotalMetricSnapshot(const MetricLockGuard& l) const { assertMetricLockLocked(l); return *_totalMetrics; } /** While accessing snapshots you should have the metric lock. */ - const MetricSnapshot& getMetricSnapshot( - const MetricLockGuard&, - uint32_t period, bool getInProgressSet = false) const; - const MetricSnapshotSet& getMetricSnapshotSet( - const MetricLockGuard&, uint32_t period) const; - bool hasTemporarySnapshot(const MetricLockGuard& l, uint32_t period) const - { return getMetricSnapshotSet(l, period).hasTemporarySnapshot(); } + const MetricSnapshot& getMetricSnapshot( const MetricLockGuard& guard, uint32_t period) const { + return getMetricSnapshot(guard, period, false); + } + const MetricSnapshot& getMetricSnapshot( const MetricLockGuard&, uint32_t period, bool getInProgressSet) const; + const MetricSnapshotSet& getMetricSnapshotSet(const MetricLockGuard&, uint32_t period) const; std::vector<uint32_t> getSnapshotPeriods(const MetricLockGuard& l) const; diff --git a/metrics/src/vespa/metrics/updatehook.h b/metrics/src/vespa/metrics/updatehook.h index 9fa0d52027e..58d9ef0d743 100644 --- a/metrics/src/vespa/metrics/updatehook.h +++ b/metrics/src/vespa/metrics/updatehook.h @@ -1,10 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include <vespa/vespalib/util/time.h> #include <mutex> namespace metrics { +using time_point = vespalib::system_time; + class MetricLockGuard { public: MetricLockGuard(std::mutex & mutex); diff --git a/metrics/src/vespa/metrics/xmlwriter.cpp b/metrics/src/vespa/metrics/xmlwriter.cpp deleted file mode 100644 index 11cb450e64d..00000000000 --- a/metrics/src/vespa/metrics/xmlwriter.cpp +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "xmlwriter.h" -#include "countmetric.h" -#include "metricset.h" -#include "metricsnapshot.h" -#include "valuemetric.h" -#include <vespa/vespalib/util/xmlstream.h> -#include <sstream> - -namespace metrics { - -XmlWriter::XmlWriter(vespalib::xml::XmlOutputStream& xos, - [[maybe_unused]] uint32_t period, int verbosity) - : _xos(xos), _verbosity(verbosity) {} - -bool -XmlWriter::visitSnapshot(const MetricSnapshot& snapshot) -{ - using namespace vespalib::xml; - _xos << XmlTag("snapshot") << XmlAttribute("name", snapshot.getName()) - << XmlAttribute("from", snapshot.getFromTime()) - << XmlAttribute("to", snapshot.getToTime()) - << XmlAttribute("period", snapshot.getPeriod()); - return true; -} - -void -XmlWriter::doneVisitingSnapshot(const MetricSnapshot&) -{ - using namespace vespalib::xml; - _xos << XmlEndTag(); -} - -bool -XmlWriter::visitMetricSet(const MetricSet& set, bool) -{ - using namespace vespalib::xml; - if (set.used() || _verbosity >= 2) { - _xos << XmlTag(set.getName(), XmlTagFlags::CONVERT_ILLEGAL_CHARACTERS); - printCommonXmlParts(set); - return true; - } - return false; -} -void -XmlWriter::doneVisitingMetricSet(const MetricSet&) { - using namespace vespalib::xml; - _xos << XmlEndTag(); -} - -bool -XmlWriter::visitCountMetric(const AbstractCountMetric& metric, bool) -{ - MetricValueClass::UP values(metric.getValues()); - if (!metric.inUse(*values) && _verbosity < 2) return true; - using namespace vespalib::xml; - std::ostringstream ost; - _xos << XmlTag(metric.getName(), XmlTagFlags::CONVERT_ILLEGAL_CHARACTERS) - << XmlAttribute(metric.sumOnAdd() - ? "count" : "value", values->toString("count")); - printCommonXmlParts(metric); - _xos << XmlEndTag(); - return true; -} - -bool -XmlWriter::visitValueMetric(const AbstractValueMetric& metric, bool) -{ - MetricValueClass::UP values(metric.getValues()); - if (!metric.inUse(*values) && _verbosity < 2) return true; - using namespace vespalib::xml; - _xos << XmlTag(metric.getName(), XmlTagFlags::CONVERT_ILLEGAL_CHARACTERS) - << XmlAttribute("average", values->getLongValue("count") == 0 - ? 0 : values->getDoubleValue("total") - / values->getDoubleValue("count")) - << XmlAttribute("last", values->toString("last")); - if (!metric.summedAverage()) { - if (values->getLongValue("count") > 0) { - _xos << XmlAttribute("min", values->toString("min")) - << XmlAttribute("max", values->toString("max")); - } - _xos << XmlAttribute("count", values->getLongValue("count")); - if (_verbosity >= 2) { - _xos << XmlAttribute("total", values->toString("total")); - } - } - printCommonXmlParts(metric); - _xos << XmlEndTag(); - return true; -} - -void -XmlWriter::printCommonXmlParts(const Metric& metric) const -{ - using namespace vespalib::xml; - const Metric::Tags& tags(metric.getTags()); - if (_verbosity >= 3 && tags.size() > 0) { - std::ostringstream ost; - // XXX print tag values as well - ost << tags[0].key(); - for (uint32_t i=1; i<tags.size(); ++i) { - ost << "," << tags[i].key(); - } - _xos << XmlAttribute("tags", ost.str()); - } - if (_verbosity >= 1 && !metric.getDescription().empty()) { - _xos << XmlAttribute("description", metric.getDescription()); - } -} - -} // metrics diff --git a/metrics/src/vespa/metrics/xmlwriter.h b/metrics/src/vespa/metrics/xmlwriter.h deleted file mode 100644 index a0e8a3efeea..00000000000 --- a/metrics/src/vespa/metrics/xmlwriter.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/metrics/metric.h> -#include <vespa/vespalib/util/xmlserializable.h> - -namespace metrics { - -class XmlWriter : public MetricVisitor { - vespalib::xml::XmlOutputStream& _xos; - int _verbosity; - -public: - XmlWriter(vespalib::xml::XmlOutputStream& xos, - uint32_t period, int verbosity); - - bool visitSnapshot(const MetricSnapshot&) override; - void doneVisitingSnapshot(const MetricSnapshot&) override; - bool visitMetricSet(const MetricSet& set, bool) override; - void doneVisitingMetricSet(const MetricSet&) override; - bool visitCountMetric(const AbstractCountMetric&, bool autoGenerated) override; - bool visitValueMetric(const AbstractValueMetric&, bool autoGenerated) override; - -private: - void printCommonXmlParts(const Metric& metric) const; -}; - -} - diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java index 27488e4027c..faa360bbcb1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java @@ -13,7 +13,6 @@ import com.yahoo.vespa.hosted.provision.persistence.CuratorDb; import java.time.Duration; import java.util.Optional; import java.util.function.Function; -import java.util.logging.Logger; /** * Thread safe class to get and set archive URI for given account and tenants. @@ -22,7 +21,6 @@ import java.util.logging.Logger; */ public class ArchiveUriManager { - private static final Logger log = Logger.getLogger(ArchiveUriManager.class.getName()); private static final Duration cacheTtl = Duration.ofMinutes(1); private final CuratorDb db; @@ -48,7 +46,14 @@ public class ArchiveUriManager { archiveUris.get().accountArchiveUris().get(node.cloudAccount()) : archiveUris.get().tenantArchiveUris().get(app.tenant())) .map(uri -> { + // TODO (freva): Remove when all URIs dont have tenant name in them anymore + String tenantSuffix = "/" + app.tenant().value() + "/"; + if (uri.endsWith(tenantSuffix)) return uri.substring(0, uri.length() - tenantSuffix.length() + 1); + return uri; + }) + .map(uri -> { StringBuilder sb = new StringBuilder(100).append(uri) + .append(app.tenant().value()).append('/') .append(app.application().value()).append('/') .append(app.instance().value()).append('/') .append(node.allocation().get().membership().cluster().id().value()).append('/'); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java index 9506bba73e7..2cc43a1eb33 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java @@ -77,6 +77,10 @@ public class Autoscaling { return peak.equals(Load.zero()); } + public boolean isPresent() { + return ! isEmpty(); + } + @Override public boolean equals(Object o) { if ( ! (o instanceof Autoscaling other)) return false; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index 674c20e25f2..69c03dbf6dc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Deployer; @@ -47,27 +48,30 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { @Override protected double maintain() { if ( ! nodeRepository().nodes().isWorking()) return 0.0; - - if ( ! nodeRepository().zone().environment().isAnyOf(Environment.dev, Environment.perf, Environment.prod)) return 1.0; - - activeNodesByApplication().forEach(this::autoscale); - return 1.0; - } - - private void autoscale(ApplicationId application, NodeList applicationNodes) { - try { - nodesByCluster(applicationNodes).forEach((clusterId, clusterNodes) -> autoscale(application, clusterId)); - } - catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Illegal arguments for " + application, e); + if (nodeRepository().zone().environment().isTest()) return 1.0; + + int attempts = 0; + int failures = 0; + for (var applicationNodes : activeNodesByApplication().entrySet()) { + for (var clusterNodes : nodesByCluster(applicationNodes.getValue()).entrySet()) { + attempts++; + if ( ! autoscale(applicationNodes.getKey(), clusterNodes.getKey())) + failures++; + } } + return asSuccessFactor(attempts, failures); } - private void autoscale(ApplicationId applicationId, ClusterSpec.Id clusterId) { + /** + * Autoscales the given cluster. + * + * @return true if an autoscaling decision was made or nothing should be done, false if there was an error + */ + private boolean autoscale(ApplicationId applicationId, ClusterSpec.Id clusterId) { try (var lock = nodeRepository().applications().lock(applicationId)) { Optional<Application> application = nodeRepository().applications().get(applicationId); - if (application.isEmpty()) return; - if (application.get().cluster(clusterId).isEmpty()) return; + if (application.isEmpty()) return true; + if (application.get().cluster(clusterId).isEmpty()) return true; Cluster cluster = application.get().cluster(clusterId).get(); NodeList clusterNodes = nodeRepository().nodes().list(Node.State.active).owner(applicationId).cluster(clusterId); @@ -79,7 +83,7 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { Autoscaling autoscaling = null; if (cluster.target().resources().isEmpty() || current.equals(cluster.target().resources().get())) { autoscaling = autoscaler.autoscale(application.get(), cluster, clusterNodes); - if ( ! autoscaling.isEmpty()) // Ignore empties we'll get from servers recently started + if ( autoscaling.isPresent() || cluster.target().isEmpty()) // Ignore empty from recently started servers cluster = cluster.withTarget(autoscaling); } @@ -94,6 +98,13 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { logAutoscaling(current, autoscaling.resources().get(), applicationId, clusterNodes.not().retired()); } } + return true; + } + catch (ApplicationLockException e) { + return false; + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Illegal arguments for " + applicationId + " cluster " + clusterId, e); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index 67c1c7359f7..5af74214648 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -267,13 +267,20 @@ public class MetricsReporter extends NodeRepositoryMaintainer { } private void updateNodeCountMetrics(NodeList nodes) { - Map<State, List<Node>> nodesByState = nodes.nodeType(NodeType.tenant).asList().stream() - .collect(Collectors.groupingBy(Node::state)); + var nodesByState = nodes.nodeType(NodeType.tenant) + .asList().stream() + .collect(Collectors.groupingBy(Node::state)); + + var hostsByState = nodes.nodeType(NodeType.host) + .asList().stream() + .collect(Collectors.groupingBy(Node::state)); // Count per state for (State state : State.values()) { - List<Node> nodesInState = nodesByState.getOrDefault(state, List.of()); - metric.set("hostedVespa." + state.name() + "Hosts", nodesInState.size(), null); + var nodesInState = nodesByState.getOrDefault(state, List.of()); + var hostsInState = hostsByState.getOrDefault(state, List.of()); + metric.set("hostedVespa." + state.name() + "Nodes", nodesInState.size(), null); + metric.set("hostedVespa." + state.name() + "Hosts", hostsInState.size(), null); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java index 8ee72d12f57..44c1c976355 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java @@ -54,10 +54,22 @@ public class ArchiveUriManagerTest { assertFalse(archiveUriManager.archiveUriFor(createNode(null, account1)).isPresent()); // URI set for this account, but not allocated assertFalse(archiveUriManager.archiveUriFor(createNode(null, account2)).isPresent()); // Not allocated assertFalse(archiveUriManager.archiveUriFor(createNode(app2, null)).isPresent()); // No URI set for this tenant or account - assertEquals("scheme://tenant-bucket/dir/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, null)).get()); - assertEquals("scheme://account-bucket/dir/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, account1)).get()); // Account has precedence + assertEquals("scheme://tenant-bucket/dir/vespa/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, null)).get()); + assertEquals("scheme://account-bucket/dir/vespa/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, account1)).get()); // Account has precedence assertFalse(archiveUriManager.archiveUriFor(createNode(app1, account2)).isPresent()); // URI set for this tenant, but is ignored because enclave account - assertEquals("scheme://tenant-bucket/dir/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, accountSystem)).get()); // URI for tenant because non-enclave acocunt + assertEquals("scheme://tenant-bucket/dir/vespa/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, accountSystem)).get()); // URI for tenant because non-enclave acocunt + } + + @Test + public void handles_uri_with_tenant_name() { + ApplicationId app1 = ApplicationId.from("vespa", "music", "main"); + ArchiveUriManager archiveUriManager = new ProvisioningTester.Builder().build().nodeRepository().archiveUriManager(); + archiveUriManager.setArchiveUri(app1.tenant(), Optional.of("scheme://tenant-bucket/vespa")); + assertEquals("scheme://tenant-bucket/vespa/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, null)).get()); + + // Archive URI ends with the tenant name + archiveUriManager.setArchiveUri(app1.tenant(), Optional.of("scheme://tenant-vespa/")); + assertEquals("scheme://tenant-vespa/vespa/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, null)).get()); } private Node createNode(ApplicationId appId, CloudAccount account) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index 44050fa747c..d7ffda542ff 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -87,7 +87,7 @@ public class MetricsReporterTest { Map<String, Number> expectedMetrics = new TreeMap<>(); expectedMetrics.put("zone.working", 1); - expectedMetrics.put("hostedVespa.provisionedHosts", 1); + expectedMetrics.put("hostedVespa.provisionedHosts", 0); expectedMetrics.put("hostedVespa.parkedHosts", 0); expectedMetrics.put("hostedVespa.readyHosts", 0); expectedMetrics.put("hostedVespa.reservedHosts", 0); @@ -97,6 +97,16 @@ public class MetricsReporterTest { expectedMetrics.put("hostedVespa.failedHosts", 0); expectedMetrics.put("hostedVespa.deprovisionedHosts", 0); expectedMetrics.put("hostedVespa.breakfixedHosts", 0); + expectedMetrics.put("hostedVespa.provisionedNodes", 1); + expectedMetrics.put("hostedVespa.parkedNodes", 0); + expectedMetrics.put("hostedVespa.readyNodes", 0); + expectedMetrics.put("hostedVespa.reservedNodes", 0); + expectedMetrics.put("hostedVespa.activeNodes", 0); + expectedMetrics.put("hostedVespa.inactiveNodes", 0); + expectedMetrics.put("hostedVespa.dirtyNodes", 0); + expectedMetrics.put("hostedVespa.failedNodes", 0); + expectedMetrics.put("hostedVespa.deprovisionedNodes", 0); + expectedMetrics.put("hostedVespa.breakfixedNodes", 0); expectedMetrics.put("hostedVespa.pendingRedeployments", 42); expectedMetrics.put("hostedVespa.docker.totalCapacityDisk", 0.0); expectedMetrics.put("hostedVespa.docker.totalCapacityMem", 0.0); @@ -207,8 +217,8 @@ public class MetricsReporterTest { MetricsReporter metricsReporter = metricsReporter(metric, tester); metricsReporter.maintain(); - assertEquals(0, metric.values.get("hostedVespa.readyHosts")); // Only tenants counts - assertEquals(2, metric.values.get("hostedVespa.reservedHosts")); + assertEquals(0, metric.values.get("hostedVespa.readyNodes")); // Only tenants counts + assertEquals(2, metric.values.get("hostedVespa.reservedNodes")); assertEquals(120.0, metric.values.get("hostedVespa.docker.totalCapacityDisk")); assertEquals(100.0, metric.values.get("hostedVespa.docker.totalCapacityMem")); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java index 80d79b036e2..03581146b9f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java @@ -45,8 +45,8 @@ public class ArchiveApiTest { "{\"message\":\"Updated archive URI for 777888999000\"}"); - tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "\"archiveUri\":\"ftp://host/dir/application3/instance3/id3/host4/\"", true); - tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com"), "\"archiveUri\":\"s3://acc-bucket/zoneapp/zoneapp/node-admin/dockerhost2/\"", true); + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "\"archiveUri\":\"ftp://host/dir/tenant3/application3/instance3/id3/host4/\"", true); + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com"), "\"archiveUri\":\"s3://acc-bucket/zoneapp/zoneapp/zoneapp/node-admin/dockerhost2/\"", true); assertFile(new Request("http://localhost:8080/nodes/v2/archive"), "archives.json"); tester.assertResponse(new Request("http://localhost:8080/nodes/v2/archive/tenant/tenant3", new byte[0], Request.Method.DELETE), "{\"message\":\"Removed archive URI for tenant3\"}"); diff --git a/parent/pom.xml b/parent/pom.xml index 2fcf5527302..273e9b0ef8f 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -1170,7 +1170,7 @@ <prometheus.client.version>0.6.0</prometheus.client.version> <protobuf.version>3.21.7</protobuf.version> <spifly.version>1.3.5</spifly.version> - <surefire.version>2.22.2</surefire.version> + <surefire.version>3.0.0-M9</surefire.version> <wiremock.version>2.35.0</wiremock.version> <zero-allocation-hashing.version>0.16</zero-allocation-hashing.version> <zookeeper.client.version>3.8.0</zookeeper.client.version> diff --git a/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h b/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h index 177d19b06f9..cd4d1686eeb 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h +++ b/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h @@ -3,13 +3,14 @@ #pragma once #include <vespa/searchlib/queryeval/begin_and_end_id.h> -#include <vespa/fastos/types.h> #include <mutex> #include <condition_variable> #include <atomic> #include <algorithm> #include <vector> +#define VESPA_DLL_LOCAL __attribute__ ((visibility("hidden"))) + namespace proton::matching { /** diff --git a/searchlib/src/vespa/searchlib/common/bitvector.h b/searchlib/src/vespa/searchlib/common/bitvector.h index 67f3e2ad502..149e57390af 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.h +++ b/searchlib/src/vespa/searchlib/common/bitvector.h @@ -3,10 +3,8 @@ #pragma once #include "bitword.h" -#include <memory> #include <vespa/vespalib/util/alloc.h> #include <vespa/vespalib/util/atomic.h> -#include <vespa/fastos/types.h> #include <algorithm> #include <cassert> diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_posting_list.h b/searchlib/src/vespa/searchlib/predicate/predicate_posting_list.h index 0bf33f1d0e5..0de9be332fa 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_posting_list.h +++ b/searchlib/src/vespa/searchlib/predicate/predicate_posting_list.h @@ -3,7 +3,8 @@ #include <memory> #include <cstdint> -#include <vespa/fastos/types.h> + +#define VESPA_DLL_LOCAL __attribute__ ((visibility("hidden"))) /** * Interface for posting lists used by PredicateSearch. @@ -25,7 +26,7 @@ protected: public: using UP = std::unique_ptr<PredicatePostingList>; - virtual ~PredicatePostingList() {} + virtual ~PredicatePostingList() = default; /* * Moves to next document after the one supplied. diff --git a/searchlib/src/vespa/searchlib/queryeval/hitcollector.h b/searchlib/src/vespa/searchlib/queryeval/hitcollector.h index fe686f3e0bf..e244c856c4a 100644 --- a/searchlib/src/vespa/searchlib/queryeval/hitcollector.h +++ b/searchlib/src/vespa/searchlib/queryeval/hitcollector.h @@ -9,7 +9,6 @@ #include <vespa/vespalib/util/sort.h> #include <algorithm> #include <vector> -#include <vespa/fastos/types.h> namespace search::queryeval { diff --git a/storage/src/tests/common/metricstest.cpp b/storage/src/tests/common/metricstest.cpp index 78fa32e24e5..7231a071319 100644 --- a/storage/src/tests/common/metricstest.cpp +++ b/storage/src/tests/common/metricstest.cpp @@ -52,8 +52,7 @@ namespace { { framework::Clock& _clock; explicit MetricClock(framework::Clock& c) : _clock(c) {} - [[nodiscard]] time_t getTime() const override { return vespalib::count_s(_clock.getMonotonicTime().time_since_epoch()); } - [[nodiscard]] time_t getTimeInMilliSecs() const override { return vespalib::count_ms(_clock.getMonotonicTime().time_since_epoch()); } + [[nodiscard]] metrics::time_point getTime() const override { return _clock.getSystemTime(); } }; } @@ -85,10 +84,7 @@ void MetricsTest::SetUp() { _metricManager->registerMetric(guard, *_topSet); } - _metricsConsumer = std::make_unique<StatusMetricConsumer>( - _node->getComponentRegister(), - *_metricManager, - "status"); + _metricsConsumer = std::make_unique<StatusMetricConsumer>(_node->getComponentRegister(), *_metricManager, "status"); _filestorMetrics = std::make_shared<FileStorMetrics>(); _filestorMetrics->initDiskMetrics(1, 1); diff --git a/storage/src/tests/storageserver/statereportertest.cpp b/storage/src/tests/storageserver/statereportertest.cpp index d6b528e5a25..3a772c1ddde 100644 --- a/storage/src/tests/storageserver/statereportertest.cpp +++ b/storage/src/tests/storageserver/statereportertest.cpp @@ -53,8 +53,7 @@ struct MetricClock : public metrics::MetricManager::Timer { framework::Clock& _clock; explicit MetricClock(framework::Clock& c) : _clock(c) {} - [[nodiscard]] time_t getTime() const override { return vespalib::count_s(_clock.getMonotonicTime().time_since_epoch()); } - [[nodiscard]] time_t getTimeInMilliSecs() const override { return vespalib::count_ms(_clock.getMonotonicTime().time_since_epoch()); } + [[nodiscard]] metrics::time_point getTime() const override { return _clock.getSystemTime(); } }; } @@ -85,11 +84,8 @@ void StateReporterTest::SetUp() { _metricManager->registerMetric(guard, *_topSet); } - _stateReporter = std::make_unique<StateReporter>( - _node->getComponentRegister(), - *_metricManager, - _generationFetcher, - "status"); + _stateReporter = std::make_unique<StateReporter>(_node->getComponentRegister(), *_metricManager, + _generationFetcher, "status"); _filestorMetrics = std::make_shared<FileStorMetrics>(); _filestorMetrics->initDiskMetrics(1, 1); @@ -125,20 +121,14 @@ vespalib::Slime slime; \ #define ASSERT_GENERATION(jsonData, component, generation) \ { \ PARSE_JSON(jsonData); \ - ASSERT_EQ( \ - generation, \ - slime.get()["config"][component]["generation"].asDouble()); \ + ASSERT_EQ(generation, slime.get()["config"][component]["generation"].asDouble()); \ } #define ASSERT_NODE_STATUS(jsonData, code, message) \ { \ PARSE_JSON(jsonData); \ - ASSERT_EQ( \ - vespalib::string(code), \ - slime.get()["status"]["code"].asString().make_string()); \ - ASSERT_EQ( \ - vespalib::string(message), \ - slime.get()["status"]["message"].asString().make_string()); \ + ASSERT_EQ(vespalib::string(code), slime.get()["status"]["code"].asString().make_string()); \ + ASSERT_EQ(vespalib::string(message), slime.get()["status"]["message"].asString().make_string()); \ } #define ASSERT_METRIC_GET_PUT(jsonData, expGetCount, expPutCount) \ @@ -148,16 +138,11 @@ vespalib::Slime slime; \ double putCount = -1; \ size_t metricCount = slime.get()["metrics"]["values"].children(); \ for (size_t j=0; j<metricCount; j++) { \ - const vespalib::string name = slime.get()["metrics"]["values"][j]["name"] \ - .asString().make_string(); \ - if (name.compare("vds.filestor.allthreads.get.count") == 0) \ - { \ - getCount = slime.get()["metrics"]["values"][j]["values"]["count"] \ - .asDouble(); \ - } else if (name.compare("vds.filestor.allthreads.put.count") == 0) \ - { \ - putCount = slime.get()["metrics"]["values"][j]["values"]["count"] \ - .asDouble(); \ + const vespalib::string name = slime.get()["metrics"]["values"][j]["name"].asString().make_string(); \ + if (name.compare("vds.filestor.allthreads.get.count") == 0) { \ + getCount = slime.get()["metrics"]["values"][j]["values"]["count"].asDouble(); \ + } else if (name.compare("vds.filestor.allthreads.put.count") == 0) { \ + putCount = slime.get()["metrics"]["values"][j]["values"]["count"].asDouble(); \ } \ } \ ASSERT_EQ(expGetCount, getCount); \ @@ -226,8 +211,7 @@ TEST_F(StateReporterTest, report_metrics) { for (uint32_t i = 0; i < 6; ++i) { _clock->addSecondsToTime(60); _metricManager->timeChangedNotification(); - while (int64_t(_metricManager->getLastProcessedTime()) < vespalib::count_s(_clock->getMonotonicTime().time_since_epoch())) - { + while (int64_t(_metricManager->getLastProcessedTime()) < vespalib::count_s(_clock->getMonotonicTime().time_since_epoch())) { std::this_thread::sleep_for(1ms); } } diff --git a/storage/src/vespa/storage/common/statusmetricconsumer.cpp b/storage/src/vespa/storage/common/statusmetricconsumer.cpp index 8eb3e9f3ab6..c6f73540605 100644 --- a/storage/src/vespa/storage/common/statusmetricconsumer.cpp +++ b/storage/src/vespa/storage/common/statusmetricconsumer.cpp @@ -5,7 +5,6 @@ #include <boost/lexical_cast.hpp> #include <vespa/metrics/jsonwriter.h> #include <vespa/metrics/textwriter.h> -#include <vespa/metrics/xmlwriter.h> #include <vespa/metrics/metricmanager.h> #include <vespa/storageapi/messageapi/storagemessage.h> #include <vespa/vespalib/stllike/asciistream.h> @@ -37,10 +36,6 @@ StatusMetricConsumer::getReportContentType(const framework::HttpUrlPath& path) c return "text/plain"; } - if (path.getAttribute("format") == "xml") { - return "application/xml"; - } - if (path.getAttribute("format") == "text") { return "text/plain"; } @@ -67,7 +62,6 @@ StatusMetricConsumer::reportStatus(std::ostream& out, LOG(debug, "Not calling update hooks as dontcallupdatehooks option has been given"); } int64_t currentTimeS(vespalib::count_s(_component.getClock().getMonotonicTime().time_since_epoch())); - bool xml = (path.getAttribute("format") == "xml"); bool json = (path.getAttribute("format") == "json"); int verbosity(path.get("verbosity", 0)); @@ -131,13 +125,7 @@ StatusMetricConsumer::reportStatus(std::ostream& out, } std::string consumer = path.getAttribute("consumer", ""); - if (xml) { - out << "<?xml version=\"1.0\"?>\n"; - vespalib::XmlOutputStream xos(out); - metrics::XmlWriter xmlWriter(xos, snapshot->getPeriod(), verbosity); - _manager.visit(metricLock, *snapshot, xmlWriter, consumer); - out << "\n"; - } else if (json) { + if (json) { vespalib::asciistream jsonStreamData; vespalib::JsonStream stream(jsonStreamData, true); stream << Object() << "metrics"; diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp index a09abb25f7a..9f8456afc37 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.cpp +++ b/storage/src/vespa/storage/storageserver/storagenode.cpp @@ -33,34 +33,36 @@ namespace storage { namespace { - using vespalib::getLastErrorString; +using vespalib::getLastErrorString; - void writePidFile(const vespalib::string& pidfile) - { - ssize_t rv = -1; - vespalib::string mypid = vespalib::make_string("%d\n", getpid()); - size_t lastSlash = pidfile.rfind('/'); - if (lastSlash != vespalib::string::npos) { - std::filesystem::create_directories(std::filesystem::path(pidfile.substr(0, lastSlash))); - } - int fd = open(pidfile.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); - if (fd != -1) { - rv = write(fd, mypid.c_str(), mypid.size()); - close(fd); - } - if (rv < 1) { - LOG(warning, "Failed to write pidfile '%s': %s", - pidfile.c_str(), getLastErrorString().c_str()); - } +void +writePidFile(const vespalib::string& pidfile) +{ + ssize_t rv = -1; + vespalib::string mypid = vespalib::make_string("%d\n", getpid()); + size_t lastSlash = pidfile.rfind('/'); + if (lastSlash != vespalib::string::npos) { + std::filesystem::create_directories(std::filesystem::path(pidfile.substr(0, lastSlash))); + } + int fd = open(pidfile.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd != -1) { + rv = write(fd, mypid.c_str(), mypid.size()); + close(fd); } + if (rv < 1) { + LOG(warning, "Failed to write pidfile '%s': %s", + pidfile.c_str(), getLastErrorString().c_str()); + } +} - void removePidFile(const vespalib::string& pidfile) - { - if (unlink(pidfile.c_str()) != 0) { - LOG(warning, "Failed to delete pidfile '%s': %s", - pidfile.c_str(), getLastErrorString().c_str()); - } +void +removePidFile(const vespalib::string& pidfile) +{ + if (unlink(pidfile.c_str()) != 0) { + LOG(warning, "Failed to delete pidfile '%s': %s", + pidfile.c_str(), getLastErrorString().c_str()); } +} } // End of anonymous namespace @@ -429,7 +431,8 @@ StorageNode::shutdown() LOG(debug, "Done shutting down node"); } -void StorageNode::configure(std::unique_ptr<StorServerConfig> config) { +void +StorageNode::configure(std::unique_ptr<StorServerConfig> config) { log_config_received(*config); // When we get config, we try to grab the config lock to ensure noone // else is doing configuration work, and then we write the new config @@ -445,7 +448,8 @@ void StorageNode::configure(std::unique_ptr<StorServerConfig> config) { } } -void StorageNode::configure(std::unique_ptr<UpgradingConfig> config) { +void +StorageNode::configure(std::unique_ptr<UpgradingConfig> config) { log_config_received(*config); { std::lock_guard configLockGuard(_configLock); @@ -457,7 +461,8 @@ void StorageNode::configure(std::unique_ptr<UpgradingConfig> config) { } } -void StorageNode::configure(std::unique_ptr<StorDistributionConfig> config) { +void +StorageNode::configure(std::unique_ptr<StorDistributionConfig> config) { log_config_received(*config); { std::lock_guard configLockGuard(_configLock); @@ -486,7 +491,8 @@ StorageNode::configure(std::unique_ptr<document::config::DocumenttypesConfig> co } } -void StorageNode::configure(std::unique_ptr<BucketspacesConfig> config) { +void +StorageNode::configure(std::unique_ptr<BucketspacesConfig> config) { log_config_received(*config); { std::lock_guard configLockGuard(_configLock); diff --git a/storage/src/vespa/storageapi/mbusprot/serializationhelper.h b/storage/src/vespa/storageapi/mbusprot/serializationhelper.h index 457a6178704..671ffbddd6f 100644 --- a/storage/src/vespa/storageapi/mbusprot/serializationhelper.h +++ b/storage/src/vespa/storageapi/mbusprot/serializationhelper.h @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/fastos/types.h> #include <vespa/document/base/globalid.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/document/util/bytebuffer.h> @@ -13,12 +12,6 @@ namespace storage::mbusprot { class SerializationHelper { public: - static int64_t getLong(document::ByteBuffer& buf) { - int64_t tmp; - buf.getLongNetwork(tmp); - return tmp; - } - static int32_t getInt(document::ByteBuffer& buf) { int32_t tmp; buf.getIntNetwork(tmp); @@ -46,26 +39,6 @@ public: return s; } - static bool getBoolean(document::ByteBuffer& buf) { - uint8_t tmp; - buf.getByte(tmp); - return (tmp == 1); - } - - static api::ReturnCode getReturnCode(document::ByteBuffer& buf) { - api::ReturnCode::Result result = (api::ReturnCode::Result) getInt(buf); - vespalib::stringref message = getString(buf); - return api::ReturnCode(result, message); - } - - static void putReturnCode(const api::ReturnCode& code, vespalib::GrowableByteBuffer& buf) - { - buf.putInt(code.getResult()); - buf.putString(code.getMessage()); - } - - static const uint32_t BUCKET_INFO_SERIALIZED_SIZE = sizeof(uint32_t) * 3; - static document::GlobalId getGlobalId(document::ByteBuffer& buf) { std::vector<char> buffer(getShort(buf)); for (uint32_t i=0; i<buffer.size(); ++i) { @@ -74,13 +47,6 @@ public: return document::GlobalId(&buffer[0]); } - static void putGlobalId(const document::GlobalId& gid, vespalib::GrowableByteBuffer& buf) - { - buf.putShort(document::GlobalId::LENGTH); - for (uint32_t i=0; i<document::GlobalId::LENGTH; ++i) { - buf.putByte(gid.get()[i]); - } - } static document::Document::UP getDocument(document::ByteBuffer& buf, const document::DocumentTypeRepo& repo) { uint32_t size = getInt(buf); diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java index 1e2c0900ff7..33e46ebc75f 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java @@ -93,7 +93,7 @@ public abstract class Maintainer implements Runnable { * * @return the degree to which the run was successful - a number between 0 (no success), to 1 (complete success). * Note that this indicates whether something is wrong, so e.g if the call did nothing because it should do - * nothing, 1.0 should be returned. + * nothing, 1.0 should be returned. */ protected abstract double maintain(); diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 2720d8786cb..8f1b687449a 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -26,6 +26,8 @@ vespa_define_module( src/apps/vespa-validate-hostname TESTS + ${VESPALIB_DIRECTIO_TESTDIR} + ${VESPALIB_PROCESS_MEMORY_STATS_TESTDIR} src/tests/alloc src/tests/approx src/tests/array @@ -38,9 +40,9 @@ vespa_define_module( src/tests/bits src/tests/box src/tests/btree - src/tests/btree/btree_store src/tests/btree/btree-scan-speed src/tests/btree/btree-stress + src/tests/btree/btree_store src/tests/clock src/tests/component src/tests/compress @@ -73,7 +75,6 @@ vespa_define_module( src/tests/datastore/unique_store src/tests/datastore/unique_store_dictionary src/tests/datastore/unique_store_string_allocator - ${VESPALIB_DIRECTIO_TESTDIR} src/tests/detect_type_benchmark src/tests/dotproduct src/tests/drop-file-from-cache @@ -86,6 +87,9 @@ vespa_define_module( src/tests/executor_idle_tracking src/tests/explore_modern_cpp src/tests/false + src/tests/fastlib/io + src/tests/fastlib/text + src/tests/fastos src/tests/fiddle src/tests/fileheader src/tests/floatingpointtype @@ -95,6 +99,7 @@ vespa_define_module( src/tests/guard src/tests/host_name src/tests/hwaccelrated + src/tests/invokeservice src/tests/io/fileutil src/tests/io/mapped_file_input src/tests/issue @@ -134,7 +139,6 @@ vespa_define_module( src/tests/printable src/tests/priority_queue src/tests/process - ${VESPALIB_PROCESS_MEMORY_STATS_TESTDIR} src/tests/programoptions src/tests/random src/tests/referencecounter @@ -144,14 +148,14 @@ vespa_define_module( src/tests/runnable_pair src/tests/rusage src/tests/sequencedtaskexecutor - src/tests/shutdownguard - src/tests/singleexecutor src/tests/sha1 src/tests/shared_operation_throttler src/tests/shared_string_repo src/tests/sharedptr + src/tests/shutdownguard src/tests/signalhandler src/tests/simple_thread_bundle + src/tests/singleexecutor src/tests/slime src/tests/slime/external_data_value src/tests/slime/summary-feature-benchmark @@ -204,17 +208,15 @@ vespa_define_module( src/tests/util/string_escape src/tests/valgrind src/tests/visit_ranges - src/tests/invokeservice src/tests/wakeup src/tests/xmlserializable src/tests/zcurve - src/tests/fastlib/io - src/tests/fastlib/text LIBS src/vespa/fastlib/io src/vespa/fastlib/text src/vespa/fastlib/text/apps + src/vespa/fastos src/vespa/vespalib src/vespa/vespalib/btree src/vespa/vespalib/component diff --git a/vespalib/src/tests/fastos/CMakeLists.txt b/vespalib/src/tests/fastos/CMakeLists.txt new file mode 100644 index 00000000000..6ea986ac0b0 --- /dev/null +++ b/vespalib/src/tests/fastos/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(fastos_file_test_app TEST + SOURCES + file_test.cpp + DEPENDS + vespalib + GTest::GTest +) +vespa_add_test(NAME fastos_file_test_app COMMAND fastos_file_test_app) diff --git a/fastos/src/tests/filetest.cpp b/vespalib/src/tests/fastos/file_test.cpp index d0f8bbfd98b..d0f8bbfd98b 100644 --- a/fastos/src/tests/filetest.cpp +++ b/vespalib/src/tests/fastos/file_test.cpp diff --git a/fastos/src/tests/hello.txt b/vespalib/src/tests/fastos/hello.txt index 62a405393a6..62a405393a6 100644 --- a/fastos/src/tests/hello.txt +++ b/vespalib/src/tests/fastos/hello.txt diff --git a/fastos/src/tests/tests.h b/vespalib/src/tests/fastos/tests.h index 9cd7a10ab48..9cd7a10ab48 100644 --- a/fastos/src/tests/tests.h +++ b/vespalib/src/tests/fastos/tests.h diff --git a/vespalib/src/vespa/fastlib/io/CMakeLists.txt b/vespalib/src/vespa/fastlib/io/CMakeLists.txt index f21cf27b21e..9f6211f3e19 100644 --- a/vespalib/src/vespa/fastlib/io/CMakeLists.txt +++ b/vespalib/src/vespa/fastlib/io/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(fastlib_io OBJECT +vespa_add_library(vespalib_fastlib_io OBJECT SOURCES bufferedfile.cpp DEPENDS diff --git a/vespalib/src/vespa/fastlib/text/CMakeLists.txt b/vespalib/src/vespa/fastlib/text/CMakeLists.txt index d6cb8c29305..06573ee814d 100644 --- a/vespalib/src/vespa/fastlib/text/CMakeLists.txt +++ b/vespalib/src/vespa/fastlib/text/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(fastlib_text OBJECT +vespa_add_library(vespalib_fastlib_text OBJECT SOURCES unicodeutil.cpp normwordfolder.cpp diff --git a/vespalib/src/vespa/fastos/CMakeLists.txt b/vespalib/src/vespa/fastos/CMakeLists.txt new file mode 100644 index 00000000000..2b7a2c3b905 --- /dev/null +++ b/vespalib/src/vespa/fastos/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(vespalib_fastos OBJECT + SOURCES + file.cpp + file_rw_ops.cpp + linux_file.cpp + unix_file.cpp + DEPENDS +) diff --git a/fastos/src/vespa/fastos/file.cpp b/vespalib/src/vespa/fastos/file.cpp index fdbacb570b4..fdbacb570b4 100644 --- a/fastos/src/vespa/fastos/file.cpp +++ b/vespalib/src/vespa/fastos/file.cpp diff --git a/fastos/src/vespa/fastos/file.h b/vespalib/src/vespa/fastos/file.h index 1cf6fee71dd..146127d4fe6 100644 --- a/fastos/src/vespa/fastos/file.h +++ b/vespalib/src/vespa/fastos/file.h @@ -10,10 +10,11 @@ #pragma once -#include "types.h" #include <cstdint> #include <string> +#define FASTOS_PREFIX(a) FastOS_##a + constexpr int FASTOS_FILE_OPEN_READ = (1<<0); constexpr int FASTOS_FILE_OPEN_WRITE = (1<<1); constexpr int FASTOS_FILE_OPEN_EXISTING = (1<<2); diff --git a/fastos/src/vespa/fastos/file_rw_ops.cpp b/vespalib/src/vespa/fastos/file_rw_ops.cpp index 79fe95b21f2..79fe95b21f2 100644 --- a/fastos/src/vespa/fastos/file_rw_ops.cpp +++ b/vespalib/src/vespa/fastos/file_rw_ops.cpp diff --git a/fastos/src/vespa/fastos/file_rw_ops.h b/vespalib/src/vespa/fastos/file_rw_ops.h index 4f7aa6f082f..4f7aa6f082f 100644 --- a/fastos/src/vespa/fastos/file_rw_ops.h +++ b/vespalib/src/vespa/fastos/file_rw_ops.h diff --git a/fastos/src/vespa/fastos/linux_file.cpp b/vespalib/src/vespa/fastos/linux_file.cpp index 6fb29782957..6fb29782957 100644 --- a/fastos/src/vespa/fastos/linux_file.cpp +++ b/vespalib/src/vespa/fastos/linux_file.cpp diff --git a/fastos/src/vespa/fastos/linux_file.h b/vespalib/src/vespa/fastos/linux_file.h index 2481b163210..2481b163210 100644 --- a/fastos/src/vespa/fastos/linux_file.h +++ b/vespalib/src/vespa/fastos/linux_file.h diff --git a/fastos/src/vespa/fastos/unix_file.cpp b/vespalib/src/vespa/fastos/unix_file.cpp index 7c4cde19125..7c4cde19125 100644 --- a/fastos/src/vespa/fastos/unix_file.cpp +++ b/vespalib/src/vespa/fastos/unix_file.cpp diff --git a/fastos/src/vespa/fastos/unix_file.h b/vespalib/src/vespa/fastos/unix_file.h index 31e45f8d2fa..31e45f8d2fa 100644 --- a/fastos/src/vespa/fastos/unix_file.h +++ b/vespalib/src/vespa/fastos/unix_file.h diff --git a/vespalib/src/vespa/vespalib/CMakeLists.txt b/vespalib/src/vespa/vespalib/CMakeLists.txt index 9e31084c067..63876d442ef 100644 --- a/vespalib/src/vespa/vespalib/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/CMakeLists.txt @@ -30,8 +30,9 @@ vespa_add_library(vespalib $<TARGET_OBJECTS:vespalib_vespalib_time> $<TARGET_OBJECTS:vespalib_vespalib_trace> $<TARGET_OBJECTS:vespalib_vespalib_util> - $<TARGET_OBJECTS:fastlib_io> - $<TARGET_OBJECTS:fastlib_text> + $<TARGET_OBJECTS:vespalib_fastlib_io> + $<TARGET_OBJECTS:vespalib_fastlib_text> + $<TARGET_OBJECTS:vespalib_fastos> INSTALL lib64 DEPENDS ${VESPA_GCC_LIB} diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp b/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp index f5d87e14802..3bdbb7a81ff 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp +++ b/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp @@ -3,7 +3,6 @@ #pragma once #include "private_helpers.hpp" -#include <vespa/fastos/types.h> namespace vespalib::hwaccelrated::avx { diff --git a/vespalib/src/vespa/vespalib/net/wakeup_pipe.cpp b/vespalib/src/vespa/vespalib/net/wakeup_pipe.cpp index 60900289e79..b8025bfcf9f 100644 --- a/vespalib/src/vespa/vespalib/net/wakeup_pipe.cpp +++ b/vespalib/src/vespa/vespalib/net/wakeup_pipe.cpp @@ -2,34 +2,40 @@ #include "wakeup_pipe.h" #include "socket_utils.h" +#include <vespa/vespalib/util/require.h> #include <unistd.h> namespace vespalib { WakeupPipe::WakeupPipe() - : _pipe() + : _reader(), + _writer() { - socketutils::nonblocking_pipe(_pipe); + int pipe[2]; + socketutils::nonblocking_pipe(pipe); + _reader.reset(pipe[0]); + _writer.reset(pipe[1]); } -WakeupPipe::~WakeupPipe() -{ - close(_pipe[0]); - close(_pipe[1]); -} +WakeupPipe::~WakeupPipe() = default; void WakeupPipe::write_token() { char token = 'T'; - [[maybe_unused]] ssize_t res = write(_pipe[1], &token, 1); + ssize_t res = _writer.write(&token, 1); + if (res < 0) { + res = -errno; + } + REQUIRE(res > 0 || res == -EAGAIN || res == -EWOULDBLOCK); } void WakeupPipe::read_tokens() { char token_trash[128]; - [[maybe_unused]] ssize_t res = read(_pipe[0], token_trash, sizeof(token_trash)); + ssize_t res = _reader.read(token_trash, sizeof(token_trash)); + REQUIRE(res > 0); } } diff --git a/vespalib/src/vespa/vespalib/net/wakeup_pipe.h b/vespalib/src/vespa/vespalib/net/wakeup_pipe.h index b52f7f9e32d..36c88b205c0 100644 --- a/vespalib/src/vespa/vespalib/net/wakeup_pipe.h +++ b/vespalib/src/vespa/vespalib/net/wakeup_pipe.h @@ -2,6 +2,8 @@ #pragma once +#include "socket_handle.h" + namespace vespalib { //----------------------------------------------------------------------------- @@ -15,11 +17,12 @@ namespace vespalib { **/ class WakeupPipe { private: - int _pipe[2]; + SocketHandle _reader; + SocketHandle _writer; public: WakeupPipe(); ~WakeupPipe(); - int get_read_fd() const { return _pipe[0]; } + int get_read_fd() const { return _reader.get(); } void write_token(); void read_tokens(); }; diff --git a/vespalib/src/vespa/vespalib/portal/http_connection.cpp b/vespalib/src/vespa/vespalib/portal/http_connection.cpp index 6ea56e2659c..3d8edf2fc2e 100644 --- a/vespalib/src/vespa/vespalib/portal/http_connection.cpp +++ b/vespalib/src/vespa/vespalib/portal/http_connection.cpp @@ -245,14 +245,16 @@ HttpConnection::handle_event(bool, bool) } void -HttpConnection::respond_with_content(const vespalib::string &content_type, - const vespalib::string &content) +HttpConnection::respond_with_content(vespalib::stringref content_type, + vespalib::stringref content) { { OutputWriter dst(_output, CHUNK_SIZE); dst.printf("HTTP/1.1 200 OK\r\n"); dst.printf("Connection: close\r\n"); - dst.printf("Content-Type: %s\r\n", content_type.c_str()); + dst.printf("Content-Type: "); + dst.write(content_type.data(), content_type.size()); + dst.printf("\r\n"); dst.printf("Content-Length: %zu\r\n", content.size()); emit_http_security_headers(dst); dst.printf("\r\n"); @@ -263,11 +265,13 @@ HttpConnection::respond_with_content(const vespalib::string &content_type, } void -HttpConnection::respond_with_error(int code, const vespalib::string &msg) +HttpConnection::respond_with_error(int code, vespalib::stringref msg) { { OutputWriter dst(_output, CHUNK_SIZE); - dst.printf("HTTP/1.1 %d %s\r\n", code, msg.c_str()); + dst.printf("HTTP/1.1 %d ", code); + dst.write(msg.data(), msg.size()); + dst.printf("\r\n"); dst.printf("Connection: close\r\n"); dst.printf("\r\n"); } diff --git a/vespalib/src/vespa/vespalib/portal/http_connection.h b/vespalib/src/vespa/vespalib/portal/http_connection.h index 03d23351e7d..8540cb87e1d 100644 --- a/vespalib/src/vespa/vespalib/portal/http_connection.h +++ b/vespalib/src/vespa/vespalib/portal/http_connection.h @@ -53,9 +53,9 @@ public: // Precondition: handshake must have been completed const net::ConnectionAuthContext &auth_context() const noexcept { return *_auth_ctx; } - void respond_with_content(const vespalib::string &content_type, - const vespalib::string &content); - void respond_with_error(int code, const vespalib::string &msg); + void respond_with_content(vespalib::stringref content_type, + vespalib::stringref content); + void respond_with_error(int code, const vespalib::stringref msg); }; } // namespace vespalib::portal diff --git a/vespalib/src/vespa/vespalib/portal/portal.cpp b/vespalib/src/vespa/vespalib/portal/portal.cpp index a98562f6504..691ddaef495 100644 --- a/vespalib/src/vespa/vespalib/portal/portal.cpp +++ b/vespalib/src/vespa/vespalib/portal/portal.cpp @@ -77,8 +77,8 @@ Portal::GetRequest::export_params() const } void -Portal::GetRequest::respond_with_content(const vespalib::string &content_type, - const vespalib::string &content) +Portal::GetRequest::respond_with_content(vespalib::stringref content_type, + vespalib::stringref content) { assert(active()); _conn->respond_with_content(content_type, content); @@ -86,7 +86,7 @@ Portal::GetRequest::respond_with_content(const vespalib::string &content_type, } void -Portal::GetRequest::respond_with_error(int code, const vespalib::string &msg) +Portal::GetRequest::respond_with_error(int code, vespalib::stringref msg) { assert(active()); _conn->respond_with_error(code, msg); diff --git a/vespalib/src/vespa/vespalib/portal/portal.h b/vespalib/src/vespa/vespalib/portal/portal.h index 314dd6e7de9..6954d800c91 100644 --- a/vespalib/src/vespa/vespalib/portal/portal.h +++ b/vespalib/src/vespa/vespalib/portal/portal.h @@ -65,9 +65,9 @@ public: bool has_param(const vespalib::string &name) const; const vespalib::string &get_param(const vespalib::string &name) const; std::map<vespalib::string, vespalib::string> export_params() const; - void respond_with_content(const vespalib::string &content_type, - const vespalib::string &content); - void respond_with_error(int code, const vespalib::string &msg); + void respond_with_content(vespalib::stringref content_type, + vespalib::stringref content); + void respond_with_error(int code, vespalib::stringref msg); const net::ConnectionAuthContext &auth_context() const noexcept; ~GetRequest(); }; |