diff options
706 files changed, 8593 insertions, 7155 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 5463c4215a0..ce11196725f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -28,6 +28,8 @@ vespa_use_default_cmake_prefix_path() SET(CMAKE_FIND_PACKAGE_SORT_ORDER NATURAL) SET(CMAKE_FIND_PACKAGE_SORT_DIRECTION DEC) +find_package(Threads REQUIRED) + find_package(LLVM REQUIRED CONFIG) message(STATUS "Found LLVM ${LLVM_PACKAGE_VERSION}") message(STATUS "Using LLVMConfig.cmake in: ${LLVM_DIR}") @@ -99,7 +101,6 @@ add_subdirectory(docprocs) add_subdirectory(document) add_subdirectory(documentapi) add_subdirectory(eval) -add_subdirectory(fastos) add_subdirectory(fbench) add_subdirectory(fileacquirer) add_subdirectory(filedistribution) diff --git a/README-cmake.md b/README-cmake.md index 214eaaa94a9..393e068d089 100644 --- a/README-cmake.md +++ b/README-cmake.md @@ -106,7 +106,6 @@ The module definition is used to specify common dependencies for every target de vespa_define_module( DEPENDS - fastos vespalib bjarnelib diff --git a/application/.gitignore b/application/.gitignore index f5d341f2123..bc385f92fa6 100644 --- a/application/.gitignore +++ b/application/.gitignore @@ -2,3 +2,4 @@ pom.xml.build /logs .preprocessed +/src/test/app-packages/*/models.generated/ diff --git a/application/pom.xml b/application/pom.xml index 2193f0fe2e3..236bcb6d81a 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -182,6 +182,12 @@ <artifactId>junit-jupiter-engine</artifactId> <scope>test</scope> </dependency> + <dependency> + <!-- Required for ContainerModelEvaluationTest --> + <groupId>com.microsoft.onnxruntime</groupId> + <artifactId>onnxruntime</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> diff --git a/application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java b/application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java index f838d7a5481..c9ff51b0d84 100644 --- a/application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java +++ b/application/src/test/java/com/yahoo/application/container/ContainerModelEvaluationTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.application.container; -import ai.vespa.modelintegration.evaluator.OnnxEvaluator; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; import com.yahoo.application.Application; import com.yahoo.application.Networking; import com.yahoo.application.container.handler.Request; @@ -40,7 +40,7 @@ public class ContainerModelEvaluationTest { @Test void testCreateApplicationInstanceWithModelEvaluation() { - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); + assumeTrue(OnnxRuntime.isRuntimeAvailable()); try (Application application = Application.fromApplicationPackage(new File("src/test/app-packages/model-evaluation"), Networking.disable)) { @@ -54,17 +54,17 @@ public class ContainerModelEvaluationTest { } { - String expected = "{\"cells\":[{\"address\":{},\"value\":2.496898}]}"; + String expected = "{\"type\":\"tensor()\",\"cells\":[{\"address\":{},\"value\":2.496898}]}"; assertResponse("http://localhost/model-evaluation/v1/xgboost_xgboost_2_2/eval?format.tensors=long", expected, jdisc); } { - String expected = "{\"cells\":[{\"address\":{},\"value\":1.9130086820218188}]}"; + String expected = "{\"type\":\"tensor()\",\"cells\":[{\"address\":{},\"value\":1.9130086820218188}]}"; assertResponse("http://localhost/model-evaluation/v1/lightgbm_regression/eval?format.tensors=long", expected, jdisc); } { - String expected = "{\"cells\":[{\"address\":{\"d0\":\"0\"},\"value\":0.3006095290184021},{\"address\":{\"d0\":\"1\"},\"value\":0.33222490549087524},{\"address\":{\"d0\":\"2\"},\"value\":0.3671652674674988}]}"; + String expected = "{\"type\":\"tensor<float>(d0[3])\",\"cells\":[{\"address\":{\"d0\":\"0\"},\"value\":0.3006095290184021},{\"address\":{\"d0\":\"1\"},\"value\":0.33222490549087524},{\"address\":{\"d0\":\"2\"},\"value\":0.36716532707214355}]}"; assertResponse("http://localhost/model-evaluation/v1/onnx_softmax_func/output/eval?format.tensors=long&input=" + inputTensor(), expected, jdisc); assertResponse("http://localhost/model-evaluation/v1/onnx_softmax_func/default.output/eval?format.tensors=long&input=" + inputTensor(), expected, jdisc); assertResponse("http://localhost/model-evaluation/v1/onnx_softmax_func/default/output/eval?format.tensors=long&input=" + inputTensor(), expected, jdisc); @@ -75,7 +75,13 @@ public class ContainerModelEvaluationTest { private void assertResponse(String url, String expectedResponse, JDisc jdisc) { try { Response response = jdisc.handleRequest(new Request(url)); - JsonTestHelper.assertJsonEquals(expectedResponse, response.getBodyAsString()); + + // Truncate JSON encoded numbers having more than 6 digits after the decimal point + String pattern = "([0-9]+\\.[0-9]{6})[0-9]*"; + String normalizedExpectedResponse = expectedResponse.replaceAll(pattern, "$1"); + String normalizedActualResponse = response.getBodyAsString().replaceAll(pattern, "$1"); + + JsonTestHelper.assertJsonEquals(normalizedExpectedResponse, normalizedActualResponse); assertEquals(200, response.getStatus()); } catch (CharacterCodingException e) { diff --git a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidatorTest.java b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidatorTest.java index b4b817da2f0..a7947aff283 100644 --- a/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidatorTest.java +++ b/athenz-identity-provider-service/src/test/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/InstanceValidatorTest.java @@ -278,7 +278,7 @@ public class InstanceValidatorTest { MockNodeFlavors flavors = new MockNodeFlavors(); List<Node> nodeList = new ArrayList<>(); for (int i = 0; i < num; i++) { - Node node = Node.create("foo" + i, IP.Config.of(Set.of("::1" + i, "::2" + i, "::3" + i), Set.of()), + Node node = Node.create("foo" + i, new IP.Config(Set.of("::1" + i, "::2" + i, "::3" + i), Set.of()), "foo" + i, flavors.getFlavorOrThrow("default"), NodeType.tenant).build(); nodeList.add(node); } diff --git a/client/CMakeLists.txt b/client/CMakeLists.txt index b73f4e153ff..4802d1656d6 100644 --- a/client/CMakeLists.txt +++ b/client/CMakeLists.txt @@ -11,6 +11,7 @@ add_custom_command(OUTPUT ${GODIR}/bin/vespa ${GODIR}/bin/vespa-wrapper add_custom_target(client_go_binaries ALL DEPENDS ${GODIR}/bin/vespa ${GODIR}/bin/vespa-wrapper) +install(PROGRAMS ${GODIR}/bin/vespa DESTINATION bin) install(PROGRAMS ${GODIR}/bin/vespa-wrapper DESTINATION libexec/vespa) install_symlink(libexec/vespa/vespa-wrapper bin/vespa-logfmt) diff --git a/client/go/go.mod b/client/go/go.mod index fc5ae544dac..d03f22a9e67 100644 --- a/client/go/go.mod +++ b/client/go/go.mod @@ -13,7 +13,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.7.0 github.com/zalando/go-keyring v0.1.1 - golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c + golang.org/x/sys v0.5.0 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) diff --git a/client/go/go.sum b/client/go/go.sum index fedf23f3e0b..084bde701ce 100644 --- a/client/go/go.sum +++ b/client/go/go.sum @@ -4,13 +4,10 @@ github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKY github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/danieljoos/wincred v1.1.0 h1:3RNcEpBg4IhIChZdFRSdlQt1QjCp1sMAPIrOnm7Yf8g= github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7hqDjlFjiygg= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= -github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= @@ -22,11 +19,8 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/pkg/browser v0.0.0-20210706143420-7d21f8c997e2 h1:acNfDZXmm28D2Yg/c3ALnZStzNaZMSagpbr96vY6Zjc= @@ -39,26 +33,17 @@ github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/zalando/go-keyring v0.1.1 h1:w2V9lcx/Uj4l+dzAf1m9s+DJ1O8ROkEHnynonHjTcYE= github.com/zalando/go-keyring v0.1.1/go.mod h1:OIC+OZ28XbmwFxU/Rp9V7eKzZjamBJwRzC8UFJH9+L8= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/client/go/internal/cli/cmd/root.go b/client/go/internal/cli/cmd/root.go index faba6bbbfd4..70e0afbcd32 100644 --- a/client/go/internal/cli/cmd/root.go +++ b/client/go/internal/cli/cmd/root.go @@ -250,6 +250,7 @@ func (c *CLI) configureCommands() { rootCmd.AddCommand(statusCmd) // status rootCmd.AddCommand(newTestCmd(c)) // test rootCmd.AddCommand(newVersionCmd(c)) // version + rootCmd.AddCommand(newVisitCmd(c)) // visit } func (c *CLI) printErr(err error, hints ...string) { @@ -263,6 +264,10 @@ func (c *CLI) printSuccess(msg ...interface{}) { fmt.Fprintln(c.Stdout, color.GreenString("Success:"), fmt.Sprint(msg...)) } +func (c *CLI) printDebug(msg ...interface{}) { + fmt.Fprintln(c.Stderr, color.CyanString("Debug:"), fmt.Sprint(msg...)) +} + func (c *CLI) printWarning(msg interface{}, hints ...string) { fmt.Fprintln(c.Stderr, color.YellowString("Warning:"), msg) for _, hint := range hints { diff --git a/client/go/internal/cli/cmd/visit.go b/client/go/internal/cli/cmd/visit.go new file mode 100644 index 00000000000..1022d74354d --- /dev/null +++ b/client/go/internal/cli/cmd/visit.go @@ -0,0 +1,348 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// vespa visit command +// Author: arnej + +package cmd + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/spf13/cobra" + "github.com/vespa-engine/vespa/client/go/internal/util" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +type visitArgs struct { + contentCluster string + fieldSet string + selection string + makeFeed bool + jsonLines bool + pretty bool + debugMode bool + chunkCount int + cli *CLI +} + +func (v *visitArgs) writeBytes(b []byte) { + v.cli.Stdout.Write(b) +} + +func (v *visitArgs) writeString(s string) { + v.writeBytes([]byte(s)) +} + +func (v *visitArgs) debugPrint(s string) { + if v.debugMode { + v.cli.printDebug(s) + } +} + +func (v *visitArgs) dumpDocuments(documents []DocumentBlob) { + comma := false + pretty := false + if v.makeFeed { + comma = true + pretty = v.pretty + } else if !v.jsonLines { + return + } + for _, value := range documents { + if pretty { + var prettyJSON bytes.Buffer + parseError := json.Indent(&prettyJSON, value.blob, "", " ") + if parseError != nil { + v.writeBytes(value.blob) + } else { + v.writeBytes(prettyJSON.Bytes()) + } + } else { + v.writeBytes(value.blob) + } + if comma { + v.writeString(",\n") + } else { + v.writeString("\n") + } + } +} + +var totalDocCount int + +func newVisitCmd(cli *CLI) *cobra.Command { + var ( + vArgs visitArgs + ) + cmd := &cobra.Command{ + Use: "visit", + Short: "Visit and print all documents in a vespa cluster", + Long: `Run visiting to retrieve all documents. + +By default prints each document received on its own line (JSON-L format). +`, + Example: `$ vespa visit # get documents from any cluster +$ vespa visit --content-cluster search # get documents from cluster named "search" +`, + Args: cobra.MaximumNArgs(0), + DisableAutoGenTag: true, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) error { + vArgs.cli = cli + service, err := documentService(cli) + if err != nil { + return err + } + result := probeHandler(service, cli) + if result.Success { + result = visitClusters(&vArgs, service) + } + if !result.Success { + return fmt.Errorf("visit failed: %s", result.Message) + } + vArgs.debugPrint(fmt.Sprintf("sum of 'documentCount': %d", totalDocCount)) + return nil + }, + } + cmd.Flags().StringVar(&vArgs.contentCluster, "content-cluster", "*", `Which content cluster to visit documents from`) + cmd.Flags().StringVar(&vArgs.fieldSet, "field-set", "", `Which fieldset to ask for`) + cmd.Flags().StringVar(&vArgs.selection, "selection", "", `select subset of cluster`) + cmd.Flags().BoolVar(&vArgs.debugMode, "debug-mode", false, `print debugging output`) + cmd.Flags().BoolVar(&vArgs.jsonLines, "json-lines", true, `output documents as JSON lines`) + cmd.Flags().BoolVar(&vArgs.makeFeed, "make-feed", false, `output JSON array suitable for vespa-feeder`) + cmd.Flags().BoolVar(&vArgs.pretty, "pretty-json", false, `format pretty JSON`) + cmd.Flags().IntVar(&vArgs.chunkCount, "chunk-count", 1000, `chunk by count`) + return cmd +} + +type HandlersInfo struct { + Handlers []struct { + HandlerId string `json:"id"` + HandlerClass string `json:"class"` + HandlerBundle string `json:"bundle"` + ServerBindings []string `json:"serverBindings"` + } `json:"handlers"` +} + +func parseHandlersOutput(r io.Reader) (*HandlersInfo, error) { + var handlersInfo HandlersInfo + codec := json.NewDecoder(r) + err := codec.Decode(&handlersInfo) + return &handlersInfo, err +} + +func probeHandler(service *vespa.Service, cli *CLI) (res util.OperationResult) { + urlPath := service.BaseURL + "/" + url, urlParseError := url.Parse(urlPath) + if urlParseError != nil { + return util.Failure("Invalid request path: '" + urlPath + "': " + urlParseError.Error()) + } + request := &http.Request{ + URL: url, + Method: "GET", + } + timeout := time.Duration(90) * time.Second + response, err := service.Do(request, timeout) + if err != nil { + return util.Failure("Request failed: " + err.Error()) + } + defer response.Body.Close() + if response.StatusCode == 200 { + handlersInfo, err := parseHandlersOutput(response.Body) + if err != nil || len(handlersInfo.Handlers) == 0 { + cli.printWarning("Could not parse JSON response from"+urlPath, err.Error()) + return util.Failure("Bad endpoint") + } + for _, h := range handlersInfo.Handlers { + if strings.HasSuffix(h.HandlerClass, "DocumentV1ApiHandler") { + for _, binding := range h.ServerBindings { + if strings.Contains(binding, "/document/v1/") { + return util.Success("handler OK") + } + } + w := fmt.Sprintf("expected /document/v1/ binding, but got: %v", h.ServerBindings) + cli.printWarning(w) + } + } + cli.printWarning("Missing /document/v1/ API; add <document-api /> to the container cluster delcaration in services.xml") + return util.Failure("Missing /document/v1 API") + } else { + return util.FailureWithPayload(service.Description()+" at "+request.URL.Host+": "+response.Status, util.ReaderToJSON(response.Body)) + } +} + +func visitClusters(vArgs *visitArgs, service *vespa.Service) (res util.OperationResult) { + clusters := []string{ + vArgs.contentCluster, + } + if vArgs.contentCluster == "*" { + clusters = probeVisit(vArgs, service) + } + if vArgs.makeFeed { + vArgs.writeString("[\n") + } + for _, c := range clusters { + vArgs.contentCluster = c + res = runVisit(vArgs, service) + if !res.Success { + return res + } + vArgs.debugPrint("Success: " + res.Message) + } + if vArgs.makeFeed { + vArgs.writeString("{}\n]\n") + } + return res +} + +func probeVisit(vArgs *visitArgs, service *vespa.Service) []string { + clusters := make([]string, 0, 3) + vvo, _ := runOneVisit(vArgs, service, "") + if vvo != nil { + msg := vvo.ErrorMsg + if strings.Contains(msg, "no content cluster '*'") { + for idx, value := range strings.Split(msg, ",") { + if idx > 0 { + parts := strings.Split(value, "'") + if len(parts) == 3 { + clusters = append(clusters, parts[1]) + } + } + } + } + } + return clusters +} + +func runVisit(vArgs *visitArgs, service *vespa.Service) (res util.OperationResult) { + vArgs.debugPrint(fmt.Sprintf("trying to visit: '%s'", vArgs.contentCluster)) + var totalDocuments int = 0 + var continuationToken string + for { + var vvo *VespaVisitOutput + vvo, res = runOneVisit(vArgs, service, continuationToken) + if !res.Success { + if vvo != nil && vvo.ErrorMsg != "" { + vArgs.cli.printWarning(vvo.ErrorMsg) + } + return res + } + vArgs.dumpDocuments(vvo.Documents) + vArgs.debugPrint(fmt.Sprintf("got %d documents", len(vvo.Documents))) + totalDocuments += len(vvo.Documents) + continuationToken = vvo.Continuation + if continuationToken == "" { + break + } + } + res.Message = fmt.Sprintf("%s [%d documents visited]", res.Message, totalDocuments) + return +} + +func quoteArgForUrl(arg string) string { + var buf strings.Builder + buf.Grow(len(arg)) + for _, r := range arg { + switch { + case 'a' <= r && r <= 'z': + buf.WriteRune(r) + case 'A' <= r && r <= 'Z': + buf.WriteRune(r) + case '0' <= r && r <= '9': + buf.WriteRune(r) + case r <= ' ' || r > '~': + buf.WriteRune('+') + default: + s := fmt.Sprintf("%s%02X", "%", r) + buf.WriteString(s) + } + } + return buf.String() +} + +func runOneVisit(vArgs *visitArgs, service *vespa.Service, contToken string) (*VespaVisitOutput, util.OperationResult) { + urlPath := service.BaseURL + "/document/v1/?cluster=" + quoteArgForUrl(vArgs.contentCluster) + if vArgs.fieldSet != "" { + urlPath = urlPath + "&fieldSet=" + quoteArgForUrl(vArgs.fieldSet) + } + if vArgs.selection != "" { + urlPath = urlPath + "&selection=" + quoteArgForUrl(vArgs.selection) + } + if contToken != "" { + urlPath = urlPath + "&continuation=" + contToken + } + if vArgs.chunkCount > 0 { + urlPath = urlPath + fmt.Sprintf("&wantedDocumentCount=%d", vArgs.chunkCount) + } + url, urlParseError := url.Parse(urlPath) + if urlParseError != nil { + return nil, util.Failure("Invalid request path: '" + urlPath + "': " + urlParseError.Error()) + } + request := &http.Request{ + URL: url, + Method: "GET", + } + timeout := time.Duration(900) * time.Second + response, err := service.Do(request, timeout) + if err != nil { + return nil, util.Failure("Request failed: " + err.Error()) + } + defer response.Body.Close() + vvo, err := parseVisitOutput(response.Body) + if response.StatusCode == 200 { + if err == nil { + totalDocCount += vvo.DocumentCount + if vvo.DocumentCount != len(vvo.Documents) { + vArgs.cli.printWarning(fmt.Sprintf("Inconsistent contents from: %v", url)) + vArgs.cli.printWarning(fmt.Sprintf("claimed count: %d", vvo.DocumentCount)) + vArgs.cli.printWarning(fmt.Sprintf("document blobs: %d", len(vvo.Documents))) + return nil, util.Failure("Inconsistent contents from document API") + } + return vvo, util.Success("visited " + vArgs.contentCluster) + } else { + return nil, util.Failure("error reading response: " + err.Error()) + } + } else if response.StatusCode/100 == 4 { + return vvo, util.FailureWithPayload("Invalid document operation: "+response.Status, util.ReaderToJSON(response.Body)) + } else { + return vvo, util.FailureWithPayload(service.Description()+" at "+request.URL.Host+": "+response.Status, util.ReaderToJSON(response.Body)) + } +} + +type DocumentBlob struct { + blob []byte +} + +func (d *DocumentBlob) UnmarshalJSON(data []byte) error { + d.blob = make([]byte, len(data)) + copy(d.blob, data) + return nil +} + +func (d *DocumentBlob) MarshalJSON() ([]byte, error) { + return d.blob, nil +} + +type VespaVisitOutput struct { + PathId string `json:"pathId"` + Documents []DocumentBlob `json:"documents"` + DocumentCount int `json:"documentCount"` + Continuation string `json:"continuation"` + ErrorMsg string `json:"message"` +} + +func parseVisitOutput(r io.Reader) (*VespaVisitOutput, error) { + codec := json.NewDecoder(r) + var parsedJson VespaVisitOutput + err := codec.Decode(&parsedJson) + if err != nil { + return nil, fmt.Errorf("could not decode JSON, error: %s", err.Error()) + } + return &parsedJson, nil +} diff --git a/client/go/internal/cli/cmd/visit_test.go b/client/go/internal/cli/cmd/visit_test.go new file mode 100644 index 00000000000..4302680b9d9 --- /dev/null +++ b/client/go/internal/cli/cmd/visit_test.go @@ -0,0 +1,142 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package cmd + +import ( + "fmt" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/vespa-engine/vespa/client/go/internal/mock" + "github.com/vespa-engine/vespa/client/go/internal/vespa" +) + +const ( + normalpre = `{"pathId":"/document/v1/","documents":[` + document1 = `{"id":"id:t:m::1","fields":{"title":"t"}}` + document2 = `{"id":"id:t:m::2","fields":{"title":"t2"}}` + document3 = `{"id":"id:t:m::3","fields":{"ar":"xyz","w":63,"title":"xyzzy","year":2000}}` + + savedresponse = `{"pathId":"/document/v1/","documents":[{"id":"id:test:music::1921492307","fields":{"title":"song","year":2010}},{"id":"id:test:music::p_try-this-clean-bonus-dvd-_music_1922003403","fields":{"artist":"xyz","weight":600000,"song":"hate","title":"xyz","year":2000}}],"documentCount":2,"continuation":"AAAACAAAAAAAAAAJAAAAAAAAAAgAAAAAAAABAAAAAAEgAAAAAAAAEAAAAAAAAAAA"}` + + saveddoc0 = `{"id":"id:test:music::1921492307","fields":{"title":"song","year":2010}}` + saveddoc1 = `{"id":"id:test:music::p_try-this-clean-bonus-dvd-_music_1922003403","fields":{"artist":"xyz","weight":600000,"song":"hate","title":"xyz","year":2000}}` + handlersResponse = `{ + "handlers" : [ { + "id" : "com.yahoo.container.usability.BindingsOverviewHandler", + "class" : "com.yahoo.container.usability.BindingsOverviewHandler", + "bundle" : "container-disc:8.0.0", + "serverBindings" : [ "http://*/" ] + }, { + "id" : "com.yahoo.document.restapi.resource.DocumentV1ApiHandler", + "class" : "com.yahoo.document.restapi.resource.DocumentV1ApiHandler", + "bundle" : "vespaclient-container-plugin:8.0.0", + "serverBindings" : [ "http://*/document/v1/*", "http://*/document/v1/*/" ] + } ] +}` + clusterStarResponse = `{"pathId":"/document/v1/","message":"Your Vespa deployment has no content cluster '*', only 'fooCC'"}` +) + +func TestQuoteFunc(t *testing.T) { + var buf []byte = make([]byte, 3) + buf[0] = 'a' + buf[2] = 'z' + for i := 0; i < 256; i++ { + buf[1] = byte(i) + s := string(buf) + res := quoteArgForUrl(s) + if i < 32 || i > 127 { + assert.Equal(t, "a+z", res) + } else { + fmt.Printf("res %3d => '%s'\n", i, res) + } + } +} + +// low-level (unit) test +func TestRunOneVisit(t *testing.T) { + withResponse := func(client *mock.HTTPClient) { + client.NextResponseString(200, savedresponse) + } + op := func(service *vespa.Service) { + vArgs := visitArgs{ + contentCluster: "fooCC", + } + vvo, res := runOneVisit(&vArgs, service, "BBBB") + assert.Equal(t, true, res.Success) + assert.Equal(t, "visited fooCC", res.Message) + assert.Equal(t, "/document/v1/", vvo.PathId) + assert.Equal(t, "", vvo.ErrorMsg) + assert.Equal(t, "AAAACAAAAAAAAAAJAAAAAAAAAAgAAAAAAAABAAAAAAEgAAAAAAAAEAAAAAAAAAAA", vvo.Continuation) + assert.Equal(t, 2, vvo.DocumentCount) + assert.Equal(t, 2, len(vvo.Documents)) + assert.Equal(t, saveddoc0, string(vvo.Documents[0].blob)) + assert.Equal(t, saveddoc1, string(vvo.Documents[1].blob)) + } + req := withMockClient(t, withResponse, op) + assert.Equal(t, "cluster=fooCC&continuation=BBBB", req.URL.RawQuery) + + op = func(service *vespa.Service) { + vArgs := visitArgs{ + contentCluster: "search", + fieldSet: "[id]", + selection: "music.year>2000", + chunkCount: 123, + } + vvo, res := runOneVisit(&vArgs, service, "asdf") + assert.Equal(t, true, res.Success) + assert.Equal(t, 2, vvo.DocumentCount) + } + req = withMockClient(t, withResponse, op) + assert.Equal(t, "cluster=search&fieldSet=%5Bid%5D&selection=music%2Eyear%3E2000&continuation=asdf&wantedDocumentCount=123", req.URL.RawQuery) +} + +func withMockClient(t *testing.T, prepCli func(*mock.HTTPClient), runOp func(*vespa.Service)) *http.Request { + client := &mock.HTTPClient{} + prepCli(client) + cli, _, _ := newTestCLI(t) + cli.httpClient = client + service, _ := documentService(cli) + runOp(service) + return client.LastRequest +} + +func TestVisitCommand(t *testing.T) { + assertVisitResults( + []string{ + "visit", + "--json-lines", + }, + t, + []string{ + normalpre + + document1 + + `],"documentCount":1,"continuation":"CAFE"}`, + normalpre + + document2 + + "," + + document3 + + `],"documentCount":2}`, + }, + "cluster=fooCC&continuation=CAFE&wantedDocumentCount=1000", + document1+"\n"+ + document2+"\n"+ + document3+"\n") +} + +func assertVisitResults(arguments []string, t *testing.T, responses []string, queryPart, output string) { + client := &mock.HTTPClient{} + client.NextResponseString(200, handlersResponse) + client.NextResponseString(400, clusterStarResponse) + for _, resp := range responses { + client.NextResponseString(200, resp) + } + cli, stdout, stderr := newTestCLI(t) + cli.httpClient = client + assert.Nil(t, cli.Run(arguments...)) + assert.Equal(t, output, stdout.String()) + assert.Equal(t, "", stderr.String()) + assert.Equal(t, queryPart, client.LastRequest.URL.RawQuery) + assert.Equal(t, "/document/v1/", client.LastRequest.URL.Path) + assert.Equal(t, "GET", client.LastRequest.Method) +} diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index 82842a8c28b..14b39867348 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -38,8 +38,8 @@ <aopalliance.version>1.0</aopalliance.version> <guava.version>27.1-jre</guava.version> <guice.version>4.2.3</guice.version> - <jackson2.version>2.13.4</jackson2.version> - <jackson-databind.version>2.13.4.2</jackson-databind.version> + <jackson2.version>2.14.2</jackson2.version> + <jackson-databind.version>2.14.2</jackson-databind.version> <javax.inject.version>1</javax.inject.version> <javax.servlet-api.version>3.1.0</javax.servlet-api.version> <javax.ws.rs-api.version>2.0.1</javax.ws.rs-api.version> diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 5707206019f..f5ad1dce4e7 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -106,6 +106,8 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { private final boolean includeSourceFiles; private final TransformerFactory transformerFactory; + private DeploymentSpec deploymentSpec = null; + /** Creates from a directory with source files included */ public static FilesApplicationPackage fromFile(File appDir) { return fromFile(appDir, false); @@ -580,6 +582,12 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { IOUtils.writeFile(metaFile, metaData.asJsonBytes()); } + @Override + public DeploymentSpec getDeploymentSpec() { + if (deploymentSpec != null) return deploymentSpec; + return deploymentSpec = parseDeploymentSpec(false); + } + private void preprocessXML(File destination, File inputXml, Zone zone) throws IOException { if ( ! inputXml.exists()) return; try { @@ -589,10 +597,9 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { instance, zone.environment(), zone.region(), - getDeployment().map(new DeploymentSpecXmlReader(false)::read) - .flatMap(spec -> spec.instance(instance)) - .map(DeploymentInstanceSpec::tags) - .orElse(Tags.empty())) + getDeploymentSpec().instance(instance) + .map(DeploymentInstanceSpec::tags) + .orElse(Tags.empty())) .run(); try (FileOutputStream outputStream = new FileOutputStream(destination)) { diff --git a/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorComplexTest.java b/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorComplexTest.java index 93e038c786a..19f1414cd10 100644 --- a/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorComplexTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/application/HostedOverrideProcessorComplexTest.java @@ -110,10 +110,7 @@ public class HostedOverrideProcessorComplexTest { private void assertOverride(InstanceName instance, Environment environment, RegionName region, String expected) throws TransformerException { ApplicationPackage app = FilesApplicationPackage.fromFile(new File(servicesFile).getParentFile()); Document inputDoc = Xml.getDocument(app.getServices()); - Tags tags = app.getDeployment() - .map(new DeploymentSpecXmlReader(false)::read) - .flatMap(spec -> spec.instance(instance).map(DeploymentInstanceSpec::tags)) - .orElse(Tags.empty()); + Tags tags = app.getDeploymentSpec().instance(instance).map(DeploymentInstanceSpec::tags).orElse(Tags.empty()); Document newDoc = new OverrideProcessor(instance, environment, region, tags).process(inputDoc); assertEquals(expected, Xml.documentAsString(newDoc, true)); } diff --git a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java index 6c83b2029ad..4742d275918 100644 --- a/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java +++ b/config-application-package/src/test/java/com/yahoo/config/model/application/provider/FilesApplicationPackageTest.java @@ -84,6 +84,7 @@ public class FilesApplicationPackageTest { assertFalse(new File(appDir, "deployment.xml").exists()); FilesApplicationPackage app = FilesApplicationPackage.fromFile(appDir); assertFalse(app.getDeployment().isPresent()); + assertTrue(app.getDeploymentSpec().isEmpty()); } @Test @@ -93,6 +94,7 @@ public class FilesApplicationPackageTest { assertTrue(deployment.exists()); FilesApplicationPackage app = FilesApplicationPackage.fromFile(appDir); assertTrue(app.getDeployment().isPresent()); + assertFalse(app.getDeploymentSpec().isEmpty()); assertFalse(app.getMajorVersion().isPresent()); assertEquals(IOUtils.readAll(app.getDeployment().get()), IOUtils.readAll(new FileReader(deployment))); } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java index 0ce09c454a0..b6a183c06ab 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java @@ -2,6 +2,7 @@ package com.yahoo.config.application.api; import com.yahoo.component.Version; +import com.yahoo.config.application.api.xml.DeploymentSpecXmlReader; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; @@ -128,20 +129,7 @@ public interface ApplicationPackage { /** Returns the major version this application is valid for, or empty if it is valid for all versions */ default Optional<Integer> getMajorVersion() { - if (getDeployment().isEmpty()) return Optional.empty(); - - Element deployElement = XML.getDocument(getDeployment().get()).getDocumentElement(); - if (deployElement == null) return Optional.empty(); - - String majorVersionString = deployElement.getAttribute("major-version"); - if (majorVersionString == null || majorVersionString.isEmpty()) - return Optional.empty(); - try { - return Optional.of(Integer.parseInt(majorVersionString)); - } - catch (NumberFormatException e) { - throw new IllegalArgumentException("major-version must be an integer number, not '" + majorVersionString + "'"); - } + return getDeploymentSpec().majorVersion(); } /** @@ -168,6 +156,19 @@ public interface ApplicationPackage { String getServicesSource(); Optional<Reader> getDeployment(); + + /** + * Returns the parsed deployment spec of this, + * without validating it, and without reparsing on each request. + */ + DeploymentSpec getDeploymentSpec(); + + default DeploymentSpec parseDeploymentSpec(boolean validate) { + return getDeployment() + .map(new DeploymentSpecXmlReader(validate)::read) + .orElse(DeploymentSpec.empty); + } + Optional<Reader> getValidationOverrides(); List<ComponentInfo> getComponentsInfo(Version vespaVersion); @@ -226,6 +227,7 @@ public interface ApplicationPackage { /** * Readers for all the schema files. + * * @return a collection of readers for schemas */ Collection<NamedReader> getSchemas(); @@ -235,10 +237,9 @@ public interface ApplicationPackage { * application package. This is the entry point for the multi environment application package support. This method * will not mutate the existing application package. * - * @param zone A valid {@link Zone} instance, used to decide which parts of services to keep and remove - * @param logger A {@link DeployLogger} to add output that will be returned to the user - * - * @return A new application package instance pointing to a new location + * @param zone a valid {@link Zone} instance, used to decide which parts of services to keep and remove + * @param logger a {@link DeployLogger} to add output that will be returned to the user + * @return a new application package instance pointing to a new location */ default ApplicationPackage preprocess(Zone zone, DeployLogger logger) throws IOException { throw new UnsupportedOperationException("This application package does not support preprocessing"); diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/Bcp.java b/config-model-api/src/main/java/com/yahoo/config/application/api/Bcp.java index af369dc2672..48464904f44 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/Bcp.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/Bcp.java @@ -86,16 +86,20 @@ public class Bcp { public static class Group { - private final Duration deadline; private final List<RegionMember> members; + private final Set<RegionName> memberRegions; + private final Duration deadline; public Group(List<RegionMember> members, Duration deadline) { this.members = List.copyOf(members); + this.memberRegions = members.stream().map(member -> member.region()).collect(Collectors.toSet()); this.deadline = deadline; } public List<RegionMember> members() { return members; } + public Set<RegionName> memberRegions() { return memberRegions; } + /** * Returns the max time until the other regions must be able to handle the additional traffic * when a region becomes unreachable, which by default is Duration.ZERO. diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java index 41644ebc87d..699010417bf 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java @@ -88,6 +88,8 @@ public class DeploymentSpec { validateBcp(); } + public boolean isEmpty() { return this == empty; } + /** Throw an IllegalArgumentException if the total delay exceeds 24 hours */ private void validateTotalDelay(List<Step> steps) { long totalDelaySeconds = steps.stream().mapToLong(step -> (step.delay().getSeconds())).sum(); diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java b/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java index fd355d427a3..1582d03df9f 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java @@ -45,7 +45,7 @@ public class Endpoint { this.level = Objects.requireNonNull(level, "level must be non-null"); this.targets = List.copyOf(Objects.requireNonNull(targets, "targets must be non-null")); if (endpointId().length() > endpointMaxLength || !endpointPattern.matcher(endpointId()).matches()) { - throw new IllegalArgumentException("Endpoint ID must be all lowercase, alphanumeric, with no consecutive dashes, " + + throw new IllegalArgumentException("Endpoint id must be all lowercase, alphanumeric, with no consecutive dashes, " + "of length 1 to 12, and begin with a character; but got '" + endpointId() + "'"); } if (targets.isEmpty()) throw new IllegalArgumentException("targets must be non-empty"); diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java index be6be5566a8..d04bb7ecfe0 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java @@ -46,8 +46,10 @@ import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; @@ -136,8 +138,10 @@ public class DeploymentSpecXmlReader { List<Step> steps = new ArrayList<>(); List<Endpoint> applicationEndpoints = new ArrayList<>(); + Bcp defaultBcp; if ( ! containsTag(instanceTag, root)) { // deployment spec skipping explicit instance -> "default" instance steps.addAll(readInstanceContent("default", root, new HashMap<>(), root)); + defaultBcp = Bcp.empty(); } else { if (XML.getChildren(root).stream().anyMatch(child -> child.getTagName().equals(prodTag))) @@ -154,6 +158,7 @@ public class DeploymentSpecXmlReader { } } readEndpoints(root, Optional.empty(), steps, applicationEndpoints, Map.of()); + defaultBcp = readBcp(root, Optional.empty(), steps, List.of(), Map.of()); } return new DeploymentSpec(steps, @@ -162,7 +167,7 @@ public class DeploymentSpecXmlReader { stringAttribute(athenzServiceAttribute, root).map(AthenzService::from), stringAttribute(cloudAccountAttribute, root).map(CloudAccount::from), applicationEndpoints, - readBcp(root), + defaultBcp, xmlForm, deprecatedElements); } @@ -210,6 +215,8 @@ public class DeploymentSpecXmlReader { List<Endpoint> endpoints = new ArrayList<>(); Map<ClusterSpec.Id, Map<ZoneId, ZoneEndpoint>> zoneEndpoints = new LinkedHashMap<>(); readEndpoints(instanceElement, Optional.of(instanceNameString), steps, endpoints, zoneEndpoints); + Bcp bcp = readBcp(instanceElement, Optional.of(instanceNameString), steps, endpoints, zoneEndpoints); + validateEndpoints(endpoints); // Build and return instances with these values Instant now = clock.instant(); @@ -230,11 +237,19 @@ public class DeploymentSpecXmlReader { notifications, endpoints, zoneEndpoints, - readBcp(instanceElement), + bcp, now)) .toList(); } + private void validateEndpoints(List<Endpoint> endpoints) { + Set<String> endpointIds = new HashSet<>(); + for (Endpoint endpoint : endpoints) { + if ( ! endpointIds.add(endpoint.endpointId())) + illegal("Endpoint id '" + endpoint.endpointId() + "' is specified multiple times"); + } + } + private List<Step> readSteps(Element stepTag, Map<String, String> prodAttributes, Element parentTag) { if (stepTag.getTagName().equals(instanceTag)) return new ArrayList<>(readInstanceContent(stepTag.getAttribute(idAttribute), stepTag, prodAttributes, parentTag)); @@ -329,140 +344,161 @@ public class DeploymentSpecXmlReader { if (endpointsElement == null) return; Endpoint.Level level = instance.isEmpty() ? Endpoint.Level.application : Endpoint.Level.instance; - Map<String, Endpoint> endpointsById = new LinkedHashMap<>(); Map<String, Map<RegionName, List<ZoneEndpoint>>> endpointsByZone = new LinkedHashMap<>(); - XML.getChildren(endpointsElement, endpointTag).stream() // Read zone settings first. - .sorted(comparingInt(endpoint -> getZoneEndpointType(endpoint, level).isPresent() ? 0 : 1)) - .forEach(endpointElement -> { - String containerId = requireStringAttribute("container-id", endpointElement); - Optional<String> endpointId = stringAttribute("id", endpointElement); - Optional<String> zoneEndpointType = getZoneEndpointType(endpointElement, level); - String msgPrefix = (level == Endpoint.Level.application ? "Application-level" : "Instance-level") + - " endpoint '" + endpointId.orElse(Endpoint.DEFAULT_ID) + "': "; - - if (zoneEndpointType.isPresent() && endpointId.isPresent()) - illegal(msgPrefix + "cannot declare 'id' with type 'zone' or 'private'"); - - String invalidChild = level == Endpoint.Level.application ? "region" : "instance"; - if ( ! XML.getChildren(endpointElement, invalidChild).isEmpty()) - illegal(msgPrefix + "invalid element '" + invalidChild + "'"); - - boolean enabled = XML.attribute("enabled", endpointElement) - .map(value -> { - if (zoneEndpointType.isEmpty() || ! zoneEndpointType.get().equals("zone")) - illegal(msgPrefix + "only endpoints of type 'zone' can specify 'enabled'"); - - return switch (value) { - case "true" -> true; - case "false" -> false; - default -> throw new IllegalArgumentException(msgPrefix + "invalid 'enabled' value; must be 'true' or 'false'"); - }; - }).orElse(true); - - List<AllowedUrn> allowedUrns = new ArrayList<>(); - for (var allow : XML.getChildren(endpointElement, "allow")) { - if (zoneEndpointType.isEmpty() || ! zoneEndpointType.get().equals("private")) - illegal(msgPrefix + "only endpoints of type 'private' can specify 'allow' children"); - - switch (requireStringAttribute("with", allow)) { - case "aws-private-link" -> allowedUrns.add(new AllowedUrn(AccessType.awsPrivateLink, requireStringAttribute("arn", allow))); - case "gcp-service-connect" -> allowedUrns.add(new AllowedUrn(AccessType.gcpServiceConnect, requireStringAttribute("project", allow))); - default -> illegal("Private endpoint for container-id '" + containerId + "': " + - "invalid attribute 'with': '" + requireStringAttribute("with", allow) + "'"); - } - } + for (Element endpointElement : XML.getChildren(endpointsElement, endpointTag).stream() // Read zone settings first. + .sorted(comparingInt(endpoint -> getZoneEndpointType(endpoint, level).isPresent() ? 0 : 1)).toList()) { + Optional<Endpoint> endpoint = readEndpoint(parent, endpointElement, level, instance, steps, List.of(), endpointsByZone); + endpoint.ifPresent(e -> endpoints.add(e)); + } + validateAndConsolidate(endpointsByZone, zoneEndpoints); + } - List<Endpoint.Target> targets = new ArrayList<>(); - if (level == Endpoint.Level.application) { - Optional<String> endpointRegion = stringAttribute("region", endpointElement); - int weightSum = 0; - for (var instanceElement : XML.getChildren(endpointElement, "instance")) { - String instanceName = instanceElement.getTextContent(); - if (instanceName == null || instanceName.isBlank()) illegal(msgPrefix + "empty 'instance' element"); - Optional<String> instanceRegion = stringAttribute("region", instanceElement); - if (endpointRegion.isPresent() == instanceRegion.isPresent()) - illegal(msgPrefix + "'region' attribute must be declared on either <endpoint> or <instance> tag"); - String weightFromAttribute = requireStringAttribute("weight", instanceElement); - int weight; - try { - weight = Integer.parseInt(weightFromAttribute); - } catch (NumberFormatException e) { - throw new IllegalArgumentException(msgPrefix + "invalid weight value '" + weightFromAttribute + "'"); - } - weightSum += weight; - targets.add(new Endpoint.Target(RegionName.from(endpointRegion.orElseGet(instanceRegion::get)), - InstanceName.from(instanceName), - weight)); - } - if (weightSum == 0) illegal(msgPrefix + "sum of all weights must be positive, got " + weightSum); - } else { - if (stringAttribute("region", endpointElement).isPresent()) illegal(msgPrefix + "invalid 'region' attribute"); - Set<RegionName> regions = new LinkedHashSet<>(); - for (var regionElement : XML.getChildren(endpointElement, "region")) { - String region = regionElement.getTextContent(); - if (region == null || region.isBlank()) - illegal(msgPrefix + "empty 'region' element"); - if ( zoneEndpointType.isEmpty() - && Stream.of(RegionName.from(region), null) - .map(endpointsByZone.getOrDefault(containerId, new HashMap<>())::get) - .flatMap(maybeEndpoints -> maybeEndpoints == null ? Stream.empty() : maybeEndpoints.stream()) - .anyMatch(endpoint -> ! endpoint.isPublicEndpoint())) - illegal(msgPrefix + "targets zone endpoint in '" + region + "' with 'enabled' set to 'false'"); - if ( ! regions.add(RegionName.from(region))) - illegal(msgPrefix + "duplicate 'region' element: '" + region + "'"); - } + /** + * @param parentElement + * @param endpointElement + * @param level decide what this method is reading TODO: Split into different methods instead + * @param instance the instance this applies to, or empty if it does not apply to an instance (application endpoints) + * @param steps + * @param forRegions the regions this applies to (for bcp), or empty (otherwise) to read this from "region" subelements + * @param endpointsByZone a map containing any zone endpoints read by this + * @return the endpoint read, unless it is added to endspointsByZone instead *sob* + */ + static Optional<Endpoint> readEndpoint(Element parentElement, + Element endpointElement, + Endpoint.Level level, + Optional<String> instance, + List<Step> steps, + Collection<RegionName> forRegions, + Map<String, Map<RegionName, List<ZoneEndpoint>>> endpointsByZone) { + String containerId = requireStringAttribute("container-id", endpointElement); + Optional<String> endpointId = stringAttribute("id", endpointElement); + Optional<String> zoneEndpointType = getZoneEndpointType(endpointElement, level); + String msgPrefix = (level == Endpoint.Level.application ? "Application-level" : "Instance-level") + + " endpoint '" + endpointId.orElse(Endpoint.DEFAULT_ID) + "': "; + + if (zoneEndpointType.isPresent() && endpointId.isPresent()) + illegal(msgPrefix + "cannot declare 'id' with type 'zone' or 'private'"); + + String invalidChild = level == Endpoint.Level.application ? "region" : "instance"; + if ( ! XML.getChildren(endpointElement, invalidChild).isEmpty()) + illegal(msgPrefix + "invalid element '" + invalidChild + "'"); + + boolean enabled = XML.attribute("enabled", endpointElement) + .map(value -> { + if (zoneEndpointType.isEmpty() || ! zoneEndpointType.get().equals("zone")) + illegal(msgPrefix + "only endpoints of type 'zone' can specify 'enabled'"); + + return switch (value) { + case "true" -> true; + case "false" -> false; + default -> throw new IllegalArgumentException(msgPrefix + "invalid 'enabled' value; must be 'true' or 'false'"); + }; + }).orElse(true); + + List<AllowedUrn> allowedUrns = new ArrayList<>(); + for (var allow : XML.getChildren(endpointElement, "allow")) { + if (zoneEndpointType.isEmpty() || ! zoneEndpointType.get().equals("private")) + illegal(msgPrefix + "only endpoints of type 'private' can specify 'allow' children"); + + switch (requireStringAttribute("with", allow)) { + case "aws-private-link" -> allowedUrns.add(new AllowedUrn(AccessType.awsPrivateLink, requireStringAttribute("arn", allow))); + case "gcp-service-connect" -> allowedUrns.add(new AllowedUrn(AccessType.gcpServiceConnect, requireStringAttribute("project", allow))); + default -> illegal("Private endpoint for container-id '" + containerId + "': " + + "invalid attribute 'with': '" + requireStringAttribute("with", allow) + "'"); + } + } - if (zoneEndpointType.isPresent()) { - if (regions.isEmpty()) regions.add(null); - ZoneEndpoint endpoint = switch (zoneEndpointType.get()) { - case "zone" -> new ZoneEndpoint(enabled, false, List.of()); - case "private" -> new ZoneEndpoint(true, true, allowedUrns); // Doesn't turn off public visibility. - default -> throw new IllegalArgumentException("unsupported zone endpoint type '" + zoneEndpointType.get() + "'"); - }; - for (RegionName region : regions) endpointsByZone.computeIfAbsent(containerId, __ -> new LinkedHashMap<>()) - .computeIfAbsent(region, __ -> new ArrayList<>()) - .add(endpoint); - } - else { - if (regions.isEmpty()) { - // No explicit targets given for instance level endpoint. Include all declared, enabled regions by default - List<RegionName> declared = - steps.stream() - .filter(step -> step.concerns(Environment.prod)) - .flatMap(step -> step.zones().stream()) - .flatMap(zone -> zone.region().stream()) - .toList(); - if (declared.isEmpty()) illegal(msgPrefix + "no declared regions to target"); - - declared.stream().filter(region -> Stream.of(region, null) - .map(endpointsByZone.getOrDefault(containerId, new HashMap<>())::get) - .flatMap(maybeEndpoints -> maybeEndpoints == null ? Stream.empty() : maybeEndpoints.stream()) - .allMatch(ZoneEndpoint::isPublicEndpoint)) - .forEach(regions::add); - } - if (regions.isEmpty()) illegal(msgPrefix + "all eligible zone endpoints have 'enabled' set to 'false'"); - InstanceName instanceName = instance.map(InstanceName::from).get(); - for (RegionName region : regions) targets.add(new Target(region, instanceName, 1)); + List<Endpoint.Target> targets = new ArrayList<>(); + if (level == Endpoint.Level.application) { + if ( ! forRegions.isEmpty()) throw new IllegalStateException("Illegal combination"); + Optional<String> endpointRegion = stringAttribute("region", endpointElement); + int weightSum = 0; + for (var instanceElement : XML.getChildren(endpointElement, "instance")) { + String instanceName = instanceElement.getTextContent(); + if (instanceName == null || instanceName.isBlank()) illegal(msgPrefix + "empty 'instance' element"); + Optional<String> instanceRegion = stringAttribute("region", instanceElement); + if (endpointRegion.isPresent() == instanceRegion.isPresent()) + illegal(msgPrefix + "'region' attribute must be declared on either <endpoint> or <instance> tag"); + String weightFromAttribute = requireStringAttribute("weight", instanceElement); + int weight; + try { + weight = Integer.parseInt(weightFromAttribute); + } catch (NumberFormatException e) { + throw new IllegalArgumentException(msgPrefix + "invalid weight value '" + weightFromAttribute + "'"); } + weightSum += weight; + targets.add(new Endpoint.Target(RegionName.from(endpointRegion.orElseGet(instanceRegion::get)), + InstanceName.from(instanceName), + weight)); + } + if (weightSum == 0) illegal(msgPrefix + "sum of all weights must be positive, got " + weightSum); + } else { + if (stringAttribute("region", endpointElement).isPresent()) illegal(msgPrefix + "invalid 'region' attribute"); + + Set<RegionName> regions = new LinkedHashSet<>(forRegions); + List<Element> regionElements = XML.getChildren(endpointElement, "region"); + if ( ! regions.isEmpty() && ! regionElements.isEmpty()) + illegal("Endpoints in <" + parentElement.getTagName() + "> cannot contain <region> children"); + for (var regionElement : XML.getChildren(endpointElement, "region")) { + String region = regionElement.getTextContent(); + if (region == null || region.isBlank()) + illegal(msgPrefix + "empty 'region' element"); + if ( zoneEndpointType.isEmpty() + && Stream.of(RegionName.from(region), null) + .map(endpointsByZone.getOrDefault(containerId, new HashMap<>())::get) + .flatMap(maybeEndpoints -> maybeEndpoints == null ? Stream.empty() : maybeEndpoints.stream()) + .anyMatch(endpoint -> ! endpoint.isPublicEndpoint())) + illegal(msgPrefix + "targets zone endpoint in '" + region + "' with 'enabled' set to 'false'"); + if ( ! regions.add(RegionName.from(region))) + illegal(msgPrefix + "duplicate 'region' element: '" + region + "'"); } - if (zoneEndpointType.isEmpty()) { - Endpoint endpoint = new Endpoint(endpointId.orElse(Endpoint.DEFAULT_ID), containerId, level, targets); - if (endpointsById.containsKey(endpoint.endpointId())) { - illegal("Endpoint ID '" + endpoint.endpointId() + "' is specified multiple times"); + if (zoneEndpointType.isPresent()) { + if (regions.isEmpty()) regions.add(null); + ZoneEndpoint endpoint = switch (zoneEndpointType.get()) { + case "zone" -> new ZoneEndpoint(enabled, false, List.of()); + case "private" -> new ZoneEndpoint(true, true, allowedUrns); // Doesn't turn off public visibility. + default -> throw new IllegalArgumentException("unsupported zone endpoint type '" + zoneEndpointType.get() + "'"); + }; + for (RegionName region : regions) endpointsByZone.computeIfAbsent(containerId, __ -> new LinkedHashMap<>()) + .computeIfAbsent(region, __ -> new ArrayList<>()) + .add(endpoint); + } + else { + if (regions.isEmpty()) { + // No explicit targets given for instance level endpoint. Include all declared, enabled regions by default + List<RegionName> declared = + steps.stream() + .filter(step -> step.concerns(Environment.prod)) + .flatMap(step -> step.zones().stream()) + .flatMap(zone -> zone.region().stream()) + .toList(); + if (declared.isEmpty()) illegal(msgPrefix + "no declared regions to target"); + + declared.stream().filter(region -> Stream.of(region, null) + .map(endpointsByZone.getOrDefault(containerId, new HashMap<>())::get) + .flatMap(maybeEndpoints -> maybeEndpoints == null ? Stream.empty() : maybeEndpoints.stream()) + .allMatch(ZoneEndpoint::isPublicEndpoint)) + .forEach(regions::add); } - endpointsById.put(endpoint.endpointId(), endpoint); + if (regions.isEmpty()) illegal(msgPrefix + "all eligible zone endpoints have 'enabled' set to 'false'"); + InstanceName instanceName = instance.map(InstanceName::from).get(); + for (RegionName region : regions) targets.add(new Target(region, instanceName, 1)); } - }); - endpoints.addAll(endpointsById.values()); - validateAndConsolidate(endpointsByZone, zoneEndpoints); + } + + if (zoneEndpointType.isEmpty()) + return Optional.of(new Endpoint(endpointId.orElse(Endpoint.DEFAULT_ID), containerId, level, targets)); + return Optional.empty(); } - static Bcp readBcp(Element element) { - Element bcpElement = XML.getChild(element, "bcp"); + static Bcp readBcp(Element parent, Optional<String> instance, List<Step> steps, + List<Endpoint> endpoints, Map<ClusterSpec.Id, Map<ZoneId, ZoneEndpoint>> zoneEndpoints) { + Element bcpElement = XML.getChild(parent, "bcp"); if (bcpElement == null) return Bcp.empty(); List<Bcp.Group> groups = new ArrayList<>(); + Map<String, Map<RegionName, List<ZoneEndpoint>>> endpointsByZone = new LinkedHashMap<>(); for (Element groupElement : XML.getChildren(bcpElement, "group")) { List<Bcp.RegionMember> regions = new ArrayList<>(); for (Element regionElement : XML.getChildren(groupElement, "region")) { @@ -470,9 +506,22 @@ public class DeploymentSpecXmlReader { double fraction = toDouble(XML.attribute("fraction", regionElement).orElse(null), "fraction").orElse(1.0); regions.add(new Bcp.RegionMember(region, fraction)); } + for (Element endpointElement : XML.getChildren(groupElement, "endpoint")) { + if (instance.isEmpty()) illegal("The default <bcp> element at the root cannot define endpoints"); + Optional<Endpoint> endpoint = readEndpoint(groupElement, + endpointElement, + Endpoint.Level.instance, + instance, + steps, + regions.stream().map(r -> r.region()).toList(), + endpointsByZone); + endpoint.ifPresent(e -> endpoints.add(e)); + } + Duration deadline = XML.attribute("deadline", groupElement).map(value -> toDuration(value, "deadline")).orElse(Duration.ZERO); groups.add(new Bcp.Group(regions, deadline)); } + validateAndConsolidate(endpointsByZone, zoneEndpoints); return new Bcp(groups); } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 04de4d2e277..d5c074a191d 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -111,6 +111,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"tokle"}) default boolean useRestrictedDataPlaneBindings() { return false; } @ModelFeatureFlag(owners = {"arnej","baldersheim"}, removeAfter = "8.110") default boolean useOldJdiscContainerStartup() { return false; } @ModelFeatureFlag(owners = {"tokle, bjorncs"}, removeAfter = "8.108") default boolean enableDataPlaneFilter() { return true; } + @ModelFeatureFlag(owners = {"arnej, bjorncs"}) default boolean enableGlobalPhase() { return false; } //Below are all flags that must be kept until 7 is out of the door @ModelFeatureFlag(owners = {"arnej"}, removeAfter="7.last") default boolean ignoreThreadStackSizes() { return false; } diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java index afa7e3e502b..89b7318739e 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java @@ -1294,22 +1294,22 @@ public class DeploymentSpecTest { @Test public void invalidEndpoints() { assertInvalidEndpoints("<endpoint id='FOO' container-id='qrs'/>", - "Endpoint ID must be all lowercase, alphanumeric, with no consecutive dashes, of length 1 to 12, and begin with a character; but got 'FOO'"); + "Endpoint id must be all lowercase, alphanumeric, with no consecutive dashes, of length 1 to 12, and begin with a character; but got 'FOO'"); assertInvalidEndpoints("<endpoint id='123' container-id='qrs'/>", - "Endpoint ID must be all lowercase, alphanumeric, with no consecutive dashes, of length 1 to 12, and begin with a character; but got '123'"); + "Endpoint id must be all lowercase, alphanumeric, with no consecutive dashes, of length 1 to 12, and begin with a character; but got '123'"); assertInvalidEndpoints("<endpoint id='foo!' container-id='qrs'/>", - "Endpoint ID must be all lowercase, alphanumeric, with no consecutive dashes, of length 1 to 12, and begin with a character; but got 'foo!'"); + "Endpoint id must be all lowercase, alphanumeric, with no consecutive dashes, of length 1 to 12, and begin with a character; but got 'foo!'"); assertInvalidEndpoints("<endpoint id='foo.bar' container-id='qrs'/>", - "Endpoint ID must be all lowercase, alphanumeric, with no consecutive dashes, of length 1 to 12, and begin with a character; but got 'foo.bar'"); + "Endpoint id must be all lowercase, alphanumeric, with no consecutive dashes, of length 1 to 12, and begin with a character; but got 'foo.bar'"); assertInvalidEndpoints("<endpoint id='foo--bar' container-id='qrs'/>", - "Endpoint ID must be all lowercase, alphanumeric, with no consecutive dashes, of length 1 to 12, and begin with a character; but got 'foo--bar'"); + "Endpoint id must be all lowercase, alphanumeric, with no consecutive dashes, of length 1 to 12, and begin with a character; but got 'foo--bar'"); assertInvalidEndpoints("<endpoint id='foo-' container-id='qrs'/>", - "Endpoint ID must be all lowercase, alphanumeric, with no consecutive dashes, of length 1 to 12, and begin with a character; but got 'foo-'"); + "Endpoint id must be all lowercase, alphanumeric, with no consecutive dashes, of length 1 to 12, and begin with a character; but got 'foo-'"); assertInvalidEndpoints("<endpoint id='foooooooooooo' container-id='qrs'/>", - "Endpoint ID must be all lowercase, alphanumeric, with no consecutive dashes, of length 1 to 12, and begin with a character; but got 'foooooooooooo'"); + "Endpoint id must be all lowercase, alphanumeric, with no consecutive dashes, of length 1 to 12, and begin with a character; but got 'foooooooooooo'"); assertInvalidEndpoints("<endpoint id='foo' container-id='qrs'/><endpoint id='foo' container-id='qrs'/>", - "Endpoint ID 'foo' is specified multiple times"); + "Endpoint id 'foo' is specified multiple times"); assertInvalidEndpoints("<endpoint id='default' type='zone' container-id='foo' />", "Instance-level endpoint 'default': cannot declare 'id' with type 'zone' or 'private'"); assertInvalidEndpoints("<endpoint id='default' type='private' container-id='foo' />", diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithBcpTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithBcpTest.java index 77aadc88be8..a78f53e2084 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithBcpTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithBcpTest.java @@ -6,6 +6,7 @@ import org.junit.Test; import java.io.StringReader; import java.time.Duration; +import java.util.List; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; @@ -100,7 +101,9 @@ public class DeploymentSpecWithBcpTest { </bcp> </deployment> """); + var spec = DeploymentSpec.fromXml(r); + var betaBcp = spec.requireInstance("beta").bcp().orElse(spec.bcp()); assertEquals(1, betaBcp.groups().size()); var betaGroup = betaBcp.groups().get(0); @@ -241,6 +244,164 @@ public class DeploymentSpecWithBcpTest { } } + @Test + public void endpointsDefinedInBcp() { + StringReader r = new StringReader(""" + <deployment version='1.0'> + <instance id='beta'> + <prod> + <region>us-east1</region> + <region>us-east2</region> + </prod> + <bcp> + <group> + <endpoint id="foo" container-id="bar"/> + <region>us-east1</region> + <region>us-east2</region> + </group> + </bcp> + </instance> + <instance id='main'> + <prod> + <region>us-east1</region> + <region>us-east2</region> + <region>us-central1</region> + <region>us-west1</region> + <region>us-west2</region> + </prod> + <bcp> + <group> + <endpoint id="east" container-id="bar"/> + <region>us-east1</region> + <region>us-east2</region> + <region fraction="0.3">us-central1</region> + </group> + <group> + <endpoint id="west" container-id="bar"/> + <region>us-west1</region> + <region>us-west2</region> + <region fraction="0.7">us-central1</region> + </group> + </bcp> + </instance> + </deployment> + """); + + var spec = DeploymentSpec.fromXml(r); + + var betaEndpoints = spec.requireInstance("beta").endpoints(); + assertEquals(1, betaEndpoints.size()); + assertEquals("foo", betaEndpoints.get(0).endpointId()); + assertEquals("bar", betaEndpoints.get(0).containerId()); + assertEquals(List.of(RegionName.from("us-east1"), RegionName.from("us-east2")), + betaEndpoints.get(0).regions()); + + var mainEndpoints = spec.requireInstance("main").endpoints(); + assertEquals(2, mainEndpoints.size()); + assertEquals("east", mainEndpoints.get(0).endpointId()); + assertEquals(List.of(RegionName.from("us-east1"), RegionName.from("us-east2"), RegionName.from("us-central1")), + mainEndpoints.get(0).regions()); + assertEquals("west", mainEndpoints.get(1).endpointId()); + assertEquals(List.of(RegionName.from("us-west1"), RegionName.from("us-west2"), RegionName.from("us-central1")), + mainEndpoints.get(1).regions()); + } + + @Test + public void endpointsDefinedInBcpImplicitInstance() { + StringReader r = new StringReader(""" + <deployment version='1.0'> + <prod> + <region>us-east1</region> + <region>us-east2</region> + <region>us-central1</region> + <region>us-west1</region> + <region>us-west2</region> + </prod> + <bcp> + <group> + <endpoint id="east" container-id="bar"/> + <region>us-east1</region> + <region>us-east2</region> + <region fraction="0.3">us-central1</region> + </group> + <group> + <endpoint id="west" container-id="bar"/> + <region>us-west1</region> + <region>us-west2</region> + <region fraction="0.7">us-central1</region> + </group> + </bcp> + </deployment> + """); + + var spec = DeploymentSpec.fromXml(r); + + var mainEndpoints = spec.requireInstance("default").endpoints(); + assertEquals(2, mainEndpoints.size()); + assertEquals("east", mainEndpoints.get(0).endpointId()); + assertEquals(List.of(RegionName.from("us-east1"), RegionName.from("us-east2"), RegionName.from("us-central1")), + mainEndpoints.get(0).regions()); + assertEquals("west", mainEndpoints.get(1).endpointId()); + assertEquals(List.of(RegionName.from("us-west1"), RegionName.from("us-west2"), RegionName.from("us-central1")), + mainEndpoints.get(1).regions()); + } + + @Test + public void endpointsDefinedInBcpValidation1() { + StringReader r = new StringReader(""" + <deployment version='1.0'> + <instance id='beta'> + <prod> + <region>us-east1</region> + <region>us-east2</region> + </prod> + </instance> + <bcp> + <group> + <endpoint id="foo" container-id="bar"/> + <region>us-east1</region> + <region>us-east2</region> + </group> + </bcp> + </deployment> + """); + try { + DeploymentSpec.fromXml(r); + } + catch (IllegalArgumentException e) { + assertEquals("The default <bcp> element at the root cannot define endpoints", Exceptions.toMessageString(e)); + } + } + + @Test + public void endpointsDefinedInBcpValidation2() { + StringReader r = new StringReader(""" + <deployment version='1.0'> + <instance id='beta'> + <prod> + <region>us-east1</region> + <region>us-east2</region> + </prod> + <bcp> + <group> + <region>us-east1</region> + <region>us-east2</region> + <endpoint id="foo" container-id="bar"> + <region>us-east1</region> + </endpoint> + </group> + </bcp> + </instance> + </deployment> + """); + try { + DeploymentSpec.fromXml(r); + } + catch (IllegalArgumentException e) { + assertEquals("Endpoints in <group> cannot contain <region> children", Exceptions.toMessageString(e)); + } + + } private void assertTwoRegions(DeploymentSpec spec) { var bcp = spec.requireInstance("default").bcp().orElse(spec.bcp()); assertEquals(1, bcp.groups().size()); diff --git a/config-model/.gitignore b/config-model/.gitignore index 6edd041cbe8..7e7e597675d 100644 --- a/config-model/.gitignore +++ b/config-model/.gitignore @@ -4,5 +4,6 @@ /target /src/test/integration/*/copy/ /src/test/integration/*/models.generated/ +/src/test/derived/*/models.generated/ *.cfg.actual /var/ diff --git a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java index 7c805417739..f2d63f3bba4 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java @@ -102,21 +102,6 @@ public class ApplicationConfigProducerRoot extends TreeConfigProducer<AnyConfigP } /** - * Returns the Service with the given id, or null if no such - * configId exists or if it belongs to a non-Service ConfigProducer. - * - * @param configId The configId, e.g. "search.0/tld.0" - * @return Service with the given configId - */ - public Service getService(String configId) { - ConfigProducer cp = getConfigProducer(configId); - if (cp == null || !(cp instanceof Service)) { - return null; - } - return (Service) cp; - } - - /** * Adds the descendant (at any depth level), so it can be looked up * on configId in the Map. * diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java index 2685570b444..051591baa75 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java @@ -51,7 +51,7 @@ public abstract class ConfigModel { * * @param configModelRepo The ConfigModelRepo of the system model */ - public void prepare(ConfigModelRepo configModelRepo, DeployState deployState) { return; } + public void prepare(ConfigModelRepo configModelRepo, DeployState deployState) { } /** * <p>Returns whether this model must be maintained in memory for serving config requests. diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java index d9918168266..13d87b852e4 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModelContext.java @@ -7,8 +7,11 @@ import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AnyConfigProducer; import com.yahoo.config.model.producer.TreeConfigProducer; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.vespa.model.VespaModel; +import java.time.Duration; +import java.util.Comparator; import java.util.stream.Stream; /** @@ -67,6 +70,18 @@ public final class ConfigModelContext { return ConfigModelContext.create(deployState, vespaModel, configModelRepoAdder, parent, producerId); } + /** Returns a cluster info builder pre-populated with info known in this context. */ + public ClusterInfo.Builder clusterInfo() { + var instance = getApplicationPackage().getDeploymentSpec().instance(properties().applicationId().instance()); + if ( ! instance.isPresent()) return new ClusterInfo.Builder(); + var maxDeadline = instance.get().bcp().groups().stream() + .filter(group -> group.memberRegions().contains(properties().zone().region())) + .map(group -> group.deadline()) + .min(Comparator.comparing(deadline -> deadline)) + .orElse(Duration.ofMinutes(0)); + return new ClusterInfo.Builder().bcpDeadline(maxDeadline); + } + /** * Create an application context from a parent producer and an id. * diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java index 45f64182b2a..8c72b5c0237 100644 --- a/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelBuilder.java @@ -88,10 +88,9 @@ public abstract class ConfigModelBuilder<MODEL extends ConfigModel> extends Abst @Override public boolean equals(Object other) { - if (!(other instanceof ConfigModelBuilder)) { + if (!(other instanceof ConfigModelBuilder<?> otherBuilder)) { return false; } - ConfigModelBuilder<?> otherBuilder = (ConfigModelBuilder<?>) other; List<ConfigModelId> thisIds = this.handlesElements(); List<ConfigModelId> otherIds = otherBuilder.handlesElements(); if (thisIds.size() != otherIds.size()) { diff --git a/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelId.java b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelId.java index cd283866550..e1970b001e1 100644 --- a/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelId.java +++ b/config-model/src/main/java/com/yahoo/config/model/builder/xml/ConfigModelId.java @@ -42,8 +42,7 @@ public class ConfigModelId implements Comparable<ConfigModelId> { @Override public boolean equals(Object object) { - if (!(object instanceof ConfigModelId)) return false; - ConfigModelId other = (ConfigModelId)object; + if (!(object instanceof ConfigModelId other)) return false; return this.name.equals(other.name) && this.version.equals(other.version); } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 1813e183a60..b8d63ba3778 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -24,6 +24,7 @@ import com.yahoo.config.model.api.ValidationParameters; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.provision.HostsXmlProvisioner; +import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.provision.SingleNodeProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.DockerImage; @@ -77,7 +78,7 @@ public class DeployState implements ConfigDefinitionStore { private final ModelContext.Properties properties; private final Version vespaVersion; private final Set<ContainerEndpoint> endpoints; - private final Zone zone; + private final Zone zone; // TODO: Zone is set separately both here and in properties private final QueryProfiles queryProfiles; private final SemanticRules semanticRules; private final ImportedMlModels importedModels; diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 49194a5d1bb..ecbb990f096 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -137,6 +137,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean useRestrictedDataPlaneBindings() { return useRestrictedDataPlaneBindings; } @Override public Optional<CloudAccount> cloudAccount() { return cloudAccount; } @Override public boolean allowUserFilters() { return allowUserFilters; } + @Override public boolean enableGlobalPhase() { return true; } // Enable global-phase by default for unit tests only public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) { this.sharedStringRepoNoReclaim = sharedStringRepoNoReclaim; diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/AnyConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/AnyConfigProducer.java index cd21fccd855..547e81354eb 100644 --- a/config-model/src/main/java/com/yahoo/config/model/producer/AnyConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/producer/AnyConfigProducer.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.model.producer; -import com.yahoo.api.annotations.Beta; import com.yahoo.config.ConfigInstance; import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.deploy.DeployState; @@ -16,11 +15,8 @@ import com.yahoo.vespa.model.HostSystem; import com.yahoo.vespa.model.Service; import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.admin.monitoring.Monitoring; - import java.io.Serializable; -import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; diff --git a/config-model/src/main/java/com/yahoo/config/model/producer/TreeConfigProducer.java b/config-model/src/main/java/com/yahoo/config/model/producer/TreeConfigProducer.java index 012bffaf7f6..30f9cd202ff 100644 --- a/config-model/src/main/java/com/yahoo/config/model/producer/TreeConfigProducer.java +++ b/config-model/src/main/java/com/yahoo/config/model/producer/TreeConfigProducer.java @@ -3,19 +3,14 @@ package com.yahoo.config.model.producer; import com.yahoo.api.annotations.Beta; import com.yahoo.config.model.ApplicationConfigProducerRoot; -import com.yahoo.vespa.model.ConfigProducer; import com.yahoo.vespa.model.Service; import com.yahoo.vespa.model.SimpleConfigProducer; import com.yahoo.vespa.model.utils.FreezableMap; -import java.io.PrintStream; -import java.io.Serializable; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; /** * Superclass for all producers with children. diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java index b351073bd25..5ea22ee4d25 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/Hosts.java @@ -36,9 +36,6 @@ public class Hosts { for (Host host : hosts) hostsBuilder.put(host.hostname(), host); this.hosts = hostsBuilder.build(); - - // Don't limit zk connections on non-hosted systems - System.setProperty("zookeeper.vespa.clients", ""); } /** Throw IllegalArgumentException if host aliases breaks invariants */ diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index dd6087eefc7..41697e61bf2 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -129,6 +129,8 @@ public class InMemoryProvisioner implements HostProvisioner { this.retiredHostNames = Set.of(retiredHostNames); } + public Provisioned provisioned() { return provisioned; } + /** May affect e.g. the number of nodes/cluster. */ public InMemoryProvisioner setEnvironment(Environment environment) { this.environment = environment; diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index 9ee279c68d3..3b715c63105 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -6,6 +6,7 @@ import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.ComponentInfo; +import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; @@ -57,12 +58,14 @@ public class MockApplicationPackage implements ApplicationPackage { private final List<String> schemas; private final Map<Path, MockApplicationFile> files; private final String schemaDir; - private final Optional<String> deploymentSpec; + private final Optional<String> deploymentSpecString; private final Optional<String> validationOverrides; private final boolean failOnValidateXml; private final QueryProfileRegistry queryProfileRegistry; private final ApplicationMetaData applicationMetaData; + private DeploymentSpec deploymentSpec = null; + protected MockApplicationPackage(File root, String hosts, String services, List<String> schemas, Map<Path, MockApplicationFile> files, String schemaDir, @@ -74,7 +77,7 @@ public class MockApplicationPackage implements ApplicationPackage { this.schemas = schemas; this.files = files; this.schemaDir = schemaDir; - this.deploymentSpec = Optional.ofNullable(deploymentSpec); + this.deploymentSpecString = Optional.ofNullable(deploymentSpec); this.validationOverrides = Optional.ofNullable(validationOverrides); this.failOnValidateXml = failOnValidateXml; queryProfileRegistry = new QueryProfileXMLReader().read(asNamedReaderList(queryProfileType), @@ -102,6 +105,12 @@ public class MockApplicationPackage implements ApplicationPackage { } @Override + public DeploymentSpec getDeploymentSpec() { + if (deploymentSpec != null) return deploymentSpec; + return deploymentSpec = parseDeploymentSpec(false); + } + + @Override public Reader getHosts() { if (hostsS == null) return null; return new StringReader(hostsS); @@ -183,7 +192,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public Optional<Reader> getDeployment() { - return deploymentSpec.map(StringReader::new); + return deploymentSpecString.map(StringReader::new); } @Override @@ -215,13 +224,6 @@ public class MockApplicationPackage implements ApplicationPackage { return new MockApplicationPackage.Builder().withHosts(emptyHosts).withServices(emptyServices).build(); } - public static ApplicationPackage fromSearchDefinitionDirectory(String dir) { - return new MockApplicationPackage.Builder() - .withEmptyHosts() - .withEmptyServices() - .withSchemaDir(dir).build(); - } - // TODO: It might work to just merge this and the above public static ApplicationPackage fromSearchDefinitionAndRootDirectory(String dir) { return new MockApplicationPackage.Builder() diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java index 7b2aaa32136..365434b9de5 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockRoot.java @@ -7,7 +7,6 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ConfigModelRepo; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AnyConfigProducer; -import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.vespa.model.ConfigProducer; import com.yahoo.vespa.model.HostSystem; diff --git a/config-model/src/main/java/com/yahoo/config/model/test/ModelBuilderAddingAccessControlFilter.java b/config-model/src/main/java/com/yahoo/config/model/test/ModelBuilderAddingAccessControlFilter.java index 840d781ac9c..234aecc6228 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/ModelBuilderAddingAccessControlFilter.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/ModelBuilderAddingAccessControlFilter.java @@ -45,8 +45,7 @@ public class ModelBuilderAddingAccessControlFilter } private static void addFilterToContainerCluster(ContainerModel containerModel) { - if (!(containerModel.getCluster() instanceof ApplicationContainerCluster)) return; - ApplicationContainerCluster cluster = (ApplicationContainerCluster) containerModel.getCluster(); + if (!(containerModel.getCluster() instanceof ApplicationContainerCluster cluster)) return; Http http = cluster.getHttp(); if (http.getAccessControl().isPresent()) { Chain<Filter> chain = http.getFilterChains() diff --git a/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java b/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java index fd98d21dcd5..7aca60bb930 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/TestDriver.java @@ -21,7 +21,6 @@ import java.util.List; * xml string and returns a config producer that can be use to test getConfig. * * @author Ulf Lilleengen - * @since 5.1.20 */ @Beta public class TestDriver { diff --git a/config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java b/config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java index c1fd8e4646d..f243c4635c7 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/TestRoot.java @@ -13,7 +13,6 @@ import java.util.List; * Test utility class that provides many methods for inspecting the state of a completely built model * * @author Ulf Lilleengen - * @since 5.1 */ @Beta public class TestRoot { diff --git a/config-model/src/main/java/com/yahoo/config/model/test/TestUtil.java b/config-model/src/main/java/com/yahoo/config/model/test/TestUtil.java index c05d7bf4942..24c418a9d2f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/TestUtil.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/TestUtil.java @@ -4,8 +4,6 @@ package com.yahoo.config.model.test; import com.yahoo.collections.CollectionUtil; import com.yahoo.config.model.builder.xml.XmlHelper; import org.w3c.dom.Element; -import org.xml.sax.InputSource; - import java.io.StringReader; import java.util.ArrayList; import java.util.Arrays; @@ -35,7 +33,4 @@ public class TestUtil { return String.join("\n", lines); } - private static InputSource inputSource(String str) { - return new InputSource(new StringReader(str)); - } } diff --git a/config-model/src/main/java/com/yahoo/schema/OnnxModel.java b/config-model/src/main/java/com/yahoo/schema/OnnxModel.java index ae6f1fd96e4..6baaea6ea05 100644 --- a/config-model/src/main/java/com/yahoo/schema/OnnxModel.java +++ b/config-model/src/main/java/com/yahoo/schema/OnnxModel.java @@ -3,6 +3,7 @@ package com.yahoo.schema; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.model.ml.OnnxModelInfo; +import com.yahoo.searchlib.rankingexpression.Reference; import java.util.Collections; import java.util.HashMap; @@ -44,11 +45,37 @@ public class OnnxModel extends DistributableResource { addInputNameMapping(onnxName, vespaName, true); } + private String validateInputSource(String source) { + var optRef = Reference.simple(source); + if (optRef.isPresent()) { + Reference ref = optRef.get(); + // input can be one of: + // attribute(foo), query(foo), constant(foo) + if (FeatureNames.isSimpleFeature(ref)) { + return ref.toString(); + } + // or a function (evaluated by backend) + if (ref.isSimple() && "rankingExpression".equals(ref.name())) { + var arg = ref.simpleArgument(); + if (arg.isPresent()) { + return ref.toString(); + } + } + } else { + // otherwise it must be an identifier + Reference ref = Reference.fromIdentifier(source); + return ref.toString(); + } + // invalid input source + throw new IllegalArgumentException("invalid input for ONNX model " + getName() + ": " + source); + } + public void addInputNameMapping(String onnxName, String vespaName, boolean overwrite) { Objects.requireNonNull(onnxName, "Onnx name cannot be null"); Objects.requireNonNull(vespaName, "Vespa name cannot be null"); + String source = validateInputSource(vespaName); if (overwrite || ! inputMap.containsKey(onnxName)) { - inputMap.put(onnxName, vespaName); + inputMap.put(onnxName, source); } } @@ -59,8 +86,10 @@ public class OnnxModel extends DistributableResource { public void addOutputNameMapping(String onnxName, String vespaName, boolean overwrite) { Objects.requireNonNull(onnxName, "Onnx name cannot be null"); Objects.requireNonNull(vespaName, "Vespa name cannot be null"); + // output name must be a valid identifier: + var ref = Reference.fromIdentifier(vespaName); if (overwrite || ! outputMap.containsKey(onnxName)) { - outputMap.put(onnxName, vespaName); + outputMap.put(onnxName, ref.toString()); } } diff --git a/config-model/src/main/java/com/yahoo/schema/RankProfile.java b/config-model/src/main/java/com/yahoo/schema/RankProfile.java index ad6eb038058..7cb0a088f5f 100644 --- a/config-model/src/main/java/com/yahoo/schema/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/RankProfile.java @@ -1019,6 +1019,9 @@ public class RankProfile implements Cloneable { var recorder = new InputRecorder(needInputs); recorder.transform(globalPhaseRanking.function().getBody(), context); for (String input : needInputs) { + if (input.startsWith("constant(") || input.startsWith("query(")) { + continue; + } try { addMatchFeatures(new FeatureList(input)); } catch (com.yahoo.searchlib.rankingexpression.parser.ParseException e) { diff --git a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java index b0f63ebb732..4e7988a2006 100644 --- a/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/InputRecorder.java @@ -3,16 +3,13 @@ package com.yahoo.schema.expressiontransforms; import com.yahoo.schema.FeatureNames; import com.yahoo.schema.RankProfile; -import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.Reference; -import com.yahoo.searchlib.rankingexpression.parser.ParseException; import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.searchlib.rankingexpression.transform.ExpressionTransformer; -import java.io.StringReader; import java.util.Set; /** @@ -86,13 +83,7 @@ public class InputRecorder extends ExpressionTransformer<RankProfileTransformCon throw new IllegalArgumentException("missing onnx model: " + arg); } for (String onnxInput : model.getInputMap().values()) { - var reader = new StringReader(onnxInput); - try { - var asExpression = new RankingExpression(reader); - transform(asExpression.getRoot(), context); - } catch (ParseException e) { - throw new IllegalArgumentException("illegal onnx input '" + onnxInput + "': " + e.getMessage()); - } + neededInputs.add(onnxInput); } return; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Host.java b/config-model/src/main/java/com/yahoo/vespa/model/Host.java index 7dbab87fac0..047a6ef9bd5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/Host.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/Host.java @@ -34,8 +34,6 @@ public final class Host extends TreeConfigProducer<AnyConfigProducer> implements Objects.requireNonNull(hostname, "The host name of a host cannot be null"); this.runsConfigServer = runsConfigServer; this.hostname = hostname; - if (parent instanceof HostSystem) - ((HostSystem)parent).checkName(hostname); } public static Host createConfigServerHost(HostSystem hostSystem, String hostname) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java b/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java index 125966ae91d..2f704db1862 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostPorts.java @@ -22,11 +22,7 @@ public class HostPorts { public final static int BASE_PORT = 19100; final static int MAX_PORTS = 799; - private DeployLogger deployLogger = new DeployLogger() { - public void log(Level level, String message) { - System.err.println("deploy log["+level+"]: "+message); - } - }; + private DeployLogger deployLogger = (level, message) -> System.err.println("deploy log["+level+"]: "+message); private final Map<Integer, NetworkPortRequestor> portDB = new LinkedHashMap<>(); @@ -98,7 +94,7 @@ public class HostPorts { /** Allocate a specific port number for a service */ public int requireNetworkPort(int port, NetworkPortRequestor service, String suffix) { - reservePort(service, port, suffix); + reservePort(service, port); String servType = service.getServiceType(); String configId = service.getConfigId(); portFinder.use(new NetworkPorts.Allocation(port, servType, configId, suffix)); @@ -119,18 +115,13 @@ public class HostPorts { return requireNetworkPort(port, service, suffix); } - /** Convenience method to allocate a preferred or required port number for a service */ - public int wantNetworkPort(int port, NetworkPortRequestor service, String suffix, boolean forceRequired) { - return forceRequired ? requireNetworkPort(port, service, suffix) : wantNetworkPort(port, service, suffix); - } - /** Allocate a dynamic port number for a service */ public int allocateNetworkPort(NetworkPortRequestor service, String suffix) { String servType = service.getServiceType(); String configId = service.getConfigId(); int fallback = nextAvailableNetworkPort(); int port = portFinder.findPort(new NetworkPorts.Allocation(fallback, servType, configId, suffix), hostname); - reservePort(service, port, suffix); + reservePort(service, port); portFinder.use(new NetworkPorts.Allocation(port, servType, configId, suffix)); return port; } @@ -161,7 +152,7 @@ public class HostPorts { * @param service the service that wishes to reserve the port. * @param port the port to be reserved. */ - void reservePort(NetworkPortRequestor service, int port, String suffix) { + void reservePort(NetworkPortRequestor service, int port) { if (portDB.containsKey(port)) { portAlreadyReserved(service, port); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java index 53e2ce0e652..00a1078b294 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostSystem.java @@ -10,6 +10,7 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.ProvisionLogger; + import java.net.UnknownHostException; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -53,10 +54,11 @@ public class HostSystem extends TreeConfigProducer<Host> { this.isHosted = isHosted; } - void checkName(String hostname) { + String checkHostname(String hostname) { + if (isHosted) return hostname; // Done in node-repo instead + if (doCheckIp) { - // Bad DNS config in a hosted system isn't actionable by the tenant, so we log any warnings internally - BiConsumer<Level, String> logFunction = isHosted ? deployLogger::log : deployLogger::logApplicationPackage; + BiConsumer<Level, String> logFunction = deployLogger::logApplicationPackage; // Give a warning if the host does not exist try { var inetAddr = java.net.InetAddress.getByName(hostname); @@ -69,6 +71,7 @@ public class HostSystem extends TreeConfigProducer<Host> { logFunction.accept(Level.WARNING, "Unable to lookup IP address of host: " + hostname); } } + return hostname; } @Override @@ -86,10 +89,10 @@ public class HostSystem extends TreeConfigProducer<Host> { } private HostResource addNewHost(HostSpec hostSpec) { - Host host = Host.createHost(this, hostSpec.hostname()); - HostResource hostResource = new HostResource(host, hostSpec); + String hostname = checkHostname(hostSpec.hostname()); + HostResource hostResource = new HostResource(Host.createHost(this, hostname), hostSpec); hostSpec.networkPorts().ifPresent(np -> hostResource.ports().addNetworkPorts(np)); - hostname2host.put(host.getHostname(), hostResource); + hostname2host.put(hostname, hostResource); return hostResource; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 4e40bd768bf..dd1579eae17 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -24,7 +24,6 @@ import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AnyConfigProducer; -import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; import com.yahoo.config.model.producer.UserConfigRepo; import com.yahoo.config.provision.AllocatedHosts; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index 12bce7a72a4..d69ddd1c5fd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -24,6 +24,7 @@ import com.yahoo.vespa.model.PortAllocBridge; import com.yahoo.vespa.model.container.Container; import com.yahoo.vespa.model.container.component.AccessLogComponent; +import java.time.Duration; import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; @@ -196,4 +197,6 @@ public class MetricsProxyContainer extends Container implements return ""; } + @Override public Optional<String> getPreShutdownCommand() { return Optional.of(prepareStopCommand(Duration.ofMinutes(6))); } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java index 925f390aad2..d6f0df8e051 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java @@ -92,6 +92,9 @@ public class VespaMetricSet { addMetric(metrics, "vds.server.network.server.insecure-connections-established"); addMetric(metrics, "vds.server.network.tls-connections-broken"); addMetric(metrics, "vds.server.network.failed-tls-config-reloads"); + // C++ capability metrics + addMetric(metrics, "vds.server.network.rpc-capability-checks-failed"); + addMetric(metrics, "vds.server.network.status-capability-checks-failed"); // C++ Fnet metrics addMetric(metrics, "vds.server.fnet.num-connections"); @@ -334,6 +337,8 @@ public class VespaMetricSet { private static Set<Metric> getSearchNodeMetrics() { Set<Metric> metrics = new LinkedHashSet<>(); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_CONFIG_GENERATION.last()); + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_TOTAL.last()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_READY.last()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_ACTIVE.last()); @@ -409,19 +414,7 @@ public class VespaMetricSet { addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_ACCEPTED.rate()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_WAKEUPS.rate()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_UTILIZATION, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_INVERTER_QUEUESIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_INVERTER_ACCEPTED.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_INVERTER_WAKEUPS.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_INVERTER_UTILIZATION, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_WRITER_QUEUESIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_WRITER_ACCEPTED.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_WRITER_WAKEUPS.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_WRITER_UTILIZATION, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_ATTRIBUTE_FIELD_WRITER_QUEUESIZE, EnumSet.of(max, sum, count)); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_ATTRIBUTE_FIELD_WRITER_ACCEPTED.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_ATTRIBUTE_FIELD_WRITER_WAKEUPS.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_ATTRIBUTE_FIELD_WRITER_UTILIZATION, EnumSet.of(max, sum, count)); - + // lid space addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_LID_BLOAT_FACTOR.average()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_LID_FRAGMENTATION_FACTOR.average()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java index 198f6b88798..04faff688f8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java @@ -214,7 +214,10 @@ public class RankSetupValidator extends Validator { if (line.startsWith("debug\t")) continue; try { LogMessage logMessage = LogMessage.parseNativeFormat(line); - message.append(logMessage.getLevel()).append(": ").append(logMessage.getPayload()).append("\n"); + message.append(logMessage.getLevel()) + .append(": ") + .append(logMessage.getPayload().replace("\\n", "\n\t")) + .append("\n"); } catch (InvalidLogFormatException e) { message.append(line).append("\n"); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java index b9ed8a3c97c..36ebbe41637 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/UserConfigBuilder.java @@ -38,7 +38,7 @@ public class UserConfigBuilder { ConfigDefinitionKey key = DomConfigPayloadBuilder.parseConfigName(element); Optional<ConfigDefinition> def = configDefinitionStore.getConfigDefinition(key); - if ( ! def.isPresent()) { // TODO: Fail instead of warn + if (def.isEmpty()) { // TODO: Fail instead of warn logger.logApplicationPackage(Level.WARNING, "Unable to find config definition '" + key.asFileName() + "'. Please ensure that the name is spelled correctly, and that the def file is included in a bundle."); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/VespaModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/VespaModelBuilder.java index 3ad81ef9f57..c6844619457 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/VespaModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/VespaModelBuilder.java @@ -1,12 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder; -import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.ConfigModelRepo; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AnyConfigProducer; import com.yahoo.config.model.producer.TreeConfigProducer; -import com.yahoo.config.model.ApplicationConfigProducerRoot; /** * Base class for classes capable of building vespa model. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index 567ccbfa88b..80000e54b1b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -123,7 +123,8 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { ClusterSpec.Type.admin, ClusterSpec.Id.from(clusterId), context.getDeployLogger(), - false) + false, + context.clusterInfo().build()) .keySet(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java deleted file mode 100644 index 3491f219a8e..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomClientProviderBuilder.java +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.builder.xml.dom; - -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.config.model.producer.AnyConfigProducer; -import com.yahoo.config.model.producer.TreeConfigProducer; -import com.yahoo.text.XML; -import com.yahoo.vespa.model.container.ApplicationContainerCluster; -import com.yahoo.vespa.model.container.component.Handler; -import com.yahoo.vespa.model.container.component.UserBindingPattern; -import org.w3c.dom.Element; - -/** - * @author gjoranv - * @since 5.1.6 - */ -public class DomClientProviderBuilder extends DomHandlerBuilder { - - public DomClientProviderBuilder(ApplicationContainerCluster cluster) { - super(cluster); - } - - @Override - protected Handler doBuild(DeployState deployState, TreeConfigProducer<AnyConfigProducer> parent, Element clientElement) { - Handler client = createHandler(clientElement); - - for (Element binding : XML.getChildren(clientElement, "binding")) - client.addClientBindings(UserBindingPattern.fromPattern(XML.getValue(binding))); - - for (Element serverBinding : XML.getChildren(clientElement, "serverBinding")) - client.addServerBindings(UserBindingPattern.fromPattern(XML.getValue(serverBinding))); - - DomComponentBuilder.addChildren(deployState, parent, clientElement, client); - - return client; - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java index 5d17619b526..70bb80ec314 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomConfigPayloadBuilder.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.collections.Tuple2; -import com.yahoo.config.ConfigurationRuntimeException; import com.yahoo.config.FileReference; import com.yahoo.config.ModelReference; import com.yahoo.config.UrlReference; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java index 3cf8ec7375f..ed53a1d2267 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomHandlerBuilder.java @@ -38,7 +38,7 @@ public class DomHandlerBuilder extends VespaDomBuilder.DomConfigProducerBuilderB VIP_HANDLER_BINDING); private final ApplicationContainerCluster cluster; - private OptionalInt portBindingOverride; + private final OptionalInt portBindingOverride; public DomHandlerBuilder(ApplicationContainerCluster cluster) { this(cluster, OptionalInt.empty()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomRoutingBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomRoutingBuilder.java index 92b24f3f7ac..14a5b13d8d2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomRoutingBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomRoutingBuilder.java @@ -2,18 +2,20 @@ package com.yahoo.vespa.model.builder.xml.dom; import com.yahoo.config.application.Xml; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.model.builder.xml.ConfigModelBuilder; import com.yahoo.config.model.builder.xml.ConfigModelId; -import com.yahoo.messagebus.routing.*; +import com.yahoo.messagebus.routing.ApplicationSpec; +import com.yahoo.messagebus.routing.HopSpec; +import com.yahoo.messagebus.routing.RouteSpec; +import com.yahoo.messagebus.routing.RoutingSpec; +import com.yahoo.messagebus.routing.RoutingTableSpec; import com.yahoo.text.XML; -import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.vespa.model.routing.Routing; import org.w3c.dom.Element; import org.w3c.dom.Node; import org.w3c.dom.NodeList; - -import java.util.Arrays; import java.util.List; /** @@ -29,7 +31,7 @@ public class DomRoutingBuilder extends ConfigModelBuilder<Routing> { @Override public List<ConfigModelId> handlesElements() { - return Arrays.asList(ConfigModelId.fromName("routing")); + return List.of(ConfigModelId.fromName("routing")); } // Overrides ConfigModelBuilder. @@ -71,7 +73,7 @@ public class DomRoutingBuilder extends ConfigModelBuilder<Routing> { * @param element The element to base the route config on. */ private static void addRoutingTable(RoutingSpec routing, Element element) { - boolean verify = element.hasAttribute("verify") ? Boolean.valueOf(element.getAttribute("verify")) : true; + boolean verify = shouldVerify(element); RoutingTableSpec table = new RoutingTableSpec(element.getAttribute("protocol"), verify); NodeList children = element.getChildNodes(); @@ -94,7 +96,7 @@ public class DomRoutingBuilder extends ConfigModelBuilder<Routing> { * @return The corresponding route spec. */ private static RouteSpec createRouteSpec(Element element) { - boolean verify = element.hasAttribute("verify") ? Boolean.valueOf(element.getAttribute("verify")) : true; + boolean verify = shouldVerify(element); RouteSpec route = new RouteSpec(element.getAttribute("name"), verify); String hops = element.getAttribute("hops"); int from = 0; @@ -123,9 +125,9 @@ public class DomRoutingBuilder extends ConfigModelBuilder<Routing> { * @return The corresponding hop spec. */ private static HopSpec createHopSpec(Element element) { - boolean verify = element.hasAttribute("verify") ? Boolean.valueOf(element.getAttribute("verify")) : true; + boolean verify = shouldVerify(element); HopSpec hop = new HopSpec(element.getAttribute("name"), element.getAttribute("selector"), verify); - if (Boolean.valueOf(element.getAttribute("ignore-result"))) { + if (Boolean.parseBoolean(element.getAttribute("ignore-result"))) { hop.setIgnoreResult(true); } NodeList children = element.getElementsByTagName("recipient"); @@ -135,4 +137,8 @@ public class DomRoutingBuilder extends ConfigModelBuilder<Routing> { } return hop; } + + private static boolean shouldVerify(Element element) { + return !element.hasAttribute("verify") || Boolean.parseBoolean(element.getAttribute("verify")); + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java index b5fa451fa0b..c968e31325a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.builder.xml.dom; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.collections.Pair; import com.yahoo.component.Version; @@ -266,8 +267,9 @@ public class NodesSpecification { ClusterSpec.Type clusterType, ClusterSpec.Id clusterId, DeployLogger logger, - boolean stateful) { - return provision(hostSystem, clusterType, clusterId, ZoneEndpoint.defaultEndpoint, logger, stateful); + boolean stateful, + ClusterInfo clusterInfo) { + return provision(hostSystem, clusterType, clusterId, ZoneEndpoint.defaultEndpoint, logger, stateful, clusterInfo); } public Map<HostResource, ClusterMembership> provision(HostSystem hostSystem, @@ -275,7 +277,8 @@ public class NodesSpecification { ClusterSpec.Id clusterId, ZoneEndpoint zoneEndpoint, DeployLogger logger, - boolean stateful) { + boolean stateful, + ClusterInfo info) { if (combinedId.isPresent()) clusterType = ClusterSpec.Type.combined; ClusterSpec cluster = ClusterSpec.request(clusterType, clusterId) @@ -286,7 +289,7 @@ public class NodesSpecification { .loadBalancerSettings(zoneEndpoint) .stateful(stateful) .build(); - return hostSystem.allocateHosts(cluster, Capacity.from(min, max, groupSize, required, canFail, cloudAccount), logger); + return hostSystem.allocateHosts(cluster, Capacity.from(min, max, groupSize, required, canFail, cloudAccount, info), logger); } private static Pair<NodeResources, NodeResources> nodeResources(ModelElement nodesElement) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java index 8398df6f5ac..07e879f0e9c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/VespaDomBuilder.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.model.builder.xml.dom; import ai.vespa.validation.Validation; -import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.ConfigModelRepo; import com.yahoo.config.model.builder.xml.XmlHelper; @@ -24,12 +23,8 @@ import com.yahoo.vespa.model.container.docproc.ContainerDocproc; import com.yahoo.vespa.model.content.Content; import com.yahoo.vespa.model.search.SearchCluster; import org.w3c.dom.Element; - -import java.util.HashSet; import java.util.LinkedHashMap; -import java.util.LinkedHashSet; import java.util.Map; -import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Pattern; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index 784902e2427..bc410670d5e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -12,11 +12,10 @@ import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.LogctlSpec; import com.yahoo.vespa.model.container.component.SimpleComponent; +import java.time.Duration; import java.util.List; import java.util.Optional; -import static com.yahoo.vespa.defaults.Defaults.getDefaults; - /** * A container that is typically used by container clusters set up from the user application. * @@ -95,11 +94,6 @@ public final class ApplicationContainer extends Container implements return featureFlags.jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type.container); } - @Override - public Optional<String> getPreShutdownCommand() { - int preshutdownTimeoutSeconds = 360; - int rpcTimeoutSeconds = preshutdownTimeoutSeconds + 10; - String rpcParams = "-t " + rpcTimeoutSeconds + " tcp/localhost:" + getRpcPort() + " prepareStop d:" + preshutdownTimeoutSeconds; - return Optional.of(getDefaults().underVespaHome("bin/vespa-rpc-invoke") + " " + rpcParams); - } + @Override public Optional<String> getPreShutdownCommand() { return Optional.of(prepareStopCommand(Duration.ofMinutes(6))); } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index 6a6f4583eae..be9f3fa894f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -29,6 +29,7 @@ import com.yahoo.vespa.model.container.http.Http; import com.yahoo.vespa.model.container.http.JettyHttpServer; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; +import java.time.Duration; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -38,6 +39,7 @@ import java.util.Optional; import static com.yahoo.container.QrConfig.Filedistributor; import static com.yahoo.container.QrConfig.Rpc; +import static com.yahoo.vespa.defaults.Defaults.getDefaults; /** * Note about components: In general, all components should belong to the cluster and not the container. However, @@ -387,6 +389,12 @@ public abstract class Container extends AbstractService implements return dimensions; } + protected String prepareStopCommand(Duration timeout) { + long rpcTimeoutSeconds = timeout.toSeconds() + 10; + String rpcParams = "-t " + rpcTimeoutSeconds + " tcp/localhost:" + getRpcPort() + " prepareStop d:" + timeout.toSeconds(); + return getDefaults().underVespaHome("bin/vespa-rpc-invoke") + " " + rpcParams; + } + private boolean messageBusEnabled() { return containerCluster().isPresent() && containerCluster().get().messageBusEnabled(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java index 49292bd6df7..0be3c825614 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java @@ -26,9 +26,9 @@ public class ContainerModelEvaluation implements OnnxModelsConfig.Producer, RankingExpressionsConfig.Producer { - private final static String EVALUATION_BUNDLE_NAME = "model-evaluation"; - private final static String INTEGRATION_BUNDLE_NAME = "model-integration"; - private final static String ONNXRUNTIME_BUNDLE_NAME = "container-onnxruntime.jar"; + public final static String EVALUATION_BUNDLE_NAME = "model-evaluation"; + public final static String INTEGRATION_BUNDLE_NAME = "model-integration"; + public final static String ONNXRUNTIME_BUNDLE_NAME = "container-onnxruntime.jar"; private final static String EVALUATOR_NAME = ModelsEvaluator.class.getName(); private final static String REST_HANDLER_NAME = "ai.vespa.models.handler.ModelsEvaluationHandler"; @@ -44,6 +44,7 @@ public class ContainerModelEvaluation implements public ContainerModelEvaluation(ApplicationContainerCluster cluster, RankProfileList rankProfileList) { this.rankProfileList = Objects.requireNonNull(rankProfileList, "rankProfileList cannot be null"); cluster.addSimpleComponent(EVALUATOR_NAME, null, EVALUATION_BUNDLE_NAME); + cluster.addSimpleComponent("ai.vespa.modelintegration.evaluator.OnnxRuntime", null, INTEGRATION_BUNDLE_NAME); cluster.addComponent(ContainerModelEvaluation.getHandler()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java index 29b1edc1397..19df9a4064f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/PlatformBundles.java @@ -10,6 +10,10 @@ import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; +import static com.yahoo.vespa.model.container.ContainerModelEvaluation.EVALUATION_BUNDLE_NAME; +import static com.yahoo.vespa.model.container.ContainerModelEvaluation.INTEGRATION_BUNDLE_NAME; +import static com.yahoo.vespa.model.container.ContainerModelEvaluation.ONNXRUNTIME_BUNDLE_NAME; + /** * NOTE: Stable ordering of bundles in config is handled by {@link ContainerCluster#addPlatformBundle(Path)} * @@ -53,7 +57,10 @@ public class PlatformBundles { public static final Set<Path> SEARCH_AND_DOCPROC_BUNDLES = toBundlePaths( SEARCH_AND_DOCPROC_BUNDLE, "docprocs", - "linguistics-components" + "linguistics-components", + EVALUATION_BUNDLE_NAME, + INTEGRATION_BUNDLE_NAME, + ONNXRUNTIME_BUNDLE_NAME ); private static Set<Path> toBundlePaths(String... bundleNames) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index 86c48407775..5d4ec598250 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -1,12 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.search; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.container.QrSearchersConfig; import com.yahoo.prelude.semantics.SemanticRulesConfig; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.search.pagetemplates.PageTemplatesConfig; import com.yahoo.search.query.profile.config.QueryProfilesConfig; +import com.yahoo.search.ranking.RankProfilesEvaluatorFactory; import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; @@ -44,18 +46,22 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> private final ApplicationContainerCluster owningCluster; private final List<SearchCluster> searchClusters = new LinkedList<>(); + private final boolean globalPhase; private QueryProfiles queryProfiles; private SemanticRules semanticRules; private PageTemplates pageTemplates; - public ContainerSearch(ApplicationContainerCluster cluster, SearchChains chains) { + public ContainerSearch(DeployState deployState, ApplicationContainerCluster cluster, SearchChains chains) { super(chains); + this.globalPhase = deployState.featureFlags().enableGlobalPhase(); this.owningCluster = cluster; owningCluster.addComponent(Component.fromClassAndBundle(CompiledQueryProfileRegistry.class, SEARCH_AND_DOCPROC_BUNDLE)); owningCluster.addComponent(Component.fromClassAndBundle(com.yahoo.search.schema.SchemaInfo.class, SEARCH_AND_DOCPROC_BUNDLE)); owningCluster.addComponent(Component.fromClassAndBundle(SearchStatusExtension.class, SEARCH_AND_DOCPROC_BUNDLE)); + owningCluster.addComponent(Component.fromClassAndBundle(RankProfilesEvaluatorFactory.class, SEARCH_AND_DOCPROC_BUNDLE)); + owningCluster.addComponent(Component.fromClassAndBundle(com.yahoo.search.ranking.GlobalPhaseRanker.class, SEARCH_AND_DOCPROC_BUNDLE)); cluster.addSearchAndDocprocBundles(); } @@ -68,9 +74,18 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> /** Adds a Dispatcher component to the owning container cluster for each search cluster */ private void initializeDispatchers(Collection<SearchCluster> searchClusters) { for (SearchCluster searchCluster : searchClusters) { - if ( ! ( searchCluster instanceof IndexedSearchCluster)) continue; - var dispatcher = new DispatcherComponent((IndexedSearchCluster)searchCluster); - owningCluster.addComponent(dispatcher); + if (searchCluster instanceof IndexedSearchCluster indexed) { + var dispatcher = new DispatcherComponent(indexed); + owningCluster.addComponent(dispatcher); + if (globalPhase) { + for (var documentDb : indexed.getDocumentDbs()) { + var factory = new RankProfilesEvaluatorComponent(documentDb); + if (! owningCluster.getComponentsMap().containsKey(factory.getComponentId())) { + owningCluster.addComponent(factory); + } + } + } + } } } @@ -143,6 +158,7 @@ public class ContainerSearch extends ContainerSubsystem<SearchChains> if ( ! (sys instanceof IndexedSearchCluster)) { scB.storagecluster(new QrSearchersConfig.Searchcluster.Storagecluster.Builder(). routespec(((StreamingSearchCluster)sys).getStorageRouteSpec())); + scB.globalphase(globalPhase); } builder.searchcluster(scB); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/RankProfilesEvaluatorComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RankProfilesEvaluatorComponent.java new file mode 100644 index 00000000000..75a2802ee53 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/RankProfilesEvaluatorComponent.java @@ -0,0 +1,49 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.container.search; + +import com.yahoo.config.model.producer.AnyConfigProducer; +import com.yahoo.osgi.provider.model.ComponentModel; +import com.yahoo.search.ranking.RankProfilesEvaluator; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; +import com.yahoo.vespa.model.container.ContainerModelEvaluation; +import com.yahoo.vespa.model.container.PlatformBundles; +import com.yahoo.vespa.model.container.component.Component; +import com.yahoo.vespa.model.search.DocumentDatabase; + +public class RankProfilesEvaluatorComponent + extends Component<AnyConfigProducer, ComponentModel> + implements + RankProfilesConfig.Producer, + RankingConstantsConfig.Producer, + RankingExpressionsConfig.Producer, + OnnxModelsConfig.Producer +{ + private final DocumentDatabase ddb; + + public RankProfilesEvaluatorComponent(DocumentDatabase db) { + super(toComponentModel(db.getSchemaName())); + ddb = db; + } + + private static ComponentModel toComponentModel(String p) { + String myComponentId = "ranking-expression-evaluator." + p; + return new ComponentModel(myComponentId, + RankProfilesEvaluator.class.getName(), + PlatformBundles.SEARCH_AND_DOCPROC_BUNDLE); + } + + @Override + public void getConfig(RankProfilesConfig.Builder builder) { ddb.getConfig(builder); } + + @Override + public void getConfig(RankingExpressionsConfig.Builder builder) { ddb.getConfig(builder); } + + @Override + public void getConfig(RankingConstantsConfig.Builder builder) { ddb.getConfig(builder); } + + @Override + public void getConfig(OnnxModelsConfig.Builder builder) { ddb.getConfig(builder); } +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index ceba3864296..36d34b99223 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.xml; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.Version; @@ -345,10 +346,10 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private void addDeploymentSpecConfig(ApplicationContainerCluster cluster, ConfigModelContext context, DeployLogger deployLogger) { if ( ! context.getDeployState().isHosted()) return; - Optional<DeploymentSpec> deploymentSpec = app.getDeployment().map(DeploymentSpec::fromXml); + DeploymentSpec deploymentSpec = app.getDeploymentSpec(); if (deploymentSpec.isEmpty()) return; - for (var deprecatedElement : deploymentSpec.get().deprecatedElements()) { + for (var deprecatedElement : deploymentSpec.deprecatedElements()) { deployLogger.logApplicationPackage(WARNING, deprecatedElement.humanReadableString()); } @@ -358,8 +359,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { context.getDeployState().getProperties().ztsUrl(), context.getDeployState().getProperties().athenzDnsSuffix(), context.getDeployState().zone(), - deploymentSpec.get()); - addRotationProperties(cluster, context.getDeployState().zone(), context.getDeployState().getEndpoints(), deploymentSpec.get()); + deploymentSpec); + addRotationProperties(cluster, context.getDeployState().zone(), context.getDeployState().getEndpoints(), deploymentSpec); } private void addRotationProperties(ApplicationContainerCluster cluster, Zone zone, Set<ContainerEndpoint> endpoints, DeploymentSpec spec) { @@ -732,7 +733,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { SearchChains searchChains = new DomSearchChainsBuilder() .build(deployState, containerCluster, producerSpec); - ContainerSearch containerSearch = new ContainerSearch(containerCluster, searchChains); + ContainerSearch containerSearch = new ContainerSearch(deployState, containerCluster, searchChains); applyApplicationPackageDirectoryConfigs(deployState.getApplicationPackage(), containerSearch); containerSearch.setQueryProfiles(deployState.getQueryProfiles()); @@ -863,10 +864,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { InstanceName instance = context.properties().applicationId().instance(); ZoneId zone = ZoneId.from(context.properties().zone().environment(), context.properties().zone().region()); - DeploymentSpec spec = context.getApplicationPackage().getDeployment() - .map(new DeploymentSpecXmlReader(false)::read) - .orElse(DeploymentSpec.empty); - return spec.zoneEndpoint(instance, zone, cluster); + return context.getApplicationPackage().getDeploymentSpec().zoneEndpoint(instance, zone, cluster); } private static Map<String, String> getEnvironmentVariables(Element environmentVariables) { @@ -924,22 +922,15 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { HostSystem hostSystem = cluster.hostSystem(); if (deployState.isHosted()) { // request just enough nodes to satisfy environment capacity requirement - ClusterSpec clusterSpec = ClusterSpec.request(ClusterSpec.Type.container, - ClusterSpec.Id.from(cluster.getName())) - .vespaVersion(deployState.getWantedNodeVespaVersion()) - .dockerImageRepository(deployState.getWantedDockerImageRepo()) - .build(); int nodeCount = deployState.zone().environment().isProduction() ? 2 : 1; - deployState.getDeployLogger().logApplicationPackage(Level.INFO, "Using " + nodeCount + - " nodes in " + cluster); - ClusterResources resources = new ClusterResources(nodeCount, 1, NodeResources.unspecified()); - Capacity capacity = Capacity.from(resources, - resources, - IntRange.empty(), - false, - !deployState.getProperties().isBootstrap(), - context.getDeployState().getProperties().cloudAccount()); - var hosts = hostSystem.allocateHosts(clusterSpec, capacity, log); + deployState.getDeployLogger().logApplicationPackage(Level.INFO, "Using " + nodeCount + " nodes in " + cluster); + var nodesSpec = NodesSpecification.dedicated(nodeCount, context); + var hosts = nodesSpec.provision(hostSystem, + ClusterSpec.Type.container, + ClusterSpec.Id.from(cluster.getName()), + deployState.getDeployLogger(), + false, + context.clusterInfo().build()); return createNodesFromHosts(hosts, cluster, context.getDeployState()); } else { @@ -956,15 +947,15 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private List<ApplicationContainer> createNodesFromNodeCount(ApplicationContainerCluster cluster, Element containerElement, Element nodesElement, ConfigModelContext context) { try { - NodesSpecification nodesSpecification = NodesSpecification.from(new ModelElement(nodesElement), context); - ClusterSpec.Id clusterId = ClusterSpec.Id.from(cluster.name()); - ZoneEndpoint zoneEndpoint = zoneEndpoint(context, clusterId); + var nodesSpecification = NodesSpecification.from(new ModelElement(nodesElement), context); + var clusterId = ClusterSpec.Id.from(cluster.name()); Map<HostResource, ClusterMembership> hosts = nodesSpecification.provision(cluster.getRoot().hostSystem(), ClusterSpec.Type.container, clusterId, - zoneEndpoint, + zoneEndpoint(context, clusterId), log, - getZooKeeper(containerElement) != null); + getZooKeeper(containerElement) != null, + context.clusterInfo().build()); return createNodesFromHosts(hosts, cluster, context.getDeployState()); } catch (IllegalArgumentException e) { @@ -998,7 +989,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { StorageGroup.provisionHosts(nodeSpecification, referenceId, cluster.getRoot().hostSystem(), - context.getDeployLogger()); + context); return createNodesFromHosts(hosts, cluster, context.getDeployState()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java index 0abd7212017..76403d369dd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ModelIdResolver.java @@ -3,6 +3,9 @@ package com.yahoo.vespa.model.container.xml; import com.yahoo.text.XML; import org.w3c.dom.Element; + +import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.stream.Collectors; @@ -14,10 +17,22 @@ import java.util.stream.Collectors; */ public class ModelIdResolver { - private static final Map<String, String> providedModels = - Map.of("minilm-l6-v2", "https://data.vespa.oath.cloud/onnx_models/sentence_all_MiniLM_L6_v2.onnx", - "mpnet-base-v2", "https://data.vespa.oath.cloud/onnx_models/sentence-all-mpnet-base-v2.onnx", - "bert-base-uncased", "https://data.vespa.oath.cloud/onnx_models/bert-base-uncased-vocab.txt"); + private static Map<String, String> setupProvidedModels() { + Map<String, String> models = new HashMap<>(); + models.put("minilm-l6-v2", "https://data.vespa.oath.cloud/onnx_models/sentence_all_MiniLM_L6_v2.onnx"); + models.put("mpnet-base-v2", "https://data.vespa.oath.cloud/onnx_models/sentence-all-mpnet-base-v2.onnx"); + models.put("bert-base-uncased", "https://data.vespa.oath.cloud/onnx_models/bert-base-uncased-vocab.txt"); + models.put("flan-t5-vocab", "https://data.vespa.oath.cloud/onnx_models/flan-t5-spiece.model"); + models.put("flan-t5-small-encoder", "https://data.vespa.oath.cloud/onnx_models/flan-t5-small-encoder-model.onnx"); + models.put("flan-t5-small-decoder", "https://data.vespa.oath.cloud/onnx_models/flan-t5-small-decoder-model.onnx"); + models.put("flan-t5-base-encoder", "https://data.vespa.oath.cloud/onnx_models/flan-t5-base-encoder-model.onnx"); + models.put("flan-t5-base-decoder", "https://data.vespa.oath.cloud/onnx_models/flan-t5-base-decoder-model.onnx"); + models.put("flan-t5-large-encoder", "https://data.vespa.oath.cloud/onnx_models/flan-t5-large-encoder-model.onnx"); + models.put("flan-t5-large-decoder", "https://data.vespa.oath.cloud/onnx_models/flan-t5-large-decoder-model.onnx"); + return Collections.unmodifiableMap(models); + } + + private static final Map<String, String> providedModels = setupProvidedModels(); /** * Finds any config values of type 'model' below the given config element and diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java index 31ec764fbde..52b2ce06dfe 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/StorageGroup.java @@ -187,10 +187,15 @@ public class StorageGroup { public static Map<HostResource, ClusterMembership> provisionHosts(NodesSpecification nodesSpecification, String clusterIdString, - HostSystem hostSystem, - DeployLogger logger) { + HostSystem hostSystem, + ConfigModelContext context) { ClusterSpec.Id clusterId = ClusterSpec.Id.from(clusterIdString); - return nodesSpecification.provision(hostSystem, ClusterSpec.Type.content, clusterId, logger, true); + return nodesSpecification.provision(hostSystem, + ClusterSpec.Type.content, + clusterId, + context.getDeployLogger(), + true, + context.clusterInfo().build()); } public static class Builder { @@ -203,7 +208,9 @@ public class StorageGroup { this.context = context; } - public StorageGroup buildRootGroup(DeployState deployState, RedundancyBuilder redundancyBuilder, ContentCluster owner) { + public StorageGroup buildRootGroup(DeployState deployState, + RedundancyBuilder redundancyBuilder, + ContentCluster owner) { try { if (owner.isHosted()) validateRedundancyAndGroups(deployState.zone().environment()); @@ -219,7 +226,7 @@ public class StorageGroup { GroupBuilder groupBuilder = collectGroup(owner.isHosted(), group, nodes, null, null); StorageGroup storageGroup = owner.isHosted() - ? groupBuilder.buildHosted(deployState, owner, Optional.empty()) + ? groupBuilder.buildHosted(deployState, owner, Optional.empty(), context) : groupBuilder.buildNonHosted(deployState, owner, Optional.empty()); Redundancy redundancy = redundancyBuilder.build(owner.isHosted(), storageGroup.subgroups.size(), @@ -334,12 +341,18 @@ public class StorageGroup { * @param parent the parent storage group, or empty if this is the root group * @return the storage group build by this */ - public StorageGroup buildHosted(DeployState deployState, ContentCluster owner, Optional<GroupBuilder> parent) { + public StorageGroup buildHosted(DeployState deployState, + ContentCluster owner, + Optional<GroupBuilder> parent, + ConfigModelContext context) { if (storageGroup.getIndex() != null) throw new IllegalArgumentException("Specifying individual groups is not supported on hosted applications"); Map<HostResource, ClusterMembership> hostMapping = nodeRequirement.isPresent() ? - provisionHosts(nodeRequirement.get(), owner.getStorageCluster().getClusterName(), owner.getRoot().hostSystem(), deployState.getDeployLogger()) : + provisionHosts(nodeRequirement.get(), + owner.getStorageCluster().getClusterName(), + owner.getRoot().hostSystem(), + context) : Collections.emptyMap(); Map<Optional<ClusterSpec.Group>, Map<HostResource, ClusterMembership>> hostGroups = collectAllocatedSubgroups(hostMapping); @@ -362,7 +375,7 @@ public class StorageGroup { storageGroup.nodes.add(createStorageNode(deployState, owner, host.getKey(), storageGroup, host.getValue())); } for (GroupBuilder subGroup : subGroups) { - storageGroup.subgroups.add(subGroup.buildHosted(deployState, owner, Optional.of(this))); + storageGroup.subgroups.add(subGroup.buildHosted(deployState, owner, Optional.of(this), context)); } } return storageGroup; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 137e19e7d86..7f4fc4cd89d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -150,7 +150,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem if (e != null) setupDocumentProcessing(c, e); } else if (c.persistenceFactory != null) { - throw new IllegalArgumentException("The specified content engine requires the <documents> element to be specified."); + throw new IllegalArgumentException("The <documents> element is mandatory in content cluster '" + clusterId + "'"); } ModelElement tuning = contentElement.child("tuning"); @@ -333,7 +333,8 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem ClusterSpec.Type.admin, ClusterSpec.Id.from(clusterName), context.getDeployLogger(), - true) + true, + context.clusterInfo().build()) .keySet(); admin.setClusterControllers(createClusterControllers(new ClusterControllerCluster(admin, "standalone", deployState), hosts, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/utils/Duration.java b/config-model/src/main/java/com/yahoo/vespa/model/utils/Duration.java index 5bb6d8cf6bf..f65b7a62f05 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/utils/Duration.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/utils/Duration.java @@ -20,8 +20,9 @@ import java.util.regex.Pattern; * Default is seconds. */ public class Duration { - private static Pattern pattern = Pattern.compile("([0-9\\.]+)\\s*([a-z]+)?"); - private static Map<String, Integer> unitMultiplier = new HashMap<>(); + + private static final Pattern pattern = Pattern.compile("([0-9\\.]+)\\s*([a-z]+)?"); + private static final Map<String, Integer> unitMultiplier = new HashMap<>(); static { unitMultiplier.put("s", 1000); unitMultiplier.put("d", 1000 * 3600 * 24); diff --git a/config-model/src/main/resources/schema/deployment.rnc b/config-model/src/main/resources/schema/deployment.rnc index d63b8885a57..cc9db471120 100644 --- a/config-model/src/main/resources/schema/deployment.rnc +++ b/config-model/src/main/resources/schema/deployment.rnc @@ -22,6 +22,7 @@ StepExceptInstance = BlockChange* & Notifications? & Endpoints? & + Bcp? & Test? & Staging? & Dev? & @@ -170,3 +171,18 @@ Endpoint = element endpoint { Endpoints = element endpoints { Endpoint+ } + +Bcp = element bcp { + Group+ +} + +Group = element group { + attribute deadline { xsd:string }? & + Endpoint* & + MemberRegion+ +} + +MemberRegion = element region { + attribute fraction { xsd:double }? & + text +} diff --git a/config-model/src/test/derived/globalphase_onnx_inside/.gitignore b/config-model/src/test/derived/globalphase_onnx_inside/.gitignore new file mode 100644 index 00000000000..d9609d7b326 --- /dev/null +++ b/config-model/src/test/derived/globalphase_onnx_inside/.gitignore @@ -0,0 +1 @@ +models.generated diff --git a/config-model/src/test/derived/globalphase_onnx_inside/files/ax_plus_b.onnx b/config-model/src/test/derived/globalphase_onnx_inside/files/ax_plus_b.onnx new file mode 100644 index 00000000000..17282d13dc3 --- /dev/null +++ b/config-model/src/test/derived/globalphase_onnx_inside/files/ax_plus_b.onnx @@ -0,0 +1,23 @@ +:© + +matrix_X +vector_AXA"MatMul + +XA +vector_Bvector_Y"AddlrZ +matrix_X + + +Z +vector_A + + +Z +vector_B + + +b +vector_Y + + +B
\ No newline at end of file diff --git a/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg b/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg new file mode 100644 index 00000000000..35bb1ccc3d2 --- /dev/null +++ b/config-model/src/test/derived/globalphase_onnx_inside/rank-profiles.cfg @@ -0,0 +1,34 @@ +rankprofile[].name "default" +rankprofile[].fef.property[].name "rankingExpression(handicap).rankingScript" +rankprofile[].fef.property[].value "query(yy)" +rankprofile[].fef.property[].name "rankingExpression(handicap).type" +rankprofile[].fef.property[].value "tensor(d0[2])" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "rankingExpression(firstphase)" +rankprofile[].fef.property[].name "rankingExpression(firstphase).rankingScript" +rankprofile[].fef.property[].value "reduce(attribute(aa), sum)" +rankprofile[].fef.property[].name "vespa.rank.globalphase" +rankprofile[].fef.property[].value "rankingExpression(globalphase)" +rankprofile[].fef.property[].name "rankingExpression(globalphase).rankingScript" +rankprofile[].fef.property[].value "reduce(constant(ww) * (onnx(inside).foobar - rankingExpression(handicap)), sum)" +rankprofile[].fef.property[].name "vespa.match.feature" +rankprofile[].fef.property[].value "attribute(aa)" +rankprofile[].fef.property[].name "vespa.globalphase.rerankcount" +rankprofile[].fef.property[].value "13" +rankprofile[].fef.property[].name "vespa.type.attribute.aa" +rankprofile[].fef.property[].value "tensor(d1[3])" +rankprofile[].fef.property[].name "vespa.type.query.bb" +rankprofile[].fef.property[].value "tensor(d0[2])" +rankprofile[].fef.property[].name "vespa.type.query.yy" +rankprofile[].fef.property[].value "tensor(d0[2])" +rankprofile[].name "unranked" +rankprofile[].fef.property[].name "vespa.rank.firstphase" +rankprofile[].fef.property[].value "value(0)" +rankprofile[].fef.property[].name "vespa.hitcollector.heapsize" +rankprofile[].fef.property[].value "0" +rankprofile[].fef.property[].name "vespa.hitcollector.arraysize" +rankprofile[].fef.property[].value "0" +rankprofile[].fef.property[].name "vespa.dump.ignoredefaultfeatures" +rankprofile[].fef.property[].value "true" +rankprofile[].fef.property[].name "vespa.type.attribute.aa" +rankprofile[].fef.property[].value "tensor(d1[3])" diff --git a/config-model/src/test/derived/globalphase_onnx_inside/test.sd b/config-model/src/test/derived/globalphase_onnx_inside/test.sd new file mode 100644 index 00000000000..c38e318ce6b --- /dev/null +++ b/config-model/src/test/derived/globalphase_onnx_inside/test.sd @@ -0,0 +1,42 @@ +schema test { + + document test { + field aa type tensor(d1[3]) { + indexing: attribute + } + } + + constant xx { + file: files/const_xx.json + type: tensor(d0[2],d1[3]) + } + constant ww { + file: files/const_ww.json + type: tensor(d0[2]) + } + + rank-profile default { + inputs { + query(bb) tensor(d0[2]) + query(yy) tensor(d0[2]) + } + onnx-model inside { + file: files/ax_plus_b.onnx + input vector_A: attribute(aa) + input matrix_X: constant(xx) + input vector_B: query(bb) + output vector_Y: foobar + } + first-phase { + expression: sum(attribute(aa)) + } + function handicap() { + expression: query(yy) + } + global-phase { + rerank-count: 13 + expression: sum(constant(ww) * (onnx(inside).foobar - handicap)) + } + } + +} diff --git a/config-model/src/test/derived/rankingexpression/rank-profiles.cfg b/config-model/src/test/derived/rankingexpression/rank-profiles.cfg index ea8bc5f77e6..e3947e9e46f 100644 --- a/config-model/src/test/derived/rankingexpression/rank-profiles.cfg +++ b/config-model/src/test/derived/rankingexpression/rank-profiles.cfg @@ -410,8 +410,6 @@ rankprofile[].fef.property[].value "rankingExpression(globalphase)" rankprofile[].fef.property[].name "rankingExpression(globalphase).rankingScript" rankprofile[].fef.property[].value "rankingExpression(myplus) + reduce(rankingExpression(mymul), sum) + firstPhase" rankprofile[].fef.property[].name "vespa.match.feature" -rankprofile[].fef.property[].value "query(fromq)" -rankprofile[].fef.property[].name "vespa.match.feature" rankprofile[].fef.property[].value "firstPhase" rankprofile[].fef.property[].name "vespa.match.feature" rankprofile[].fef.property[].value "attribute(t1)" diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/HostsXmlProvisionerTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/HostsXmlProvisionerTest.java index 05b8681b5fa..d92fea24d51 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/HostsXmlProvisionerTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/HostsXmlProvisionerTest.java @@ -70,8 +70,6 @@ public class HostsXmlProvisionerTest { assertEquals(3, map.size()); assertCorrectNumberOfHosts(map, 3); assertTrue(map.keySet().containsAll(aliases)); - - assertEquals("", System.getProperty("zookeeper.vespa.clients")); } @Test diff --git a/config-model/src/test/java/com/yahoo/schema/derived/GlobalPhaseOnnxModelsTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/GlobalPhaseOnnxModelsTestCase.java new file mode 100644 index 00000000000..2ff33dd70d8 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/schema/derived/GlobalPhaseOnnxModelsTestCase.java @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.schema.parser.ParseException; +import org.junit.jupiter.api.Test; +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Tests exporting with global-phase and ONNX models + * + * @author arnej + */ +public class GlobalPhaseOnnxModelsTestCase extends AbstractExportingTestCase { + + @Test + void testModelInRankProfile() throws IOException, ParseException { + assertCorrectDeriving("globalphase_onnx_inside"); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java new file mode 100644 index 00000000000..0abb153696c --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java @@ -0,0 +1,85 @@ +package com.yahoo.vespa.model; + +import com.yahoo.config.model.NullConfigModelRegistry; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.provision.InMemoryProvisioner; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author bratseth + */ +public class ClusterInfoTest { + + @Test + void bcp_deadline_is_passed_in_cluster_info() throws Exception { + var servicesXml = """ + <services version='1.0'> + <container id='testcontainer' version='1.0'> + <nodes count='3'/> + </container> + <content id='testcontent' version='1.0'> + <redundancy>2</redundancy> + <documents/> + </content> + </services> + """; + + var deploymentXml = """ + <deployment version='1.0'> + <prod> + <region>us-west-1</region> + <region>us-east-1</region> + </prod> + <bcp> + <group deadline='30m'> + <region fraction='0.5'>us-east-1</region> + <region>us-west-1</region> + </group> + <group> + <region fraction='0.5'>us-east-1</region> + </group> + </bcp> + </deployment> + """; + + var requestedInUsEast1 = requestedCapacityIn("us-east-1", servicesXml, deploymentXml); + assertEquals(Duration.ofMinutes(0), requestedInUsEast1.get(new ClusterSpec.Id("testcontainer")).clusterInfo().bcpDeadline()); + assertEquals(Duration.ofMinutes(0), requestedInUsEast1.get(new ClusterSpec.Id("testcontent")).clusterInfo().bcpDeadline()); + + var requestedInUsWest1 = requestedCapacityIn("us-west-1", servicesXml, deploymentXml); + assertEquals(Duration.ofMinutes(30), requestedInUsWest1.get(new ClusterSpec.Id("testcontainer")).clusterInfo().bcpDeadline()); + assertEquals(Duration.ofMinutes(30), requestedInUsWest1.get(new ClusterSpec.Id("testcontent")).clusterInfo().bcpDeadline()); + } + + private Map<ClusterSpec.Id, Capacity> requestedCapacityIn(String region, String servicesXml, String deploymentXml) throws Exception { + var applicationPackage = new MockApplicationPackage.Builder() + .withServices(servicesXml) + .withDeploymentSpec(deploymentXml) + .build(); + + var provisioner = new InMemoryProvisioner(10, true); + var deployState = new DeployState.Builder() + .applicationPackage(applicationPackage) + .zone(new Zone(Environment.prod, RegionName.from(region))) + .properties(new TestProperties().setHostedVespa(true) + .setZone(new Zone(Environment.prod, RegionName.from(region)))) + .modelHostProvisioner(provisioner) + .provisioned(provisioner.provisioned()) + .build(); + new VespaModel(new NullConfigModelRegistry(), deployState); + return deployState.provisioned().all(); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/HostPortsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/HostPortsTest.java index a138ef71b2f..4a3cae37570 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/HostPortsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/HostPortsTest.java @@ -30,7 +30,7 @@ public class HostPortsTest { void next_available_baseport_is_BASE_PORT_plus_one_when_one_port_has_been_reserved() { HostPorts host = new HostPorts("myhostname"); MockRoot root = new MockRoot(); - host.reservePort(new TestService(root, 1), HostPorts.BASE_PORT, "foo"); + host.reservePort(new TestService(root, 1), HostPorts.BASE_PORT); assertThat(host.nextAvailableBaseport(1), is(HostPorts.BASE_PORT + 1)); } @@ -40,12 +40,12 @@ public class HostPortsTest { MockRoot root = new MockRoot(); for (int p = HostPorts.BASE_PORT; p < HostPorts.BASE_PORT + HostPorts.MAX_PORTS; p += 2) { - host.reservePort(new TestService(root, 1), p, "foo"); + host.reservePort(new TestService(root, 1), p); } assertThat(host.nextAvailableBaseport(2), is(0)); try { - host.reservePort(new TestService(root, 2), HostPorts.BASE_PORT, "bar"); + host.reservePort(new TestService(root, 2), HostPorts.BASE_PORT); } catch (RuntimeException e) { assertThat(e.getMessage(), containsString("Too many ports are reserved")); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/CloudAccountChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/CloudAccountChangeValidatorTest.java index fcc8c82a6e9..a8a063cb5fb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/CloudAccountChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/CloudAccountChangeValidatorTest.java @@ -1,5 +1,6 @@ package com.yahoo.vespa.model.application.validation.change; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.model.deploy.DeployState; @@ -57,7 +58,8 @@ class CloudAccountChangeValidatorTest { IntRange.empty(), false, false, - Optional.of(cloudAccount).filter(account -> !account.isUnspecified())); + Optional.of(cloudAccount).filter(account -> !account.isUnspecified()), + ClusterInfo.empty()); } private static VespaModel model(Provisioned provisioned) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 6ac4dbd17e3..cad36f5574c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -835,7 +835,7 @@ public class ContentBuilderTest extends DomBuilderTest { " </group>" + "</content>"); }); - assertTrue(exception.getMessage().contains("The specified content engine requires the <documents> element to be specified.")); + assertEquals("The <documents> element is mandatory in content cluster 'a'", exception.getMessage()); } private ProtonConfig getProtonConfig(ContentCluster content) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index ce2d5ca2da5..5973ef56962 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -485,7 +485,7 @@ public class ContainerClusterTest { if (isCombinedCluster) cluster.setHostClusterId("test-content-cluster"); cluster.setMemoryPercentage(memoryPercentage); - cluster.setSearch(new ContainerSearch(cluster, new SearchChains(cluster, "search-chain"))); + cluster.setSearch(new ContainerSearch(root.getDeployState(), cluster, new SearchChains(cluster, "search-chain"))); return cluster; } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java index 063f8f3109e..5b6c7b97875 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.ml; -import ai.vespa.modelintegration.evaluator.OnnxEvaluator; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; import ai.vespa.models.evaluation.FunctionEvaluator; import ai.vespa.models.evaluation.ModelsEvaluator; import com.yahoo.tensor.Tensor; @@ -21,7 +21,7 @@ public class ModelsEvaluatorTest { void testModelsEvaluator() { // Assumption fails but test passes on Intel macs // Assumption fails and test fails on ARM64 - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); + assumeTrue(OnnxRuntime.isRuntimeAvailable()); ModelsEvaluator modelsEvaluator = ModelsEvaluatorTester.create("src/test/cfg/application/stateless_eval"); assertEquals(3, modelsEvaluator.models().size()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/EmbedderTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/EmbedderTestCase.java index 509d8527bf5..e7f8086e554 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/EmbedderTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/EmbedderTestCase.java @@ -24,6 +24,7 @@ import java.nio.charset.StandardCharsets; import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; public class EmbedderTestCase { @@ -91,8 +92,7 @@ public class EmbedderTestCase { " </config>" + "</component>"; assertTransformThrows(embedder, - "Unknown model id 'my_model_id' on 'transformerModel'. " + - "Available models are [bert-base-uncased, minilm-l6-v2, mpnet-base-v2]", + "Unknown model id 'my_model_id' on 'transformerModel'", true); } @@ -194,7 +194,7 @@ public class EmbedderTestCase { ModelIdResolver.resolveModelIds(createElement(embedder), hosted); fail("Expected exception was not thrown: " + expectedMessage); } catch (IllegalArgumentException e) { - assertEquals(expectedMessage, e.getMessage()); + assertTrue(e.getMessage().contains(expectedMessage), "Expected error message not found"); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java index caf0d22d44e..fc70a65b394 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.ml; -import ai.vespa.modelintegration.evaluator.OnnxEvaluator; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; import ai.vespa.models.evaluation.Model; import ai.vespa.models.evaluation.ModelsEvaluator; import ai.vespa.models.handler.ModelsEvaluationHandler; @@ -27,7 +27,10 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assumptions.assumeTrue; /** @@ -60,7 +63,7 @@ public class ModelEvaluationTest { @Test void testMl_serving() throws IOException { - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); + assumeTrue(OnnxRuntime.isRuntimeAvailable()); Path appDir = Path.fromString("src/test/cfg/application/ml_serving"); Path storedAppDir = appDir.append("copy"); try { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java index a731e9c7ccc..b0fe2c09227 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/StatelessOnnxEvaluationTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.ml; -import ai.vespa.modelintegration.evaluator.OnnxEvaluator; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; import ai.vespa.models.evaluation.FunctionEvaluator; import ai.vespa.models.evaluation.Model; import ai.vespa.models.evaluation.ModelsEvaluator; @@ -45,7 +45,7 @@ public class StatelessOnnxEvaluationTest { @Test void testStatelessOnnxModelNameCollision() { - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); + assumeTrue(OnnxRuntime.isRuntimeAvailable()); Path appDir = Path.fromString("src/test/cfg/application/onnx_name_collision"); try { ImportedModelTester tester = new ImportedModelTester("onnx", appDir); @@ -66,7 +66,7 @@ public class StatelessOnnxEvaluationTest { @Test void testStatelessOnnxModelEvaluation() throws Exception { - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); + assumeTrue(OnnxRuntime.isRuntimeAvailable()); Path appDir = Path.fromString("src/test/cfg/application/onnx"); Path storedAppDir = appDir.append("copy"); try { @@ -91,7 +91,7 @@ public class StatelessOnnxEvaluationTest { @Test void testStatelessOnnxModelEvaluationWithGpu() { - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); + assumeTrue(OnnxRuntime.isRuntimeAvailable()); NodeResources resources = new NodeResources(4, 16, 125, 10, NodeResources.DiskSpeed.fast, NodeResources.StorageType.local, NodeResources.Architecture.x86_64, diff --git a/config-model/src/test/schema-test-files/deployment-with-instances.xml b/config-model/src/test/schema-test-files/deployment-with-instances.xml index f37ff9f6cc6..0c3409533d1 100644 --- a/config-model/src/test/schema-test-files/deployment-with-instances.xml +++ b/config-model/src/test/schema-test-files/deployment-with-instances.xml @@ -30,6 +30,18 @@ </endpoint> <endpoint container-id="bar" /> </endpoints> + <bcp> + <group deadline="60m"> + <endpoint id="foo" container-id="baz"/> + <region>us-west-1</region> + <region fraction="0.5">us-central-1</region> + </group> + <group> + <region>us-north-1</region> + <region>us-south-2</region> + <region fraction="0.5">us-central-1</region> + </group> + </bcp> </instance> <delay hours='2'/> diff --git a/config-model/src/test/schema-test-files/deployment.xml b/config-model/src/test/schema-test-files/deployment.xml index 38145a1ac74..bc0f070d88c 100644 --- a/config-model/src/test/schema-test-files/deployment.xml +++ b/config-model/src/test/schema-test-files/deployment.xml @@ -26,4 +26,16 @@ </endpoint> <endpoint container-id="bar" /> </endpoints> + <bcp> + <group deadline="60m"> + <endpoint id="foo" container-id="baz"/> + <region>us-west-1</region> + <region fraction="0.5">us-central-1</region> + </group> + <group> + <region>us-north-1</region> + <region>us-south-2</region> + <region fraction="0.5">us-central-1</region> + </group> + </bcp> </deployment> diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java index 2477d19d46c..c92715e5d4f 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java @@ -20,6 +20,7 @@ public final class Capacity { private final boolean canFail; private final NodeType type; private final Optional<CloudAccount> cloudAccount; + private final ClusterInfo clusterInfo; private Capacity(ClusterResources min, ClusterResources max, @@ -27,7 +28,8 @@ public final class Capacity { boolean required, boolean canFail, NodeType type, - Optional<CloudAccount> cloudAccount) { + Optional<CloudAccount> cloudAccount, + ClusterInfo clusterInfo) { validate(min); validate(max); if (max.smallerThan(min)) @@ -42,6 +44,7 @@ public final class Capacity { this.canFail = canFail; this.type = type; this.cloudAccount = Objects.requireNonNull(cloudAccount); + this.clusterInfo = clusterInfo; } private static void validate(ClusterResources resources) { @@ -77,12 +80,14 @@ public final class Capacity { return cloudAccount; } + public ClusterInfo clusterInfo() { return clusterInfo; } + public Capacity withLimits(ClusterResources min, ClusterResources max) { return withLimits(min, max, IntRange.empty()); } public Capacity withLimits(ClusterResources min, ClusterResources max, IntRange groupSize) { - return new Capacity(min, max, groupSize, required, canFail, type, cloudAccount); + return new Capacity(min, max, groupSize, required, canFail, type, cloudAccount, clusterInfo); } @Override @@ -98,25 +103,31 @@ public final class Capacity { /** Create a non-required, failable capacity request */ public static Capacity from(ClusterResources min, ClusterResources max) { - return from(min, max, IntRange.empty(), false, true, Optional.empty()); + return from(min, max, IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty()); } public static Capacity from(ClusterResources resources, boolean required, boolean canFail) { return from(resources, required, canFail, NodeType.tenant); } - // TODO: Remove after February 2023 + // TODO: Remove after March 2023 + public static Capacity from(ClusterResources min, ClusterResources max, IntRange groupSize, boolean required, boolean canFail, Optional<CloudAccount> cloudAccount) { + return new Capacity(min, max, groupSize, required, canFail, NodeType.tenant, cloudAccount, ClusterInfo.empty()); + } + + // TODO: Remove after March 2023 public static Capacity from(ClusterResources min, ClusterResources max, boolean required, boolean canFail) { - return new Capacity(min, max, IntRange.empty(), required, canFail, NodeType.tenant, Optional.empty()); + return new Capacity(min, max, IntRange.empty(), required, canFail, NodeType.tenant, Optional.empty(), ClusterInfo.empty()); } - // TODO: Remove after February 2023 + // TODO: Remove after March 2023 public static Capacity from(ClusterResources min, ClusterResources max, boolean required, boolean canFail, Optional<CloudAccount> cloudAccount) { - return new Capacity(min, max, IntRange.empty(), required, canFail, NodeType.tenant, cloudAccount); + return new Capacity(min, max, IntRange.empty(), required, canFail, NodeType.tenant, cloudAccount, ClusterInfo.empty()); } - public static Capacity from(ClusterResources min, ClusterResources max, IntRange groupSize, boolean required, boolean canFail, Optional<CloudAccount> cloudAccount) { - return new Capacity(min, max, groupSize, required, canFail, NodeType.tenant, cloudAccount); + public static Capacity from(ClusterResources min, ClusterResources max, IntRange groupSize, boolean required, boolean canFail, + Optional<CloudAccount> cloudAccount, ClusterInfo clusterInfo) { + return new Capacity(min, max, groupSize, required, canFail, NodeType.tenant, cloudAccount, clusterInfo); } /** Creates this from a node type */ @@ -125,7 +136,7 @@ public final class Capacity { } private static Capacity from(ClusterResources resources, boolean required, boolean canFail, NodeType type) { - return new Capacity(resources, resources, IntRange.empty(), required, canFail, type, Optional.empty()); + return new Capacity(resources, resources, IntRange.empty(), required, canFail, type, Optional.empty(), ClusterInfo.empty()); } } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java new file mode 100644 index 00000000000..fe8acb0c3c0 --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java @@ -0,0 +1,61 @@ +package com.yahoo.config.provision; + +import java.time.Duration; +import java.util.Objects; + +/** + * Auxiliary information about a cluster, provided by the config model to the node repo during a + * capacity request. + * + * @author bratseth + */ +public class ClusterInfo { + + private static final ClusterInfo empty = new ClusterInfo.Builder().build(); + + private final Duration bcpDeadline; + + private ClusterInfo(Builder builder) { + this.bcpDeadline = builder.bcpDeadline; + } + + public Duration bcpDeadline() { return bcpDeadline; } + + public static ClusterInfo empty() { return empty; } + + public boolean isEmpty() { return this.equals(empty); } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! (o instanceof ClusterInfo other)) return false; + if ( ! other.bcpDeadline.equals(this.bcpDeadline)) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(bcpDeadline); + } + + @Override + public String toString() { + return "cluster info: [bcp deadline: " + bcpDeadline + "]"; + } + + public static class Builder { + + private Duration bcpDeadline = Duration.ofMinutes(0); + + public Builder bcpDeadline(Duration duration) { + this.bcpDeadline = Objects.requireNonNull(duration); + return this; + } + + public ClusterInfo build() { + return new ClusterInfo(this); + } + + } + +} diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/WireguardKey.java b/config-provisioning/src/main/java/com/yahoo/config/provision/WireguardKey.java index 37218a42c70..8f6494d8f74 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/WireguardKey.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/WireguardKey.java @@ -1,7 +1,9 @@ package com.yahoo.config.provision; import ai.vespa.validation.PatternedStringWrapper; +import com.google.common.io.CharStreams; +import java.util.UUID; import java.util.regex.Pattern; /** @@ -27,4 +29,9 @@ public class WireguardKey extends PatternedStringWrapper<WireguardKey> { public String toString() { return "Wireguard key '" + value() + "'"; } + + public static WireguardKey generateRandomForTesting() { + var str = UUID.randomUUID().toString().replace("-", ""); + return new WireguardKey(str + "12345678900="); + } } diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/CapacityTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/CapacityTest.java index 89c0e98b076..a7614bbc016 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/CapacityTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/CapacityTest.java @@ -21,7 +21,8 @@ public class CapacityTest { IntRange.empty(), false, true, - Optional.empty()); + Optional.empty(), + ClusterInfo.empty()); assertValidationFailure(new ClusterResources(4, 2, new NodeResources(1, 2, 3, 4)), new ClusterResources(2, 2, new NodeResources(1, 2, 3, 4))); assertValidationFailure(new ClusterResources(4, 4, new NodeResources(1, 2, 3, 4)), @@ -41,7 +42,7 @@ public class CapacityTest { private void assertValidationFailure(ClusterResources min, ClusterResources max) { try { - Capacity.from(min, max, IntRange.empty(), false, true, Optional.empty()); + Capacity.from(min, max, IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty()); fail("Expected exception with min " + min + " and max " + max); } catch (IllegalArgumentException e) { diff --git a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java index f67e0442468..c6a320b19d9 100644 --- a/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java +++ b/config-proxy/src/main/java/com/yahoo/vespa/config/proxy/ConfigProxyRpcServer.java @@ -13,6 +13,7 @@ import com.yahoo.jrt.Supervisor; import com.yahoo.jrt.Target; import com.yahoo.jrt.TargetWatcher; import com.yahoo.security.tls.Capability; +import com.yahoo.security.tls.CapabilitySet; import com.yahoo.vespa.config.JRTMethods; import com.yahoo.vespa.config.RawConfig; import com.yahoo.vespa.config.protocol.JRTServerConfigRequest; @@ -82,6 +83,7 @@ public class ConfigProxyRpcServer implements Runnable, TargetWatcher { .requireCapabilities(Capability.CONFIGPROXY__CONFIG_API)); supervisor.addMethod(new Method("ping", "", "i", this::ping) + .requireCapabilities(CapabilitySet.none()) .methodDesc("ping") .returnDesc(0, "ret code", "return code, 0 is OK")); supervisor.addMethod(new Method("listCachedConfig", "", "S", diff --git a/config/CMakeLists.txt b/config/CMakeLists.txt index 1c04aa3eaa8..561ddbc078c 100644 --- a/config/CMakeLists.txt +++ b/config/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalib vespalog fnet diff --git a/config/src/apps/vespa-get-config/getconfig.cpp b/config/src/apps/vespa-get-config/getconfig.cpp index 74380390095..03a049044ae 100644 --- a/config/src/apps/vespa-get-config/getconfig.cpp +++ b/config/src/apps/vespa-get-config/getconfig.cpp @@ -12,6 +12,7 @@ #include <vespa/config/common/configresponse.h> #include <vespa/config/common/trace.h> #include <vespa/vespalib/util/signalhandler.h> +#include <cinttypes> #include <unistd.h> #include <sstream> diff --git a/config/src/tests/file_acquirer/file_acquirer_test.cpp b/config/src/tests/file_acquirer/file_acquirer_test.cpp index 8449a33a782..7ea6556c074 100644 --- a/config/src/tests/file_acquirer/file_acquirer_test.cpp +++ b/config/src/tests/file_acquirer/file_acquirer_test.cpp @@ -4,14 +4,12 @@ #include <vespa/fnet/frt/supervisor.h> #include <vespa/fnet/frt/rpcrequest.h> #include <vespa/fnet/transport.h> -#include <vespa/fastos/thread.h> #include <vespa/vespalib/util/stringfmt.h> using namespace config; struct ServerFixture : FRT_Invokable { fnet::frt::StandaloneFRT server; - FastOS_ThreadPool threadPool; FNET_Transport transport; FRT_Supervisor &orb; vespalib::string spec; @@ -26,14 +24,13 @@ struct ServerFixture : FRT_Invokable { ServerFixture() : server(), - threadPool(), transport(), orb(server.supervisor()) { init_rpc(); orb.Listen(0); spec = vespalib::make_string("tcp/localhost:%d", orb.GetListenPort()); - transport.Start(&threadPool); + transport.Start(); } void RPC_waitFor(FRT_RPCRequest *req) { diff --git a/config/src/tests/frtconnectionpool/frtconnectionpool.cpp b/config/src/tests/frtconnectionpool/frtconnectionpool.cpp index 9c3498a92a1..834b1797ed0 100644 --- a/config/src/tests/frtconnectionpool/frtconnectionpool.cpp +++ b/config/src/tests/frtconnectionpool/frtconnectionpool.cpp @@ -4,7 +4,6 @@ #include <vespa/config/frt/frtconnectionpool.h> #include <vespa/fnet/frt/error.h> #include <vespa/fnet/transport.h> -#include <vespa/fastos/thread.h> #include <sstream> #include <set> #include <unistd.h> @@ -14,7 +13,6 @@ using namespace config; class Test : public vespalib::TestApp { private: static ServerSpec::HostSpecList _sources; - FastOS_ThreadPool _threadPool; FNET_Transport _transport; void verifyAllSourcesInRotation(FRTConnectionPool& sourcePool); public: @@ -33,10 +31,9 @@ public: Test::Test() : vespalib::TestApp(), - _threadPool(), _transport() { - _transport.Start(&_threadPool); + _transport.Start(); } Test::~Test() { diff --git a/config/src/tests/functiontest/functiontest.cpp b/config/src/tests/functiontest/functiontest.cpp index 333645176a0..9d6605be9c1 100644 --- a/config/src/tests/functiontest/functiontest.cpp +++ b/config/src/tests/functiontest/functiontest.cpp @@ -7,6 +7,7 @@ #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/testkit/test_kit.h> #include <fstream> +#include <cinttypes> #include <vespa/log/log.h> diff --git a/config/src/vespa/config/file_acquirer/file_acquirer.cpp b/config/src/vespa/config/file_acquirer/file_acquirer.cpp index 048e9544dab..4ee5b4aa4a9 100644 --- a/config/src/vespa/config/file_acquirer/file_acquirer.cpp +++ b/config/src/vespa/config/file_acquirer/file_acquirer.cpp @@ -5,7 +5,6 @@ #include <vespa/fnet/frt/target.h> #include <vespa/fnet/frt/rpcrequest.h> #include <vespa/fnet/transport.h> -#include <vespa/fastos/thread.h> #include <vespa/log/log.h> LOG_SETUP(".config.file_acquirer"); diff --git a/config/src/vespa/config/frt/frtconfigagent.cpp b/config/src/vespa/config/frt/frtconfigagent.cpp index b5bcb3591f0..15b605ad690 100644 --- a/config/src/vespa/config/frt/frtconfigagent.cpp +++ b/config/src/vespa/config/frt/frtconfigagent.cpp @@ -4,6 +4,7 @@ #include <vespa/config/common/trace.h> #include <vespa/config/common/configresponse.h> #include <vespa/config/common/iconfigholder.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".config.frt.frtconfigagent"); diff --git a/config/src/vespa/config/frt/frtconnectionpool.cpp b/config/src/vespa/config/frt/frtconnectionpool.cpp index 21d6f0dbe90..b73a28483e6 100644 --- a/config/src/vespa/config/frt/frtconnectionpool.cpp +++ b/config/src/vespa/config/frt/frtconnectionpool.cpp @@ -5,7 +5,6 @@ #include <vespa/vespalib/util/size_literals.h> #include <vespa/fnet/frt/supervisor.h> #include <vespa/fnet/transport.h> -#include <vespa/fastos/thread.h> #include <vespa/log/log.h> LOG_SETUP(".config.frt.frtconnectionpool"); @@ -167,14 +166,12 @@ FRTConnectionPool::getScheduler() { return _supervisor->GetScheduler(); } -FRTConnectionPoolWithTransport::FRTConnectionPoolWithTransport(std::unique_ptr<FastOS_ThreadPool> threadPool, - std::unique_ptr<FNET_Transport> transport, +FRTConnectionPoolWithTransport::FRTConnectionPoolWithTransport(std::unique_ptr<FNET_Transport> transport, const ServerSpec & spec, const TimingValues & timingValues) - : _threadPool(std::move(threadPool)), - _transport(std::move(transport)), - _connectionPool(std::make_unique<FRTConnectionPool>(*_transport, spec, timingValues)) + : _transport(std::move(transport)), + _connectionPool(std::make_unique<FRTConnectionPool>(*_transport, spec, timingValues)) { - _transport->Start(_threadPool.get()); + _transport->Start(); } FRTConnectionPoolWithTransport::~FRTConnectionPoolWithTransport() diff --git a/config/src/vespa/config/frt/frtconnectionpool.h b/config/src/vespa/config/frt/frtconnectionpool.h index 564c6506159..5d97f2ae338 100644 --- a/config/src/vespa/config/frt/frtconnectionpool.h +++ b/config/src/vespa/config/frt/frtconnectionpool.h @@ -8,7 +8,6 @@ #include <map> class FNET_Transport; -class FastOS_ThreadPool; namespace config { @@ -103,8 +102,7 @@ public: class FRTConnectionPoolWithTransport : public ConnectionFactory { public: - FRTConnectionPoolWithTransport(std::unique_ptr<FastOS_ThreadPool> threadPool, - std::unique_ptr<FNET_Transport> transport, + FRTConnectionPoolWithTransport(std::unique_ptr<FNET_Transport> transport, const ServerSpec & spec, const TimingValues & timingValues); FRTConnectionPoolWithTransport(const FRTConnectionPoolWithTransport&) = delete; FRTConnectionPoolWithTransport& operator=(const FRTConnectionPoolWithTransport&) = delete; @@ -113,7 +111,6 @@ public: void syncTransport() override { _connectionPool->syncTransport(); } Connection* getCurrent() override { return _connectionPool->getCurrent(); } private: - std::unique_ptr<FastOS_ThreadPool> _threadPool; std::unique_ptr<FNET_Transport> _transport; std::unique_ptr<FRTConnectionPool> _connectionPool; }; diff --git a/config/src/vespa/config/set/configsetsource.cpp b/config/src/vespa/config/set/configsetsource.cpp index 41886a12d01..b84f6411855 100644 --- a/config/src/vespa/config/set/configsetsource.cpp +++ b/config/src/vespa/config/set/configsetsource.cpp @@ -4,6 +4,7 @@ #include <vespa/config/print/asciiconfigwriter.h> #include <vespa/config/common/iconfigholder.h> #include <vespa/config/common/exceptions.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".config.set.configsetsource"); diff --git a/config/src/vespa/config/subscription/configsubscriptionset.cpp b/config/src/vespa/config/subscription/configsubscriptionset.cpp index 4f7bf64f603..9c09a508fff 100644 --- a/config/src/vespa/config/subscription/configsubscriptionset.cpp +++ b/config/src/vespa/config/subscription/configsubscriptionset.cpp @@ -6,6 +6,7 @@ #include <vespa/config/common/misc.h> #include <vespa/config/common/iconfigmanager.h> #include <vespa/config/common/iconfigcontext.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".config.subscription.configsubscriptionset"); diff --git a/config/src/vespa/config/subscription/sourcespec.cpp b/config/src/vespa/config/subscription/sourcespec.cpp index c05f639f9ba..0ab0806885f 100644 --- a/config/src/vespa/config/subscription/sourcespec.cpp +++ b/config/src/vespa/config/subscription/sourcespec.cpp @@ -13,7 +13,6 @@ #include <vespa/config/print/asciiconfigwriter.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/fnet/transport.h> -#include <vespa/fastos/thread.h> #include <cassert> namespace config { @@ -127,8 +126,7 @@ ServerSpec::createSourceFactory(const TimingValues & timingValues) const { const auto vespaVersion = VespaVersion::getCurrentVersion(); return std::make_unique<FRTSourceFactory>( - std::make_unique<FRTConnectionPoolWithTransport>(std::make_unique<FastOS_ThreadPool>(), - std::make_unique<FNET_Transport>(), + std::make_unique<FRTConnectionPoolWithTransport>(std::make_unique<FNET_Transport>(), *this, timingValues), timingValues, _traceLevel, vespaVersion, _compressionType); } diff --git a/configd/src/apps/sentinel/config-owner.cpp b/configd/src/apps/sentinel/config-owner.cpp index 90ceb705dc7..1fe361ef739 100644 --- a/configd/src/apps/sentinel/config-owner.cpp +++ b/configd/src/apps/sentinel/config-owner.cpp @@ -2,6 +2,7 @@ #include "config-owner.h" #include <vespa/config/subscription/configsubscriber.hpp> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".sentinel.config-owner"); @@ -9,7 +10,6 @@ LOG_SETUP(".sentinel.config-owner"); namespace config::sentinel { ConfigOwner::ConfigOwner() = default; - ConfigOwner::~ConfigOwner() = default; void diff --git a/configd/src/apps/sentinel/connectivity.cpp b/configd/src/apps/sentinel/connectivity.cpp index b9641cc19e0..9cd8d5c985a 100644 --- a/configd/src/apps/sentinel/connectivity.cpp +++ b/configd/src/apps/sentinel/connectivity.cpp @@ -75,10 +75,9 @@ void classifyConnFails(ConnectivityMap &connectivityMap, LOG_ASSERT(cmIter != connectivityMap.end()); OutwardCheckContext cornerContext(goodNeighborSpecs.size(), nameToCheck, portToCheck, rpcServer.orb()); ConnectivityMap cornerProbes; - int ping_timeout = 1000; + int ping_timeout = 1000 + 50 * goodNeighborSpecs.size(); for (const auto & hp : goodNeighborSpecs) { cornerProbes.try_emplace(hp.first, spec(hp), cornerContext, ping_timeout); - ping_timeout += 20; } cornerContext.latch.await(); size_t numReportsUp = 0; @@ -154,10 +153,9 @@ Connectivity::checkConnectivity(RpcServer &rpcServer) { rpcServer.getPort(), rpcServer.orb()); ConnectivityMap connectivityMap; - int ping_timeout = 1000; + int ping_timeout = 1000 + 50 * _checkSpecs.size(); for (const auto &host_and_port : _checkSpecs) { connectivityMap.try_emplace(host_and_port.first, spec(host_and_port), checkContext, ping_timeout); - ping_timeout += 20; } checkContext.latch.await(); classifyConnFails(connectivityMap, _checkSpecs, rpcServer); diff --git a/configd/src/apps/sentinel/logctl.cpp b/configd/src/apps/sentinel/logctl.cpp index 94759d4f102..057178cec2e 100644 --- a/configd/src/apps/sentinel/logctl.cpp +++ b/configd/src/apps/sentinel/logctl.cpp @@ -5,7 +5,7 @@ #include <sys/types.h> #include <sys/wait.h> #include <unistd.h> -#include <stdio.h> +#include <cstring> #include <vespa/log/log.h> LOG_SETUP(".sentinel.logctl"); diff --git a/configd/src/apps/sentinel/model-owner.cpp b/configd/src/apps/sentinel/model-owner.cpp index 5dd7a033933..93024911f4e 100644 --- a/configd/src/apps/sentinel/model-owner.cpp +++ b/configd/src/apps/sentinel/model-owner.cpp @@ -3,6 +3,7 @@ #include "model-owner.h" #include <vespa/config/common/exceptions.h> #include <vespa/config/subscription/configsubscriber.hpp> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".sentinel.model-owner"); diff --git a/configd/src/apps/sentinel/report-connectivity.cpp b/configd/src/apps/sentinel/report-connectivity.cpp index 8417384dda9..d059980aae9 100644 --- a/configd/src/apps/sentinel/report-connectivity.cpp +++ b/configd/src/apps/sentinel/report-connectivity.cpp @@ -22,9 +22,9 @@ ReportConnectivity::ReportConnectivity(FRT_RPCRequest *req, int timeout_ms, FRT_ auto map = Connectivity::specsFrom(cfg.value()); LOG(debug, "making connectivity report for %zd peers", map.size()); _remaining = map.size(); + timeout_ms += 50 * map.size(); for (const auto & [ hostname, port ] : map) { _checks.emplace_back(std::make_unique<PeerCheck>(*this, hostname, port, orb, timeout_ms)); - timeout_ms += 20; } } else { _parentRequest->SetError(FRTE_RPC_METHOD_FAILED, "failed getting model config"); diff --git a/configdefinitions/CMakeLists.txt b/configdefinitions/CMakeLists.txt index c374e93904e..80b53d1dc0a 100644 --- a/configdefinitions/CMakeLists.txt +++ b/configdefinitions/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalib config_cloudconfig diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 9865cda7bc9..8a4d523a6e4 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -4,8 +4,8 @@ package com.yahoo.vespa.config.server; import ai.vespa.http.DomainName; import ai.vespa.http.HttpURL; import ai.vespa.http.HttpURL.Query; -import ai.vespa.util.http.hc5.VespaHttpClientBuilder; import ai.vespa.util.http.hc5.DefaultHttpClientBuilder; +import ai.vespa.util.http.hc5.VespaHttpClientBuilder; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.component.Version; import com.yahoo.component.annotation.Inject; @@ -94,10 +94,8 @@ import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.orchestrator.Orchestrator; import org.apache.hc.client5.http.classic.methods.HttpGet; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; -import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; import org.apache.hc.core5.http.HttpHeaders; import org.apache.hc.core5.http.message.BasicHeader; - import java.io.File; import java.io.IOException; import java.io.InputStream; @@ -229,7 +227,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye // Should be used by tests only (first constructor in this class makes sure we use injectable components where possible) public static class Builder { private TenantRepository tenantRepository; - private Optional<Provisioner> hostProvisioner; private HttpProxy httpProxy = new HttpProxy(new SimpleHttpFetcher(Duration.ofSeconds(30))); private EndpointsChecker endpointsChecker = __ -> { throw new UnsupportedOperationException(); }; private Clock clock = Clock.systemUTC(); @@ -252,18 +249,6 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye return this; } - public Builder withProvisioner(Provisioner provisioner) { - if (this.hostProvisioner != null) throw new IllegalArgumentException("provisioner already set in builder"); - this.hostProvisioner = Optional.ofNullable(provisioner); - return this; - } - - public Builder withHostProvisionerProvider(HostProvisionerProvider hostProvisionerProvider) { - if (this.hostProvisioner != null) throw new IllegalArgumentException("provisioner already set in builder"); - this.hostProvisioner = hostProvisionerProvider.getHostProvisioner(); - return this; - } - public Builder withHttpProxy(HttpProxy httpProxy) { this.httpProxy = httpProxy; return this; @@ -316,7 +301,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye public ApplicationRepository build() { return new ApplicationRepository(tenantRepository, - hostProvisioner, + tenantRepository.hostProvisionerProvider().getHostProvisioner(), InfraDeployerProvider.empty().getInfraDeployer(), configConvergenceChecker, httpProxy, diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigActivationListener.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigActivationListener.java index f7e9e270b9c..e52089f5400 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigActivationListener.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ConfigActivationListener.java @@ -4,35 +4,17 @@ package com.yahoo.vespa.config.server; import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.config.server.application.ApplicationSet; -import java.util.Collection; - /** * A ConfigActivationListener is used to signal to a component that config has been - * activated. It only exists because the RpcServer cannot distinguish between a - * successful activation of a new application and an activation of the same application. + * activated for an application or that an application has been removed. It only exists + * because the RpcServer cannot distinguish between a successful activation of a new + * application and an activation of the same application. * * @author Ulf Lilleengen */ public interface ConfigActivationListener { /** - * Signals the listener that hosts used by a particular tenant. - * - * @param applicationId application id - * @param newHosts a {@link Collection} of hosts used by tenant. - */ - void hostsUpdated(ApplicationId applicationId, Collection<String> newHosts); - - /** - * Verifies that given hosts are available for use by tenant. - * - * @param applicationId application id - * @param newHosts a {@link java.util.Collection} of hosts that tenant wants to allocate. - * @throws java.lang.IllegalArgumentException if one or more of the hosts are in use by another tenant. - */ - void verifyHostsAreAvailable(ApplicationId applicationId, Collection<String> newHosts); - - /** * Configs has been activated for an application: Either an application * has been deployed for the first time, or it has been externally or internally redeployed. * diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java b/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java index a614e251dca..89f7755729c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/GetConfigContext.java @@ -38,9 +38,15 @@ public class GetConfigContext { return new GetConfigContext(app, handler, trace); } + public static GetConfigContext empty() { + return new GetConfigContext(null, null, null); + } + public static GetConfigContext testContext(ApplicationId app) { return new GetConfigContext(app, null, null); } + + public boolean isEmpty() { return app == null && requestHandler == null && trace == null; } /** * Helper to produce a log preamble with the tenant and app id diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationSet.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationSet.java index ad50117e327..5650c2e7e15 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationSet.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationSet.java @@ -1,16 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.application; +import com.yahoo.component.Version; import com.yahoo.config.model.api.HostInfo; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.component.Version; - import java.time.Instant; -import java.util.*; -import java.util.stream.Collectors; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Optional; /** - * Immutable set of {@link Application}s with the same {@link ApplicationId}. With methods for getting defaults. + * Immutable set of {@link Application}s with the same {@link ApplicationId}, applications have difference vespa versions. * * @author vegard */ @@ -42,8 +44,17 @@ public final class ApplicationSet { latestVersion = this.applications.keySet().stream().max(Version::compareTo).get(); } + public static ApplicationSet fromList(List<Application> applications) { + return new ApplicationSet(applications); + } + + // For testing + public static ApplicationSet from(Application application) { + return fromList(List.of(application)); + } + /** - * Returns the specified version, or the latest if not specified, or if the given version is not + * Returns an application for the specified version, or the latest if not specified, or if the given version is not * available and the latest is a permissible substitution. * * @throws VersionDoesNotExistException if the specified version is not available and the latest version is not @@ -68,21 +79,13 @@ public final class ApplicationSet { return Optional.empty(); } - /** Returns the application for the given version, if available */ + /** Returns the application for the given version, or empty if not found */ public Optional<Application> get(Version version) { return Optional.ofNullable(applications.get(version)); } public ApplicationId getId() { return applicationId; } - public static ApplicationSet fromList(List<Application> applications) { - return new ApplicationSet(applications); - } - - public static ApplicationSet from(Application application) { - return fromList(List.of(application)); - } - public Collection<String> getAllHosts() { return applications.values().stream() .flatMap(app -> app.getModel().getHosts().stream() @@ -91,9 +94,7 @@ public final class ApplicationSet { } public void updateHostMetrics() { - for (Application application : applications.values()) { - application.updateHostMetrics(application.getModel().getHosts().size()); - } + applications.values().forEach(app -> app.updateHostMetrics(app.getModel().getHosts().size())); } public long getApplicationGeneration() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 5831cb3e75f..8f3bc83984a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -216,10 +216,10 @@ public class TenantApplications implements RequestHandler, HostValidator { } private void notifyConfigActivationListeners(ApplicationSet applicationSet) { - if (applicationSet.getAllApplications().isEmpty()) throw new IllegalArgumentException("application set cannot be empty"); + List<Application> applications = applicationSet.getAllApplications(); + if (applications.isEmpty()) throw new IllegalArgumentException("application set cannot be empty"); - configActivationListener.hostsUpdated(applicationSet.getAllApplications().get(0).toApplicationInfo().getApplicationId(), - applicationSet.getAllHosts()); + hostRegistry.update(applications.get(0).getId(), applicationSet.getAllHosts()); configActivationListener.configActivated(applicationSet); } @@ -253,7 +253,7 @@ public class TenantApplications implements RequestHandler, HostValidator { if (hasApplication(applicationId)) { applicationMapper.remove(applicationId); - hostRegistry.removeHostsForKey(applicationId); + hostRegistry.removeHosts(applicationId); configActivationListenersOnRemove(applicationId); tenantMetricUpdater.setApplications(applicationMapper.numApplications()); metrics.removeMetricUpdater(Metrics.createDimensions(applicationId)); @@ -277,17 +277,17 @@ public class TenantApplications implements RequestHandler, HostValidator { } private void configActivationListenersOnRemove(ApplicationId applicationId) { - configActivationListener.hostsUpdated(applicationId, hostRegistry.getHostsForKey(applicationId)); + hostRegistry.removeHosts(applicationId); configActivationListener.applicationRemoved(applicationId); } private void setActiveApp(ApplicationSet applicationSet) { - ApplicationId id = applicationSet.getId(); + ApplicationId applicationId = applicationSet.getId(); Collection<String> hostsForApp = applicationSet.getAllHosts(); - hostRegistry.update(id, hostsForApp); + hostRegistry.update(applicationId, hostsForApp); applicationSet.updateHostMetrics(); tenantMetricUpdater.setApplications(applicationMapper.numApplications()); - applicationMapper.register(id, applicationSet); + applicationMapper.register(applicationId, applicationSet); } @Override @@ -377,7 +377,7 @@ public class TenantApplications implements RequestHandler, HostValidator { @Override public ApplicationId resolveApplicationId(String hostName) { - return hostRegistry.getKeyForHost(hostName); + return hostRegistry.getApplicationId(hostName); } @Override @@ -399,11 +399,11 @@ public class TenantApplications implements RequestHandler, HostValidator { @Override public void verifyHosts(ApplicationId applicationId, Collection<String> newHosts) { hostRegistry.verifyHosts(applicationId, newHosts); - configActivationListener.verifyHostsAreAvailable(applicationId, newHosts); } + // TODO: Duplicate of resolveApplicationId() above public ApplicationId getApplicationIdForHostName(String hostname) { - return hostRegistry.getKeyForHost(hostname); + return hostRegistry.getApplicationId(hostname); } public TenantFileSystemDirs getTenantFileSystemDirs() { return tenantFileSystemDirs; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index df449ca017b..66a5bc5a023 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -1,13 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.deploy; -import com.google.common.base.Supplier; -import com.google.common.base.Suppliers; import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.provision.ActivationContext; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; @@ -29,11 +28,14 @@ import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.session.Session; import com.yahoo.vespa.config.server.session.SessionRepository; import com.yahoo.vespa.config.server.tenant.Tenant; +import com.yahoo.yolean.concurrent.Memoized; + import java.time.Clock; import java.time.Duration; import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -113,8 +115,6 @@ public class Deployment implements com.yahoo.config.provision.Deployment { deleteSession(); throw e; } - - waitForResourcesOrTimeout(params, session, provisioner); } /** Activates this. If it is not already prepared, this will call prepare first. */ @@ -125,6 +125,8 @@ public class Deployment implements com.yahoo.config.provision.Deployment { validateSessionStatus(session); PrepareParams params = this.params.get(); + waitForResourcesOrTimeout(params, session, provisioner); + ApplicationId applicationId = session.getApplicationId(); try (ActionTimer timer = applicationRepository.timerFor(applicationId, "deployment.activateMillis")) { TimeoutBudget timeoutBudget = params.getTimeoutBudget(); @@ -263,7 +265,7 @@ public class Deployment implements com.yahoo.config.provision.Deployment { // Use supplier because we shouldn't/can't create this before validateSessionStatus() for prepared deployments, // memoize because we want to create this once for unprepared deployments - return Suppliers.memoize(() -> { + return new Memoized<>(() -> { TimeoutBudget timeoutBudget = new TimeoutBudget(clock, timeout); PrepareParams.Builder params = new PrepareParams.Builder() @@ -288,20 +290,19 @@ public class Deployment implements com.yahoo.config.provision.Deployment { Set<HostSpec> preparedHosts = session.getAllocatedHosts().getHosts(); ActivationContext context = new ActivationContext(session.getSessionId()); - ProvisionLock lock = new ProvisionLock(session.getApplicationId(), () -> {}); - AtomicReference<TransientException> lastException = new AtomicReference<>(); + AtomicReference<Exception> lastException = new AtomicReference<>(); while (true) { params.getTimeoutBudget().assertNotTimedOut( () -> "Timeout exceeded while waiting for application resources of '" + session.getApplicationId() + "'" + Optional.ofNullable(lastException.get()).map(e -> ". Last exception: " + e.getMessage()).orElse("")); - try { + try (ProvisionLock lock = provisioner.get().lock(session.getApplicationId())) { // Call to activate to make sure that everything is ready, but do not commit the transaction ApplicationTransaction transaction = new ApplicationTransaction(lock, new NestedTransaction()); provisioner.get().activate(preparedHosts, context, transaction); return; - } catch (TransientException e) { + } catch (ApplicationLockException | TransientException e) { lastException.set(e); try { Thread.sleep(durationBetweenResourceReadyChecks.toMillis()); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 1d80740ffe0..ad5423f0a94 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -202,6 +202,7 @@ public class ModelContextImpl implements ModelContext { private final int rpc_events_before_wakeup; private final boolean useRestrictedDataPlaneBindings; private final int heapPercentage; + private final boolean enableGlobalPhase; public FeatureFlags(FlagSource source, ApplicationId appId, Version version) { this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -246,6 +247,7 @@ public class ModelContextImpl implements ModelContext { this.queryDispatchWarmup = flagValue(source, appId, version, PermanentFlags.QUERY_DISPATCH_WARMUP); this.useRestrictedDataPlaneBindings = flagValue(source, appId, version, Flags.RESTRICT_DATA_PLANE_BINDINGS); this.heapPercentage = flagValue(source, appId, version, PermanentFlags.HEAP_SIZE_PERCENTAGE); + this.enableGlobalPhase = flagValue(source, appId, version, Flags.ENABLE_GLOBAL_PHASE); } @Override public int heapSizePercentage() { return heapPercentage; } @@ -298,6 +300,7 @@ public class ModelContextImpl implements ModelContext { return defVal; } @Override public boolean useRestrictedDataPlaneBindings() { return useRestrictedDataPlaneBindings; } + @Override public boolean enableGlobalPhase() { return enableGlobalPhase; } private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java index 85b30f4d303..b9118602058 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/filedistribution/FileDirectory.java @@ -23,6 +23,7 @@ import java.nio.channels.FileChannel; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; +import java.time.Clock; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.logging.Level; @@ -153,6 +154,8 @@ public class FileDirectory extends AbstractComponent { return true; } + // update last modified time so that maintainer deleting unused file references considers this as recently used + destinationDir.setLastModified(Clock.systemUTC().instant().toEpochMilli()); log.log(Level.FINE, "Directory for file reference '" + fileReference.value() + "' already exists and has all content"); return false; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java b/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java index b89f3bba835..e6f2452b693 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/host/HostRegistry.java @@ -23,19 +23,19 @@ public class HostRegistry implements HostValidator { private static final Logger log = Logger.getLogger(HostRegistry.class.getName()); - private final Map<String, ApplicationId> host2KeyMap = new ConcurrentHashMap<>(); + private final Map<String, ApplicationId> host2ApplicationId = new ConcurrentHashMap<>(); - public ApplicationId getKeyForHost(String hostName) { - return host2KeyMap.get(hostName); + public ApplicationId getApplicationId(String hostName) { + return host2ApplicationId.get(hostName); } public synchronized void update(ApplicationId key, Collection<String> newHosts) { verifyHosts(key, newHosts); - Collection<String> currentHosts = getHostsForKey(key); + Collection<String> currentHosts = getHosts(key); log.log(Level.FINE, () -> "Setting hosts for key '" + key + "', " + "newHosts: " + newHosts + ", " + "currentHosts: " + currentHosts); - Collection<String> removedHosts = getRemovedHosts(newHosts, currentHosts); + Collection<String> removedHosts = findRemovedHosts(newHosts, currentHosts); removeHosts(removedHosts); addHosts(key, newHosts); } @@ -45,49 +45,49 @@ public class HostRegistry implements HostValidator { for (String host : newHosts) { if (hostAlreadyTaken(host, applicationId)) { throw new IllegalArgumentException("'" + applicationId + "' tried to allocate host '" + host + - "', but the host is already taken by '" + host2KeyMap.get(host) + "'"); + "', but the host is already taken by '" + host2ApplicationId.get(host) + "'"); } } } - public synchronized void removeHostsForKey(ApplicationId key) { - host2KeyMap.entrySet().removeIf(entry -> entry.getValue().equals(key)); + public synchronized void removeHosts(ApplicationId key) { + host2ApplicationId.entrySet().removeIf(entry -> entry.getValue().equals(key)); } - public synchronized void removeHostsForKey(TenantName key) { - host2KeyMap.entrySet().removeIf(entry -> entry.getValue().tenant().equals(key)); + public synchronized void removeHosts(TenantName key) { + host2ApplicationId.entrySet().removeIf(entry -> entry.getValue().tenant().equals(key)); } public synchronized Collection<String> getAllHosts() { - return Collections.unmodifiableCollection(new ArrayList<>(host2KeyMap.keySet())); + return Collections.unmodifiableCollection(new ArrayList<>(host2ApplicationId.keySet())); } - public synchronized Collection<String> getHostsForKey(ApplicationId key) { - return host2KeyMap.entrySet().stream() - .filter(entry -> entry.getValue().equals(key)) - .map(Map.Entry::getKey) - .collect(Collectors.toSet()); + public synchronized Collection<String> getHosts(ApplicationId key) { + return host2ApplicationId.entrySet().stream() + .filter(entry -> entry.getValue().equals(key)) + .map(Map.Entry::getKey) + .collect(Collectors.toSet()); } private boolean hostAlreadyTaken(String host, ApplicationId key) { - return host2KeyMap.containsKey(host) && !key.equals(host2KeyMap.get(host)); + return host2ApplicationId.containsKey(host) && !key.equals(host2ApplicationId.get(host)); } - private static Collection<String> getRemovedHosts(Collection<String> newHosts, Collection<String> previousHosts) { + private static Collection<String> findRemovedHosts(Collection<String> newHosts, Collection<String> previousHosts) { return Collections2.filter(previousHosts, host -> !newHosts.contains(host)); } private void removeHosts(Collection<String> removedHosts) { for (String host : removedHosts) { log.log(Level.FINE, () -> "Removing " + host); - host2KeyMap.remove(host); + host2ApplicationId.remove(host); } } private void addHosts(ApplicationId key, Collection<String> newHosts) { for (String host : newHosts) { log.log(Level.FINE, () -> "Adding " + host); - host2KeyMap.put(host, key); + host2ApplicationId.put(host, key); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java index 9229fb88b40..5547156721a 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/HostProvisionerProvider.java @@ -42,11 +42,6 @@ public class HostProvisionerProvider { } // for testing - public static HostProvisionerProvider withProvisioner(Provisioner provisioner, boolean hostedVespa) { - return withProvisioner(provisioner, new ConfigserverConfig(new ConfigserverConfig.Builder().hostedVespa(hostedVespa))); - } - - // for testing public static HostProvisionerProvider withProvisioner(Provisioner provisioner, ConfigserverConfig config) { ComponentRegistry<Provisioner> registry = new ComponentRegistry<>(); registry.register(ComponentId.createAnonymousComponentId("foobar"), provisioner); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java index 1c419ce047a..5e7fff8c922 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/GetConfigProcessor.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.config.server.rpc; import com.yahoo.cloud.config.SentinelConfig; -import com.yahoo.collections.Pair; import com.yahoo.component.Version; import com.yahoo.config.provision.TenantName; import com.yahoo.container.di.config.ApplicationBundlesConfig; @@ -37,12 +36,10 @@ class GetConfigProcessor implements Runnable { private static final Logger log = Logger.getLogger(GetConfigProcessor.class.getName()); private static final String localHostName = HostName.getLocalhost(); - private static final PayloadChecksums emptyApplicationBundlesConfigChecksums = PayloadChecksums.fromPayload(Payload.from(ConfigPayload.fromInstance(new ApplicationBundlesConfig.Builder().build()))); private final JRTServerConfigRequest request; - /* True only when this request has expired its server timeout and we need to respond to the client */ private final boolean forceResponse; private final RpcServer rpcServer; @@ -75,13 +72,13 @@ class GetConfigProcessor implements Runnable { } // TODO: Increment statistics (Metrics) failed counters when requests fail - public Pair<GetConfigContext, Long> getConfig(JRTServerConfigRequest request) { + public Optional<DelayedConfig> resolveConfig(JRTServerConfigRequest request) { // Request has already been detached if ( ! request.validateParameters()) { // Error code is set in verifyParameters if parameters are not OK. log.log(Level.WARNING, "Parameters for request " + request + " did not validate: " + request.errorCode() + " : " + request.errorMessage()); respond(request); - return null; + return Optional.empty(); } Trace trace = request.getRequestTrace(); debugLog(trace, "GetConfigProcessor.run() on " + localHostName); @@ -92,13 +89,13 @@ class GetConfigProcessor implements Runnable { // fabricate an empty request to cause the sentinel to stop all running services if (rpcServer.canReturnEmptySentinelConfig() && rpcServer.allTenantsLoaded() && tenant.isEmpty() && isSentinelConfigRequest(request)) { returnEmpty(request); - return null; + return Optional.empty(); } GetConfigContext context = rpcServer.createGetConfigContext(tenant, request, trace); - if (context == null || ! context.requestHandler().hasApplication(context.applicationId(), Optional.empty())) { + if (context.isEmpty() || ! context.requestHandler().hasApplication(context.applicationId(), Optional.empty())) { handleError(request, APPLICATION_NOT_LOADED, "No application exists"); - return null; + return Optional.empty(); } logPre = TenantRepository.logPre(context.applicationId()); @@ -109,7 +106,7 @@ class GetConfigProcessor implements Runnable { if ( ! context.requestHandler().hasApplication(context.applicationId(), vespaVersion)) { handleError(request, ErrorCode.UNKNOWN_VESPA_VERSION, "Unknown Vespa version in request: " + printableVespaVersion(vespaVersion)); - return null; + return Optional.empty(); } ConfigResponse config; @@ -117,14 +114,14 @@ class GetConfigProcessor implements Runnable { config = rpcServer.resolveConfig(request, context, vespaVersion); } catch (UnknownConfigDefinitionException e) { handleError(request, ErrorCode.UNKNOWN_DEFINITION, "Unknown config definition " + request.getConfigKey()); - return null; + return Optional.empty(); } catch (UnknownConfigIdException e) { handleError(request, ErrorCode.ILLEGAL_CONFIGID, "Illegal config id " + request.getConfigKey().getConfigId()); - return null; + return Optional.empty(); } catch (Throwable e) { log.log(Level.SEVERE, "Unexpected error handling config request", e); handleError(request, ErrorCode.INTERNAL_ERROR, "Internal error " + e.getMessage()); - return null; + return Optional.empty(); } // config == null is not an error, but indicates that the config will be returned later. @@ -135,7 +132,7 @@ class GetConfigProcessor implements Runnable { && ! context.requestHandler().compatibleWith(vespaVersion, context.applicationId()) // ... with a runtime version incompatible with the deploying version ... && ! emptyApplicationBundlesConfigChecksums.matches(config.getPayloadChecksums())) { // ... and there actually are incompatible user bundles, then return no config: handleError(request, ErrorCode.INCOMPATIBLE_VESPA_VERSION, "Version " + printableVespaVersion(vespaVersion) + " is binary incompatible with the latest deployed version"); - return null; + return Optional.empty(); } // debugLog(trace, "config response before encoding:" + config.toString()); @@ -144,23 +141,24 @@ class GetConfigProcessor implements Runnable { respond(request); } else { debugLog(trace, "delaying response " + request.getShortDescription()); - return new Pair<>(context, config != null ? config.getGeneration() : 0); + return Optional.of(new DelayedConfig(context, config != null ? config.getGeneration() : 0)); } - return null; + return Optional.empty(); } @Override public void run() { - Pair<GetConfigContext, Long> delayed = getConfig(request); + Optional<DelayedConfig> delayed = resolveConfig(request); - if (delayed != null) { - rpcServer.delayResponse(request, delayed.getFirst()); - if (rpcServer.hasNewerGeneration(delayed.getFirst().applicationId(), delayed.getSecond())) { + delayed.ifPresent(d -> { + GetConfigContext context = d.context(); + rpcServer.delayResponse(request, context); + if (rpcServer.hasNewerGeneration(context.applicationId(), d.generation())) { // This will ensure that if the config activation train left the station while I was boarding, // another train will immediately be scheduled. - rpcServer.configActivated(delayed.getFirst().applicationId()); + rpcServer.configActivated(context.applicationId()); } - } + }); } private boolean isSentinelConfigRequest(JRTServerConfigRequest request) { @@ -194,4 +192,6 @@ class GetConfigProcessor implements Runnable { } } + private record DelayedConfig(GetConfigContext context, long generation) {} + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java index be4738258d8..7a1b2d2aeef 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/RpcServer.java @@ -44,11 +44,9 @@ import com.yahoo.vespa.filedistribution.FileDownloader; import com.yahoo.vespa.filedistribution.FileReceiver; import com.yahoo.vespa.filedistribution.FileReferenceData; import com.yahoo.vespa.filedistribution.FileReferenceDownload; - import java.nio.ByteBuffer; import java.time.Duration; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; @@ -320,17 +318,6 @@ public class RpcServer implements Runnable, ConfigActivationListener, TenantList } @Override - public void hostsUpdated(ApplicationId applicationId, Collection<String> newHosts) { - log.log(Level.FINE, () -> "Updating hosts in tenant host registry '" + hostRegistry + "' with " + newHosts); - hostRegistry.update(applicationId, newHosts); - } - - @Override - public void verifyHostsAreAvailable(ApplicationId applicationId, Collection<String> newHosts) { - hostRegistry.verifyHosts(applicationId, newHosts); - } - - @Override public void applicationRemoved(ApplicationId applicationId) { superModelRequestHandler.removeApplication(applicationId); configActivated(applicationId); @@ -350,8 +337,9 @@ public class RpcServer implements Runnable, ConfigActivationListener, TenantList */ Optional<TenantName> resolveTenant(JRTServerConfigRequest request, Trace trace) { if ("*".equals(request.getConfigKey().getConfigId())) return Optional.of(ApplicationId.global().tenant()); + String hostname = request.getClientHostName(); - ApplicationId applicationId = hostRegistry.getKeyForHost(hostname); + ApplicationId applicationId = hostRegistry.getApplicationId(hostname); if (applicationId == null) { if (GetConfigProcessor.logDebug(trace)) { String message = "Did not find tenant for host '" + hostname + "', using " + TenantName.defaultName() + @@ -418,7 +406,7 @@ public class RpcServer implements Runnable, ConfigActivationListener, TenantList metrics.incUnknownHostRequests(); trace.trace(TRACELEVEL, msg); log.log(Level.WARNING, msg); - return null; + return GetConfigContext.empty(); } RequestHandler handler = requestHandler.get(); ApplicationId applicationId = handler.resolveApplicationId(request.getClientHostName()); @@ -445,7 +433,6 @@ public class RpcServer implements Runnable, ConfigActivationListener, TenantList log.log(Level.FINE, () -> TenantRepository.logPre(tenant) + "Tenant deleted, removing request handler and cleaning host registry"); tenants.remove(tenant); - hostRegistry.removeHostsForKey(tenant); } @Override diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java index 536a446df2f..21f7354401f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/rpc/security/MultiTenantRpcAuthorizer.java @@ -106,7 +106,7 @@ public class MultiTenantRpcAuthorizer implements RpcAuthorizer { return; // global config access ok } else { String hostname = configRequest.getClientHostName(); - ApplicationId applicationId = hostRegistry.getKeyForHost(hostname); + ApplicationId applicationId = hostRegistry.getApplicationId(hostname); if (applicationId == null) { if (isConfigKeyForSentinelConfig(configKey)) { return; // config processor will return empty sentinel config for unknown nodes diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index ffafbb8827e..06c3aa5330e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -270,8 +270,7 @@ public class SessionPreparer { // Validate after doing our own preprocessing on these two files ApplicationMetaData meta = applicationPackage.getMetaData(); InstanceName instance = meta.getApplicationId().instance(); - Tags tags = applicationPackage.getDeployment().map(new DeploymentSpecXmlReader(false)::read) - .flatMap(spec -> spec.instance(instance)) + Tags tags = applicationPackage.getDeploymentSpec().instance(instance) .map(DeploymentInstanceSpec::tags) .orElse(Tags.empty()); if (servicesXml.exists()) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java index 69d13bf2dea..4833a62aa37 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TenantRepository.java @@ -401,6 +401,7 @@ public class TenantRepository { } private void notifyRemovedTenant(TenantName name) { + hostRegistry.removeHosts(name); tenantListener.onTenantDelete(name); } @@ -618,4 +619,6 @@ public class TenantRepository { public com.yahoo.vespa.curator.Curator getCurator() { return curator; } + public HostProvisionerProvider hostProvisionerProvider() { return hostProvisionerProvider; } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java index fbf14fbdb8c..6d6901136a6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackage.java @@ -6,6 +6,7 @@ import com.yahoo.component.Version; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationMetaData; import com.yahoo.config.application.api.ComponentInfo; +import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.codegen.DefParser; @@ -55,6 +56,8 @@ public class ZKApplicationPackage extends AbstractApplicationPackage { public static final String allocatedHostsNode = "allocatedHosts"; private final ApplicationMetaData metaData; + private DeploymentSpec deploymentSpec = null; + public ZKApplicationPackage(AddFileInterface fileManager, Curator curator, Path sessionPath, int maxNodeSize) { verifyAppPath(curator, sessionPath); zkApplication = new ZKApplication(curator, sessionPath, maxNodeSize); @@ -73,6 +76,12 @@ public class ZKApplicationPackage extends AbstractApplicationPackage { return Optional.of(readAllocatedHosts()); } + @Override + public DeploymentSpec getDeploymentSpec() { + if (deploymentSpec != null) return deploymentSpec; + return deploymentSpec = parseDeploymentSpec(false); + } + /** * Reads allocated hosts at the given node. * diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 498940f8a63..253b0d7c101 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -13,7 +13,6 @@ import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Deployment; -import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NetworkPorts; @@ -38,6 +37,7 @@ import com.yahoo.vespa.config.server.deploy.TenantFileSystemDirs; import com.yahoo.vespa.config.server.filedistribution.FileDirectory; import com.yahoo.vespa.config.server.filedistribution.MockFileDistributionFactory; import com.yahoo.vespa.config.server.http.v2.PrepareResult; +import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.LocalSession; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.session.Session; @@ -79,7 +79,6 @@ import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; /** * @author hmusum @@ -125,21 +124,21 @@ public class ApplicationRepositoryTest { .build(); flagSource = new InMemoryFlagSource(); fileDirectory = new FileDirectory(configserverConfig); + provisioner = new MockProvisioner(); tenantRepository = new TestTenantRepository.Builder() .withClock(clock) .withConfigserverConfig(configserverConfig) .withCurator(curator) .withFileDistributionFactory(new MockFileDistributionFactory(configserverConfig)) .withFlagSource(flagSource) + .withHostProvisionerProvider(HostProvisionerProvider.withProvisioner(provisioner, configserverConfig)) .build(); tenantRepository.addTenant(TenantRepository.HOSTED_VESPA_TENANT); tenantRepository.addTenant(tenant1); tenantRepository.addTenant(tenant2); orchestrator = new OrchestratorMock(); - provisioner = new MockProvisioner(); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(provisioner) .withConfigserverConfig(configserverConfig) .withOrchestrator(orchestrator) .withLogRetriever(new MockLogRetriever()) @@ -177,7 +176,6 @@ public class ApplicationRepositoryTest { public void prepareAndActivateWithRestart() { applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(provisioner) .withConfigserverConfig(configserverConfig) .withOrchestrator(orchestrator) .withLogRetriever(new MockLogRetriever()) @@ -188,8 +186,7 @@ public class ApplicationRepositoryTest { prepareAndActivate(testAppJdiscOnly); PrepareResult result = prepareAndActivate(testAppJdiscOnlyRestart); assertTrue(result.configChangeActions().getRefeedActions().isEmpty()); - assertTrue(result.configChangeActions().getRestartActions().isEmpty()); - assertEquals(HostFilter.hostname("mytesthost2"), provisioner.lastRestartFilter()); + assertFalse(result.configChangeActions().getRestartActions().isEmpty()); } @Test @@ -197,7 +194,6 @@ public class ApplicationRepositoryTest { applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) .withOrchestrator(orchestrator) - .withProvisioner(null) .build(); prepareAndActivate(testAppJdiscOnly); @@ -285,7 +281,6 @@ public class ApplicationRepositoryTest { applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(provisioner) .withOrchestrator(orchestrator) .withClock(clock) .build(); @@ -329,20 +324,16 @@ public class ApplicationRepositoryTest { File sessionFile = new File(tenantFileSystemDirs.sessionsPath(), String.valueOf(sessionId)); assertTrue(sessionFile.exists()); - // Delete app and verify that it has been deleted from repos and provisioner and no application set exists + // Delete app and verify that it has been deleted from repos and no application set exists assertTrue(applicationRepository.delete(applicationId())); assertTrue(applicationRepository.getActiveSession(applicationId()).isEmpty()); assertEquals(Optional.empty(), sessionRepository.getRemoteSession(sessionId).applicationSet()); - assertEquals(1, provisioner.removeCount()); - assertEquals(tenant().getName(), provisioner.lastApplicationId().tenant()); - assertEquals(applicationId(), provisioner.lastApplicationId()); assertTrue(curator.exists(sessionNode)); assertEquals(Session.Status.DELETE.name(), Utf8.toString(curator.getData(sessionNode.append("sessionState")).get())); assertTrue(sessionFile.exists()); - // Deleting a non-existent application still attempts to remove resources + // Deleting a non-existent application will return false assertFalse(applicationRepository.delete(applicationId())); - assertEquals(2, provisioner.removeCount()); } { @@ -358,46 +349,9 @@ public class ApplicationRepositoryTest { // Delete app with id fooId, should not affect original app assertTrue(applicationRepository.delete(fooId)); - assertEquals(fooId, provisioner.lastApplicationId()); - assertNotNull(applicationRepository.getActiveSession(applicationId())); - - assertTrue(applicationRepository.delete(applicationId())); - } - - // If delete fails, a retry should work if the failure is transient and zookeeper state should be consistent - { - long sessionId = deployApp(testApp).sessionId(); - assertNotNull(sessionRepository.getRemoteSession(sessionId)); - assertNotNull(applicationRepository.getActiveSession(applicationId())); - assertEquals(sessionId, applicationRepository.getActiveSession(applicationId()).get().getSessionId()); - assertNotNull(applicationRepository.getApplication(applicationId())); - - provisioner.failureOnRemove(true); - try { - applicationRepository.delete(applicationId()); - fail("Should fail with RuntimeException"); - } catch (RuntimeException e) { - // ignore - } - assertNotNull(sessionRepository.getRemoteSession(sessionId)); assertNotNull(applicationRepository.getActiveSession(applicationId())); - assertEquals(sessionId, applicationRepository.getActiveSession(applicationId()).get().getSessionId()); - // Delete should work when there is no failure anymore - provisioner.failureOnRemove(false); assertTrue(applicationRepository.delete(applicationId())); - - // Session should be in state DELETE - Path sessionNode = sessionRepository.getSessionPath(sessionId); - assertEquals(Session.Status.DELETE.name(), Utf8.toString(curator.getData(sessionNode.append("sessionState")).get())); - assertNotNull(sessionRepository.getRemoteSession(sessionId)); // session still exists - assertTrue(applicationRepository.getActiveSession(applicationId()).isEmpty()); // but it is not active - try { - applicationRepository.getApplication(applicationId()); - fail("Should fail with NotFoundException, application should not exist"); - } catch (NotFoundException e) { - // ignore - } } } @@ -522,7 +476,6 @@ public class ApplicationRepositoryTest { MockMetric actual = new MockMetric(); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(provisioner) .withOrchestrator(orchestrator) .withMetric(actual) .withClock(new ManualClock()) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockProvisioner.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockProvisioner.java index 0ba3a6d883c..8effd9b6dfe 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/MockProvisioner.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockProvisioner.java @@ -22,15 +22,7 @@ import java.util.List; */ public class MockProvisioner implements Provisioner { - private boolean activated = false; - private int removeCount = 0; - private boolean restarted = false; - private ApplicationId lastApplicationId; - private Collection<HostSpec> lastHosts; - private HostFilter lastRestartFilter; - private boolean transientFailureOnPrepare = false; - private boolean failureOnRemove = false; private HostProvisioner hostProvisioner = null; public MockProvisioner hostProvisioner(HostProvisioner hostProvisioner) { @@ -43,10 +35,6 @@ public class MockProvisioner implements Provisioner { return this; } - public void failureOnRemove(boolean failureOnRemove) { - this.failureOnRemove = failureOnRemove; - } - @Override public List<HostSpec> prepare(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity, ProvisionLogger logger) { if (hostProvisioner != null) { @@ -60,25 +48,14 @@ public class MockProvisioner implements Provisioner { @Override public void activate(Collection<HostSpec> hosts, ActivationContext context, ApplicationTransaction transaction) { - activated = true; - lastApplicationId = transaction.application(); - lastHosts = hosts; } @Override public void remove(ApplicationTransaction transaction) { - if (failureOnRemove) - throw new IllegalStateException("Unable to remove " + transaction.application()); - - removeCount++; - lastApplicationId = transaction.application(); } @Override public void restart(ApplicationId application, HostFilter filter) { - restarted = true; - lastApplicationId = application; - lastRestartFilter = filter; } @Override @@ -86,28 +63,4 @@ public class MockProvisioner implements Provisioner { return new ProvisionLock(application, () -> {}); } - public Collection<HostSpec> lastHosts() { - return lastHosts; - } - - public boolean activated() { - return activated; - } - - public int removeCount() { - return removeCount; - } - - public boolean restarted() { - return restarted; - } - - public ApplicationId lastApplicationId() { - return lastApplicationId; - } - - public HostFilter lastRestartFilter() { - return lastRestartFilter; - } - } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java index 85e64b4a32d..728f3e8510f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java @@ -203,15 +203,6 @@ public class TenantApplicationsTest { } @Override - public void hostsUpdated(ApplicationId applicationId, Collection<String> newHosts) { - tenantHosts.put(applicationId.tenant().value(), newHosts); - } - - @Override - public void verifyHostsAreAvailable(ApplicationId applicationId, Collection<String> newHosts) { - } - - @Override public void applicationRemoved(ApplicationId applicationId) { removed.incrementAndGet(); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java index ab527833803..c716219f86b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/DeployTester.java @@ -296,7 +296,8 @@ public class DeployTester { .withZone(zone) .withFlagSource(flagSource); - if (configserverConfig.hostedVespa()) builder.withHostProvisionerProvider(HostProvisionerProvider.withProvisioner(provisioner, true)); + if (configserverConfig.hostedVespa()) + builder.withHostProvisionerProvider(HostProvisionerProvider.withProvisioner(provisioner, configserverConfig)); TenantRepository tenantRepository = builder.build(); tenantRepository.addTenant(tenantName); @@ -306,7 +307,6 @@ public class DeployTester { .withConfigserverConfig(configserverConfig) .withOrchestrator(new OrchestratorMock()) .withClock(clock) - .withProvisioner(provisioner) .withConfigConvergenceChecker(configConvergenceChecker) .withFlagSource(flagSource) .build(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/host/HostRegistryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/host/HostRegistryTest.java index df00d28134f..646017a498e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/host/HostRegistryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/host/HostRegistryTest.java @@ -24,23 +24,23 @@ public class HostRegistryTest { @Test public void old_hosts_are_removed() { HostRegistry reg = new HostRegistry(); - assertNull(reg.getKeyForHost("foo.com")); + assertNull(reg.getApplicationId("foo.com")); reg.update(foo, List.of("foo.com", "bar.com", "baz.com")); assertGetKey(reg, "foo.com", foo); assertGetKey(reg, "bar.com", foo); assertGetKey(reg, "baz.com", foo); assertEquals(3, reg.getAllHosts().size()); reg.update(foo, List.of("bar.com", "baz.com")); - assertNull(reg.getKeyForHost("foo.com")); + assertNull(reg.getApplicationId("foo.com")); assertGetKey(reg, "bar.com", foo); assertGetKey(reg, "baz.com", foo); assertEquals(2, reg.getAllHosts().size()); assertTrue(reg.getAllHosts().containsAll(List.of("bar.com", "baz.com"))); - reg.removeHostsForKey(foo); + reg.removeHosts(foo); assertTrue(reg.getAllHosts().isEmpty()); - assertNull(reg.getKeyForHost("foo.com")); - assertNull(reg.getKeyForHost("bar.com")); + assertNull(reg.getApplicationId("foo.com")); + assertNull(reg.getApplicationId("bar.com")); } @Test @@ -74,9 +74,9 @@ public class HostRegistryTest { HostRegistry reg = new HostRegistry(); List<String> hosts = new ArrayList<>(List.of("foo.com", "bar.com", "baz.com")); reg.update(foo, hosts); - assertEquals(3, reg.getHostsForKey(foo).size()); + assertEquals(3, reg.getHosts(foo).size()); hosts.remove(2); - assertEquals(3, reg.getHostsForKey(foo).size()); + assertEquals(3, reg.getHosts(foo).size()); } @Test @@ -90,8 +90,8 @@ public class HostRegistryTest { } private void assertGetKey(HostRegistry reg, String host, ApplicationId expectedKey) { - assertNotNull(reg.getKeyForHost(host)); - assertEquals(expectedKey, reg.getKeyForHost(host)); + assertNotNull(reg.getApplicationId(host)); + assertEquals(expectedKey, reg.getApplicationId(host)); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java index 279f3a237e8..6cfdab1257f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpGetConfigHandlerTest.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.TenantRepository; @@ -16,7 +15,6 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; - import java.io.File; import java.io.IOException; import java.util.Collections; @@ -59,7 +57,6 @@ public class HttpGetConfigHandlerTest { tenantRepository.addTenant(tenant); ApplicationRepository applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(new MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .withConfigserverConfig(configserverConfig) .build(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java index 520b4d0edc5..79881d07b25 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/HttpListConfigsHandlerTest.java @@ -8,7 +8,6 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.HttpListConfigsHandler.ListConfigsResponse; import com.yahoo.vespa.config.server.session.PrepareParams; @@ -18,7 +17,6 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; - import java.io.File; import java.io.IOException; import java.util.HashSet; @@ -64,7 +62,6 @@ public class HttpListConfigsHandlerTest { tenantRepository.addTenant(tenant); ApplicationRepository applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(new MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .withConfigserverConfig(configserverConfig) .build(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java index 052f39c9e1f..60ee3299de5 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationContentHandlerTest.java @@ -9,7 +9,6 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.Response; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.ContentHandlerTestBase; import com.yahoo.vespa.config.server.session.PrepareParams; @@ -21,7 +20,6 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; - import java.io.File; import java.io.IOException; @@ -63,7 +61,6 @@ public class ApplicationContentHandlerTest extends ContentHandlerTestBase { applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(new MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .withConfigserverConfig(configserverConfig) .build(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index c270b4559f9..e8c4d819c31 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -87,9 +87,7 @@ import static com.yahoo.vespa.config.server.http.v2.ApplicationHandler.HttpServi import static com.yahoo.yolean.Exceptions.uncheck; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -136,14 +134,13 @@ public class ApplicationHandlerTest { .withClock(clock) .withConfigserverConfig(configserverConfig) .withFileDistributionFactory(new MockFileDistributionFactory(configserverConfig)) - .withHostProvisionerProvider(HostProvisionerProvider.withProvisioner(provisioner, false)) + .withHostProvisionerProvider(HostProvisionerProvider.withProvisioner(provisioner, configserverConfig)) .withModelFactoryRegistry(new ModelFactoryRegistry(modelFactories)) .build(); tenantRepository.addTenant(mytenantName); orchestrator = new OrchestratorMock(); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(provisioner) .withOrchestrator(orchestrator) .withClock(clock) .withTesterClient(testerClient) @@ -350,11 +347,9 @@ public class ApplicationHandlerTest { @Test public void testRestart() throws Exception { - applicationRepository.deploy(testApp, prepareParams(applicationId)); - assertFalse(provisioner.restarted()); + var result = applicationRepository.deploy(testApp, prepareParams(applicationId)); + assertTrue(result.configChangeActions().getRestartActions().isEmpty()); restart(applicationId, Zone.defaultZone()); - assertTrue(provisioner.restarted()); - assertEquals(applicationId, provisioner.lastApplicationId()); } @Test @@ -378,7 +373,6 @@ public class ApplicationHandlerTest { HttpProxy mockHttpProxy = mock(HttpProxy.class); ApplicationRepository applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withHostProvisionerProvider(HostProvisionerProvider.empty()) .withOrchestrator(orchestrator) .withTesterClient(testerClient) .withHttpProxy(mockHttpProxy) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java index fbc5e87c329..ba1d69c13dd 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HostHandlerTest.java @@ -11,7 +11,6 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.Response; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; @@ -22,7 +21,6 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; - import java.io.File; import java.io.IOException; @@ -58,7 +56,6 @@ public class HostHandlerTest { tenantRepository.addTenant(mytenant); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(new MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .withConfigserverConfig(configserverConfig) .build(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java index 9aae64cb884..a0b5b879e45 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpGetConfigHandlerTest.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpConfigRequest; @@ -23,7 +22,6 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; - import java.io.File; import java.io.IOException; import java.util.Collections; @@ -60,15 +58,13 @@ public class HttpGetConfigHandlerTest { .configDefinitionsDir(temporaryFolder.newFolder().getAbsolutePath()) .fileReferencesDir(temporaryFolder.newFolder().getAbsolutePath()) .build(); - MockProvisioner provisioner = new MockProvisioner(); TenantRepository tenantRepository = new TestTenantRepository.Builder() .withConfigserverConfig(configserverConfig) - .withHostProvisionerProvider(HostProvisionerProvider.withProvisioner(provisioner, false)) + .withHostProvisionerProvider(HostProvisionerProvider.empty()) .build(); tenantRepository.addTenant(tenant); ApplicationRepository applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(provisioner) .withOrchestrator(new OrchestratorMock()) .withConfigserverConfig(configserverConfig) .build(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java index 2ee1064f614..3762e52ae62 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/HttpListConfigsHandlerTest.java @@ -11,7 +11,6 @@ import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; @@ -23,7 +22,6 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; - import java.io.File; import java.io.IOException; import java.util.HashSet; @@ -71,7 +69,6 @@ public class HttpListConfigsHandlerTest { tenantRepository.addTenant(tenant); ApplicationRepository applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(new MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .withConfigserverConfig(configserverConfig) .build(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java index 1c71ef0b7fb..d7e6273352b 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionActiveHandlerTest.java @@ -19,6 +19,7 @@ import com.yahoo.vespa.config.server.http.HandlerTest; import com.yahoo.vespa.config.server.http.HttpErrorResponse; import com.yahoo.vespa.config.server.model.TestModelFactory; import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry; +import com.yahoo.vespa.config.server.provision.HostProvisionerProvider; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.session.Session; import com.yahoo.vespa.config.server.tenant.Tenant; @@ -74,11 +75,11 @@ public class SessionActiveHandlerTest { TenantRepository tenantRepository = new TestTenantRepository.Builder() .withConfigserverConfig(configserverConfig) .withModelFactoryRegistry(new ModelFactoryRegistry(List.of((modelFactory)))) + .withHostProvisionerProvider(HostProvisionerProvider.withProvisioner(provisioner, configserverConfig)) .build(); tenantRepository.addTenant(tenantName); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(provisioner) .withOrchestrator(new OrchestratorMock()) .withClock(clock) .withConfigserverConfig(configserverConfig) @@ -164,8 +165,6 @@ public class SessionActiveHandlerTest { "/environment/" + "prod" + "/region/" + "default" + "/instance/" + "default")); - assertTrue(provisioner.activated()); - assertEquals(1, provisioner.lastHosts().size()); } private SessionActiveHandler createHandler() { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java index 7c2e0be0c3a..b17f80fd510 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionContentHandlerTest.java @@ -11,7 +11,6 @@ import com.yahoo.jdisc.Response; import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.ContentHandlerTestBase; import com.yahoo.vespa.config.server.http.SessionHandlerTest; @@ -24,7 +23,6 @@ import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; - import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; @@ -63,7 +61,6 @@ public class SessionContentHandlerTest extends ContentHandlerTestBase { ApplicationRepository applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(new MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .withConfigserverConfig(configserverConfig) .build(); @@ -186,7 +183,6 @@ public class SessionContentHandlerTest extends ContentHandlerTestBase { SessionContentHandler.testContext(), new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(new MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .withClock(Clock.systemUTC()) .build() diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java index 2e86f5e0538..04531fbb2e0 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionCreateHandlerTest.java @@ -1,15 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.http.v2; +import ai.vespa.http.HttpURL.Path; import com.yahoo.cloud.config.ConfigserverConfig; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import ai.vespa.http.HttpURL.Path; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.application.CompressedApplicationInputStreamTest; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.HttpErrorResponse; @@ -23,7 +22,6 @@ import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; - import java.io.ByteArrayInputStream; import java.io.File; import java.io.FileInputStream; @@ -84,7 +82,6 @@ public class SessionCreateHandlerTest extends SessionHandlerTest { .build(); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(new MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .build(); tenantRepository.addTenant(tenant); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java index de6073bb1ea..765523177a9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java @@ -16,7 +16,6 @@ import com.yahoo.jdisc.http.HttpRequest; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.TimeoutBudget; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.http.HttpErrorResponse; @@ -30,7 +29,6 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; - import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -82,7 +80,6 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { tenantRepository.addTenant(tenant); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(new MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .withClock(clock) .withConfigserverConfig(configserverConfig) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java index b8bd35a564a..b39050250f9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/TenantHandlerTest.java @@ -10,7 +10,6 @@ import com.yahoo.container.jdisc.HttpRequestBuilder; import com.yahoo.jdisc.http.HttpRequest.Method; import com.yahoo.restapi.RestApiTestDriver; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.session.PrepareParams; import com.yahoo.vespa.config.server.tenant.TenantRepository; @@ -20,7 +19,6 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; - import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -58,7 +56,6 @@ public class TenantHandlerTest { .build(); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(new MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .withConfigserverConfig(configserverConfig) .build(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java index a2dc0216b72..5fb92e1f66f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/maintenance/MaintainerTester.java @@ -41,13 +41,12 @@ class MaintainerTester { .build(); tenantRepository = new TestTenantRepository.Builder() .withClock(clock) - .withHostProvisionerProvider(HostProvisionerProvider.withProvisioner(provisioner, true)) + .withHostProvisionerProvider(HostProvisionerProvider.withProvisioner(provisioner, configserverConfig)) .withConfigserverConfig(configserverConfig) .withModelFactoryRegistry(new ModelFactoryRegistry(List.of(new DeployTester.CountingModelFactory(clock)))) .build(); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(provisioner) .withOrchestrator(new OrchestratorMock()) .withLogRetriever(new MockLogRetriever()) .withClock(clock) diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java index 8607fc0e2dc..9190cbc0d8a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcServerTest.java @@ -38,10 +38,10 @@ import java.io.File; import java.io.IOException; import java.util.Optional; +import static com.yahoo.vespa.config.server.rpc.RpcServer.ChunkedFileReceiver.createMetaRequest; import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.gzip; import static com.yahoo.vespa.filedistribution.FileReferenceData.CompressionType.lz4; import static com.yahoo.vespa.filedistribution.FileReferenceData.Type.compressed; -import static com.yahoo.vespa.config.server.rpc.RpcServer.ChunkedFileReceiver.createMetaRequest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -84,7 +84,7 @@ public class RpcServerTest { public void testEmptySentinelConfigWhenAppDeletedOnHostedVespa() throws IOException, InterruptedException { ConfigserverConfig.Builder configBuilder = new ConfigserverConfig.Builder().canReturnEmptySentinelConfig(true); try (RpcTester tester = new RpcTester(applicationId, temporaryFolder, configBuilder)) { - tester.rpcServer().onTenantDelete(tenantName); + tester.hostRegistry.removeHosts(applicationId); tester.rpcServer().onTenantsLoaded(); JRTClientConfigRequest clientReq = createSentinelRequest(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java index b29edd480ad..8770970308a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/rpc/RpcTester.java @@ -12,7 +12,6 @@ import com.yahoo.jrt.Transport; import com.yahoo.test.ManualClock; import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.MemoryGenerationCounter; -import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.PortRangeAllocator; import com.yahoo.vespa.config.server.SuperModelManager; import com.yahoo.vespa.config.server.SuperModelRequestHandler; @@ -55,7 +54,7 @@ public class RpcTester implements AutoCloseable { private final ApplicationId applicationId; private final TenantName tenantName; private final TenantRepository tenantRepository; - private final HostRegistry hostRegistry = new HostRegistry(); + final HostRegistry hostRegistry = new HostRegistry(); private final ApplicationRepository applicationRepository; private final List<Integer> allocatedPorts = new ArrayList<>(); @@ -87,7 +86,6 @@ public class RpcTester implements AutoCloseable { applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) .withConfigserverConfig(configserverConfig) - .withProvisioner(new MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .build(); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index c7113bbf803..cc6cd4d86e9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -335,7 +335,9 @@ public class SessionPreparerTest { @Test(expected = LoadBalancerServiceException.class) public void require_that_conflict_is_returned_when_creating_load_balancer_fails() throws IOException { - preparer = createPreparer(HostProvisionerProvider.withProvisioner(new MockProvisioner().transientFailureOnPrepare(), true)); + var configserverConfig = new ConfigserverConfig.Builder().hostedVespa(true).build(); + MockProvisioner provisioner = new MockProvisioner().transientFailureOnPrepare(); + preparer = createPreparer(HostProvisionerProvider.withProvisioner(provisioner, configserverConfig)); var params = new PrepareParams.Builder().applicationId(applicationId("test")).build(); prepare(new File("src/test/resources/deploy/hosted-app"), params); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java index a5360fbc01c..83ada4122c2 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionRepositoryTest.java @@ -19,7 +19,6 @@ import com.yahoo.io.reader.NamedReader; import com.yahoo.path.Path; import com.yahoo.text.Utf8; import com.yahoo.vespa.config.server.ApplicationRepository; -import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.application.ApplicationSet; import com.yahoo.vespa.config.server.application.OrchestratorMock; import com.yahoo.vespa.config.server.filedistribution.MockFileDistributionFactory; @@ -103,7 +102,6 @@ public class SessionRepositoryTest { tenantRepository.addTenant(SessionRepositoryTest.tenantName); applicationRepository = new ApplicationRepository.Builder() .withTenantRepository(tenantRepository) - .withProvisioner(new MockProvisioner()) .withOrchestrator(new OrchestratorMock()) .withFlagSource(flagSource) .build(); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java index 823466603b1..9af1bbb875e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TenantRepositoryTest.java @@ -13,7 +13,6 @@ import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.config.server.ConfigServerDB; -import com.yahoo.vespa.config.server.MockProvisioner; import com.yahoo.vespa.config.server.MockSecretStore; import com.yahoo.vespa.config.server.ServerCache; import com.yahoo.vespa.config.server.TestConfigDefinitionRepo; @@ -219,7 +218,7 @@ public class TenantRepositoryTest { flagSource, new InThreadExecutorService(), new MockSecretStore(), - HostProvisionerProvider.withProvisioner(new MockProvisioner(), false), + HostProvisionerProvider.empty(), configserverConfig, new ConfigServerDB(configserverConfig), Zone.defaultZone(), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java index 8f11e171ebe..f3a2c42852f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/zookeeper/ZKApplicationPackageTest.java @@ -101,7 +101,9 @@ public class ZKApplicationPackageTest { assertEquals("6.0.1", readInfo.getHosts().iterator().next().version().get().toString()); assertEquals(dockerImage, readInfo.getHosts().iterator().next().dockerImageRepo().get().asString()); assertTrue(zkApp.getDeployment().isPresent()); + assertFalse(zkApp.getDeploymentSpec().isEmpty()); assertEquals("mydisc", DeploymentSpec.fromXml(zkApp.getDeployment().get()).requireInstance("default").globalServiceId().get()); + assertEquals("mydisc", zkApp.getDeploymentSpec().requireInstance("default").globalServiceId().get()); } private void feed(com.yahoo.vespa.curator.Curator zk, File dirToFeed) throws IOException { diff --git a/configutil/CMakeLists.txt b/configutil/CMakeLists.txt index 4ec9ad13648..a1d699ae009 100644 --- a/configutil/CMakeLists.txt +++ b/configutil/CMakeLists.txt @@ -2,7 +2,6 @@ vespa_define_module( DEPENDS vespadefaults - fastos config_cloudconfig vbench vespalib diff --git a/container-core/abi-spec.json b/container-core/abi-spec.json index 3d5b9e8d59e..f9b146bc669 100644 --- a/container-core/abi-spec.json +++ b/container-core/abi-spec.json @@ -249,12 +249,15 @@ }, "com.yahoo.container.handler.LogHandler" : { "superClass" : "com.yahoo.container.jdisc.ThreadedHttpRequestHandler", - "interfaces" : [ ], + "interfaces" : [ + "com.yahoo.container.jdisc.utils.CapabilityRequiringRequestHandler" + ], "attributes" : [ "public" ], "methods" : [ "public void <init>(java.util.concurrent.Executor, com.yahoo.container.core.LogHandlerConfig)", + "public com.yahoo.security.tls.Capability requiredCapability(com.yahoo.container.jdisc.RequestView)", "public com.yahoo.container.jdisc.AsyncHttpResponse handle(com.yahoo.container.jdisc.HttpRequest)", "public bridge synthetic com.yahoo.container.jdisc.HttpResponse handle(com.yahoo.container.jdisc.HttpRequest)" ], diff --git a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java index 72a399744f3..4c0f85d5521 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java +++ b/container-core/src/main/java/com/yahoo/container/handler/LogHandler.java @@ -5,9 +5,12 @@ import com.yahoo.component.annotation.Inject; import com.yahoo.container.core.LogHandlerConfig; import com.yahoo.container.jdisc.AsyncHttpResponse; import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.RequestView; import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; +import com.yahoo.container.jdisc.utils.CapabilityRequiringRequestHandler; import com.yahoo.jdisc.handler.CompletionHandler; import com.yahoo.jdisc.handler.ContentChannel; +import com.yahoo.security.tls.Capability; import java.io.OutputStream; import java.time.Instant; @@ -15,7 +18,7 @@ import java.util.Optional; import java.util.concurrent.Executor; import java.util.logging.Level; -public class LogHandler extends ThreadedHttpRequestHandler { +public class LogHandler extends ThreadedHttpRequestHandler implements CapabilityRequiringRequestHandler { private final LogReader logReader; private static final long MB = 1024 * 1024; @@ -30,6 +33,8 @@ public class LogHandler extends ThreadedHttpRequestHandler { this.logReader = logReader; } + @Override public Capability requiredCapability(RequestView __) { return Capability.LOGSERVER_API; } + @Override public AsyncHttpResponse handle(HttpRequest request) { Instant from = Optional.ofNullable(request.getProperty("from")) diff --git a/container-core/src/main/java/com/yahoo/metrics/SearchNodeMetrics.java b/container-core/src/main/java/com/yahoo/metrics/SearchNodeMetrics.java index 42d0bee2dc6..ed38a4b2ba3 100644 --- a/container-core/src/main/java/com/yahoo/metrics/SearchNodeMetrics.java +++ b/container-core/src/main/java/com/yahoo/metrics/SearchNodeMetrics.java @@ -7,6 +7,8 @@ import java.util.List; */ public enum SearchNodeMetrics implements VespaMetrics { + CONTENT_PROTON_CONFIG_GENERATION("content.proton.config.generation", Unit.VERSION, "The oldest config generation used by this search node"), + CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_TOTAL("content.proton.documentdb.documents.total", Unit.DOCUMENT, "The total number of documents in this documents db (ready + not-ready)"), CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_READY("content.proton.documentdb.documents.ready", Unit.DOCUMENT, "The number of ready documents in this document db"), CONTENT_PROTON_DOCUMENTDB_DOCUMENTS_ACTIVE("content.proton.documentdb.documents.active", Unit.DOCUMENT, "The number of active / searchable documents in this document db"), @@ -82,18 +84,6 @@ public enum SearchNodeMetrics implements VespaMetrics { CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_ACCEPTED("content.proton.documentdb.threading_service.summary.accepted", Unit.TASK, "Number of accepted threading service summary tasks"), CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_WAKEUPS("content.proton.documentdb.threading_service.summary.wakeups", Unit.WAKEUP, "Number of times a threading service summary worker thread has been woken up"), CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_UTILIZATION("content.proton.documentdb.threading_service.summary.utilization", Unit.FRACTION, "Ratio of time the threading service summary worker threads has been active"), - CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_INVERTER_QUEUESIZE("content.proton.documentdb.threading_service.index_field_inverter.queuesize", Unit.TASK, "Size of threading service inverter task queue"), - CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_INVERTER_ACCEPTED("content.proton.documentdb.threading_service.index_field_inverter.accepted", Unit.TASK, "Number of accepted threading service inverter tasks"), - CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_INVERTER_WAKEUPS("content.proton.documentdb.threading_service.index_field_inverter.wakeups", Unit.WAKEUP, "Number of times a threading service inverter worker thread has been woken up"), - CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_INVERTER_UTILIZATION("content.proton.documentdb.threading_service.index_field_inverter.utilization", Unit.FRACTION, "Ratio of time the threading service inverter worker threads has been active"), - CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_WRITER_QUEUESIZE("content.proton.documentdb.threading_service.index_field_writer.queuesize", Unit.TASK, "Size of threading service index field writer task queue"), - CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_WRITER_ACCEPTED("content.proton.documentdb.threading_service.index_field_writer.accepted", Unit.TASK, "Number of accepted threading service index field writer tasks"), - CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_WRITER_WAKEUPS("content.proton.documentdb.threading_service.index_field_writer.wakeups", Unit.WAKEUP, "Number of times a threading service index field writer worker thread has been woken up"), - CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_FIELD_WRITER_UTILIZATION("content.proton.documentdb.threading_service.index_field_writer.utilization", Unit.FRACTION, "Ratio of time the threading service index field writer worker threads has been active"), - CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_ATTRIBUTE_FIELD_WRITER_QUEUESIZE("content.proton.documentdb.threading_service.attribute_field_writer.queuesize", Unit.TASK, "Size of threading service attribute field writer task queue"), - CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_ATTRIBUTE_FIELD_WRITER_ACCEPTED("content.proton.documentdb.threading_service.attribute_field_writer.accepted", Unit.TASK, "Number of accepted threading service attribute field writer tasks"), - CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_ATTRIBUTE_FIELD_WRITER_WAKEUPS("content.proton.documentdb.threading_service.attribute_field_writer.wakeups", Unit.WAKEUP, "Number of times a threading service attribute field writer worker thread has been woken up"), - CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_ATTRIBUTE_FIELD_WRITER_UTILIZATION("content.proton.documentdb.threading_service.attribute_field_writer.utilization", Unit.FRACTION, "Ratio of time the threading service attribute field writer worker threads has been active"), // lid space CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_LID_BLOAT_FACTOR("content.proton.documentdb.ready.lid_space.lid_bloat_factor", Unit.FRACTION, "The bloat factor of this lid space, indicating the total amount of holes in the allocated lid space ((lid_limit - used_lids) / lid_limit)"), diff --git a/container-core/src/main/resources/configdefinitions/container.qr-searchers.def b/container-core/src/main/resources/configdefinitions/container.qr-searchers.def index e89ee40d792..034e7fc442b 100644 --- a/container-core/src/main/resources/configdefinitions/container.qr-searchers.def +++ b/container-core/src/main/resources/configdefinitions/container.qr-searchers.def @@ -71,3 +71,6 @@ searchcluster[].indexingmode enum { REALTIME, STREAMING } default=REALTIME ## Storage cluster route to use for search cluster if indexingmode is streaming. searchcluster[].storagecluster.routespec string default="" + +## Enable global phase ranking +searchcluster[].globalphase bool default=false diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index d7600cac927..a95e0061992 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -243,8 +243,8 @@ <error-prone-annotations.version>2.18.0</error-prone-annotations.version> <guava.version>27.1-jre</guava.version> <guice.version>4.2.3</guice.version> - <jackson2.version>2.13.4</jackson2.version> - <jackson-databind.version>2.13.4.2</jackson-databind.version> + <jackson2.version>2.14.2</jackson2.version> + <jackson-databind.version>2.14.2</jackson-databind.version> <javax.inject.version>1</javax.inject.version> <javax.servlet-api.version>3.1.0</javax.servlet-api.version> <javax.ws.rs-api.version>2.0.1</javax.ws.rs-api.version> diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index 729aebf2fc2..08d8d54bc53 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -20,6 +20,7 @@ import com.yahoo.search.Searcher; import com.yahoo.search.config.ClusterConfig; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.query.ParameterParser; +import com.yahoo.search.ranking.GlobalPhaseRanker; import com.yahoo.search.result.ErrorMessage; import com.yahoo.search.schema.SchemaInfo; import com.yahoo.search.searchchain.Execution; @@ -64,6 +65,8 @@ public class ClusterSearcher extends Searcher { private final VespaBackEndSearcher server; private final Executor executor; + private final GlobalPhaseRanker globalPhaseHelper; + private final boolean enableGlobalPhase; @Inject public ClusterSearcher(ComponentId id, @@ -73,10 +76,12 @@ public class ClusterSearcher extends Searcher { DocumentdbInfoConfig documentDbConfig, SchemaInfo schemaInfo, ComponentRegistry<Dispatcher> dispatchers, + GlobalPhaseRanker globalPhaseHelper, VipStatus vipStatus, VespaDocumentAccess access) { super(id); this.executor = executor; + this.globalPhaseHelper = globalPhaseHelper; int searchClusterIndex = clusterConfig.clusterId(); searchClusterName = clusterConfig.clusterName(); QrSearchersConfig.Searchcluster searchClusterConfig = getSearchClusterConfigFromClusterName(qrsConfig, searchClusterName); @@ -101,6 +106,7 @@ public class ClusterSearcher extends Searcher { server = searchDispatch(searchClusterIndex, searchClusterName, uniqueServerId, docSumParams, documentDbConfig, schemaInfo, dispatchers); } + enableGlobalPhase = searchClusterConfig.globalphase(); } private static QrSearchersConfig.Searchcluster getSearchClusterConfigFromClusterName(QrSearchersConfig config, String name) { @@ -159,7 +165,10 @@ public class ClusterSearcher extends Searcher { maxQueryCacheTimeout = DEFAULT_MAX_QUERY_CACHE_TIMEOUT; server = searcher; this.executor = executor; + this.globalPhaseHelper = null; + this.enableGlobalPhase = false; } + /** Do not use, for internal testing purposes only. **/ ClusterSearcher(Set<String> schemas) { this(schemas, null, null); @@ -169,7 +178,7 @@ public class ClusterSearcher extends Searcher { public void fill(com.yahoo.search.Result result, String summaryClass, Execution execution) { Query query = result.getQuery(); - VespaBackEndSearcher searcher = server; + Searcher searcher = server; if (searcher != null) { if (query.getTimeLeft() > 0) { searcher.fill(result, summaryClass, execution); @@ -190,7 +199,7 @@ public class ClusterSearcher extends Searcher { public Result search(Query query, Execution execution) { validateQueryTimeout(query); validateQueryCache(query); - VespaBackEndSearcher searcher = server; + Searcher searcher = server; if (searcher == null) { return new Result(query, ErrorMessage.createNoBackendsInService("Could not search")); } @@ -228,8 +237,21 @@ public class ClusterSearcher extends Searcher { } else { String docType = schemas.iterator().next(); query.getModel().setRestrict(docType); - return searcher.search(query, execution); + return perSchemaSearch(searcher, query, execution); + } + } + + private Result perSchemaSearch(Searcher searcher, Query query, Execution execution) { + Set<String> restrict = query.getModel().getRestrict(); + if (restrict.size() != 1) { + throw new IllegalStateException("perSchemaSearch must always be called with 1 schema, got: " + restrict.size()); + } + String schema = restrict.iterator().next(); + Result result = searcher.search(query, execution); + if (globalPhaseHelper != null && enableGlobalPhase) { + globalPhaseHelper.process(query, result, schema); } + return result; } private static void processResult(Query query, FutureTask<Result> task, Result mergedResult) { @@ -248,12 +270,12 @@ public class ClusterSearcher extends Searcher { Set<String> schemas = resolveSchemas(query, execution.context().getIndexFacts()); List<Query> queries = createQueries(query, schemas); if (queries.size() == 1) { - return searcher.search(queries.get(0), execution); + return perSchemaSearch(searcher, queries.get(0), execution); } else { Result mergedResult = new Result(query); List<FutureTask<Result>> pending = new ArrayList<>(queries.size()); for (Query q : queries) { - FutureTask<Result> task = new FutureTask<>(() -> searcher.search(q, execution)); + FutureTask<Result> task = new FutureTask<>(() -> perSchemaSearch(searcher, q, execution)); try { executor.execute(task); pending.add(task); diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java index 898e348db92..2d6e059342e 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java @@ -10,6 +10,8 @@ import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.searchchain.Execution; +import com.yahoo.search.yql.MinimalQueryInserter; +import com.yahoo.yolean.chain.After; /** * Recursively replaces all instances of OrItems with WeakAndItems if the query property weakand.replace is true. @@ -17,6 +19,7 @@ import com.yahoo.search.searchchain.Execution; * * @author karowan */ +@After(MinimalQueryInserter.EXTERNAL_YQL) public class WeakAndReplacementSearcher extends Searcher { static final CompoundName WEAKAND_REPLACE = new CompoundName("weakAnd.replace"); static final CompoundName WAND_HITS = new CompoundName("wand.hits"); diff --git a/container-search/src/main/java/com/yahoo/search/ranking/Evaluator.java b/container-search/src/main/java/com/yahoo/search/ranking/Evaluator.java new file mode 100644 index 00000000000..d2edb776c92 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/Evaluator.java @@ -0,0 +1,14 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import com.yahoo.tensor.Tensor; + +import java.util.Collection; + +interface Evaluator { + Collection<String> needInputs(); + + Evaluator bind(String name, Tensor value); + + double evaluateScore(); +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java new file mode 100644 index 00000000000..b72f81f1439 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/GlobalPhaseRanker.java @@ -0,0 +1,92 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import ai.vespa.models.evaluation.FunctionEvaluator; +import ai.vespa.models.evaluation.Model; +import com.yahoo.component.annotation.Inject; +import com.yahoo.search.Query; +import com.yahoo.search.Result; +import com.yahoo.search.ranking.RankProfilesEvaluator.GlobalPhaseData; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; +import com.yahoo.tensor.Tensor; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.logging.Logger; +import java.util.function.Supplier; + +public class GlobalPhaseRanker { + + private static final Logger logger = Logger.getLogger(GlobalPhaseRanker.class.getName()); + private final RankProfilesEvaluatorFactory factory; + + @Inject + public GlobalPhaseRanker(RankProfilesEvaluatorFactory factory) { + this.factory = factory; + logger.info("using factory: " + factory); + } + + public void process(Query query, Result result, String schema) { + var proxy = factory.proxyForSchema(schema); + String rankProfile = query.getRanking().getProfile(); + var optData = proxy.getGlobalPhaseData(rankProfile); + if (optData.isEmpty()) + return; + GlobalPhaseData data = optData.get(); + var functionEvaluatorSource = data.functionEvaluatorSource(); + var prepared = findFromQuery(query, data.needInputs()); + Supplier<Evaluator> supplier = () -> { + var evaluator = functionEvaluatorSource.get(); + var simple = new SimpleEvaluator(evaluator); + for (var entry : prepared) { + simple.bind(entry.name(), entry.value()); + } + return simple; + }; + int rerankCount = data.rerankCount(); + if (rerankCount < 0) + rerankCount = 100; + ResultReranker.rerankHits(result, new HitRescorer(supplier), rerankCount); + } + + record NameAndValue(String name, Tensor value) { } + + /* do this only once per query: */ + List<NameAndValue> findFromQuery(Query query, List<String> needInputs) { + List<NameAndValue> result = new ArrayList<>(); + var ranking = query.getRanking(); + var rankFeatures = ranking.getFeatures(); + var rankProps = ranking.getProperties().asMap(); + for (String needed : needInputs) { + var optRef = com.yahoo.searchlib.rankingexpression.Reference.simple(needed); + if (optRef.isEmpty()) continue; + var ref = optRef.get(); + if (ref.name().equals("constant")) { + // XXX in theory, we should be able to avoid this + result.add(new NameAndValue(needed, null)); + continue; + } + if (ref.isSimple() && ref.name().equals("query")) { + String queryFeatureName = ref.simpleArgument().get(); + // searchers are recommended to place query features here: + var feature = rankFeatures.getTensor(queryFeatureName); + if (feature.isPresent()) { + result.add(new NameAndValue(needed, feature.get())); + } else { + // but other ways of setting query features end up in the properties: + var objList = rankProps.get(queryFeatureName); + if (objList != null && objList.size() == 1 && objList.get(0) instanceof Tensor t) { + result.add(new NameAndValue(needed, t)); + } + } + } + } + return result; + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/HitRescorer.java b/container-search/src/main/java/com/yahoo/search/ranking/HitRescorer.java new file mode 100644 index 00000000000..ebdbbb693f1 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/HitRescorer.java @@ -0,0 +1,56 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import com.yahoo.search.result.FeatureData; +import com.yahoo.search.result.Hit; + +import java.util.function.Supplier; +import java.util.logging.Logger; + +class HitRescorer { + + private static final Logger logger = Logger.getLogger(HitRescorer.class.getName()); + + private final Supplier<Evaluator> evaluatorSource; + + public HitRescorer(Supplier<Evaluator> evaluatorSource) { + this.evaluatorSource = evaluatorSource; + } + + boolean rescoreHit(Hit hit) { + var features = hit.getField("matchfeatures"); + if (features instanceof FeatureData matchFeatures) { + var scorer = evaluatorSource.get(); + for (String argName : scorer.needInputs()) { + var asTensor = matchFeatures.getTensor(argName); + if (asTensor == null) { + asTensor = matchFeatures.getTensor(alternate(argName)); + } + if (asTensor != null) { + scorer.bind(argName, asTensor); + } else { + logger.warning("Missing match-feature for Evaluator argument: " + argName); + return false; + } + } + double newScore = scorer.evaluateScore(); + hit.setRelevance(newScore); + return true; + } else { + logger.warning("Hit without match-features: " + hit); + return false; + } + } + + private static final String RE_PREFIX = "rankingExpression("; + private static final String RE_SUFFIX = ")"; + private static final int RE_PRE_LEN = RE_PREFIX.length(); + private static final int RE_SUF_LEN = RE_SUFFIX.length(); + + static String alternate(String argName) { + if (argName.startsWith(RE_PREFIX) && argName.endsWith(RE_SUFFIX)) { + return argName.substring(RE_PRE_LEN, argName.length() - RE_SUF_LEN); + } + return argName; + } +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluator.java b/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluator.java new file mode 100644 index 00000000000..2ca91a3ea91 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluator.java @@ -0,0 +1,97 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.search.ranking; + +import ai.vespa.models.evaluation.FunctionEvaluator; +import ai.vespa.models.evaluation.Model; +import ai.vespa.models.evaluation.ModelsEvaluator; +import com.yahoo.api.annotations.Beta; +import com.yahoo.component.AbstractComponent; +import com.yahoo.component.annotation.Inject; +import com.yahoo.filedistribution.fileacquirer.FileAcquirer; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Supplier; +import java.util.logging.Logger; + +/** + * proxy for model-evaluation components + * @author arnej + */ +@Beta +public class RankProfilesEvaluator extends AbstractComponent { + + private final ModelsEvaluator evaluator; + private static final Logger logger = Logger.getLogger(RankProfilesEvaluator.class.getName()); + + @Inject + public RankProfilesEvaluator( + RankProfilesConfig rankProfilesConfig, + RankingConstantsConfig constantsConfig, + RankingExpressionsConfig expressionsConfig, + OnnxModelsConfig onnxModelsConfig, + FileAcquirer fileAcquirer) + { + this.evaluator = new ModelsEvaluator( + rankProfilesConfig, + constantsConfig, + expressionsConfig, + onnxModelsConfig, + fileAcquirer); + extractGlobalPhaseData(rankProfilesConfig); + } + + public Model modelForRankProfile(String rankProfile) { + var m = evaluator.models().get(rankProfile); + if (m == null) { + throw new IllegalArgumentException("unknown rankprofile: " + rankProfile); + } + return m; + } + + public FunctionEvaluator evaluatorForFunction(String rankProfile, String functionName) { + return modelForRankProfile(rankProfile).evaluatorOf(functionName); + } + + static record GlobalPhaseData(Supplier<FunctionEvaluator> functionEvaluatorSource, + int rerankCount, + List<String> needInputs) {} + + private Map<String, GlobalPhaseData> profilesWithGlobalPhase = new HashMap<>(); + + Optional<GlobalPhaseData> getGlobalPhaseData(String rankProfile) { + return Optional.ofNullable(profilesWithGlobalPhase.get(rankProfile)); + } + + private void extractGlobalPhaseData(RankProfilesConfig rankProfilesConfig) { + for (var rp : rankProfilesConfig.rankprofile()) { + String name = rp.name(); + Supplier<FunctionEvaluator> functionEvaluatorSource = null; + int rerankCount = -1; + List<String> needInputs = null; + + for (var prop : rp.fef().property()) { + if (prop.name().equals("vespa.globalphase.rerankcount")) { + rerankCount = Integer.valueOf(prop.value()); + } + if (prop.name().equals("vespa.rank.globalphase")) { + var model = modelForRankProfile(name); + functionEvaluatorSource = () -> model.evaluatorOf("globalphase"); + var evaluator = functionEvaluatorSource.get(); + needInputs = List.copyOf(evaluator.function().arguments()); + } + } + if (functionEvaluatorSource != null && needInputs != null) { + profilesWithGlobalPhase.put(name, new GlobalPhaseData(functionEvaluatorSource, rerankCount, needInputs)); + } + } + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluatorFactory.java b/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluatorFactory.java new file mode 100644 index 00000000000..edb05ed9788 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/RankProfilesEvaluatorFactory.java @@ -0,0 +1,40 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.search.ranking; + +import com.yahoo.api.annotations.Beta; +import com.yahoo.component.annotation.Inject; +import com.yahoo.component.provider.ComponentRegistry; + +/** + * factory for model-evaluation proxies + * @author arnej + */ +@Beta +public class RankProfilesEvaluatorFactory { + + private final ComponentRegistry<RankProfilesEvaluator> registry; + + @Inject + public RankProfilesEvaluatorFactory(ComponentRegistry<RankProfilesEvaluator> registry) { + this.registry = registry; + } + + public RankProfilesEvaluator proxyForSchema(String schemaName) { + var component = registry.getComponent("ranking-expression-evaluator." + schemaName); + if (component == null) { + throw new IllegalArgumentException("ranking expression evaluator for schema '" + schemaName + "' not found"); + } + return component; + } + + public String toString() { + var buf = new StringBuilder(); + buf.append(this.getClass().getName()).append(" containing: ["); + for (var id : registry.allComponentsById().keySet()) { + buf.append(" ").append(id.toString()); + } + buf.append(" ]"); + return buf.toString(); + } +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/ResultReranker.java b/container-search/src/main/java/com/yahoo/search/ranking/ResultReranker.java new file mode 100644 index 00000000000..11b3fa7390a --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/ResultReranker.java @@ -0,0 +1,91 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import com.yahoo.search.Result; +import com.yahoo.search.result.Hit; +import com.yahoo.search.result.HitGroup; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.logging.Logger; + +class ResultReranker { + + private static final Logger logger = Logger.getLogger(ResultReranker.class.getName()); + + // scale and adjust the score according to the range + // of the original and final score values to avoid that + // a score from the backend is larger than finalScores_low + static class Ranges { + private double initialScores_high = -Double.MAX_VALUE; + private double initialScores_low = Double.MAX_VALUE; + private double finalScores_high = -Double.MAX_VALUE; + private double finalScores_low = Double.MAX_VALUE; + + boolean valid() { + return (initialScores_high >= initialScores_low + && + finalScores_high >= finalScores_low); + } + void withInitialScore(double score) { + if (score < initialScores_low) initialScores_low = score; + if (score > initialScores_high) initialScores_high = score; + } + void withFinalScore(double score) { + if (score < finalScores_low) finalScores_low = score; + if (score > finalScores_high) finalScores_high = score; + } + private double initialRange() { + double r = initialScores_high - initialScores_low; + if (r < 1.0) r = 1.0; + return r; + } + private double finalRange() { + double r = finalScores_high - finalScores_low; + if (r < 1.0) r = 1.0; + return r; + } + double scale() { return finalRange() / initialRange(); } + double bias() { return finalScores_low - initialScores_low * scale(); } + } + + static void rerankHits(Result result, HitRescorer hitRescorer, int rerankCount) { + List<Hit> hitsToRescore = new ArrayList<>(); + // consider doing recursive iteration explicitly instead of using deepIterator? + for (var iterator = result.hits().deepIterator(); iterator.hasNext();) { + Hit hit = iterator.next(); + if (hit.isMeta() || hit instanceof HitGroup) { + continue; + } + // what about hits inside grouping results? + // they are inside GroupingListHit, we won't recurse into it; so we won't see them. + hitsToRescore.add(hit); + } + // we can't be 100% certain that hits were sorted according to relevance: + hitsToRescore.sort(Comparator.naturalOrder()); + var ranges = new Ranges(); + for (var iterator = hitsToRescore.iterator(); rerankCount > 0 && iterator.hasNext(); ) { + Hit hit = iterator.next(); + double oldScore = hit.getRelevance().getScore(); + boolean didRerank = hitRescorer.rescoreHit(hit); + if (didRerank) { + ranges.withInitialScore(oldScore); + ranges.withFinalScore(hit.getRelevance().getScore()); + --rerankCount; + iterator.remove(); + } + } + // if any hits are left in the list, they need rescaling: + if (ranges.valid()) { + double scale = ranges.scale(); + double bias = ranges.bias(); + for (Hit hit : hitsToRescore) { + double oldScore = hit.getRelevance().getScore(); + hit.setRelevance(oldScore * scale + bias); + } + } + result.hits().sort(); + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java b/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java new file mode 100644 index 00000000000..f247eab1649 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/search/ranking/SimpleEvaluator.java @@ -0,0 +1,51 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.search.ranking; + +import ai.vespa.models.evaluation.FunctionEvaluator; +import com.yahoo.search.result.FeatureData; +import com.yahoo.search.result.Hit; +import com.yahoo.tensor.Tensor; + +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +class SimpleEvaluator implements Evaluator { + + private final FunctionEvaluator evaluator; + private final Set<String> neededInputs; + + public SimpleEvaluator(FunctionEvaluator prototype) { + this.evaluator = prototype; + this.neededInputs = new HashSet<String>(prototype.function().arguments()); + } + + @Override + public Collection<String> needInputs() { return List.copyOf(neededInputs); } + + @Override + public SimpleEvaluator bind(String name, Tensor value) { + if (value != null) evaluator.bind(name, value); + neededInputs.remove(name); + return this; + } + + @Override + public double evaluateScore() { + return evaluator.evaluate().asDouble(); + } + + @Override + public String toString() { + var buf = new StringBuilder(); + buf.append("SimpleEvaluator("); + buf.append(evaluator.function().toString()); + buf.append(")["); + for (String arg : neededInputs) { + buf.append("{").append(arg).append("}"); + } + buf.append("]"); + return buf.toString(); + } +} diff --git a/fastos/src/tests/mazeserver.cpp b/container-search/src/main/java/com/yahoo/search/ranking/package-info.java index b62e98645f2..a86a5c1e52f 100644 --- a/fastos/src/tests/mazeserver.cpp +++ b/container-search/src/main/java/com/yahoo/search/ranking/package-info.java @@ -1,4 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#define DO_MAZE_SERVER 1 -#include "sockettest.cpp" +@ExportPackage +package com.yahoo.search.ranking; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java index 5df8d2e5444..06ae9923dae 100644 --- a/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/cluster/ClusterSearcherTestCase.java @@ -464,6 +464,7 @@ public class ClusterSearcherTestCase { documentDbConfig.build(), new SchemaInfo(List.of(schema.build()), Map.of()), dispatchers, + null, vipStatus, null); } diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index 33cd2f48d46..33f840c7af0 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -44,11 +44,9 @@ import com.yahoo.search.query.Sorting.Order; import com.yahoo.search.query.Sorting.UcaSorter; import com.yahoo.search.query.parser.Parsable; import com.yahoo.search.query.parser.ParserEnvironment; -import com.yahoo.search.query.parser.ParserFactory; import com.yahoo.search.searchchain.Execution; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; import java.util.ArrayList; import java.util.Collection; @@ -68,7 +66,6 @@ public class YqlParserTestCase { private final YqlParser parser = new YqlParser(new ParserEnvironment()); @Test - @Timeout(120_000) void failsGracefullyOnMissingQuoteEscapingAndSubsequentUnicodeCharacter() { assertParseFail("select * from bar where rank(ids contains 'http://en.wikipedia.org/wiki/Hors_d'Å“uvre') limit 10", new IllegalInputException("com.yahoo.search.yql.ProgramCompileException: query:L1:79 token recognition error at: 'Å“'")); @@ -976,16 +973,9 @@ public class YqlParserTestCase { assertEquals(4, terms.size()); for (IndexedItem term : terms) { switch (term.getIndexedString()) { - case "a": - case "c": - assertFalse(((Item) term).isRanked()); - break; - case "b": - case "d": - assertTrue(((Item) term).isRanked()); - break; - default: - fail(); + case "a", "c" -> assertFalse(((Item) term).isRanked()); + case "b", "d" -> assertTrue(((Item) term).isRanked()); + default -> fail(); } } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveUriUpdate.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveUriUpdate.java new file mode 100644 index 00000000000..e6dec99b84c --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/archive/ArchiveUriUpdate.java @@ -0,0 +1,43 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.archive; + +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.TenantName; + +import java.net.URI; +import java.util.Optional; + +/** + * Represents an operation to update or unset the archive URI value for a given tenant or cloud account. + * + * @author freva + */ +public class ArchiveUriUpdate { + private final Optional<TenantName> tenantName; + private final Optional<CloudAccount> cloudAccount; + private final Optional<URI> archiveUri; + + private ArchiveUriUpdate(Optional<TenantName> tenantName, Optional<CloudAccount> cloudAccount, Optional<URI> archiveUri) { + this.tenantName = tenantName; + this.cloudAccount = cloudAccount; + this.archiveUri = archiveUri; + } + + public Optional<TenantName> tenantName() { return tenantName; } + public Optional<CloudAccount> cloudAccount() { return cloudAccount; } + public Optional<URI> archiveUri() { return archiveUri; } + + public static ArchiveUriUpdate setArchiveUriFor(TenantName tenantName, URI archiveUri) { + return new ArchiveUriUpdate(Optional.of(tenantName), Optional.empty(), Optional.of(archiveUri)); + } + public static ArchiveUriUpdate deleteArchiveUriFor(TenantName tenantName) { + return new ArchiveUriUpdate(Optional.of(tenantName), Optional.empty(), Optional.empty()); + } + + public static ArchiveUriUpdate setArchiveUriFor(CloudAccount cloudAccount, URI archiveUri) { + return new ArchiveUriUpdate(Optional.empty(), Optional.of(cloudAccount), Optional.of(archiveUri)); + } + public static ArchiveUriUpdate deleteArchiveUriFor(CloudAccount cloudAccount) { + return new ArchiveUriUpdate(Optional.empty(), Optional.of(cloudAccount), Optional.empty()); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactoryMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactoryMock.java index 54fda58d19c..c4194315922 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactoryMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/AthenzClientFactoryMock.java @@ -39,7 +39,7 @@ public class AthenzClientFactoryMock extends AbstractComponent implements Athenz @Override public ZtsClient createZtsClient() { - return new ZtsClientMock(athenz); + return new ZtsClientMock(athenz, createZmsClient()); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java index d3e74965c4b..3ca0fdd0f23 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZtsClientMock.java @@ -5,10 +5,12 @@ import com.yahoo.security.Pkcs10Csr; import com.yahoo.vespa.athenz.api.AthenzAccessToken; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.AthenzRole; import com.yahoo.vespa.athenz.api.AwsRole; import com.yahoo.vespa.athenz.api.AwsTemporaryCredentials; import com.yahoo.vespa.athenz.api.ZToken; +import com.yahoo.vespa.athenz.client.zms.ZmsClient; import com.yahoo.vespa.athenz.client.zts.Identity; import com.yahoo.vespa.athenz.client.zts.InstanceIdentity; import com.yahoo.vespa.athenz.client.zts.ZtsClient; @@ -17,6 +19,7 @@ import java.security.KeyPair; import java.security.cert.X509Certificate; import java.time.Duration; import java.util.List; +import java.util.Optional; import java.util.logging.Level; import java.util.logging.Logger; @@ -27,9 +30,14 @@ public class ZtsClientMock implements ZtsClient { private static final Logger log = Logger.getLogger(ZtsClientMock.class.getName()); private final AthenzDbMock athenz; + private final Optional<ZmsClient> zmsClient; public ZtsClientMock(AthenzDbMock athenz) { + this(athenz, null); + } + public ZtsClientMock(AthenzDbMock athenz, ZmsClient zmsClient) { this.athenz = athenz; + this.zmsClient = Optional.ofNullable(zmsClient); } @Override @@ -98,6 +106,12 @@ public class ZtsClientMock implements ZtsClient { } @Override + public boolean hasAccess(AthenzResourceName resource, String action, AthenzIdentity identity) { + return zmsClient.orElseThrow(UnsupportedOperationException::new) + .hasAccess(resource, action, identity); + } + + @Override public void close() { } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ArchiveUris.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ArchiveUris.java new file mode 100644 index 00000000000..a0f6955b59f --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ArchiveUris.java @@ -0,0 +1,15 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.configserver; + +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.TenantName; + +import java.net.URI; +import java.util.Map; + +/** + * @author freva + */ +public record ArchiveUris(Map<TenantName, URI> tenantArchiveUris, Map<CloudAccount, URI> accountArchiveUris) { + public static final ArchiveUris EMPTY = new ArchiveUris(Map.of(), Map.of()); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeFilter.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeFilter.java index 7b209d231c4..796ce5da449 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeFilter.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeFilter.java @@ -20,14 +20,16 @@ public class NodeFilter { private final boolean includeDeprovisioned; private final Set<Node.State> states; private final Set<HostName> hostnames; + private final Set<HostName> parentHostnames; private final Set<ApplicationId> applications; private NodeFilter(boolean includeDeprovisioned, Set<Node.State> states, Set<HostName> hostnames, - Set<ApplicationId> applications) { + Set<HostName> parentHostnames, Set<ApplicationId> applications) { this.includeDeprovisioned = includeDeprovisioned; // Uses Guava Set to preserve insertion order this.states = ImmutableSet.copyOf(Objects.requireNonNull(states)); this.hostnames = ImmutableSet.copyOf(Objects.requireNonNull(hostnames)); + this.parentHostnames = ImmutableSet.copyOf(Objects.requireNonNull(parentHostnames)); this.applications = ImmutableSet.copyOf(Objects.requireNonNull(applications)); if (!includeDeprovisioned && states.contains(Node.State.deprovisioned)) { throw new IllegalArgumentException("Must include deprovisioned nodes when matching deprovisioned state"); @@ -46,12 +48,16 @@ public class NodeFilter { return hostnames; } + public Set<HostName> parentHostnames() { + return parentHostnames; + } + public Set<ApplicationId> applications() { return applications; } public NodeFilter includeDeprovisioned(boolean includeDeprovisioned) { - return new NodeFilter(includeDeprovisioned, states, hostnames, applications); + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications); } public NodeFilter states(Node.State... states) { @@ -59,7 +65,7 @@ public class NodeFilter { } public NodeFilter states(Set<Node.State> states) { - return new NodeFilter(includeDeprovisioned, states, hostnames, applications); + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications); } public NodeFilter hostnames(HostName... hostnames) { @@ -67,7 +73,15 @@ public class NodeFilter { } public NodeFilter hostnames(Set<HostName> hostnames) { - return new NodeFilter(includeDeprovisioned, states, hostnames, applications); + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications); + } + + public NodeFilter parentHostnames(HostName... parentHostnames) { + return parentHostnames(ImmutableSet.copyOf(parentHostnames)); + } + + public NodeFilter parentHostnames(Set<HostName> parentHostnames) { + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications); } public NodeFilter applications(ApplicationId... applications) { @@ -75,12 +89,12 @@ public class NodeFilter { } public NodeFilter applications(Set<ApplicationId> applications) { - return new NodeFilter(includeDeprovisioned, states, hostnames, applications); + return new NodeFilter(includeDeprovisioned, states, hostnames, parentHostnames, applications); } /** A filter which matches all nodes, except deprovisioned ones */ public static NodeFilter all() { - return new NodeFilter(false, Set.of(), Set.of(), Set.of()); + return new NodeFilter(false, Set.of(), Set.of(), Set.of(), Set.of()); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java index 1768de8d012..4c5a67626ea 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java @@ -5,15 +5,10 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveUriUpdate; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.ApplicationPatch; -import javax.ws.rs.HeaderParam; -import javax.ws.rs.POST; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import java.net.URI; import java.util.List; import java.util.Map; @@ -48,14 +43,11 @@ public interface NodeRepository { /** Get node statistics such as cost and load from given zone */ NodeRepoStats getStats(ZoneId zone); - /** Get all archive URLs found in zone */ - Map<TenantName, URI> getArchiveUris(ZoneId zone); + /** Get all archive URIs found in zone */ + ArchiveUris getArchiveUris(ZoneId zone); - /** Update archive URL for given tenant */ - void setArchiveUri(ZoneId zone, TenantName tenantName, URI archiveUri); - - /** Remove archive URL for given tenant */ - void removeArchiveUri(ZoneId zone, TenantName tenantName); + /** Update some archive URI in the given zone */ + void updateArchiveUri(ZoneId zone, ArchiveUriUpdate archiveUriUpdate); /** Upgrade all nodes of given type to a new version */ void upgrade(ZoneId zone, NodeType type, Version version, boolean allowDowngrade); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ArchiveList.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ArchiveList.java index 274d07bfc3b..172523eb261 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ArchiveList.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/ArchiveList.java @@ -16,6 +16,9 @@ public class ArchiveList { @JsonProperty("tenant") public String tenant; + @JsonProperty("account") + public String account; + @JsonProperty("uri") public String uri; } diff --git a/controller-server/pom.xml b/controller-server/pom.xml index 1a8a68be9e0..0aec7b18a97 100644 --- a/controller-server/pom.xml +++ b/controller-server/pom.xml @@ -125,7 +125,7 @@ <dependency> <groupId>commons-fileupload</groupId> <artifactId>commons-fileupload</artifactId> - <version>1.4</version> + <version>1.5</version> </dependency> <dependency> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java index 6a493f3f5ed..65320a25984 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/athenz/impl/AthenzFacade.java @@ -309,7 +309,7 @@ public class AthenzFacade implements AccessControl { } private boolean lookupAccess(AccessTuple t) { - boolean result = zmsClient.hasAccess(AthenzResourceName.fromString(t.resource), t.action, t.identity); + boolean result = ztsClient.hasAccess(AthenzResourceName.fromString(t.resource), t.action, t.identity); log("getAccess(action=%s, resource=%s, principal=%s) = %b", t.action, t.resource, t.identity, result); return result; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java index 0c8a50fa821..ddb1365f2de 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java @@ -5,12 +5,13 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveUriUpdate; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ArchiveUris; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.archive.CuratorArchiveBucketDb; import com.yahoo.yolean.Exceptions; -import java.net.URI; import java.time.Duration; import java.util.HashMap; import java.util.HashSet; @@ -56,17 +57,19 @@ public class ArchiveUriUpdater extends ControllerMaintainer { int failures = 0; for (ZoneId zone : tenantsByZone.keySet()) { try { - Map<TenantName, URI> zoneArchiveUris = nodeRepository.getArchiveUris(zone); + ArchiveUris zoneArchiveUris = nodeRepository.getArchiveUris(zone); for (TenantName tenant : tenantsByZone.get(zone)) { archiveBucketDb.archiveUriFor(zone, tenant, true) - .filter(uri -> !uri.equals(zoneArchiveUris.get(tenant))) - .ifPresent(uri -> nodeRepository.setArchiveUri(zone, tenant, uri)); + .filter(uri -> !uri.equals(zoneArchiveUris.tenantArchiveUris().get(tenant))) + .ifPresent(uri -> nodeRepository.updateArchiveUri(zone, ArchiveUriUpdate.setArchiveUriFor(tenant, uri))); } - zoneArchiveUris.keySet().stream() + zoneArchiveUris.tenantArchiveUris().keySet().stream() .filter(tenant -> !tenantsByZone.get(zone).contains(tenant)) - .forEach(tenant -> nodeRepository.removeArchiveUri(zone, tenant)); + .forEach(tenant -> nodeRepository.updateArchiveUri(zone, ArchiveUriUpdate.deleteArchiveUriFor(tenant))); + + // TODO (freva): Update account archive URIs } catch (Exception e) { log.log(Level.WARNING, "Failed to update archive URI in " + zone + ". Retrying in " + interval() + ". Error: " + Exceptions.toMessageString(e)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java index cf9db1517a0..f6029eade37 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentInfoMaintainer.java @@ -1,6 +1,5 @@ package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; @@ -9,7 +8,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeReposi import com.yahoo.yolean.Exceptions; import java.time.Duration; -import java.util.stream.Stream; +import java.util.Collection; /** * This pulls application deployment information from the node repo on all config servers, @@ -28,29 +27,35 @@ public class DeploymentInfoMaintainer extends ControllerMaintainer { @Override protected double maintain() { - controller().applications().asList().stream() - .flatMap(this::mapApplicationToInstances) - .flatMap(this::mapInstanceToDeployments) - .forEach(this::updateDeploymentInfo); - return 1.0; - } - - private Stream<Instance> mapApplicationToInstances(Application application) { - return application.instances().values().stream(); + int attempts = 0; + int failures = 0; + for (var application : controller().applications().asList()) { + for (var instance : application.instances().values()) { + for (var deployment : instanceDeployments(instance)) { + attempts++; + if ( ! updateDeploymentInfo(deployment)) + failures++; + } + } + } + return asSuccessFactor(attempts, failures); } - private Stream<DeploymentId> mapInstanceToDeployments(Instance instance) { + private Collection<DeploymentId> instanceDeployments(Instance instance) { return instance.deployments().keySet().stream() .filter(zoneId -> !zoneId.environment().isTest()) - .map(zoneId -> new DeploymentId(instance.id(), zoneId)); + .map(zoneId -> new DeploymentId(instance.id(), zoneId)) + .toList(); } - private void updateDeploymentInfo(DeploymentId id) { + private boolean updateDeploymentInfo(DeploymentId id) { try { controller().applications().deploymentInfo().put(id, nodeRepository.getApplication(id.zoneId(), id.applicationId())); + return true; } catch (ConfigServerException e) { log.info("Could not retrieve deployment info for " + id + ": " + Exceptions.toMessageString(e)); + return false; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index d49cb244e47..82b3141e503 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -24,7 +24,6 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.OptionalInt; import java.util.Random; import java.util.Set; import java.util.function.UnaryOperator; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java index 45b3e4ef5dd..da0fa890960 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VcmrMaintainer.java @@ -192,17 +192,15 @@ public class VcmrMaintainer extends ControllerMaintainer { } if (shouldRetire(changeRequest, hostAction)) { - if (!node.wantToRetire()) { + if (!wantToRetireRecursive(zoneId, node)) { LOG.info(Text.format("Retiring %s due to %s", node.hostname().value(), changeRequest.getChangeRequestSource().getId())); // TODO: Remove try/catch once retirement is stabilized try { setWantToRetire(zoneId, node, true); } catch (Exception e) { LOG.warning("Failed to retire host " + node.hostname() + ": " + Exceptions.toMessageString(e)); - // Check if retirement actually failed - if (!nodeRepository.getNode(zoneId, node.hostname().value()).wantToRetire()) { - return hostAction; - } + // Will retry next maintenance run + return hostAction; } } return hostAction.withState(State.RETIRING); @@ -225,6 +223,13 @@ public class VcmrMaintainer extends ControllerMaintainer { return hostAction; } + // Determines if a host and all its children are retiring + private boolean wantToRetireRecursive(ZoneId zoneId, Node node) { + var children = nodeRepository.list(zoneId, NodeFilter.all().parentHostnames(node.hostname())); + return node.wantToRetire() && + children.stream().allMatch(Node::wantToRetire); + } + // Dirty host iff the parked host was retired by this maintainer private void recycleNode(ZoneId zoneId, Node node, HostAction hostAction) { if (hostAction.getState() == State.RETIRED && diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java index 1c76f58a6b2..82dc333d178 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java @@ -9,6 +9,7 @@ import com.yahoo.text.Text; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer; import com.yahoo.vespa.hosted.controller.api.integration.organization.MailerException; @@ -76,7 +77,7 @@ public class Notifier { } private boolean dispatchEnabled(NotificationSource source) { - return Flags.NOTIFICATION_DISPATCH_FLAG.bindTo(flagSource) + return PermanentFlags.NOTIFICATION_DISPATCH_FLAG.bindTo(flagSource) .with(FetchVector.Dimension.TENANT_ID, source.tenant().value()) .value(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 7fcad017569..6d3c15c2d57 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -596,7 +596,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { var mergedAddress = updateTenantInfoAddress(inspector.field("address"), info.address()); var mergedInfo = info - .withName(getString(inspector.field("tenant").field("name"), info.name())) + .withName(getString(inspector.field("tenant").field("company"), info.name())) .withWebsite(getString(inspector.field("tenant").field("website"), info.website())) .withContact(mergedContact) .withAddress(mergedAddress); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index e3c7dc79575..ce60e0054c4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -218,7 +218,7 @@ class JobControllerApiHandlerHelper { Cursor responseObject = slime.setObject(); Optional<Run> run = jobs.last(id, type).flatMap(last -> jobs.active(last.id())); if (run.isPresent()) { - jobs.abort(run.get().id(), "aborted by " + request.getJDiscRequest().getUserPrincipal()); + jobs.abort(run.get().id(), "aborted by " + request.getJDiscRequest().getUserPrincipal().getName()); responseObject.setString("message", "Aborting " + run.get().id()); } else diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index 37ef85b991b..297997365b0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.integration; import com.yahoo.collections.Pair; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; @@ -11,8 +12,10 @@ import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveUriUpdate; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Application; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationStats; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ArchiveUris; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Load; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; @@ -22,6 +25,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.TargetVers import com.yahoo.vespa.hosted.controller.api.integration.noderepository.ApplicationPatch; import java.net.URI; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -41,7 +45,7 @@ public class NodeRepositoryMock implements NodeRepository { private final Map<ZoneId, TargetVersions> targetVersions = new ConcurrentHashMap<>(); private final Map<DeploymentId, Pair<Double, Double>> trafficFractions = new ConcurrentHashMap<>(); private final Map<DeploymentClusterId, BcpGroupInfo> bcpGroupInfos = new ConcurrentHashMap<>(); - private final Map<ZoneId, Map<TenantName, URI>> archiveUris = new ConcurrentHashMap<>(); + private final Map<ZoneId, ArchiveUris> archiveUris = new ConcurrentHashMap<>(); private boolean allowPatching = true; private boolean hasSpareCapacity = false; @@ -118,18 +122,26 @@ public class NodeRepositoryMock implements NodeRepository { } @Override - public Map<TenantName, URI> getArchiveUris(ZoneId zone) { - return Map.copyOf(archiveUris.getOrDefault(zone, Map.of())); + public ArchiveUris getArchiveUris(ZoneId zone) { + return archiveUris.getOrDefault(zone, ArchiveUris.EMPTY); } @Override - public void setArchiveUri(ZoneId zone, TenantName tenantName, URI archiveUri) { - archiveUris.computeIfAbsent(zone, z -> new ConcurrentHashMap<>()).put(tenantName, archiveUri); - } - - @Override - public void removeArchiveUri(ZoneId zone, TenantName tenantName) { - Optional.ofNullable(archiveUris.get(zone)).ifPresent(map -> map.remove(tenantName)); + public void updateArchiveUri(ZoneId zone, ArchiveUriUpdate update) { + archiveUris.compute(zone, (z, prev) -> { + prev = prev == null ? ArchiveUris.EMPTY : prev; + if (update.tenantName().isPresent()) { + Map<TenantName, URI> updated = new HashMap<>(prev.tenantArchiveUris()); + update.archiveUri().ifPresentOrElse(uri -> updated.put(update.tenantName().get(), uri), + () -> updated.remove(update.tenantName().get())); + return new ArchiveUris(updated, prev.accountArchiveUris()); + } else { + Map<CloudAccount, URI> updated = new HashMap<>(prev.accountArchiveUris()); + update.archiveUri().ifPresentOrElse(uri -> updated.put(update.cloudAccount().get(), uri), + () -> updated.remove(update.cloudAccount().get())); + return new ArchiveUris(prev.tenantArchiveUris(), updated); + } + }); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java index 8c44b39691c..de62a2fc48c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdaterTest.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.yahoo.component.Version; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveBucket; +import com.yahoo.vespa.hosted.controller.api.integration.archive.ArchiveUriUpdate; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.SystemApplication; @@ -62,7 +62,7 @@ public class ArchiveUriUpdaterTest { private void assertArchiveUris(Map<TenantName, String> expectedUris, ZoneId zone) { Map<TenantName, String> actualUris = tester.controller().serviceRegistry().configServer().nodeRepository() - .getArchiveUris(zone).entrySet().stream() + .getArchiveUris(zone).tenantArchiveUris().entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toString())); assertEquals(expectedUris, actualUris); } @@ -76,7 +76,7 @@ public class ArchiveUriUpdaterTest { private void setArchiveUriInNodeRepo(Map<TenantName, String> archiveUris, ZoneId zone) { NodeRepository nodeRepository = tester.controller().serviceRegistry().configServer().nodeRepository(); - archiveUris.forEach((tenant, uri) -> nodeRepository.setArchiveUri(zone, tenant, URI.create(uri))); + archiveUris.forEach((tenant, uri) -> nodeRepository.updateArchiveUri(zone, ArchiveUriUpdate.setArchiveUriFor(tenant, URI.create(uri)))); } private void deploy(DeploymentContext application, ZoneId zone) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java index 834777abb62..04da0105e99 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDbTest.java @@ -12,6 +12,7 @@ import com.yahoo.test.ManualClock; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing; @@ -79,7 +80,7 @@ public class NotificationsDbTest { private final ManualClock clock = new ManualClock(Instant.ofEpochSecond(12345)); private final MockCuratorDb curatorDb = new MockCuratorDb(SystemName.Public); private final MockMailer mailer = new MockMailer(); - private final FlagSource flagSource = new InMemoryFlagSource().withBooleanFlag(Flags.NOTIFICATION_DISPATCH_FLAG.id(), true); + private final FlagSource flagSource = new InMemoryFlagSource().withBooleanFlag(PermanentFlags.NOTIFICATION_DISPATCH_FLAG.id(), true); private final NotificationsDb notificationsDb = new NotificationsDb(clock, curatorDb, new Notifier(curatorDb, new ZoneRegistryMock(SystemName.cd), mailer, flagSource)); @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java index 0c031a13e6f..96edba27c6f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/notification/NotifierTest.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; @@ -58,7 +59,7 @@ public class NotifierTest { @Test void dispatch() throws IOException { var mailer = new MockMailer(); - var flagSource = new InMemoryFlagSource().withBooleanFlag(Flags.NOTIFICATION_DISPATCH_FLAG.id(), true); + var flagSource = new InMemoryFlagSource().withBooleanFlag(PermanentFlags.NOTIFICATION_DISPATCH_FLAG.id(), true); var notifier = new Notifier(curatorDb, new ZoneRegistryMock(SystemName.cd), mailer, flagSource); var notification = new Notification(Instant.now(), Notification.Type.testPackage, Notification.Level.warning, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java index 8a37bb560e2..41622e669e6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java @@ -77,7 +77,7 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { .roles(Set.of(Role.administrator(tenantName))); tester.assertResponse(updateRequest, "{\"message\":\"Tenant info updated\"}", 200); - tester.assertResponse(request, "{\"contact\":{\"name\":\"Some Name\",\"email\":\"foo@example.com\",\"emailVerified\":false},\"tenant\":{\"company\":\"\",\"website\":\"https://example.com/\"}}", 200); + tester.assertResponse(request, "{\"contact\":{\"name\":\"Some Name\",\"email\":\"foo@example.com\",\"emailVerified\":false},\"tenant\":{\"company\":\"Scoober, Inc.\",\"website\":\"https://example.com/\"}}", 200); } @Test diff --git a/dist/vespa.spec b/dist/vespa.spec index 56b849a8135..58c2a18d3c1 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -54,8 +54,7 @@ BuildRequires: gcc-toolset-12-libatomic-devel BuildRequires: maven BuildRequires: maven-openjdk17 BuildRequires: vespa-pybind11-devel -BuildRequires: python3-pytest -BuildRequires: python36-devel +BuildRequires: python38-devel BuildRequires: glibc-langpack-en %endif %if 0%{?el9} @@ -95,6 +94,7 @@ BuildRequires: vespa-gtest = 1.11.0 BuildRequires: vespa-lz4-devel >= 1.9.4-1 BuildRequires: vespa-onnxruntime-devel = 1.13.1 BuildRequires: vespa-protobuf-devel = 3.21.7 +%define _use_vespa_protobuf 1 BuildRequires: vespa-libzstd-devel >= 1.5.2-1 %endif %if 0%{?el9} @@ -106,6 +106,7 @@ BuildRequires: vespa-lz4-devel >= 1.9.4-1 BuildRequires: vespa-onnxruntime-devel = 1.13.1 BuildRequires: vespa-libzstd-devel >= 1.5.2-1 BuildRequires: vespa-protobuf-devel = 3.21.7 +%define _use_vespa_protobuf 1 BuildRequires: llvm-devel BuildRequires: boost-devel >= 1.75 BuildRequires: gtest-devel @@ -210,7 +211,7 @@ Requires: %{name}-tools = %{version}-%{release} # Ugly workaround because vespamalloc/src/vespamalloc/malloc/mmap.cpp uses the private # _dl_sym function. # Exclude automated requires for libraries in /opt/vespa-deps/lib64. -%global __requires_exclude ^lib(c\\.so\\.6\\(GLIBC_PRIVATE\\)|pthread\\.so\\.0\\(GLIBC_PRIVATE\\)|(icui18n|icuuc|lz4|protobuf|zstd|onnxruntime%{?_use_vespa_openssl:|crypto|ssl}%{?_use_vespa_openblas:|openblas}%{?_use_vespa_re2:|re2}%{?_use_vespa_xxhash:|xxhash}%{?_use_vespa_gtest:|(gtest|gmock)(_main)?})\\.so\\.[0-9.]*\\([A-Za-z._0-9]*\\))\\(64bit\\)$ +%global __requires_exclude ^lib(c\\.so\\.6\\(GLIBC_PRIVATE\\)|pthread\\.so\\.0\\(GLIBC_PRIVATE\\)|(lz4%{?_use_vespa_protobuf:|protobuf}|zstd|onnxruntime%{?_use_vespa_openssl:|crypto|ssl}%{?_use_vespa_openblas:|openblas}%{?_use_vespa_re2:|re2}%{?_use_vespa_xxhash:|xxhash}%{?_use_vespa_gtest:|(gtest|gmock)(_main)?})\\.so\\.[0-9.]*\\([A-Za-z._0-9]*\\))\\(64bit\\)$ %description @@ -375,7 +376,7 @@ Summary: Vespa - The open big data serving engine - ann-benchmark Requires: %{name}-base-libs = %{version}-%{release} Requires: %{name}-libs = %{version}-%{release} %if 0%{?el8} -Requires: python36 +Requires: python38 %endif %if 0%{?el9} Requires: python3 @@ -688,7 +689,6 @@ fi %endif %dir %{_prefix} %dir %{_prefix}/lib64 -%{_prefix}/lib64/libfastos.so %{_prefix}/lib64/libfnet.so %{_prefix}/lib64/libvespadefaults.so %{_prefix}/lib64/libvespalib.so @@ -700,7 +700,6 @@ fi %endif %dir %{_prefix} %{_prefix}/lib64 -%exclude %{_prefix}/lib64/libfastos.so %exclude %{_prefix}/lib64/libfnet.so %exclude %{_prefix}/lib64/libvespadefaults.so %exclude %{_prefix}/lib64/libvespalib.so diff --git a/document/CMakeLists.txt b/document/CMakeLists.txt index 88dbe2816d9..e1e4d8ff5cc 100644 --- a/document/CMakeLists.txt +++ b/document/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib config_cloudconfig diff --git a/document/src/main/java/com/yahoo/document/datatypes/BoolFieldValue.java b/document/src/main/java/com/yahoo/document/datatypes/BoolFieldValue.java index 199ca199667..dc5cf609381 100644 --- a/document/src/main/java/com/yahoo/document/datatypes/BoolFieldValue.java +++ b/document/src/main/java/com/yahoo/document/datatypes/BoolFieldValue.java @@ -94,11 +94,10 @@ public class BoolFieldValue extends FieldValue { @Override public boolean equals(Object o) { if (this == o) return true; - if ( ! (o instanceof BoolFieldValue)) return false; + if ( ! (o instanceof BoolFieldValue other)) return false; if ( ! super.equals(o)) return false; - BoolFieldValue that = (BoolFieldValue) o; - return (value == that.value); + return (value == other.value); } @Override diff --git a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java index b7a3589a4d0..4c4d9c78c8e 100644 --- a/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java +++ b/document/src/test/java/com/yahoo/document/select/DocumentSelectorTestCase.java @@ -775,7 +775,7 @@ public class DocumentSelectorTestCase { @Test public void testInheritance() throws ParseException { - var s=new DocumentSelector("parent.parentField = \"parentValue\""); + new DocumentSelector("parent.parentField = \"parentValue\""); List<DocumentPut> documents = createDocs(); assertEquals(Result.TRUE, evaluate("test", documents.get(0))); assertEquals("Matching on type is exact", Result.FALSE, evaluate("parent", documents.get(0))); diff --git a/document/src/tests/documentselectparsertest.cpp b/document/src/tests/documentselectparsertest.cpp index ce4b69419a3..5e14a080f27 100644 --- a/document/src/tests/documentselectparsertest.cpp +++ b/document/src/tests/documentselectparsertest.cpp @@ -673,8 +673,9 @@ TEST_F(DocumentSelectParserTest, operators_1) // Inherited doctypes PARSE("testdoctype2", *_doc[4], True); PARSE("testdoctype2", *_doc[3], False); - PARSE("testdoctype1", *_doc[4], False); // testdoctype2 inherits testdoctype1, but we use exact matching for types - PARSE("testdoctype1.headerval = 10", *_doc[4], Invalid); + PARSE("testdoctype1", *_doc[4], False); // testdoctype2 inherits testdoctype1, but we use exact matching for "standalone" doctype matches. + PARSE("testdoctype1.headerval = 10", *_doc[4], True); // But _field lookups_ use is-a type matching semantics. + PARSE("testdoctype2.headerval = 10", *_doc[4], True); // Exact type match with parent field also works transparently } TEST_F(DocumentSelectParserTest, operators_2) diff --git a/document/src/tests/documentupdatetestcase.cpp b/document/src/tests/documentupdatetestcase.cpp index 40f398ee93e..7a5d88d1013 100644 --- a/document/src/tests/documentupdatetestcase.cpp +++ b/document/src/tests/documentupdatetestcase.cpp @@ -1,14 +1,17 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/document/test/fieldvalue_helpers.h> -#include <vespa/document/base/testdocman.h> +#include <vespa/document/annotation/spanlist.h> #include <vespa/document/base/exceptions.h> -#include <vespa/document/datatype/tensor_data_type.h> +#include <vespa/document/base/testdocman.h> #include <vespa/document/datatype/documenttype.h> +#include <vespa/document/datatype/tensor_data_type.h> #include <vespa/document/fieldvalue/fieldvalues.h> #include <vespa/document/fieldvalue/tensorfieldvalue.h> #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/documenttyperepo.h> +#include <vespa/document/repo/fixedtyperepo.h> +#include <vespa/document/serialization/vespadocumentdeserializer.h> #include <vespa/document/serialization/vespadocumentserializer.h> #include <vespa/document/update/addvalueupdate.h> #include <vespa/document/update/arithmeticvalueupdate.h> @@ -26,8 +29,8 @@ #include <vespa/document/util/bytebuffer.h> #include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/tensor_spec.h> -#include <vespa/eval/eval/value.h> #include <vespa/eval/eval/test/value_compare.h> +#include <vespa/eval/eval/value.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exception.h> #include <vespa/vespalib/util/exceptions.h> @@ -1329,4 +1332,68 @@ TEST(DocumentUpdateTest, array_element_update_for_invalid_index_is_ignored) EXPECT_EQ(array_value, *result_array); } +struct UpdateToEmptyDocumentFixture { + std::unique_ptr<DocumentTypeRepo> repo; + const DocumentType& doc_type; + FixedTypeRepo fixed_repo; + + UpdateToEmptyDocumentFixture() + : repo(make_repo()), + doc_type(*repo->getDocumentType("test")), + fixed_repo(*repo, doc_type) + { + } + + std::unique_ptr<DocumentTypeRepo> make_repo() { + config_builder::DocumenttypesConfigBuilderHelper builder; + builder.document(222, "test", + Struct("test.header").addField("text", DataType::T_STRING), + Struct("test.body")); + return std::make_unique<DocumentTypeRepo>(builder.config()); + } + + Document::UP make_empty_doc() { + vespalib::nbostream stream; + { + Document doc(doc_type, DocumentId("id:test:test::0")); + VespaDocumentSerializer serializer(stream); + serializer.write(doc); + } + // This simulates that the document is read from e.g. the document store + return std::make_unique<Document>(*repo, stream); + } + + DocumentUpdate::UP make_update() { + auto text = std::make_unique<StringFieldValue>("hello world"); + auto span_list_up = std::make_unique<SpanList>(); + auto span_list = span_list_up.get(); + auto tree = std::make_unique<SpanTree>("my_span_tree", std::move(span_list_up)); + tree->annotate(span_list->add(std::make_unique<Span>(0, 5)), *AnnotationType::TERM); + tree->annotate(span_list->add(std::make_unique<Span>(6, 3)), *AnnotationType::TERM); + StringFieldValue::SpanTrees trees; + trees.push_back(std::move(tree)); + text->setSpanTrees(trees, fixed_repo); + + auto result = std::make_unique<DocumentUpdate>(*repo, doc_type, DocumentId("id:test:test::0")); + result->addUpdate(FieldUpdate(doc_type.getField("text")) + .addUpdate(std::make_unique<AssignValueUpdate>(std::move(text)))); + return result; + } +}; + +TEST(DocumentUpdateTest, string_field_annotations_can_be_deserialized_after_assign_update_to_empty_document) +{ + UpdateToEmptyDocumentFixture f; + auto doc = f.make_empty_doc(); + auto update = f.make_update(); + update->applyTo(*doc); + auto fv = doc->getValue("text"); + auto& text = dynamic_cast<StringFieldValue&>(*fv); + // This uses both the DocumentTypeRepo and DocumentType in order to deserialize the annotations. + auto tree = text.getSpanTrees(); + EXPECT_EQ("hello world", text.getValue()); + ASSERT_EQ(1, tree.size()); + ASSERT_EQ(2, tree[0]->numAnnotations()); +} + } // namespace document diff --git a/document/src/vespa/document/select/valuenodes.cpp b/document/src/vespa/document/select/valuenodes.cpp index 8102a944ff0..b3052cc07e2 100644 --- a/document/src/vespa/document/select/valuenodes.cpp +++ b/document/src/vespa/document/select/valuenodes.cpp @@ -20,10 +20,20 @@ LOG_SETUP(".document.select.valuenode"); namespace document::select { namespace { - bool documentTypeEqualsName(const DocumentType& type, vespalib::stringref name) - { - return (type.getName() == name); + +[[nodiscard]] bool document_type_is_a(const DocumentType& type, vespalib::stringref name) { + if (type.getName() == name) { + return true; + } + // No exact match; try to recursively match name against any types inherited from. + for (const auto* parent : type.getInheritedTypes()) { + if (document_type_is_a(*parent, name)) { + return true; + } } + return false; +} + } InvalidValueNode::InvalidValueNode(vespalib::stringref name) @@ -409,7 +419,7 @@ FieldValueNode::getValue(const Context& context) const const Document& doc = *context._doc; - if (!documentTypeEqualsName(doc.getType(), _doctype)) { + if (!document_type_is_a(doc.getType(), _doctype)) { return std::make_unique<InvalidValue>(); } // Imported fields can only be meaningfully evaluated inside Proton, so we @@ -473,7 +483,7 @@ FieldValueNode::traceValue(const Context &context, std::ostream& out) const return defaultTrace(getValue(context), out); } const Document &doc(*context._doc); - if (!documentTypeEqualsName(doc.getType(), _doctype)) { + if (!document_type_is_a(doc.getType(), _doctype)) { out << "Document is of type " << doc.getType() << " which isn't a " << _doctype << " document, thus resolving invalid.\n"; return std::make_unique<InvalidValue>(); diff --git a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp index bbe4f5373cb..8b75c8758ee 100644 --- a/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp +++ b/document/src/vespa/document/serialization/vespadocumentdeserializer.cpp @@ -86,6 +86,7 @@ VespaDocumentDeserializer::readDocument(Document &value) { value.getFields().reset(); } value.setRepo(_repo.getDocumentTypeRepo()); + value.getFields().setDocumentType(value.getType()); FixedTypeRepo repo(_repo.getDocumentTypeRepo(), value.getType()); VarScope<FixedTypeRepo> repo_scope(_repo, repo); diff --git a/documentapi/CMakeLists.txt b/documentapi/CMakeLists.txt index 9261bcf9114..beeda4afeb4 100644 --- a/documentapi/CMakeLists.txt +++ b/documentapi/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog config_cloudconfig vespalib diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.cpp index 70f6fc24d83..43082d9dbae 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/externpolicy.cpp @@ -7,7 +7,6 @@ #include <vespa/slobrok/sbmirror.h> #include <vespa/fnet/transport.h> #include <vespa/fnet/frt/supervisor.h> -#include <vespa/fastos/thread.h> #include <vespa/log/log.h> LOG_SETUP(".externpolicy"); diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/externslobrokpolicy.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/externslobrokpolicy.cpp index 34d2b6d3369..b2b648545cc 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/externslobrokpolicy.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/externslobrokpolicy.cpp @@ -9,7 +9,6 @@ #include <vespa/slobrok/sbmirror.h> #include <vespa/fnet/frt/supervisor.h> #include <vespa/fnet/transport.h> -#include <vespa/fastos/thread.h> #include <thread> using slobrok::api::IMirrorAPI; diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/mirror_with_all.cpp b/documentapi/src/vespa/documentapi/messagebus/policies/mirror_with_all.cpp index 0841c2ed32b..d425bfb6679 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/mirror_with_all.cpp +++ b/documentapi/src/vespa/documentapi/messagebus/policies/mirror_with_all.cpp @@ -5,17 +5,15 @@ #include <vespa/slobrok/sbmirror.h> #include <vespa/fnet/frt/supervisor.h> #include <vespa/fnet/transport.h> -#include <vespa/fastos/thread.h> namespace documentapi { MirrorAndStuff::MirrorAndStuff(const slobrok::ConfiguratorFactory & config) - : _threadPool(std::make_unique<FastOS_ThreadPool>()), - _transport(std::make_unique<FNET_Transport>()), - _orb(std::make_unique<FRT_Supervisor>(_transport.get())), - _mirror(std::make_unique<slobrok::api::MirrorAPI>(*_orb, config)) + : _transport(std::make_unique<FNET_Transport>()), + _orb(std::make_unique<FRT_Supervisor>(_transport.get())), + _mirror(std::make_unique<slobrok::api::MirrorAPI>(*_orb, config)) { - _transport->Start(_threadPool.get()); + _transport->Start(); } MirrorAndStuff::~MirrorAndStuff() { diff --git a/documentapi/src/vespa/documentapi/messagebus/policies/mirror_with_all.h b/documentapi/src/vespa/documentapi/messagebus/policies/mirror_with_all.h index 05571c4420e..ed5dd459768 100644 --- a/documentapi/src/vespa/documentapi/messagebus/policies/mirror_with_all.h +++ b/documentapi/src/vespa/documentapi/messagebus/policies/mirror_with_all.h @@ -4,7 +4,6 @@ #include <memory> -class FastOS_ThreadPool; class FNET_Transport; class FRT_Supervisor; namespace slobrok { class ConfiguratorFactory; } @@ -18,7 +17,6 @@ public: ~MirrorAndStuff(); slobrok::api::IMirrorAPI * mirror() { return _mirror.get(); } private: - std::unique_ptr<FastOS_ThreadPool> _threadPool; std::unique_ptr<FNET_Transport> _transport; std::unique_ptr<FRT_Supervisor> _orb; std::unique_ptr<slobrok::api::IMirrorAPI> _mirror; diff --git a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java index 680853ef687..bd2b057835c 100644 --- a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java +++ b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java @@ -47,8 +47,10 @@ import com.yahoo.vespa.documentgen.test.Book; import com.yahoo.vespa.documentgen.test.Book.Ss0; import com.yahoo.vespa.documentgen.test.Book.Ss1; import com.yahoo.vespa.documentgen.test.Common; +import com.yahoo.vespa.documentgen.test.Common2; import com.yahoo.vespa.documentgen.test.ConcreteDocumentFactory; import com.yahoo.vespa.documentgen.test.Music; +import com.yahoo.vespa.documentgen.test.Music2; import com.yahoo.vespa.documentgen.test.Music3; import com.yahoo.vespa.documentgen.test.Music4; import com.yahoo.vespa.documentgen.test.Parent; @@ -1024,5 +1026,36 @@ public class DocumentGenPluginTest { assertTrue(docType.hasImportedField("my_foo")); assertFalse(docType.hasImportedField("some_field_that_does_not_exist")); } + + @Test + public void subtypes_are_tagged_as_inheriting_supertypes() { + // music -> common + assertTrue(Music.type.isA("common")); + assertTrue(Music.type.inherits(Common.type)); + // ... but not common2 + assertFalse(Music.type.inherits(Common2.type)); + + // music3 -> (music2 -> common), common2 + assertTrue(Music3.type.isA("common")); + assertTrue(Music3.type.isA("common2")); + assertTrue(Music3.type.isA("music2")); + assertTrue(Music3.type.inherits(Common.type)); + assertTrue(Music3.type.inherits(Common2.type)); + assertTrue(Music3.type.inherits(Music2.type)); + // ... but not parent + assertFalse(Music3.type.isA("parent")); + assertFalse(Music3.type.inherits(Parent.type)); + + // music4 -> music3 -> (music2 -> common), common2 + assertTrue(Music4.type.inherits(Common.type)); + assertTrue(Music4.type.inherits(Common2.type)); + assertTrue(Music4.type.inherits(Music2.type)); + assertTrue(Music4.type.inherits(Music3.type)); + // ... but not music + assertFalse(Music4.type.inherits(Music.type)); + + // parent has no explicit inheritance + assertFalse(Parent.type.isA("common")); + } } diff --git a/eval/src/vespa/eval/eval/fast_value.hpp b/eval/src/vespa/eval/eval/fast_value.hpp index 2eaefa3670c..47f99d19055 100644 --- a/eval/src/vespa/eval/eval/fast_value.hpp +++ b/eval/src/vespa/eval/eval/fast_value.hpp @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "value_builder_factory.h" -#include "fast_addr_map.h" +#include "fast_value_index.h" #include "inline_operation.h" #include <vespa/eval/instruction/generic_join.h> #include <vespa/vespalib/stllike/hashtable.hpp> @@ -12,16 +12,6 @@ namespace vespalib::eval { //----------------------------------------------------------------------------- -// This is the class instructions will look for when optimizing sparse -// operations by calling inline functions directly. -struct FastValueIndex final : Value::Index { - FastAddrMap map; - FastValueIndex(size_t num_mapped_dims_in, const StringIdVector &labels, size_t expected_subspaces_in) - : map(num_mapped_dims_in, labels, expected_subspaces_in) {} - size_t size() const override { return map.size(); } - std::unique_ptr<View> create_view(ConstArrayRef<size_t> dims) const override; -}; - inline bool is_fast(const Value::Index &index) { return (std::type_index(typeid(index)) == std::type_index(typeid(FastValueIndex))); } diff --git a/eval/src/vespa/eval/eval/fast_value_index.h b/eval/src/vespa/eval/eval/fast_value_index.h new file mode 100644 index 00000000000..edf96490db6 --- /dev/null +++ b/eval/src/vespa/eval/eval/fast_value_index.h @@ -0,0 +1,24 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "value.h" +#include "fast_addr_map.h" + +namespace vespalib::eval { + +/* + * Tensor value index, used to map labels to dense subspace indexes. + * + * This is the class instructions will look for when optimizing sparse + * operations by calling inline functions directly. + */ +struct FastValueIndex final : Value::Index { + FastAddrMap map; + FastValueIndex(size_t num_mapped_dims_in, const StringIdVector &labels, size_t expected_subspaces_in) + : map(num_mapped_dims_in, labels, expected_subspaces_in) {} + size_t size() const override { return map.size(); } + std::unique_ptr<View> create_view(ConstArrayRef<size_t> dims) const override; +}; + +} diff --git a/fastos/.gitignore b/fastos/.gitignore deleted file mode 100644 index 54e2680a6d8..00000000000 --- a/fastos/.gitignore +++ /dev/null @@ -1,11 +0,0 @@ -*.ilk -*.pdb -.Build_completed -.Dist_completed -.Install_completed -.PreBuild_completed -bin -include -lib -update.log -Makefile diff --git a/fastos/CMakeLists.txt b/fastos/CMakeLists.txt deleted file mode 100644 index c17752e234c..00000000000 --- a/fastos/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_define_module( - LIBS - src/vespa/fastos - - TESTS - src/tests -) diff --git a/fastos/OWNERS b/fastos/OWNERS deleted file mode 100644 index 912f61c59b8..00000000000 --- a/fastos/OWNERS +++ /dev/null @@ -1,2 +0,0 @@ -baldersheim -arnej27959 diff --git a/fastos/README b/fastos/README deleted file mode 100644 index ed9afabffb8..00000000000 --- a/fastos/README +++ /dev/null @@ -1,4 +0,0 @@ -Old OS abstraction layer - -obsolete, to be replaced with implementations using -standard C++14 threads and newer unix networking APIs diff --git a/fastos/src/.gitignore b/fastos/src/.gitignore deleted file mode 100644 index 2e8e6fd906a..00000000000 --- a/fastos/src/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/Makefile.ini -/config_command.sh -/project.dsw diff --git a/fastos/src/tests/.gitignore b/fastos/src/tests/.gitignore deleted file mode 100644 index ccb5f0210ab..00000000000 --- a/fastos/src/tests/.gitignore +++ /dev/null @@ -1,17 +0,0 @@ -/Makefile -/backtracetest -/backtracetest.log -/filetest -/filetest.log -/processtest -/processtest.log -/sockettest -/sockettest.log -/threadtest -/threadtest.log -/timetest -/timetest.log -/typetest -/typetest.log -/usecputest -*test_app diff --git a/fastos/src/tests/CMakeLists.txt b/fastos/src/tests/CMakeLists.txt deleted file mode 100644 index 3bf68a88e79..00000000000 --- a/fastos/src/tests/CMakeLists.txt +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(fastos_filetest_app TEST - SOURCES - filetest.cpp - DEPENDS - fastos -) -vespa_add_test(NAME fastos_filetest_app NO_VALGRIND COMMAND fastos_filetest_app) -vespa_add_executable(fastos_thread_stats_test_app TEST - SOURCES - thread_stats_test.cpp - DEPENDS - fastos -) -vespa_add_test(NAME fastos_thread_stats_test_app NO_VALGRIND COMMAND fastos_thread_stats_test_app) -vespa_add_executable(fastos_thread_joinwait_test_app TEST - SOURCES - thread_joinwait_test.cpp - DEPENDS - fastos -) -vespa_add_test(NAME fastos_thread_joinwait_test_app NO_VALGRIND COMMAND fastos_thread_joinwait_test_app) -vespa_add_executable(fastos_threadtest_app TEST - SOURCES - threadtest.cpp - DEPENDS - fastos -) -vespa_add_test(NAME fastos_threadtest_app NO_VALGRIND COMMAND fastos_threadtest_app) -vespa_add_executable(fastos_typetest_app TEST - SOURCES - typetest.cpp - DEPENDS - fastos -) -vespa_add_test(NAME fastos_typetest_app NO_VALGRIND COMMAND fastos_typetest_app) diff --git a/fastos/src/tests/coretest2.cpp b/fastos/src/tests/coretest2.cpp deleted file mode 100644 index bd93623922b..00000000000 --- a/fastos/src/tests/coretest2.cpp +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - - - - -static void -bomb(void) -{ - char *p; - - p = nullptr; - *p = 4; -} - -void *BomberRun(void *arg) -{ - (void) arg; - bomb(); - return nullptr; -}; - -static int -bombMain(void) -{ - pthread_t thread; - void *ret; - - (void) pthread_create(&thread, nullptr, BomberRun, nullptr); - ret = nullptr; - (void) pthread_join(thread, &ret); - return (0); -} - - -int -main(int argc, char **argv) -{ - (void) argc; - (void) argv; - return bombMain(); -} diff --git a/fastos/src/tests/job.h b/fastos/src/tests/job.h deleted file mode 100644 index 4546cfe1daa..00000000000 --- a/fastos/src/tests/job.h +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <mutex> -#include <condition_variable> - -enum JobCode -{ - PRINT_MESSAGE_AND_WAIT3MSEC, - INCREASE_NUMBER, - WAIT_FOR_BREAK_FLAG, - WAIT_FOR_THREAD_TO_FINISH, - TEST_ID, - SILENTNOP, - NOP -}; - -class Job -{ -private: - Job(const Job &); - Job &operator=(const Job&); - -public: - JobCode code; - char *message; - std::mutex *mutex; - std::condition_variable *condition; - FastOS_ThreadInterface *otherThread, *ownThread; - std::atomic<int> result; - FastOS_ThreadId _threadId; - - Job() - : code(NOP), - message(nullptr), - mutex(nullptr), - condition(nullptr), - otherThread(nullptr), - ownThread(nullptr), - result(-1), - _threadId() - { - } - - ~Job() - { - if(message != nullptr) - free(message); - } -}; diff --git a/fastos/src/tests/performancetest.cpp b/fastos/src/tests/performancetest.cpp deleted file mode 100644 index f566979957a..00000000000 --- a/fastos/src/tests/performancetest.cpp +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <stdlib.h> - -#include "tests.h" - -void PerformanceTest (char *buffer); - -int main (int argc, char **argv) -{ - (void)argc; - (void)argv; - - void (*test)(char *buffer) = PerformanceTest; - - test(nullptr); - return 0; -} - -void PerformanceTest (char *buffer) -{ - // Cause exception - *static_cast<char *>(nullptr) = 'e'; - -#if 1 - FastOS_File file("test.txt"); - - if(file.OpenReadOnly()) - { - file.Read(buffer, 20); - file.Write2(buffer, 20); - file.Read(buffer, 20); - file.Write2(buffer, 20); - file.Read(buffer, 20); - file.Write2(buffer, 20); - } -#else - - int filedes = open("test.txt", O_RDONLY, 0664); - - if(filedes != -1) - { - write(filedes, buffer, 20); - read(filedes, buffer, 20); - write(filedes, buffer, 20); - read(filedes, buffer, 20); - write(filedes, buffer, 20); - read(filedes, buffer, 20); - write(filedes, buffer, 20); - - close(filedes); - } -#endif -} - diff --git a/fastos/src/tests/thread_joinwait_test.cpp b/fastos/src/tests/thread_joinwait_test.cpp deleted file mode 100644 index 6c7e8a7dc3c..00000000000 --- a/fastos/src/tests/thread_joinwait_test.cpp +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "tests.h" -#include "job.h" -#include "thread_test_base.hpp" - -class Thread_JoinWait_Test : public ThreadTestBase -{ - int Main () override; - - void SingleThreadJoinWaitMultipleTest(int variant) - { - bool rc=false; - - char testName[300]; - - snprintf(testName, sizeof(testName), "Single Thread Join Wait Multiple Test %d", variant); - TestHeader(testName); - - FastOS_ThreadPool pool; - - const int testThreads=5; - int lastThreadNum = testThreads-1; - int i; - - Job jobs[testThreads]; - - std::mutex jobMutex; - - // The mutex is used to pause the first threads until we have created - // the last one. - jobMutex.lock(); - - for(i=0; i<lastThreadNum; i++) - { - jobs[i].code = WAIT_FOR_THREAD_TO_FINISH; - jobs[i].mutex = &jobMutex; - jobs[i].ownThread = pool.NewThread(this, static_cast<void *>(&jobs[i])); - - rc = (jobs[i].ownThread != nullptr); - Progress(rc, "Creating Thread %d", i+1); - - if(!rc) - break; - } - - if (rc) - { - jobs[lastThreadNum].code = (((variant & 2) != 0) ? NOP : PRINT_MESSAGE_AND_WAIT3MSEC); - jobs[lastThreadNum].message = strdup("This is the thread that others wait for."); - - FastOS_ThreadInterface *lastThread; - - lastThread = pool.NewThread(this, - static_cast<void *> - (&jobs[lastThreadNum])); - - rc = (lastThread != nullptr); - Progress(rc, "Creating last thread"); - - if (rc) - { - for(i=0; i<lastThreadNum; i++) { - jobs[i].otherThread = lastThread; - } - } - } - - jobMutex.unlock(); - - if ((variant & 1) != 0) - { - for (i=0; i<lastThreadNum; i++) - { - Progress(true, "Waiting for thread %d to finish using Join()", i+1); - jobs[i].ownThread->Join(); - Progress(true, "Thread %d finished.", i+1); - } - } - - Progress(true, "Closing pool."); - pool.Close(); - Progress(true, "Pool closed."); - PrintSeparator(); - } - -}; - -int Thread_JoinWait_Test::Main () -{ - printf("grep for the string '%s' to detect failures.\n\n", failString); - time_t before = time(0); - - SingleThreadJoinWaitMultipleTest(0); - { time_t now = time(0); printf("[%ld seconds]\n", now-before); before = now; } - SingleThreadJoinWaitMultipleTest(1); - { time_t now = time(0); printf("[%ld seconds]\n", now-before); before = now; } - SingleThreadJoinWaitMultipleTest(2); - { time_t now = time(0); printf("[%ld seconds]\n", now-before); before = now; } - SingleThreadJoinWaitMultipleTest(3); - { time_t now = time(0); printf("[%ld seconds]\n", now-before); before = now; } - SingleThreadJoinWaitMultipleTest(2); - { time_t now = time(0); printf("[%ld seconds]\n", now-before); before = now; } - SingleThreadJoinWaitMultipleTest(1); - { time_t now = time(0); printf("[%ld seconds]\n", now-before); before = now; } - SingleThreadJoinWaitMultipleTest(0); - { time_t now = time(0); printf("[%ld seconds]\n", now-before); before = now; } - - printf("END OF TEST (%s)\n", _argv[0]); - return allWasOk() ? 0 : 1; -} - -int main (int argc, char **argv) -{ - Thread_JoinWait_Test app; - setvbuf(stdout, nullptr, _IOLBF, 8192); - return app.Entry(argc, argv); -} diff --git a/fastos/src/tests/thread_stats_test.cpp b/fastos/src/tests/thread_stats_test.cpp deleted file mode 100644 index 40c1199135c..00000000000 --- a/fastos/src/tests/thread_stats_test.cpp +++ /dev/null @@ -1,129 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "tests.h" -#include "job.h" -#include "thread_test_base.hpp" - -class Thread_Stats_Test : public ThreadTestBase -{ - void ThreadStatsTest () - { - int inactiveThreads; - int activeThreads; - int startedThreads; - - TestHeader("Thread Statistics Test"); - - FastOS_ThreadPool pool; - Job job[2]; - - inactiveThreads = pool.GetNumInactiveThreads(); - Progress(inactiveThreads == 0, "Initial inactive threads = %d", inactiveThreads); - activeThreads = pool.GetNumActiveThreads(); - Progress(activeThreads == 0, "Initial active threads = %d", activeThreads); - startedThreads = pool.GetNumStartedThreads(); - Progress(startedThreads == 0, "Initial started threads = %d", startedThreads); - - job[0].code = WAIT_FOR_BREAK_FLAG; - job[0].ownThread = pool.NewThread(this, static_cast<void *>(&job[0])); - - inactiveThreads = pool.GetNumInactiveThreads(); - Progress(inactiveThreads == 0, "Inactive threads = %d", inactiveThreads); - activeThreads = pool.GetNumActiveThreads(); - Progress(activeThreads == 1, "Active threads = %d", activeThreads); - startedThreads = pool.GetNumStartedThreads(); - Progress(startedThreads == 1, "Started threads = %d", startedThreads); - - job[1].code = WAIT_FOR_BREAK_FLAG; - job[1].ownThread = pool.NewThread(this, static_cast<void *>(&job[1])); - - inactiveThreads = pool.GetNumInactiveThreads(); - Progress(inactiveThreads == 0, "Inactive threads = %d", inactiveThreads); - activeThreads = pool.GetNumActiveThreads(); - Progress(activeThreads == 2, "Active threads = %d", activeThreads); - startedThreads = pool.GetNumStartedThreads(); - Progress(startedThreads == 2, "Started threads = %d", startedThreads); - - Progress(true, "Setting breakflag on threads..."); - job[0].ownThread->SetBreakFlag(); - job[1].ownThread->SetBreakFlag(); - - job[0].ownThread->Join(); - job[1].ownThread->Join(); - while (pool.GetNumInactiveThreads() != 2) { - std::this_thread::sleep_for(1ms); - } - - inactiveThreads = pool.GetNumInactiveThreads(); - Progress(inactiveThreads == 2, "Inactive threads = %d", inactiveThreads); - activeThreads = pool.GetNumActiveThreads(); - Progress(activeThreads == 0, "Active threads = %d", activeThreads); - startedThreads = pool.GetNumStartedThreads(); - Progress(startedThreads == 2, "Started threads = %d", startedThreads); - - Progress(true, "Repeating process in the same pool..."); - - job[0].code = WAIT_FOR_BREAK_FLAG; - job[0].ownThread = pool.NewThread(this, static_cast<void *>(&job[0])); - - inactiveThreads = pool.GetNumInactiveThreads(); - Progress(inactiveThreads == 1, "Inactive threads = %d", inactiveThreads); - activeThreads = pool.GetNumActiveThreads(); - Progress(activeThreads == 1, "Active threads = %d", activeThreads); - startedThreads = pool.GetNumStartedThreads(); - Progress(startedThreads == 3, "Started threads = %d", startedThreads); - - job[1].code = WAIT_FOR_BREAK_FLAG; - job[1].ownThread = pool.NewThread(this, static_cast<void *>(&job[1])); - - inactiveThreads = pool.GetNumInactiveThreads(); - Progress(inactiveThreads == 0, "Inactive threads = %d", inactiveThreads); - activeThreads = pool.GetNumActiveThreads(); - Progress(activeThreads == 2, "Active threads = %d", activeThreads); - startedThreads = pool.GetNumStartedThreads(); - Progress(startedThreads == 4, "Started threads = %d", startedThreads); - - Progress(true, "Setting breakflag on threads..."); - job[0].ownThread->SetBreakFlag(); - job[1].ownThread->SetBreakFlag(); - - job[0].ownThread->Join(); - job[1].ownThread->Join(); - while (pool.GetNumInactiveThreads() != 2) { - std::this_thread::sleep_for(1ms); - } - - inactiveThreads = pool.GetNumInactiveThreads(); - Progress(inactiveThreads == 2, "Inactive threads = %d", inactiveThreads); - activeThreads = pool.GetNumActiveThreads(); - Progress(activeThreads == 0, "Active threads = %d", activeThreads); - startedThreads = pool.GetNumStartedThreads(); - Progress(startedThreads == 4, "Started threads = %d", startedThreads); - - pool.Close(); - Progress(true, "Pool closed."); - - PrintSeparator(); - } - - int Main () override; -}; - -int Thread_Stats_Test::Main () -{ - printf("grep for the string '%s' to detect failures.\n\n", failString); - time_t before = time(0); - - ThreadStatsTest(); - { time_t now = time(0); printf("[%ld seconds]\n", now-before); before = now; } - - printf("END OF TEST (%s)\n", _argv[0]); - return allWasOk() ? 0 : 1; -} - -int main (int argc, char **argv) -{ - Thread_Stats_Test app; - setvbuf(stdout, nullptr, _IOLBF, 8192); - return app.Entry(argc, argv); -} diff --git a/fastos/src/tests/thread_test_base.hpp b/fastos/src/tests/thread_test_base.hpp deleted file mode 100644 index eb994537f6e..00000000000 --- a/fastos/src/tests/thread_test_base.hpp +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <chrono> -#include <thread> - -static std::atomic<int64_t> number; -#define INCREASE_NUMBER_AMOUNT 10000 - -using namespace std::chrono_literals; - -class ThreadTestBase : public BaseTest, public FastOS_Runnable -{ -private: - std::mutex printMutex; - -public: - ThreadTestBase(void) - : printMutex() - { - } - virtual ~ThreadTestBase() {} - - void PrintProgress (char *string) override { - std::lock_guard<std::mutex> guard(printMutex); - BaseTest::PrintProgress(string); - } - - void Run (FastOS_ThreadInterface *thread, void *arg) override; - - void WaitForThreadsToFinish (Job *jobs, int count) { - int i; - - Progress(true, "Waiting for threads to finish..."); - for(;;) { - bool threadsFinished=true; - - for (i=0; i<count; i++) { - if (jobs[i].result == -1) { - threadsFinished = false; - break; - } - } - - std::this_thread::sleep_for(1us); - - if(threadsFinished) - break; - } - - Progress(true, "Threads finished"); - } -}; - - -void ThreadTestBase::Run (FastOS_ThreadInterface *thread, void *arg) -{ - if(arg == nullptr) - return; - - Job *job = static_cast<Job *>(arg); - char someStack[15*1024]; - - memset(someStack, 0, 15*1024); - - switch(job->code) - { - case SILENTNOP: - { - job->result = 1; - break; - } - - case NOP: - { - Progress(true, "Doing NOP"); - job->result = 1; - break; - } - - case PRINT_MESSAGE_AND_WAIT3MSEC: - { - Progress(true, "Thread printing message: [%s]", job->message); - job->result = strlen(job->message); - - std::this_thread::sleep_for(3ms); - break; - } - - case INCREASE_NUMBER: - { - int result; - - std::unique_lock<std::mutex> guard; - if(job->mutex != nullptr) { - guard = std::unique_lock<std::mutex>(*job->mutex); - } - - result = static_cast<int>(number.load(std::memory_order_relaxed)); - - int sleepOn = (INCREASE_NUMBER_AMOUNT/2) * 321/10000; - for (int i=0; i<(INCREASE_NUMBER_AMOUNT/2); i++) { - number.fetch_add(2, std::memory_order_relaxed); - - if (i == sleepOn) - std::this_thread::sleep_for(1ms); - } - - guard = std::unique_lock<std::mutex>(); - - job->result = result; // This marks the end of the thread - - break; - } - - case WAIT_FOR_BREAK_FLAG: - { - for(;;) { - std::this_thread::sleep_for(1us); - - if (thread->GetBreakFlag()) { - Progress(true, "Thread %p got breakflag", thread); - break; - } - } - break; - } - - case WAIT_FOR_THREAD_TO_FINISH: - { - std::unique_lock<std::mutex> guard; - if (job->mutex != nullptr) { - guard = std::unique_lock<std::mutex>(*job->mutex); - } - - if (job->otherThread != nullptr) - job->otherThread->Join(); - - break; - } - - case TEST_ID: - { - job->mutex->lock(); // Initially the parent threads owns the lock - job->mutex->unlock(); // It is unlocked when we should start - - FastOS_ThreadId currentId = FastOS_Thread::GetCurrentThreadId(); - - if(currentId == job->_threadId) - job->result = 1; - else - job->result = -1; - break; - } - - default: - Progress(false, "Unknown jobcode"); - break; - } -} diff --git a/fastos/src/tests/threadtest.cpp b/fastos/src/tests/threadtest.cpp deleted file mode 100644 index 563b41ac229..00000000000 --- a/fastos/src/tests/threadtest.cpp +++ /dev/null @@ -1,283 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "tests.h" -#include "job.h" -#include "thread_test_base.hpp" -#include <cstdlib> -#include <chrono> - -#define MAX_THREADS 7 - -using namespace std::chrono; -using namespace std::chrono_literals; - -class ThreadTest : public ThreadTestBase -{ - int Main () override; - - void TooManyThreadsTest () - { - TestHeader("Too Many Threads Test"); - static constexpr size_t message_size = 100; - - FastOS_ThreadPool *pool = new FastOS_ThreadPool(MAX_THREADS); - - if (Progress(pool != nullptr, "Allocating ThreadPool")) { - int i; - Job jobs[MAX_THREADS+1]; - - for (i=0; i<MAX_THREADS+1; i++) { - jobs[i].code = WAIT_FOR_BREAK_FLAG; - jobs[i].message = static_cast<char *>(malloc(message_size)); - if (jobs[i].message == nullptr) { - abort(); // GCC may infer that a potentially null ptr is passed to sprintf - } - snprintf(jobs[i].message, message_size, "Thread %d invocation", i+1); - } - - for (i=0; i<MAX_THREADS+1; i++) { - if (i==MAX_THREADS) { - while (pool->GetNumInactiveThreads() > 0); - jobs[MAX_THREADS].code = PRINT_MESSAGE_AND_WAIT3MSEC; - bool rc = (nullptr == pool->NewThread(this, static_cast<void *>(&jobs[MAX_THREADS]))); - Progress(rc, "Creating too many threads should fail."); - } else { - jobs[i].ownThread = pool->NewThread(this, static_cast<void *>(&jobs[i])); - Progress(jobs[i].ownThread != nullptr, "Creating Thread"); - } - } - for (i=0; i<MAX_THREADS; i++) { - jobs[i].ownThread->SetBreakFlag(); - } - - Progress(true, "Closing threadpool..."); - pool->Close(); - - Progress(true, "Deleting threadpool..."); - delete(pool); - } - PrintSeparator(); - } - - void CreateSingleThreadAndJoin () - { - TestHeader("Create Single Thread And Join Test"); - - FastOS_ThreadPool *pool = new FastOS_ThreadPool; - - if (Progress(pool != nullptr, "Allocating ThreadPool")) { - Job job; - - job.code = NOP; - job.result = -1; - - bool rc = (nullptr != pool->NewThread(this, &job)); - Progress(rc, "Creating Thread"); - - WaitForThreadsToFinish(&job, 1); - } - - Progress(true, "Closing threadpool..."); - pool->Close(); - - Progress(true, "Deleting threadpool..."); - delete(pool); - PrintSeparator(); - } - - void ThreadCreatePerformance (bool silent, int count, int outercount) { - int i; - int j; - bool rc; - int threadsok; - - if (!silent) - TestHeader("Thread Create Performance"); - - FastOS_ThreadPool *pool = new FastOS_ThreadPool; - - if (!silent) - Progress(pool != nullptr, "Allocating ThreadPool"); - if (pool != nullptr) { - Job *jobs = new Job[count]; - - threadsok = 0; - steady_clock::time_point start = steady_clock::now(); - for (i = 0; i < count; i++) { - jobs[i].code = SILENTNOP; - jobs[i].ownThread = pool->NewThread(this, &jobs[i]); - rc = (jobs[i].ownThread != nullptr); - if (rc) - threadsok++; - } - - for (j = 0; j < outercount; j++) { - for (i = 0; i < count; i++) { - if (jobs[i].ownThread != nullptr) - jobs[i].ownThread->Join(); - jobs[i].ownThread = pool->NewThread(this, &jobs[i]); - rc = (jobs[i].ownThread != nullptr); - if (rc) - threadsok++; - } - } - for (i = 0; i < count; i++) { - if (jobs[i].ownThread != nullptr) - jobs[i].ownThread->Join(); - } - nanoseconds used = steady_clock::now() - start; - - if (!silent) { - double timeused = used.count() / 1000000000.0; - - Progress(true, "Used time: %2.3f", timeused); - ProgressFloat(true, "Threads/s: %6.1f", - static_cast<float>(static_cast<double>(threadsok) / timeused)); - } - if (threadsok != ((outercount + 1) * count)) - Progress(false, "Only started %d of %d threads", threadsok, - (outercount + 1) * count); - - if (!silent) - Progress(true, "Closing threadpool..."); - pool->Close(); - delete [] jobs; - - if (!silent) - Progress(true, "Deleting threadpool..."); - delete(pool); - if (!silent) - PrintSeparator(); - } - } - - void ClosePoolStability(void) { - int i; - TestHeader("ThreadPool close stability test"); - for (i = 0; i < 1000; i++) { - // Progress(true, "Creating pool iteration %d", i + 1); - ThreadCreatePerformance(true, 2, 1); - } - PrintSeparator(); - } - - - - void ClosePoolTest () - { - TestHeader("Close Pool Test"); - - FastOS_ThreadPool pool; - const int closePoolThreads=9; - Job jobs[closePoolThreads]; - - number = 0; - - for(int i=0; i<closePoolThreads; i++) { - jobs[i].code = INCREASE_NUMBER; - - bool rc = (nullptr != pool.NewThread(this, static_cast<void *>(&jobs[i]))); - Progress(rc, "Creating Thread %d", i+1); - } - - Progress(true, "Waiting for threads to finish using pool.Close()..."); - pool.Close(); - Progress(true, "Pool closed."); - PrintSeparator(); - } - - void BreakFlagTest () { - TestHeader("BreakFlag Test"); - - FastOS_ThreadPool pool; - - const int breakFlagThreads=4; - - Job jobs[breakFlagThreads]; - - for (int i=0; i<breakFlagThreads; i++) { - jobs[i].code = WAIT_FOR_BREAK_FLAG; - - bool rc = (nullptr != pool.NewThread(this, static_cast<void *>(&jobs[i]))); - Progress(rc, "Creating Thread %d", i+1); - } - - Progress(true, "Waiting for threads to finish using pool.Close()..."); - pool.Close(); - Progress(true, "Pool closed."); - PrintSeparator(); - } - - void ThreadIdTest () { - constexpr int numThreads = 5; - - TestHeader ("Thread Id Test"); - - FastOS_ThreadPool pool; - Job jobs[numThreads]; - std::mutex slowStartMutex; - - slowStartMutex.lock(); // Halt all threads until we want them to run - - for (int i=0; i<numThreads; i++) { - jobs[i].code = TEST_ID; - jobs[i].result = -1; - jobs[i]._threadId = 0; - jobs[i].mutex = &slowStartMutex; - jobs[i].ownThread = pool.NewThread(this, static_cast<void *>(&jobs[i])); - bool rc=(jobs[i].ownThread != nullptr); - if (rc) { - jobs[i]._threadId = jobs[i].ownThread->GetThreadId(); - } - Progress(rc, "CreatingThread %d id:%lu", i+1, (unsigned long)(jobs[i]._threadId)); - - for (int j=0; j<i; j++) { - if (jobs[j]._threadId == jobs[i]._threadId) { - Progress(false, "Two different threads received the same thread id (%lu)", - (unsigned long)(jobs[i]._threadId)); - } - } - } - - slowStartMutex.unlock(); // Allow threads to run - - Progress(true, "Waiting for threads to finish using pool.Close()..."); - pool.Close(); - Progress(true, "Pool closed."); - - for (int i=0; i<numThreads; i++) { - Progress(jobs[i].result == 1, - "Thread %lu: ID comparison (current vs stored)", - (unsigned long)(jobs[i]._threadId)); - } - - PrintSeparator(); - } - -}; - -int ThreadTest::Main () -{ - printf("grep for the string '%s' to detect failures.\n\n", failString); - time_t before = time(0); - - ThreadIdTest(); - CreateSingleThreadAndJoin(); - TooManyThreadsTest(); - ClosePoolTest(); - BreakFlagTest(); - CreateSingleThreadAndJoin(); - ThreadCreatePerformance(false, 50, 10); - ClosePoolStability(); - { time_t now = time(0); printf("[%ld seconds]\n", now-before); before = now; } - - printf("END OF TEST (%s)\n", _argv[0]); - return allWasOk() ? 0 : 1; -} - -int main (int argc, char **argv) -{ - ThreadTest app; - setvbuf(stdout, nullptr, _IOLBF, 8192); - return app.Entry(argc, argv); -} diff --git a/fastos/src/tests/typetest.cpp b/fastos/src/tests/typetest.cpp deleted file mode 100644 index e5d7e9ceb74..00000000000 --- a/fastos/src/tests/typetest.cpp +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "tests.h" -#include <vespa/fastos/file.h> - -class TypeTest : public BaseTest -{ -private: - - void ObjectSizeTest () - { - TestHeader("Object Sizes (bytes)"); - - Progress(true, "FastOS_DirectoryScan %d", sizeof(FastOS_DirectoryScan)); - Progress(true, "FastOS_File: %d", sizeof(FastOS_File)); - Progress(true, "FastOS_Runnable %d", sizeof(FastOS_Runnable)); - Progress(true, "FastOS_StatInfo %d", sizeof(FastOS_StatInfo)); - Progress(true, "FastOS_Thread: %d", sizeof(FastOS_Thread)); - Progress(true, "FastOS_ThreadPool: %d", sizeof(FastOS_ThreadPool)); - - PrintSeparator(); - } - -public: - virtual ~TypeTest() {}; - - int Main () override - { - printf("grep for the string '%s' to detect failures.\n\n", failString); - - ObjectSizeTest(); - - PrintSeparator(); - printf("END OF TEST (%s)\n", _argv[0]); - - return allWasOk() ? 0 : 1; - } -}; - - -int main (int argc, char **argv) -{ - setvbuf(stdout, nullptr, _IOLBF, 8192); - TypeTest app; - return app.Entry(argc, argv); -} - diff --git a/fastos/src/vespa/fastos/.gitignore b/fastos/src/vespa/fastos/.gitignore deleted file mode 100644 index 004799df5b4..00000000000 --- a/fastos/src/vespa/fastos/.gitignore +++ /dev/null @@ -1,28 +0,0 @@ -*.So -*.core -*.exe -*.ilk -*.pdb -.depend -.depend.NEW -Debug -Makefile -Makefile.factory -Makefile.overrides -Makefile.pre -Release -autoconf.h -config_command.bat -config_command.sh -fastconfig -fastconfig.exe -fastos.lib -fastosconfig.h -libfastos.a -makefeatures -makemake -processtest.log -test.txt -vc60.idb -vc60.pdb -/libfastos.so.5.1 diff --git a/fastos/src/vespa/fastos/CMakeLists.txt b/fastos/src/vespa/fastos/CMakeLists.txt deleted file mode 100644 index 0e2bb51f79e..00000000000 --- a/fastos/src/vespa/fastos/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(fastos_objects OBJECT - SOURCES - file.cpp - file_rw_ops.cpp - linux_file.cpp - thread.cpp - unix_file.cpp - unix_thread.cpp -) - -vespa_add_library(fastos - SOURCES - $<TARGET_OBJECTS:fastos_objects> - INSTALL lib64 - DEPENDS - ${CMAKE_DL_LIBS} -) - -find_package(Threads REQUIRED) -target_link_libraries(fastos PUBLIC ${CMAKE_THREAD_LIBS_INIT}) diff --git a/fastos/src/vespa/fastos/thread.cpp b/fastos/src/vespa/fastos/thread.cpp deleted file mode 100644 index 9a9c3321cac..00000000000 --- a/fastos/src/vespa/fastos/thread.cpp +++ /dev/null @@ -1,363 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -//************************************************************************ -/** - * Implementation of FastOS_ThreadPool and FastOS_Thread methods. - * - * @author Oivind H. Danielsen - */ - -#include "thread.h" -#include <cstdio> -#include <cassert> - -// ---------------------------------------------------------------------- -// FastOS_ThreadPool -// ---------------------------------------------------------------------- - -FastOS_ThreadPool::FastOS_ThreadPool() : FastOS_ThreadPool(0) {} - -FastOS_ThreadPool::FastOS_ThreadPool(int maxThreads) - : _startedThreadsCount(0), - _closeFlagMutex(), - _closeCalledFlag(false), - _freeMutex(), - _liveMutex(), - _liveCond(), - _freeThreads(nullptr), - _activeThreads(nullptr), - _numFree(0), - _numActive(0), - _numTerminated(0), - _numLive(0), - _maxThreads(maxThreads) // 0 means unbounded -{ -} - -FastOS_ThreadPool::~FastOS_ThreadPool(void) -{ - Close(); -} - -void FastOS_ThreadPool::ThreadIsAboutToTerminate(FastOS_ThreadInterface *) -{ - assert(isClosed()); - - std::lock_guard<std::mutex> guard(_liveMutex); - - _numTerminated++; - _numLive--; - if (_numLive == 0) { - _liveCond.notify_all(); - } -} - - -// This is a NOP if the thread isn't active. -void FastOS_ThreadPool::FreeThread (FastOS_ThreadInterface *thread) -{ - std::lock_guard<std::mutex> guard(_freeMutex); - - if(thread->_active) { - LinkOutThread(thread, &_activeThreads); - - thread->_active = false; - _numActive--; - - LinkInThread(thread, &_freeThreads); - _numFree++; - } -} - -void FastOS_ThreadPool::LinkOutThread (FastOS_ThreadInterface *thread, FastOS_ThreadInterface **listHead) -{ - if (thread->_prev != nullptr) - thread->_prev->_next = thread->_next; - if (thread->_next != nullptr) - thread->_next->_prev = thread->_prev; - - if (thread == *listHead) - *listHead = thread->_next; -} - -void FastOS_ThreadPool::LinkInThread (FastOS_ThreadInterface *thread, FastOS_ThreadInterface **listHead) -{ - thread->_prev = nullptr; - thread->_next = *listHead; - - if (*listHead != nullptr) - (*listHead)->_prev = thread; - - *listHead = thread; -} - - -// _freeMutex is held by caller. -void FastOS_ThreadPool::ActivateThread (FastOS_ThreadInterface *thread) -{ - LinkOutThread(thread, &_freeThreads); - LinkInThread(thread, &_activeThreads); - - thread->_active = true; - _numActive++; - _startedThreadsCount++; -} - - -// Allocate a thread, either from pool of free or by 'new'. Finally, -// make this thread call parameter fcn when it becomes active. -FastOS_ThreadInterface *FastOS_ThreadPool::NewThread (FastOS_Runnable *owner, void *arg) -{ - FastOS_ThreadInterface *thread=nullptr; - - std::unique_lock<std::mutex> freeGuard(_freeMutex); - - if (!isClosed()) { - if ((thread = _freeThreads) != nullptr) { - // Reusing thread entry - _freeThreads = thread->_next; - _numFree--; - - ActivateThread(thread); - } else { - // Creating new thread entry - - if (_maxThreads != 0 && ((_numActive + _numFree) >= _maxThreads)) { - fprintf(stderr, "Error: Maximum number of threads (%d)" - " already allocated.\n", _maxThreads); - } else { - freeGuard.unlock(); - { - std::lock_guard<std::mutex> liveGuard(_liveMutex); - _numLive++; - } - thread = FastOS_Thread::CreateThread(this); - - if (thread == nullptr) { - std::lock_guard<std::mutex> liveGuard(_liveMutex); - _numLive--; - if (_numLive == 0) { - _liveCond.notify_all(); - } - } - freeGuard.lock(); - - if(thread != nullptr) - ActivateThread(thread); - } - } - } - - freeGuard.unlock(); - if(thread != nullptr) { - std::lock_guard<std::mutex> liveGuard(_liveMutex); - thread->Dispatch(owner, arg); - } - - return thread; -} - - -void FastOS_ThreadPool::BreakThreads () -{ - FastOS_ThreadInterface *thread; - - std::lock_guard<std::mutex> freeGuard(_freeMutex); - - // Notice all active threads that they should quit - for(thread=_activeThreads; thread != nullptr; thread=thread->_next) { - thread->SetBreakFlag(); - } - - // Notice all free threads that they should quit - for(thread=_freeThreads; thread != nullptr; thread=thread->_next) { - thread->SetBreakFlag(); - } -} - - -void FastOS_ThreadPool::JoinThreads () -{ - std::unique_lock<std::mutex> liveGuard(_liveMutex); - while (_numLive > 0) { - _liveCond.wait(liveGuard); - } -} - -void FastOS_ThreadPool::DeleteThreads () -{ - FastOS_ThreadInterface *thread; - - std::lock_guard<std::mutex> freeGuard(_freeMutex); - - assert(_numActive == 0); - assert(_numLive == 0); - - while((thread = _freeThreads) != nullptr) { - LinkOutThread(thread, &_freeThreads); - _numFree--; - // printf("deleting thread %p\n", thread); - delete(thread); - } - - assert(_numFree == 0); -} - -void FastOS_ThreadPool::Close () -{ - std::unique_lock<std::mutex> closeFlagGuard(_closeFlagMutex); - if (!_closeCalledFlag) { - _closeCalledFlag = true; - closeFlagGuard.unlock(); - - BreakThreads(); - JoinThreads(); - DeleteThreads(); - } -} - -bool FastOS_ThreadPool::isClosed() -{ - std::lock_guard<std::mutex> closeFlagGuard(_closeFlagMutex); - bool closed(_closeCalledFlag); - return closed; -} - -extern "C" -{ -void *FastOS_ThreadHook (void *arg) -{ - FastOS_ThreadInterface *thread = static_cast<FastOS_ThreadInterface *>(arg); - thread->Hook(); - - return nullptr; -} -}; - - -// ---------------------------------------------------------------------- -// FastOS_ThreadInterface -// ---------------------------------------------------------------------- - -void FastOS_ThreadInterface::Hook () -{ - // Loop forever doing the following: Wait on the signal _dispatched. - // When awoken, call _start_fcn with the parameters. Then zero set - // things and return this to the owner, i.e. pool of free threads - bool finished=false; - bool deleteOnCompletion = false; - - while(!finished) { - - std::unique_lock<std::mutex> dispatchedGuard(_dispatchedMutex); // BEGIN lock - while (_owner == nullptr && !(finished = _pool->isClosed())) { - _dispatchedCond.wait(dispatchedGuard); - } - - dispatchedGuard.unlock(); // END lock - - if(!finished) { - deleteOnCompletion = _owner->DeleteOnCompletion(); - _owner->Run(this, _startArg); - - dispatchedGuard.lock(); // BEGIN lock - - if (deleteOnCompletion) { - delete _owner; - } - _owner = nullptr; - _startArg = nullptr; - _breakFlag.store(false, std::memory_order_relaxed); - finished = _pool->isClosed(); - - dispatchedGuard.unlock(); // END lock - - { - std::lock_guard<std::mutex> runningGuard(_runningMutex); - _runningFlag = false; - _runningCond.notify_all(); - } - - _pool->FreeThread(this); - // printf("Thread given back to FastOS_ThreadPool: %p\n", this); - } - } - - _pool->ThreadIsAboutToTerminate(this); - - // Be sure not to touch any members from here on, as we are about - // to be deleted. -} - - -// Make this thread call parameter fcn with parameters argh -// when this becomes active. -// Restriction: _liveCond must be held by the caller. - -void FastOS_ThreadInterface::Dispatch(FastOS_Runnable *newOwner, void *arg) -{ - std::lock_guard<std::mutex> dispatchedGuard(_dispatchedMutex); - - { - std::unique_lock<std::mutex> runningGuard(_runningMutex); - while (_runningFlag) { - _runningCond.wait(runningGuard); - } - _runningFlag = true; - } - - _owner = newOwner; - _startArg = arg; - // Set _thread variable before NewThread returns - _owner->_thread.store(this, std::memory_order_release); - - // It is safe to signal after the unlock since _liveCond is still held - // so the signalled thread still exists. - // However as thread creation is infrequent and as helgrind suggest doing - // it the safe way we just do that, instead of keeping a unneccessary long - // suppressionslist. It will be long enough anyway. - - _dispatchedCond.notify_one(); -} - -void FastOS_ThreadInterface::SetBreakFlag() -{ - std::lock_guard<std::mutex> dispatchedGuard(_dispatchedMutex); - _breakFlag.store(true, std::memory_order_relaxed); - _dispatchedCond.notify_one(); -} - - -FastOS_ThreadInterface *FastOS_ThreadInterface::CreateThread(FastOS_ThreadPool *pool) -{ - FastOS_ThreadInterface *thread = new FastOS_Thread(pool); - - if(!thread->Initialize()) { - delete(thread); - thread = nullptr; - } - - return thread; -} - -void FastOS_ThreadInterface::Join () -{ - std::unique_lock<std::mutex> runningGuard(_runningMutex); - while (_runningFlag) { - _runningCond.wait(runningGuard); - } -} - - -// ---------------------------------------------------------------------- -// FastOS_Runnable -// ---------------------------------------------------------------------- - -FastOS_Runnable::FastOS_Runnable() - : _thread(nullptr) -{ -} - -FastOS_Runnable::~FastOS_Runnable() -{ - // assert(_thread == nullptr); -} diff --git a/fastos/src/vespa/fastos/thread.h b/fastos/src/vespa/fastos/thread.h deleted file mode 100644 index 2fb717403f2..00000000000 --- a/fastos/src/vespa/fastos/thread.h +++ /dev/null @@ -1,467 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -//************************************************************************ -/** - * @file - * Class definitions for FastOS_ThreadPool, FastOS_ThreadInterface and - * FastOS_Runnable. - * - * @author Oivind H. Danielsen - */ - -#pragma once - - -#include "types.h" -#include <atomic> -#include <mutex> -#include <condition_variable> - -typedef pthread_t FastOS_ThreadId; - -class FastOS_Runnable; -class FastOS_ThreadInterface; - - -/** - * This class implements an initially empty pool of threads. - * - * As threads are allocated with @ref NewThread() the number of - * threads in the pool increases. A maximum number of threads - * contained in the pool can be set using the constructor - * FastOS_ThreadPool(int maxThreads). - * - * Threads are automatically returned to the pool when they - * terminate. - */ -class FastOS_ThreadPool -{ - friend class FastOS_ThreadInterface; - -private: - int _startedThreadsCount; - std::mutex _closeFlagMutex; - bool _closeCalledFlag; - - // Always lock in this order - mutable std::mutex _freeMutex; - std::mutex _liveMutex; - std::condition_variable _liveCond; - /** - * List of free (available) threads. - */ - FastOS_ThreadInterface *_freeThreads; - - /** - * List of active (allocated) threads. - */ - FastOS_ThreadInterface *_activeThreads; - - /** - * Number of available threads in the threadpool. - * Total number of threads = free + active - */ - int _numFree; - - /** - * Number of active threads in the threadpool. - * Total number of threads = free + active - */ - int _numActive; - - /** - * Number of threads that have terminated - */ - int _numTerminated; - - /** - * Number of threads that have not been terminated - */ - int _numLive; - - /** - * Maximum number of threads in the threadpool. A value of - * zero means that there is no limit. - */ - int _maxThreads; - - /** - * Put this thread on the @ref _activeThreads list. - */ - void ActivateThread (FastOS_ThreadInterface *thread); - - /** - * Return previously active thread to the list of free thread. - */ - void FreeThread (FastOS_ThreadInterface *thread); - - /** - * A thread is informing the thread pool that it is about to - * terminate. - */ - void ThreadIsAboutToTerminate(FastOS_ThreadInterface *thread); - - /** - * Set the break flag on all threads. - */ - void BreakThreads (); - - /** - * Wait for all threads to finish. - */ - void JoinThreads (); - - /** - * Delete all threads in threadpool. - */ - void DeleteThreads (); - - /** - * Remove a thread from a list. - */ - void LinkOutThread (FastOS_ThreadInterface *thread, - FastOS_ThreadInterface **listHead); - - /** - * Add a thread to a list. Notice that a thread can be on only one - * list at a time. - */ - void LinkInThread (FastOS_ThreadInterface *thread, - FastOS_ThreadInterface **listHead); - -public: - FastOS_ThreadPool(const FastOS_ThreadPool&) = delete; - FastOS_ThreadPool& operator=(const FastOS_ThreadPool&) = delete; - FastOS_ThreadPool(int maxThreads); - /// Unlimited threads - FastOS_ThreadPool(); - - /** - * Destructor. Closes pool if necessary. - */ - virtual ~FastOS_ThreadPool(); - - - /** - * Allocate a new thread, and make this thread invoke the Run() method - * of the @ref FastOS_Runnable object [owner] with parameters [arg]. - * The thread is automatically freed (returned to the treadpool) - * when Run() returns. - * - * @param owner Instance to be invoked by new thread. - * @param arg Arguments to be passed to new thread. - * - * @return Pointer to newly created thread or nullptr on failure. - */ - FastOS_ThreadInterface *NewThread (FastOS_Runnable *owner, void *arg=nullptr); - - /** - * Close the threadpool. This involves setting the break flag on - * all active threads, and waiting for them to finish. Once Close - * is called, no more threads can be allocated from the thread - * pool. There exists no way to reopen a closed threadpool. - */ - void Close (); - - /** - * This will tell if the pool has been closed. - */ - bool isClosed(); - - /** - * Get the number of currently active threads. - * The total number of actual allocated threads is the sum of - * @ref GetNumActiveThreads() and @ref GetNumInactiveThreads(). - * @return Number of currently active threads - */ - int GetNumActiveThreads () const { - std::lock_guard<std::mutex> guard(_freeMutex); - return _numActive; - } - - /** - * Get the number of currently inactive threads. - * The total number of actual allocated threads is the sum of - * @ref GetNumActiveThreads() and @ref GetNumInactiveThreads(). - * @return Number of currently inactive threads - */ - int GetNumInactiveThreads () const { - std::lock_guard<std::mutex> guard(_freeMutex); - return _numFree; - } - - /** - * Get the number of started threads since instantiation of the thread pool. - * @return Number of threads started - */ - int GetNumStartedThreads () const { return _startedThreadsCount; } -}; - - -// Operating system thread entry point -extern "C" { - void *FastOS_ThreadHook (void *arg); -} - -/** - * This class controls each operating system thread. - * - * In most cases you would not want to create objects of this class - * directly. Use @ref FastOS_ThreadPool::NewThread() instead. - */ -class FastOS_ThreadInterface -{ - friend class FastOS_ThreadPool; - friend void *FastOS_ThreadHook (void *arg); - -private: - FastOS_ThreadInterface(const FastOS_ThreadInterface&); - FastOS_ThreadInterface& operator=(const FastOS_ThreadInterface&); - -protected: - /** - * The thread does not start (call @ref FastOS_Runnable::Run()) - * until this event has been triggered. - */ - std::mutex _dispatchedMutex; - std::condition_variable _dispatchedCond; - - FastOS_ThreadInterface *_next; - FastOS_ThreadInterface *_prev; - - /** - * A pointer to the instance which implements the interface - * @ref FastOS_Runnable. - */ - FastOS_Runnable *_owner; - - /** - * A pointer to the originating @ref FastOS_ThreadPool - */ - FastOS_ThreadPool *_pool; - - /** - * Entry point for the OS thread. The thread will sleep here - * until dispatched. - */ - void Hook (); - - /** - * Signals that thread should be dispatched. - * @param owner Instance of @ref FastOS_Runnable. - * @param arg Thread invocation arguments. - */ - void Dispatch (FastOS_Runnable *owner, void *arg); - - /** - * Initializes a thread. This includes creating the operating system - * thread handle and setting it up and making it ready to be dispatched. - * @return Boolean success/failure - */ - virtual bool Initialize ()=0; - - /** - * Used to store thread invocation arguments. These are passed along - * to @ref FastOS_Runnable::Run() when the thread is dispatched. - */ - void *_startArg; - - /** - * Create an operating system thread. In most cases you would want - * to create threads using @ref FastOS_ThreadPool::NewThread() instead. - * @param pool The threadpool which is about to contain the new thread. - * @return A new @ref FastOS_Thread or nullptr on failure. - */ - static FastOS_ThreadInterface *CreateThread(FastOS_ThreadPool *pool); - - /** - * Break flag. If true, the thread should exit. - */ - std::atomic<bool> _breakFlag; - - /** - * Is this thread active or free in the threadpool? - */ - bool _active; - - /** - * Is the thread running? This is used by @ref Join(), to wait for threads - * to finish. - */ - std::mutex _runningMutex; - std::condition_variable _runningCond; - bool _runningFlag; - -public: - /** - * Constructor. Resets internal attributes. - */ - FastOS_ThreadInterface (FastOS_ThreadPool *pool) - : _dispatchedMutex(), - _dispatchedCond(), - _next(nullptr), - _prev(nullptr), - _owner(nullptr), - _pool(pool), - _startArg(nullptr), - _breakFlag(false), - _active(false), - _runningMutex(), - _runningCond(), - _runningFlag(false) - { - } - - /** - * Destructor. - */ - virtual ~FastOS_ThreadInterface () {} - - /** - * Instruct a thread to exit. This could be used in conjunction with - * @ref GetBreakFlag() in a worker thread, to have cooperative thread - * termination. When a threadpool closes, all threads in the pool will - * have their break flag set. - */ - void SetBreakFlag (); - - /** - * Return the status of this thread's break flag. If the break flag - * is set, someone wants the thread to terminate. It is up to the - * implementor of the thread to decide whether the break flag - * should be used. - * - * In scenarios where a worker thread loops "forever" waiting for - * new jobs, the break flag should be polled in order to eventually - * exit from the loop and terminate the thread. - * - * In scenarios where a worker thread performs a task which - * always should run to completion, the break flag could be ignored - * as the thread sooner or later will terminate. - * - * When a threadpool is closed, the break flag is set on all - * threads in the pool. If a thread loops forever and chooses to - * ignore the break flag, a @ref FastOS_ThreadPool::Close() will - * never finish. (see @ref SetBreakFlag) - */ - bool GetBreakFlag () const - { - return _breakFlag.load(std::memory_order_relaxed); - } - - /** - * Wait for a thread to finish. - */ - void Join (); - - /** - * Returns the id of this thread. - */ - virtual FastOS_ThreadId GetThreadId () const noexcept = 0; -}; - - -/** - * This class gives a generic interface for invoking new threads with an object. - * - * The thread object should inherit this interface (class), and implement - * the @ref Run() method. When @ref FastOS_ThreadPool::NewThread() is - * called, the @ref Run() method of the passed instance will be invoked. - * - * Arguments could be supplied via @ref FastOS_ThreadPool::NewThread(), but - * it is also possible to supply arguments to the new thread through the - * worker thread object constructor or some other attribute-setting method - * prior to creating the thread. Choose whichever method works best for you. - * - * Example: - * @code - * // Arguments passed to the new thread. - * struct MyThreadArgs - * { - * int _something; - * char _tenChars[10]; - * }; - * - * class MyWorkerThread : public FastOS_Runnable - * { - * public: - * - * // Delete this instance upon completion - * virtual bool DeleteOnCompletion() const { return true; } - * - * virtual void Run (FastOS_ThreadInterface *thread, void *arguments) - * { - * MyThreadArgs *args = static_cast<MyThreadArgs *>(arguments); - * - * // Do some computation... - * Foo(args->_something); - * - * for(int i=0; i<30000; i++) - * { - * ... - * ... - * - * if(thread->GetBreakFlag()) - * break; - * ... - * ... - * - * } - * - * // Thread terminates... - * } - * }; - * - * - * // Example on how to create a thread using the above classes. - * void SomeClass::SomeMethod (FastOS_ThreadPool *pool) - * { - * MyWorkerThread *workerThread = new MyWorkerThread(); - * static MyThreadArgs arguments; - * - * arguments._something = 123456; - * - * // the workerThread instance will be deleted when Run completes - * // see the DeleteOnCompletion doc - * pool->NewThread(workerThread, &arguments); - * } - * @endcode - */ -class FastOS_Runnable -{ -private: - friend class FastOS_ThreadInterface; - std::atomic<FastOS_ThreadInterface*> _thread; - -public: - FastOS_Runnable(const FastOS_Runnable&) = delete; - FastOS_Runnable& operator=(const FastOS_Runnable&) = delete; - FastOS_Runnable(); - virtual ~FastOS_Runnable(); - - /** - * The DeleteOnCompletion method should be overridden to return true - * if the runnable instance should be deleted when run completes - * - * @author Nils Sandoy - * @return true iff this runnable instance should be deleted on completion - */ - virtual bool DeleteOnCompletion() const { return false; } - - /** - * When an object implementing interface @ref FastOS_Runnable is used to - * create a thread, starting the thread causes the object's @ref Run() - * method to be called in that separately executing thread. The thread - * terminates when @ref Run() returns. - * @param thisThread A thread object. - * @param arguments Supplied to @ref FastOS_ThreadPool::NewThread - */ - virtual void Run(FastOS_ThreadInterface *thisThread, void *arguments)=0; - - FastOS_ThreadInterface *GetThread() noexcept { return _thread.load(std::memory_order_acquire); } - const FastOS_ThreadInterface *GetThread() const noexcept { return _thread.load(std::memory_order_acquire); } - bool HasThread() const noexcept { return GetThread() != nullptr; } -}; - -#include <vespa/fastos/unix_thread.h> -typedef FastOS_UNIX_Thread FASTOS_PREFIX(Thread); - diff --git a/fastos/src/vespa/fastos/types.h b/fastos/src/vespa/fastos/types.h deleted file mode 100644 index 69dd3e5231c..00000000000 --- a/fastos/src/vespa/fastos/types.h +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#define FASTOS_PREFIX(a) FastOS_##a - -// New macros to support the new gcc visibility features. -#define VESPA_DLL_EXPORT __attribute__ ((visibility("default"))) -#define VESPA_DLL_LOCAL __attribute__ ((visibility("hidden"))) diff --git a/fastos/src/vespa/fastos/unix_thread.cpp b/fastos/src/vespa/fastos/unix_thread.cpp deleted file mode 100644 index 621505b7e02..00000000000 --- a/fastos/src/vespa/fastos/unix_thread.cpp +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "thread.h" - -bool FastOS_UNIX_Thread::Initialize () -{ - pthread_attr_t attr; - pthread_attr_init(&attr); - pthread_attr_setscope(&attr, PTHREAD_SCOPE_SYSTEM); - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_JOINABLE); - _handleValid = (0 == pthread_create(&_handle, &attr, FastOS_ThreadHook, this)); - pthread_attr_destroy(&attr); - - return _handleValid; -} - -FastOS_UNIX_Thread::~FastOS_UNIX_Thread() -{ - if (!_handleValid) return; - - void *value = nullptr; - pthread_join(_handle, &value); -} - -FastOS_ThreadId FastOS_UNIX_Thread::GetThreadId () const noexcept -{ - return _handle; -} - -FastOS_ThreadId FastOS_UNIX_Thread::GetCurrentThreadId () -{ - return pthread_self(); -} - -bool FastOS_UNIX_Thread::CompareThreadIds (FastOS_ThreadId a, FastOS_ThreadId b) -{ - return (pthread_equal(a, b) != 0); -} diff --git a/fastos/src/vespa/fastos/unix_thread.h b/fastos/src/vespa/fastos/unix_thread.h deleted file mode 100644 index c3c757e3fd9..00000000000 --- a/fastos/src/vespa/fastos/unix_thread.h +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** -****************************************************************************** -* @author Oivind H. Danielsen -* @date Creation date: 2000-02-02 -* @file -* Class definition for FastOS_UNIX_Thread -*****************************************************************************/ - -#pragma once - -#include "thread.h" - -class FastOS_UNIX_Thread : public FastOS_ThreadInterface -{ -protected: - pthread_t _handle; - bool _handleValid; - - bool Initialize () override; -public: - FastOS_UNIX_Thread(const FastOS_UNIX_Thread &) = delete; - FastOS_UNIX_Thread& operator=(const FastOS_UNIX_Thread &) = delete; - FastOS_UNIX_Thread(FastOS_ThreadPool *pool) - : FastOS_ThreadInterface(pool), - _handle(), - _handleValid(false) - {} - - ~FastOS_UNIX_Thread() override; - - FastOS_ThreadId GetThreadId () const noexcept override; - static bool CompareThreadIds (FastOS_ThreadId a, FastOS_ThreadId b); - static FastOS_ThreadId GetCurrentThreadId (); -}; - - diff --git a/fbench/CMakeLists.txt b/fbench/CMakeLists.txt index ff287d221ec..3f1d78a66a0 100644 --- a/fbench/CMakeLists.txt +++ b/fbench/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalib APPS diff --git a/fbench/src/httpclient/CMakeLists.txt b/fbench/src/httpclient/CMakeLists.txt index 163a68f9c98..2d5e3b32437 100644 --- a/fbench/src/httpclient/CMakeLists.txt +++ b/fbench/src/httpclient/CMakeLists.txt @@ -4,5 +4,4 @@ vespa_add_library(fbench_httpclient STATIC httpclient.cpp DEPENDS fbench_util - fastos ) diff --git a/fbench/src/test/CMakeLists.txt b/fbench/src/test/CMakeLists.txt index d13b6b82a81..c81d818ed06 100644 --- a/fbench/src/test/CMakeLists.txt +++ b/fbench/src/test/CMakeLists.txt @@ -26,6 +26,5 @@ vespa_add_executable(fbench_clientstatus_app TEST clientstatus.cpp DEPENDS fbench_util - fastos ) vespa_add_test(NAME fbench_clientstatus_app COMMAND fbench_clientstatus_app) diff --git a/fileacquirer/CMakeLists.txt b/fileacquirer/CMakeLists.txt index 13150f58ba3..cc18dc2bd84 100644 --- a/fileacquirer/CMakeLists.txt +++ b/fileacquirer/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib config_cloudconfig diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 68324f463c0..7691cf4031f 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -281,13 +281,6 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); - public static final UnboundBooleanFlag NOTIFICATION_DISPATCH_FLAG = defineFeatureFlag( - "dispatch-notifications", false, - List.of("enygaard"), "2022-05-02", "2023-03-01", - "Whether we should send notification for a given tenant", - "Takes effect immediately", - TENANT_ID); - public static final UnboundBooleanFlag ENABLE_PROXY_PROTOCOL_MIXED_MODE = defineFeatureFlag( "enable-proxy-protocol-mixed-mode", true, List.of("tokle"), "2022-05-09", "2023-03-31", @@ -338,13 +331,6 @@ public class Flags { "Takes effect immediately", CONSOLE_USER_EMAIL); - public static final UnboundBooleanFlag USE_WIREGUARD_ON_CONFIGSERVERS = defineFeatureFlag( - "use-wireguard-on-configservers", false, - List.of("andreer", "gjoranv"), "2022-09-28", "2023-04-01", - "Set up a WireGuard endpoint on config servers", - "Takes effect on configserver restart", - HOSTNAME); - public static final UnboundStringFlag CORE_ENCRYPTION_PUBLIC_KEY_ID = defineStringFlag( "core-encryption-public-key-id", "", List.of("vekterli"), "2022-11-03", "2023-05-01", @@ -352,11 +338,12 @@ public class Flags { "Takes effect on the next tick.", ZONE_ID, NODE_TYPE, HOSTNAME); - public static final UnboundStringFlag ZOOKEEPER_SNAPSHOT_METHOD = defineStringFlag( - "zookeeper-snapshot-method", "", - List.of("mpolden"), "2023-02-01", "2023-05-01", - "ZooKeeper snapshot compression method. Valid values are '', 'gz' and 'snappy'", - "Takes effect on node restart"); + public static final UnboundBooleanFlag ENABLE_GLOBAL_PHASE = defineFeatureFlag( + "enable-global-phase", false, + List.of("arnej", "bjorncs"), "2023-02-28", "2024-01-10", + "Enable global phase ranking", + "Takes effect at redeployment", + APPLICATION_ID); /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index 384fa7a4177..f128d13b7c4 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -322,14 +322,20 @@ public class PermanentFlags { APPLICATION_ID); public static final UnboundLongFlag CONFIG_SERVER_SESSION_EXPIRY_TIME = defineLongFlag( - // TODO: Lower to 3600, which is default session expiry time - "config-server-session-expiry-time", 3600 * 2, + "config-server-session-expiry-time", 3600, "Expiry time in seconds for remote sessions (session in ZooKeeper). Default should be equal to session lifetime, " + "but can be lowered if there are incidents/bugs where one needs to delete sessions", "Takes effect immediately", ZONE_ID ); + public static final UnboundBooleanFlag NOTIFICATION_DISPATCH_FLAG = defineFeatureFlag( + "dispatch-notifications", true, + "Whether we should send notification for a given tenant", + "Takes effect immediately", + TENANT_ID); + + private PermanentFlags() {} private static UnboundBooleanFlag defineFeatureFlag( diff --git a/fnet/CMakeLists.txt b/fnet/CMakeLists.txt index eeef7f63876..6d8836817e0 100644 --- a/fnet/CMakeLists.txt +++ b/fnet/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib diff --git a/fnet/INSTALL b/fnet/INSTALL deleted file mode 100644 index 9bb6c6152a4..00000000000 --- a/fnet/INSTALL +++ /dev/null @@ -1,22 +0,0 @@ -********************************* -* Compiling and Installing fnet * -********************************* - -'fnet' uses 'FastOS'. -In the following instructions, let %FASTOS_DIR% denote the install -directory for FastOS. A reasonable install directory would be: - %FASTOS_DIR% = '/usr/fastsearch/fastos' - -Install FastOS: -- checkout the fastos CVS module -- go to fastos/src/fastos -- ./configure --install-dir %FASTOS_DIR% [<config parameters>] - (run ./configure --help for help) -- make install - -Install fnet: -- checkout the fnet CVS module -- go to fnet/src -- ./configure --fastos-dir %FASTOS_DIR% [<config parameters>] - (run ./configure --fastos-dir %FASTOS_DIR% --help for help) -- make install diff --git a/fnet/README b/fnet/README deleted file mode 100644 index dab36e5b120..00000000000 --- a/fnet/README +++ /dev/null @@ -1,26 +0,0 @@ -This cvs module contains the code for FNET and FRT -FNET currently stands for 'FuNET' -FRT currently stands for 'FNET Remote Tools' - -FNET is a multi-threaded object-oriented networking library based on -FastOS. FRT implements a proprietary RPC protocol on top of FNET. - -For how to compile and install, read the INSTALL file. -For release information, read the RELEASEINFO file. - -The maintainer of this module is [havardpe@yahoo-inc.com] - -Notes from the maintainer: - -This library is work in progress. This means that some APIs may change -in new versions. I will try to keep these changes to a minimum and -also document them in the RELEASEINFO file. - -No extra effort has been added to hide things from the users of this -library. This means that you can use components like the scheduler to -implement a scheduler thread that has nothing to do with -networking. It also means that you have to expect to handle more API -changes if you are using classes not intended for direct use. It also -means that you will have access to methods you were never meant to -invoke. If this becomes a problem, I will start making things private -and the classes more 'friendly'. diff --git a/fnet/src/examples/ping/pingclient.cpp b/fnet/src/examples/ping/pingclient.cpp index 9b32e40ac83..43296df7e57 100644 --- a/fnet/src/examples/ping/pingclient.cpp +++ b/fnet/src/examples/ping/pingclient.cpp @@ -6,7 +6,6 @@ #include <vespa/fnet/connection.h> #include <examples/ping/packets.h> #include <vespa/vespalib/util/signalhandler.h> -#include <vespa/fastos/thread.h> #include <vespa/log/log.h> LOG_SETUP("pingclient"); @@ -28,7 +27,6 @@ PingClient::main(int argc, char **argv) } FNET_PacketQueue queue; - FastOS_ThreadPool pool; PingPacketFactory factory; FNET_SimplePacketStreamer streamer(&factory); FNET_Transport transport; @@ -39,7 +37,7 @@ PingClient::main(int argc, char **argv) if (argc == 3) { timeout_ms = atof(argv[2]) * 1000; } - transport.Start(&pool); + transport.Start(); uint32_t channelCnt = 0; for (uint32_t i = 0; i < 10; i++) { @@ -90,7 +88,6 @@ PingClient::main(int argc, char **argv) if (conn != nullptr) conn->SubRef(); transport.ShutDown(true); - pool.Close(); return 0; } diff --git a/fnet/src/examples/timeout/timeout.cpp b/fnet/src/examples/timeout/timeout.cpp index 41de852d48c..e0830c7cde1 100644 --- a/fnet/src/examples/timeout/timeout.cpp +++ b/fnet/src/examples/timeout/timeout.cpp @@ -5,7 +5,6 @@ #include <vespa/fnet/packetqueue.h> #include <vespa/fnet/controlpacket.h> #include <vespa/vespalib/util/signalhandler.h> -#include <vespa/fastos/thread.h> #include <vespa/vespalib/util/time.h> #include <thread> @@ -54,10 +53,9 @@ MyApp::main(int, char **) ms_double ms; clock::time_point t; FNET_PacketQueue queue; - FastOS_ThreadPool pool; FNET_Transport transport; Timeout timeout(transport.GetScheduler(), &queue); - transport.Start(&pool); + transport.Start(); // stable-state operation std::this_thread::sleep_for(100ms); @@ -90,7 +88,6 @@ MyApp::main(int, char **) fprintf(stderr, "time since timeout was scheduled: %f ms\n", ms.count()); transport.ShutDown(true); - pool.Close(); return 0; } diff --git a/fnet/src/tests/connect/connect_test.cpp b/fnet/src/tests/connect/connect_test.cpp index 681c3b7676a..9d566cf37a7 100644 --- a/fnet/src/tests/connect/connect_test.cpp +++ b/fnet/src/tests/connect/connect_test.cpp @@ -88,26 +88,25 @@ struct BlockingCryptoEngine : public CryptoEngine { struct TransportFixture : FNET_IPacketHandler, FNET_IConnectionCleanupHandler { FNET_SimplePacketStreamer streamer; - FastOS_ThreadPool pool; FNET_Transport transport; Gate conn_lost; Gate conn_deleted; - TransportFixture() : streamer(nullptr), pool(), transport(), + TransportFixture() : streamer(nullptr), transport(), conn_lost(), conn_deleted() { - transport.Start(&pool); + transport.Start(); } TransportFixture(AsyncResolver::HostResolver::SP host_resolver) - : streamer(nullptr), pool(), transport(fnet::TransportConfig().resolver(make_resolver(std::move(host_resolver)))), + : streamer(nullptr), transport(fnet::TransportConfig().resolver(make_resolver(std::move(host_resolver)))), conn_lost(), conn_deleted() { - transport.Start(&pool); + transport.Start(); } TransportFixture(CryptoEngine::SP crypto) - : streamer(nullptr), pool(), transport(fnet::TransportConfig().crypto(std::move(crypto))), + : streamer(nullptr), transport(fnet::TransportConfig().crypto(std::move(crypto))), conn_lost(), conn_deleted() { - transport.Start(&pool); + transport.Start(); } HP_RetCode HandlePacket(FNET_Packet *packet, FNET_Context) override { ASSERT_TRUE(packet->GetCommand() == FNET_ControlPacket::FNET_CMD_CHANNEL_LOST); @@ -127,7 +126,6 @@ struct TransportFixture : FNET_IPacketHandler, FNET_IConnectionCleanupHandler { } ~TransportFixture() override { transport.ShutDown(true); - pool.Close(); } }; diff --git a/fnet/src/tests/connection_spread/connection_spread_test.cpp b/fnet/src/tests/connection_spread/connection_spread_test.cpp index d65e4fb70fe..6286ce65657 100644 --- a/fnet/src/tests/connection_spread/connection_spread_test.cpp +++ b/fnet/src/tests/connection_spread/connection_spread_test.cpp @@ -6,7 +6,6 @@ #include <vespa/fnet/ipacketstreamer.h> #include <vespa/fnet/connector.h> #include <vespa/fnet/connection.h> -#include <vespa/fastos/thread.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/stringfmt.h> #include <thread> @@ -28,13 +27,12 @@ struct DummyStreamer : FNET_IPacketStreamer { struct Fixture { DummyStreamer streamer; DummyAdapter adapter; - FastOS_ThreadPool thread_pool; FNET_Transport client; FNET_Transport server; - Fixture() : streamer(), adapter(), thread_pool(), client(8), server(8) + Fixture() : streamer(), adapter(), client(8), server(8) { - ASSERT_TRUE(client.Start(&thread_pool)); - ASSERT_TRUE(server.Start(&thread_pool)); + ASSERT_TRUE(client.Start()); + ASSERT_TRUE(server.Start()); } void wait_for_components(size_t client_cnt, size_t server_cnt) { bool ok = false; @@ -49,7 +47,6 @@ struct Fixture { ~Fixture() { server.ShutDown(true); client.ShutDown(true); - thread_pool.Close(); } }; diff --git a/fnet/src/tests/frt/detach_supervisor/detach_supervisor_test.cpp b/fnet/src/tests/frt/detach_supervisor/detach_supervisor_test.cpp index 716c433ff61..6325c60413a 100644 --- a/fnet/src/tests/frt/detach_supervisor/detach_supervisor_test.cpp +++ b/fnet/src/tests/frt/detach_supervisor/detach_supervisor_test.cpp @@ -10,7 +10,6 @@ #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/time.h> -#include <vespa/fastos/thread.h> #include <thread> using namespace vespalib; @@ -19,14 +18,12 @@ using vespalib::make_string_short::fmt; CryptoEngine::SP null_crypto = std::make_shared<NullCryptoEngine>(); struct BasicFixture { - FastOS_ThreadPool thread_pool; FNET_Transport transport; - BasicFixture() : thread_pool(), transport(fnet::TransportConfig(4).crypto(null_crypto)) { - ASSERT_TRUE(transport.Start(&thread_pool)); + BasicFixture() : transport(fnet::TransportConfig(4).crypto(null_crypto)) { + ASSERT_TRUE(transport.Start()); } ~BasicFixture() { transport.ShutDown(true); - thread_pool.Close(); } }; diff --git a/fnet/src/tests/frt/parallel_rpc/parallel_rpc_test.cpp b/fnet/src/tests/frt/parallel_rpc/parallel_rpc_test.cpp index 624f5a73ae6..41bfb7d06a6 100644 --- a/fnet/src/tests/frt/parallel_rpc/parallel_rpc_test.cpp +++ b/fnet/src/tests/frt/parallel_rpc/parallel_rpc_test.cpp @@ -4,7 +4,6 @@ #include <vespa/fnet/frt/rpcrequest.h> #include <vespa/fnet/frt/target.h> #include <vespa/fnet/transport.h> -#include <vespa/fastos/thread.h> #include <vespa/vespalib/util/benchmark_timer.h> #include <vespa/vespalib/net/crypto_engine.h> #include <vespa/vespalib/net/tls/tls_crypto_engine.h> @@ -15,13 +14,12 @@ using namespace vespalib; struct Rpc : FRT_Invokable { - FastOS_ThreadPool thread_pool; FNET_Transport transport; FRT_Supervisor orb; Rpc(CryptoEngine::SP crypto, size_t num_threads, bool drop_empty) - : thread_pool(), transport(fnet::TransportConfig(num_threads).crypto(std::move(crypto)).drop_empty_buffers(drop_empty)), orb(&transport) {} + : transport(fnet::TransportConfig(num_threads).crypto(std::move(crypto)).drop_empty_buffers(drop_empty)), orb(&transport) {} void start() { - ASSERT_TRUE(transport.Start(&thread_pool)); + ASSERT_TRUE(transport.Start()); } uint32_t listen() { ASSERT_TRUE(orb.Listen(0)); @@ -32,7 +30,6 @@ struct Rpc : FRT_Invokable { } ~Rpc() override { transport.ShutDown(true); - thread_pool.Close(); } }; diff --git a/fnet/src/tests/frt/rpc/invoke.cpp b/fnet/src/tests/frt/rpc/invoke.cpp index 38f260dd202..e930c1252bf 100644 --- a/fnet/src/tests/frt/rpc/invoke.cpp +++ b/fnet/src/tests/frt/rpc/invoke.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/net/socket_spec.h> #include <vespa/vespalib/net/tls/capability_env_config.h> +#include <vespa/vespalib/net/tls/statistics.h> #include <vespa/vespalib/util/benchmark_timer.h> #include <vespa/vespalib/util/latch.h> #include <vespa/fnet/frt/supervisor.h> @@ -16,6 +17,7 @@ using vespalib::SocketSpec; using vespalib::BenchmarkTimer; +using vespalib::net::tls::CapabilityStatistics; using namespace vespalib::net::tls; constexpr double timeout = 60.0; @@ -486,6 +488,7 @@ TEST_F("request allowed by access filter invokes server method as usual", Fixtur } TEST_F("capability checking filter is enforced under mTLS unless overridden by env var", Fixture()) { + const auto cap_stats_before = CapabilityStatistics::get().snapshot(); MyReq req("capabilityRestricted"); // Requires content node cap set; disallowed f1.target().InvokeSync(req.borrow(), timeout); auto cap_mode = capability_enforcement_mode_from_env(); @@ -494,6 +497,9 @@ TEST_F("capability checking filter is enforced under mTLS unless overridden by e // Default authz rule does not give required capabilities; must fail. EXPECT_EQUAL(req.get().GetErrorCode(), FRTE_RPC_PERMISSION_DENIED); EXPECT_FALSE(f1.server_instance().restricted_method_was_invoked()); + // Permission denied should bump capability check failure statistic + const auto cap_stats = CapabilityStatistics::get().snapshot().subtract(cap_stats_before); + EXPECT_EQUAL(cap_stats.rpc_capability_checks_failed, 1u); } else { // Either no mTLS configured (implicit full capability set) or capabilities not enforced. ASSERT_FALSE(req.get().IsError()); @@ -502,11 +508,15 @@ TEST_F("capability checking filter is enforced under mTLS unless overridden by e } TEST_F("access is allowed by capability filter when peer is granted the required capability", Fixture()) { + const auto cap_stats_before = CapabilityStatistics::get().snapshot(); MyReq req("capabilityAllowed"); // Requires telemetry cap set; allowed f1.target().InvokeSync(req.borrow(), timeout); // Should always be allowed, regardless of mTLS mode or capability enforcement ASSERT_FALSE(req.get().IsError()); EXPECT_TRUE(f1.server_instance().restricted_method_was_invoked()); + // Should _not_ bump capability check failure statistic + const auto cap_stats = CapabilityStatistics::get().snapshot().subtract(cap_stats_before); + EXPECT_EQUAL(cap_stats.rpc_capability_checks_failed, 0u); } TEST_F("access is allowed by capability filter when required capability set is empty", Fixture()) { diff --git a/fnet/src/tests/sync_execute/sync_execute.cpp b/fnet/src/tests/sync_execute/sync_execute.cpp index b8fa21cf147..5d2f4097ab4 100644 --- a/fnet/src/tests/sync_execute/sync_execute.cpp +++ b/fnet/src/tests/sync_execute/sync_execute.cpp @@ -4,7 +4,6 @@ #include <vespa/vespalib/util/size_literals.h> #include <vespa/fnet/transport.h> #include <vespa/fnet/iexecutable.h> -#include <vespa/fastos/thread.h> struct DoIt : public FNET_IExecutable { vespalib::Gate gate; @@ -18,10 +17,9 @@ TEST("sync execute") { DoIt exe2; DoIt exe3; DoIt exe4; - FastOS_ThreadPool pool; FNET_Transport transport; ASSERT_TRUE(transport.execute(&exe1)); - ASSERT_TRUE(transport.Start(&pool)); + ASSERT_TRUE(transport.Start()); exe1.gate.await(); ASSERT_TRUE(transport.execute(&exe2)); transport.sync(); @@ -32,7 +30,6 @@ TEST("sync execute") { transport.sync(); transport.WaitFinished(); transport.sync(); - pool.Close(); ASSERT_TRUE(exe1.gate.getCount() == 0u); ASSERT_TRUE(exe2.gate.getCount() == 0u); ASSERT_TRUE(exe3.gate.getCount() == 0u); diff --git a/fnet/src/vespa/fnet/connection.cpp b/fnet/src/vespa/fnet/connection.cpp index 26367c904b2..e344f2a22a6 100644 --- a/fnet/src/vespa/fnet/connection.cpp +++ b/fnet/src/vespa/fnet/connection.cpp @@ -475,7 +475,7 @@ FNET_Connection::FNET_Connection(FNET_TransportThread *owner, _streamer(streamer), _serverAdapter(serverAdapter), _socket(owner->owner().create_server_crypto_socket(std::move(socket))), - _resolve_handler(nullptr), + _resolve_handler(), _context(), _state(FNET_CONNECTING), _flags(owner->owner().getConfig()), @@ -506,7 +506,7 @@ FNET_Connection::FNET_Connection(FNET_TransportThread *owner, _streamer(streamer), _serverAdapter(serverAdapter), _socket(), - _resolve_handler(nullptr), + _resolve_handler(), _context(context), _state(FNET_CONNECTING), _flags(owner->owner().getConfig()), @@ -529,6 +529,7 @@ FNET_Connection::FNET_Connection(FNET_TransportThread *owner, FNET_Connection::~FNET_Connection() { + assert(!_resolve_handler); assert(_cleanup == nullptr); _num_connections.fetch_sub(1, std::memory_order_relaxed); } diff --git a/fnet/src/vespa/fnet/frt/require_capabilities.cpp b/fnet/src/vespa/fnet/frt/require_capabilities.cpp index 6996557c91e..26504d06e0f 100644 --- a/fnet/src/vespa/fnet/frt/require_capabilities.cpp +++ b/fnet/src/vespa/fnet/frt/require_capabilities.cpp @@ -5,6 +5,7 @@ #include <vespa/fnet/connection.h> #include <vespa/vespalib/net/connection_auth_context.h> #include <vespa/vespalib/net/tls/capability_env_config.h> +#include <vespa/vespalib/net/tls/statistics.h> #include <vespa/log/bufferedlogger.h> LOG_SETUP(".fnet.frt.require_capabilities"); @@ -19,6 +20,7 @@ FRT_RequireCapabilities::allow(FRT_RPCRequest& req) const noexcept if (is_authorized) { return true; } else { + CapabilityStatistics::get().inc_rpc_capability_checks_failed(); const auto mode = capability_enforcement_mode_from_env(); if (mode == CapabilityEnforcementMode::Disable) { return true; diff --git a/fnet/src/vespa/fnet/frt/supervisor.cpp b/fnet/src/vespa/fnet/frt/supervisor.cpp index b08516c5009..966c606bf97 100644 --- a/fnet/src/vespa/fnet/frt/supervisor.cpp +++ b/fnet/src/vespa/fnet/frt/supervisor.cpp @@ -7,7 +7,6 @@ #include <vespa/fnet/transport.h> #include <vespa/fnet/transport_thread.h> #include <vespa/fnet/connector.h> -#include <vespa/fastos/thread.h> #include <vespa/vespalib/util/require.h> FNET_IPacketStreamer * @@ -291,11 +290,10 @@ FRT_Supervisor::SchedulerPtr::SchedulerPtr(FNET_TransportThread *transport_threa namespace fnet::frt { StandaloneFRT::StandaloneFRT(const TransportConfig &config) - : _threadPool(std::make_unique<FastOS_ThreadPool>()), - _transport(std::make_unique<FNET_Transport>(config)), + : _transport(std::make_unique<FNET_Transport>(config)), _supervisor(std::make_unique<FRT_Supervisor>(_transport.get())) { - REQUIRE(_transport->Start(_threadPool.get())); + REQUIRE(_transport->Start()); } StandaloneFRT::StandaloneFRT() diff --git a/fnet/src/vespa/fnet/frt/supervisor.h b/fnet/src/vespa/fnet/frt/supervisor.h index 93272e93b4a..0261c7863b9 100644 --- a/fnet/src/vespa/fnet/frt/supervisor.h +++ b/fnet/src/vespa/fnet/frt/supervisor.h @@ -13,7 +13,6 @@ namespace fnet { class TransportConfig; } class FNET_Transport; class FRT_Target; -class FastOS_ThreadPool; class FNET_Scheduler; class FRT_RPCInvoker; class FRT_IRequestWait; @@ -106,7 +105,6 @@ public: const FRT_Supervisor &supervisor() const { return *_supervisor; } void shutdown(); private: - std::unique_ptr<FastOS_ThreadPool> _threadPool; std::unique_ptr<FNET_Transport> _transport; std::unique_ptr<FRT_Supervisor> _supervisor; }; diff --git a/fnet/src/vespa/fnet/transport.cpp b/fnet/src/vespa/fnet/transport.cpp index ae864ea0821..1553fc010c0 100644 --- a/fnet/src/vespa/fnet/transport.cpp +++ b/fnet/src/vespa/fnet/transport.cpp @@ -136,6 +136,7 @@ FNET_Transport::FNET_Transport(const fnet::TransportConfig &cfg) _time_tools(cfg.time_tools()), _work_pool(std::make_unique<vespalib::ThreadStackExecutor>(1, fnet_work_pool, 1024)), _threads(), + _pool(), _config(cfg.config()) { // TODO Temporary logging to track down overspend @@ -146,7 +147,10 @@ FNET_Transport::FNET_Transport(const fnet::TransportConfig &cfg) } } -FNET_Transport::~FNET_Transport() = default; +FNET_Transport::~FNET_Transport() +{ + _pool.join(); +} void FNET_Transport::post_or_perform(vespalib::Executor::Task::UP task) @@ -266,13 +270,12 @@ FNET_Transport::WaitFinished() } bool -FNET_Transport::Start(FastOS_ThreadPool *pool) +FNET_Transport::Start() { - bool result = true; for (const auto &thread: _threads) { - result &= thread->Start(pool); + thread->Start(_pool); } - return result; + return true; } void diff --git a/fnet/src/vespa/fnet/transport.h b/fnet/src/vespa/fnet/transport.h index 3f9328296d6..d658059f0bb 100644 --- a/fnet/src/vespa/fnet/transport.h +++ b/fnet/src/vespa/fnet/transport.h @@ -7,9 +7,9 @@ #include <vespa/vespalib/net/async_resolver.h> #include <vespa/vespalib/net/crypto_engine.h> #include <vespa/vespalib/util/time.h> +#include <vespa/vespalib/util/thread.h> class FNET_TransportThread; -class FastOS_ThreadPool; class FNET_Connector; class FNET_IPacketStreamer; class FNET_IServerAdapter; @@ -111,6 +111,7 @@ private: fnet::TimeTools::SP _time_tools; std::unique_ptr<vespalib::SyncableThreadExecutor> _work_pool; Threads _threads; + vespalib::ThreadPool _pool; const FNET_Config _config; /** @@ -317,9 +318,8 @@ public: * ok. * * @return thread create status. - * @param pool threadpool that may be used to spawn new threads. **/ - bool Start(FastOS_ThreadPool *pool); + bool Start(); /** * Capture transport threads. Used for testing purposes, diff --git a/fnet/src/vespa/fnet/transport_thread.cpp b/fnet/src/vespa/fnet/transport_thread.cpp index 262cdaec190..970dc40150f 100644 --- a/fnet/src/vespa/fnet/transport_thread.cpp +++ b/fnet/src/vespa/fnet/transport_thread.cpp @@ -598,28 +598,28 @@ FNET_TransportThread::endEventLoop() { bool -FNET_TransportThread::Start(FastOS_ThreadPool *pool) +FNET_TransportThread::Start(vespalib::ThreadPool &pool) { - return (pool != nullptr && pool->NewThread(this)); + pool.start([this](){run();}); + return true; } void FNET_TransportThread::Main() { - Run(nullptr, nullptr); + run(); } void -FNET_TransportThread::Run(FastOS_ThreadInterface *thisThread, void *) +FNET_TransportThread::run() { if (!InitEventLoop()) { LOG(warning, "Transport: Run: Could not init event loop"); return; } while (EventLoopIteration()) { - if (thisThread != nullptr && thisThread->GetBreakFlag()) - ShutDown(false); + // event loop must be stopped from the outside } } diff --git a/fnet/src/vespa/fnet/transport_thread.h b/fnet/src/vespa/fnet/transport_thread.h index 1911b11a81c..1744a3d60e5 100644 --- a/fnet/src/vespa/fnet/transport_thread.h +++ b/fnet/src/vespa/fnet/transport_thread.h @@ -6,9 +6,9 @@ #include "config.h" #include "task.h" #include "packetqueue.h" -#include <vespa/fastos/thread.h> #include <vespa/vespalib/net/socket_handle.h> #include <vespa/vespalib/net/selector.h> +#include <vespa/vespalib/util/thread.h> #include <atomic> #include <mutex> #include <condition_variable> @@ -26,7 +26,7 @@ class FNET_IServerAdapter; * the network related work for the application in both client and * server aspects. **/ -class FNET_TransportThread : public FastOS_Runnable +class FNET_TransportThread { friend class FNET_IOComponent; @@ -195,7 +195,7 @@ public: * Destruct object. This should NOT be done before the transport * thread has completed it's work and raised the finished flag. **/ - ~FNET_TransportThread() override; + ~FNET_TransportThread(); /** @@ -425,7 +425,7 @@ public: * @return thread create status. * @param pool threadpool that may be used to spawn a new thread. **/ - bool Start(FastOS_ThreadPool *pool); + bool Start(vespalib::ThreadPool &pool); /** @@ -440,5 +440,5 @@ public: * This is where the transport thread lives, when started by * invoking one of the @ref Main or @ref Start methods. **/ - void Run(FastOS_ThreadInterface *thisThread, void *args) override; + void run(); }; diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java index 53acd8cbb1b..1ae25aa0cfa 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java @@ -45,6 +45,7 @@ public class ExportPackages { } } + // Make sure to update the junit integration test `ExportPackagesIT.java` if the set of exported packages is modified. private static String getExportPackages(String[] jars) throws IOException { StringBuilder out = new StringBuilder(); out.append(getSystemPackages()).append(", ") @@ -53,6 +54,7 @@ public class ExportPackages { .append("com.yahoo.jdisc.handler, ") .append("com.yahoo.jdisc.service, ") .append("com.yahoo.jdisc.statistics, ") + .append("com.yahoo.jdisc.refcount, ") .append("javax.inject;version=1.0.0, ") // TODO Vespa 9: remove. Included in guice, but not exported. Needed by container-jersey. .append("org.aopalliance.intercept, ") diff --git a/jdisc_core/src/test/resources/exportPackages.properties b/jdisc_core/src/test/resources/exportPackages.properties index 82242b1644b..b49f91842e3 100644 --- a/jdisc_core/src/test/resources/exportPackages.properties +++ b/jdisc_core/src/test/resources/exportPackages.properties @@ -1,3 +1,3 @@ #generated by com.yahoo.jdisc.core.ExportPackages #Wed Jul 20 02:55:26 CEST 2022 -exportPackages=org.osgi.framework; version\="1.10.0", org.osgi.framework.connect; version\="1.0.0", org.osgi.framework.dto; uses\:\="org.osgi.dto"; version\="1.8.0", org.osgi.framework.hooks.bundle; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.resolver; uses\:\="org.osgi.framework.wiring"; version\="1.0.0", org.osgi.framework.hooks.service; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.weaving; uses\:\="org.osgi.framework.wiring"; version\="1.1.0", org.osgi.framework.launch; uses\:\="org.osgi.framework"; version\="1.2.0", org.osgi.framework.namespace; uses\:\="org.osgi.resource"; version\="1.2.0", org.osgi.framework.startlevel; uses\:\="org.osgi.framework"; version\="1.0.0", org.osgi.framework.startlevel.dto; uses\:\="org.osgi.dto"; version\="1.0.0", org.osgi.framework.wiring; uses\:\="org.osgi.framework,org.osgi.resource"; version\="1.2.0", org.osgi.framework.wiring.dto; uses\:\="org.osgi.dto,org.osgi.resource.dto"; version\="1.3.0", org.osgi.resource; version\="1.0.1", org.osgi.resource.dto; uses\:\="org.osgi.dto"; version\="1.0.1", org.osgi.service.packageadmin; uses\:\="org.osgi.framework"; version\="1.2.1", org.osgi.service.startlevel; uses\:\="org.osgi.framework"; version\="1.1.1", org.osgi.service.url; version\="1.0.1", org.osgi.service.resolver; uses\:\="org.osgi.resource"; version\="1.1.1", org.osgi.util.tracker; uses\:\="org.osgi.framework"; version\="1.5.3", org.osgi.dto; version\="1.1.1", org.osgi.service.condition; version\="1.0.0", javax.security.auth.callback; version\="0.0.0.JavaSE_017", javax.security.auth.login; version\="0.0.0.JavaSE_017", javax.net.ssl; version\="0.0.0.JavaSE_017", java.lang.constant; version\="0.0.0.JavaSE_017", java.security.interfaces; version\="0.0.0.JavaSE_017", java.text.spi; version\="0.0.0.JavaSE_017", java.nio.channels.spi; version\="0.0.0.JavaSE_017", java.math; version\="0.0.0.JavaSE_017", java.nio.file; version\="0.0.0.JavaSE_017", java.util.concurrent.atomic; version\="0.0.0.JavaSE_017", java.security.cert; version\="0.0.0.JavaSE_017", java.security.spec; version\="0.0.0.JavaSE_017", java.nio.channels; version\="0.0.0.JavaSE_017", java.time.chrono; version\="0.0.0.JavaSE_017", javax.crypto; version\="0.0.0.JavaSE_017", java.time.zone; version\="0.0.0.JavaSE_017", java.nio.charset; version\="0.0.0.JavaSE_017", java.io; version\="0.0.0.JavaSE_017", java.util.spi; version\="0.0.0.JavaSE_017", java.net; version\="0.0.0.JavaSE_017", javax.security.cert; version\="0.0.0.JavaSE_017", java.lang.annotation; version\="0.0.0.JavaSE_017", javax.security.auth.spi; version\="0.0.0.JavaSE_017", java.util.concurrent; version\="0.0.0.JavaSE_017", java.nio.charset.spi; version\="0.0.0.JavaSE_017", javax.crypto.interfaces; version\="0.0.0.JavaSE_017", java.util; version\="0.0.0.JavaSE_017", java.security; version\="0.0.0.JavaSE_017", java.nio.file.spi; version\="0.0.0.JavaSE_017", java.nio; version\="0.0.0.JavaSE_017", java.util.jar; version\="0.0.0.JavaSE_017", javax.security.auth; version\="0.0.0.JavaSE_017", java.lang.ref; version\="0.0.0.JavaSE_017", java.util.regex; version\="0.0.0.JavaSE_017", java.net.spi; version\="0.0.0.JavaSE_017", java.lang.module; version\="0.0.0.JavaSE_017", java.lang.invoke; version\="0.0.0.JavaSE_017", java.time.format; version\="0.0.0.JavaSE_017", java.util.concurrent.locks; version\="0.0.0.JavaSE_017", java.time.temporal; version\="0.0.0.JavaSE_017", java.util.zip; version\="0.0.0.JavaSE_017", java.nio.file.attribute; version\="0.0.0.JavaSE_017", java.util.random; version\="0.0.0.JavaSE_017", java.text; version\="0.0.0.JavaSE_017", javax.crypto.spec; version\="0.0.0.JavaSE_017", java.util.stream; version\="0.0.0.JavaSE_017", java.time; version\="0.0.0.JavaSE_017", java.lang; version\="0.0.0.JavaSE_017", java.lang.runtime; version\="0.0.0.JavaSE_017", java.util.function; version\="0.0.0.JavaSE_017", javax.net; version\="0.0.0.JavaSE_017", javax.security.auth.x500; version\="0.0.0.JavaSE_017", java.lang.reflect; version\="0.0.0.JavaSE_017", javax.lang.model; version\="0.0.0.JavaSE_017", javax.annotation.processing; version\="0.0.0.JavaSE_017", javax.lang.model.element; version\="0.0.0.JavaSE_017", javax.tools; version\="0.0.0.JavaSE_017", javax.lang.model.type; version\="0.0.0.JavaSE_017", javax.lang.model.util; version\="0.0.0.JavaSE_017", java.awt.datatransfer; version\="0.0.0.JavaSE_017", java.awt.desktop; version\="0.0.0.JavaSE_017", javax.swing.plaf.synth; version\="0.0.0.JavaSE_017", java.beans; version\="0.0.0.JavaSE_017", javax.swing.text.html.parser; version\="0.0.0.JavaSE_017", javax.swing.text.rtf; version\="0.0.0.JavaSE_017", java.awt.font; version\="0.0.0.JavaSE_017", javax.imageio; version\="0.0.0.JavaSE_017", java.awt.im.spi; version\="0.0.0.JavaSE_017", java.applet; version\="0.0.0.JavaSE_017", javax.sound.midi; version\="0.0.0.JavaSE_017", java.awt.dnd; version\="0.0.0.JavaSE_017", javax.swing.text; version\="0.0.0.JavaSE_017", javax.swing.plaf.basic; version\="0.0.0.JavaSE_017", javax.swing.undo; version\="0.0.0.JavaSE_017", javax.swing.plaf; version\="0.0.0.JavaSE_017", javax.swing.filechooser; version\="0.0.0.JavaSE_017", javax.imageio.event; version\="0.0.0.JavaSE_017", javax.sound.sampled; version\="0.0.0.JavaSE_017", javax.print.attribute; version\="0.0.0.JavaSE_017", javax.print; version\="0.0.0.JavaSE_017", javax.swing.plaf.nimbus; version\="0.0.0.JavaSE_017", javax.accessibility; version\="0.0.0.JavaSE_017", java.awt.event; version\="0.0.0.JavaSE_017", javax.swing.text.html; version\="0.0.0.JavaSE_017", javax.imageio.spi; version\="0.0.0.JavaSE_017", javax.swing.border; version\="0.0.0.JavaSE_017", javax.sound.sampled.spi; version\="0.0.0.JavaSE_017", javax.imageio.plugins.bmp; version\="0.0.0.JavaSE_017", java.awt.color; version\="0.0.0.JavaSE_017", java.awt.geom; version\="0.0.0.JavaSE_017", javax.imageio.plugins.jpeg; version\="0.0.0.JavaSE_017", javax.swing.tree; version\="0.0.0.JavaSE_017", javax.imageio.plugins.tiff; version\="0.0.0.JavaSE_017", java.awt.print; version\="0.0.0.JavaSE_017", java.awt.image; version\="0.0.0.JavaSE_017", javax.imageio.metadata; version\="0.0.0.JavaSE_017", javax.swing.table; version\="0.0.0.JavaSE_017", javax.sound.midi.spi; version\="0.0.0.JavaSE_017", javax.print.attribute.standard; version\="0.0.0.JavaSE_017", javax.swing.colorchooser; version\="0.0.0.JavaSE_017", javax.swing; version\="0.0.0.JavaSE_017", java.awt.image.renderable; version\="0.0.0.JavaSE_017", javax.swing.plaf.multi; version\="0.0.0.JavaSE_017", java.awt.im; version\="0.0.0.JavaSE_017", javax.print.event; version\="0.0.0.JavaSE_017", javax.swing.plaf.metal; version\="0.0.0.JavaSE_017", java.beans.beancontext; version\="0.0.0.JavaSE_017", java.awt; version\="0.0.0.JavaSE_017", javax.imageio.stream; version\="0.0.0.JavaSE_017", javax.swing.event; version\="0.0.0.JavaSE_017", java.lang.instrument; version\="0.0.0.JavaSE_017", java.util.logging; version\="0.0.0.JavaSE_017", javax.management.timer; version\="0.0.0.JavaSE_017", javax.management; version\="0.0.0.JavaSE_017", javax.management.relation; version\="0.0.0.JavaSE_017", javax.management.loading; version\="0.0.0.JavaSE_017", javax.management.openmbean; version\="0.0.0.JavaSE_017", java.lang.management; version\="0.0.0.JavaSE_017", javax.management.remote; version\="0.0.0.JavaSE_017", javax.management.monitor; version\="0.0.0.JavaSE_017", javax.management.modelmbean; version\="0.0.0.JavaSE_017", javax.management.remote.rmi; version\="0.0.0.JavaSE_017", javax.naming.event; version\="0.0.0.JavaSE_017", javax.naming.ldap.spi; version\="0.0.0.JavaSE_017", javax.naming; version\="0.0.0.JavaSE_017", javax.naming.spi; version\="0.0.0.JavaSE_017", javax.naming.ldap; version\="0.0.0.JavaSE_017", javax.naming.directory; version\="0.0.0.JavaSE_017", java.net.http; version\="0.0.0.JavaSE_017", java.util.prefs; version\="0.0.0.JavaSE_017", java.rmi.registry; version\="0.0.0.JavaSE_017", javax.rmi.ssl; version\="0.0.0.JavaSE_017", java.rmi.dgc; version\="0.0.0.JavaSE_017", java.rmi; version\="0.0.0.JavaSE_017", java.rmi.server; version\="0.0.0.JavaSE_017", javax.script; version\="0.0.0.JavaSE_017", org.ietf.jgss; version\="0.0.0.JavaSE_017", javax.security.auth.kerberos; version\="0.0.0.JavaSE_017", javax.security.sasl; version\="0.0.0.JavaSE_017", javax.smartcardio; version\="0.0.0.JavaSE_017", java.sql; version\="0.0.0.JavaSE_017", javax.sql; version\="0.0.0.JavaSE_017", javax.sql.rowset.spi; version\="0.0.0.JavaSE_017", javax.sql.rowset.serial; version\="0.0.0.JavaSE_017", javax.sql.rowset; version\="0.0.0.JavaSE_017", javax.transaction.xa; version\="0.0.0.JavaSE_017", javax.xml; version\="0.0.0.JavaSE_017", javax.xml.transform.sax; version\="0.0.0.JavaSE_017", javax.xml.datatype; version\="0.0.0.JavaSE_017", javax.xml.catalog; version\="0.0.0.JavaSE_017", org.w3c.dom.traversal; version\="0.0.0.JavaSE_017", javax.xml.stream.events; version\="0.0.0.JavaSE_017", javax.xml.stream; version\="0.0.0.JavaSE_017", org.xml.sax; version\="0.0.0.JavaSE_017", javax.xml.transform; version\="0.0.0.JavaSE_017", javax.xml.xpath; version\="0.0.0.JavaSE_017", org.w3c.dom.events; version\="0.0.0.JavaSE_017", org.w3c.dom.ranges; version\="0.0.0.JavaSE_017", org.w3c.dom.ls; version\="0.0.0.JavaSE_017", javax.xml.stream.util; version\="0.0.0.JavaSE_017", javax.xml.namespace; version\="0.0.0.JavaSE_017", javax.xml.transform.stax; version\="0.0.0.JavaSE_017", org.xml.sax.helpers; version\="0.0.0.JavaSE_017", org.w3c.dom.views; version\="0.0.0.JavaSE_017", org.w3c.dom.bootstrap; version\="0.0.0.JavaSE_017", org.w3c.dom; version\="0.0.0.JavaSE_017", javax.xml.transform.stream; version\="0.0.0.JavaSE_017", javax.xml.transform.dom; version\="0.0.0.JavaSE_017", javax.xml.validation; version\="0.0.0.JavaSE_017", javax.xml.parsers; version\="0.0.0.JavaSE_017", org.xml.sax.ext; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.spec; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.keyinfo; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig; version\="0.0.0.JavaSE_017", com.sun.java.accessibility.util; version\="0.0.0.JavaSE_017", com.sun.tools.attach.spi; version\="0.0.0.JavaSE_017", com.sun.tools.attach; version\="0.0.0.JavaSE_017", com.sun.source.doctree; version\="0.0.0.JavaSE_017", com.sun.source.tree; version\="0.0.0.JavaSE_017", com.sun.source.util; version\="0.0.0.JavaSE_017", com.sun.tools.javac; version\="0.0.0.JavaSE_017", jdk.dynalink.beans; version\="0.0.0.JavaSE_017", jdk.dynalink.linker.support; version\="0.0.0.JavaSE_017", jdk.dynalink.support; version\="0.0.0.JavaSE_017", jdk.dynalink; version\="0.0.0.JavaSE_017", jdk.dynalink.linker; version\="0.0.0.JavaSE_017", com.sun.net.httpserver; version\="0.0.0.JavaSE_017", com.sun.net.httpserver.spi; version\="0.0.0.JavaSE_017", com.sun.jarsigner; version\="0.0.0.JavaSE_017", jdk.security.jarsigner; version\="0.0.0.JavaSE_017", jdk.javadoc.doclet; version\="0.0.0.JavaSE_017", com.sun.tools.jconsole; version\="0.0.0.JavaSE_017", com.sun.jdi.connect; version\="0.0.0.JavaSE_017", com.sun.jdi.event; version\="0.0.0.JavaSE_017", com.sun.jdi.connect.spi; version\="0.0.0.JavaSE_017", com.sun.jdi; version\="0.0.0.JavaSE_017", com.sun.jdi.request; version\="0.0.0.JavaSE_017", jdk.jfr.consumer; version\="0.0.0.JavaSE_017", jdk.jfr; version\="0.0.0.JavaSE_017", jdk.jshell; version\="0.0.0.JavaSE_017", jdk.jshell.execution; version\="0.0.0.JavaSE_017", jdk.jshell.spi; version\="0.0.0.JavaSE_017", jdk.jshell.tool; version\="0.0.0.JavaSE_017", netscape.javascript; version\="0.0.0.JavaSE_017", com.sun.management; version\="0.0.0.JavaSE_017", jdk.management.jfr; version\="0.0.0.JavaSE_017", jdk.net; version\="0.0.0.JavaSE_017", jdk.nio; version\="0.0.0.JavaSE_017", jdk.nio.mapmode; version\="0.0.0.JavaSE_017", com.sun.nio.sctp; version\="0.0.0.JavaSE_017", com.sun.security.auth.module; version\="0.0.0.JavaSE_017", com.sun.security.auth.login; version\="0.0.0.JavaSE_017", com.sun.security.auth; version\="0.0.0.JavaSE_017", com.sun.security.auth.callback; version\="0.0.0.JavaSE_017", com.sun.security.jgss; version\="0.0.0.JavaSE_017", sun.reflect; version\="0.0.0.JavaSE_017", sun.misc; version\="0.0.0.JavaSE_017", com.sun.nio.file; version\="0.0.0.JavaSE_017", jdk.swing.interop; version\="0.0.0.JavaSE_017", org.w3c.dom.stylesheets; version\="0.0.0.JavaSE_017", org.w3c.dom.html; version\="0.0.0.JavaSE_017", org.w3c.dom.xpath; version\="0.0.0.JavaSE_017", org.w3c.dom.css; version\="0.0.0.JavaSE_017", com.yahoo.jdisc, com.yahoo.jdisc.application, com.yahoo.jdisc.handler, com.yahoo.jdisc.service, com.yahoo.jdisc.statistics, javax.inject;version\=1.0.0, org.aopalliance.intercept, org.aopalliance.aop, com.google.common.annotations;version\="27.1.0",com.google.common.base;version\="27.1.0",com.google.common.cache;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent",com.google.common.collect;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.escape;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.eventbus;version\="27.1.0",com.google.common.graph;version\="27.1.0";uses\:\="com.google.common.collect",com.google.common.hash;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.html;version\="27.1.0";uses\:\="com.google.common.escape",com.google.common.io;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.graph,com.google.common.hash",com.google.common.math;version\="27.1.0",com.google.common.net;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.escape",com.google.common.primitives;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.reflect;version\="27.1.0";uses\:\="com.google.common.collect,com.google.common.io",com.google.common.util.concurrent;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent.internal",com.google.common.xml;version\="27.1.0";uses\:\="com.google.common.escape", com.google.inject;version\="1.4",com.google.inject.binder;version\="1.4",com.google.inject.matcher;version\="1.4",com.google.inject.multibindings;version\="1.4",com.google.inject.name;version\="1.4",com.google.inject.spi;version\="1.4",com.google.inject.util;version\="1.4", org.slf4j;version\=1.7.32, org.slf4j.spi;version\=1.7.32, org.slf4j.helpers;version\=1.7.32, org.slf4j.event;version\=1.7.32, org.slf4j.impl;version\=1.7.32, org.apache.commons.logging;version\=1.2, org.apache.commons.logging.impl;version\=1.2, org.apache.log4j;version\=1.2.17,org.apache.log4j.helpers;version\=1.2.17,org.apache.log4j.spi;version\=1.2.17,org.apache.log4j.xml;version\=1.2.17, com.yahoo.component.annotation;version\="1.0.0", com.yahoo.config;version\=1.0.0, com.yahoo.vespa.defaults;version\=1.0.0, ai.vespa.http;version\=1.0.0,ai.vespa.validation;version\=1.0.0,com.yahoo.binaryprefix;version\=1.0.0,com.yahoo.collections;version\=1.0.0,com.yahoo.compress;version\=1.0.0,com.yahoo.concurrent.classlock;version\=1.0.0,com.yahoo.concurrent.maintenance;version\=1.0.0,com.yahoo.concurrent;version\=1.0.0,com.yahoo.data.access.simple;version\=1.0.0,com.yahoo.data.access.slime;version\=1.0.0,com.yahoo.data.access;version\=1.0.0,com.yahoo.errorhandling;version\=1.0.0,com.yahoo.exception;version\=1.0.0,com.yahoo.geo;version\=1.0.0,com.yahoo.io.reader;version\=1.0.0,com.yahoo.io;version\=1.0.0,com.yahoo.javacc;version\=1.0.0,com.yahoo.lang;version\=1.0.0,com.yahoo.nativec;version\=1.0.0,com.yahoo.net;version\=1.0.0,com.yahoo.path;version\=1.0.0,com.yahoo.protect;version\=1.0.0,com.yahoo.reflection;version\=1.0.0,com.yahoo.slime;version\=1.0.0,com.yahoo.stream;version\=1.0.0,com.yahoo.system.execution;version\=1.0.0,com.yahoo.system;version\=1.0.0,com.yahoo.tensor.evaluation;version\=1.0.0,com.yahoo.tensor.functions;version\=1.0.0,com.yahoo.tensor.serialization;version\=1.0.0,com.yahoo.tensor;version\=1.0.0,com.yahoo.text.internal;version\=1.0.0,com.yahoo.text;version\=1.0.0,com.yahoo.time;version\=1.0.0,com.yahoo.transaction;version\=1.0.0,com.yahoo.vespa.objects;version\=1.0.0,com.yahoo.yolean.chain;version\=1.0.0,com.yahoo.yolean.concurrent;version\=1.0.0,com.yahoo.yolean.function;version\=1.0.0,com.yahoo.yolean.system;version\=1.0.0,com.yahoo.yolean.trace;version\=1.0.0,com.yahoo.yolean;version\=1.0.0, com.yahoo.log.event;version\=1.0.0,com.yahoo.log.impl;version\=1.0.0,com.yahoo.log;version\=1.0.0, javax.xml.bind;version\="2.3";uses\:\="javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.annotation;version\="2.3";uses\:\="javax.xml.bind,javax.xml.parsers,javax.xml.transform,javax.xml.transform.dom,org.w3c.dom",javax.xml.bind.annotation.adapters;version\="2.3",javax.xml.bind.attachment;version\="2.3";uses\:\="javax.activation",javax.xml.bind.helpers;version\="2.3";uses\:\="javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.util;version\="2.3";uses\:\="javax.xml.bind,javax.xml.transform.sax", com.sun.istack;version\="3.0.5";uses\:\="javax.activation,javax.xml.stream,org.xml.sax,org.xml.sax.helpers",com.sun.istack.localization;version\="3.0.5",com.sun.istack.logging;version\="3.0.5",com.sun.xml.bind;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.annotation;version\="2.3.0",com.sun.xml.bind.api;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.api.impl;version\="2.3.0",com.sun.xml.bind.marshaller;uses\:\="javax.xml.parsers,org.w3c.dom,org.xml.sax,org.xml.sax.helpers";version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;version\="2.3.0",com.sun.xml.bind.v2;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.core;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.impl,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.namespace,javax.xml.transform";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.nav";version\="2.3.0",com.sun.xml.bind.v2.model.nav;uses\:\="com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.util;uses\:\="javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.xml.bind.v2.model.annotation,javax.activation,javax.xml.bind,javax.xml.bind.annotation.adapters";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen.episode;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="javax.xml.parsers,javax.xml.transform,javax.xml.validation,javax.xml.xpath";version\="2.3.0",com.sun.xml.txw2;uses\:\="com.sun.xml.txw2.output,javax.xml.namespace";version\="2.3.0",com.sun.xml.txw2.annotation;version\="2.3.0",com.sun.xml.txw2.output;uses\:\="com.sun.xml.txw2,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.dom,javax.xml.transform.sax,javax.xml.transform.stream,org.w3c.dom,org.xml.sax,org.xml.sax.ext,org.xml.sax.helpers";version\="2.3.0", com.sun.xml.bind;uses\:\="com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.datatype,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.api;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.xml.bind,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.marshaller;version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;uses\:\="com.sun.xml.bind,javax.xml.bind.helpers,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,javax.xml.bind";version\="2.3.0",com.sun.xml.bind.v2.bytecode;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.model.runtime;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.namespace,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.istack,com.sun.xml.bind.api,com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.property,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.output;uses\:\="com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.runtime,com.sun.xml.fastinfoset.stax,javax.xml.stream,org.jvnet.staxex,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.property;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,com.sun.xml.bind.v2.runtime.unmarshaller,com.sun.xml.bind.v2.util,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect.opt;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="com.sun.xml.bind,com.sun.xml.bind.api,com.sun.xml.bind.unmarshaller,com.sun.xml.bind.util,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.reflect,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.txw2.output,javax.xml.bind,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.schemagen.xmlschema;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.namespace,javax.xml.transform.stream,org.xml.sax";version\="2.3.0", javax.activation;uses\:\="com.sun.activation.registries";version\="1.2",com.sun.activation.viewers;uses\:\="javax.activation";version\="1.2.0",com.sun.activation.registries;version\="1.2.0" +exportPackages=org.osgi.framework; version\="1.10.0", org.osgi.framework.connect; version\="1.0.0", org.osgi.framework.dto; uses\:\="org.osgi.dto"; version\="1.8.0", org.osgi.framework.hooks.bundle; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.resolver; uses\:\="org.osgi.framework.wiring"; version\="1.0.0", org.osgi.framework.hooks.service; uses\:\="org.osgi.framework"; version\="1.1.0", org.osgi.framework.hooks.weaving; uses\:\="org.osgi.framework.wiring"; version\="1.1.0", org.osgi.framework.launch; uses\:\="org.osgi.framework"; version\="1.2.0", org.osgi.framework.namespace; uses\:\="org.osgi.resource"; version\="1.2.0", org.osgi.framework.startlevel; uses\:\="org.osgi.framework"; version\="1.0.0", org.osgi.framework.startlevel.dto; uses\:\="org.osgi.dto"; version\="1.0.0", org.osgi.framework.wiring; uses\:\="org.osgi.framework,org.osgi.resource"; version\="1.2.0", org.osgi.framework.wiring.dto; uses\:\="org.osgi.dto,org.osgi.resource.dto"; version\="1.3.0", org.osgi.resource; version\="1.0.1", org.osgi.resource.dto; uses\:\="org.osgi.dto"; version\="1.0.1", org.osgi.service.packageadmin; uses\:\="org.osgi.framework"; version\="1.2.1", org.osgi.service.startlevel; uses\:\="org.osgi.framework"; version\="1.1.1", org.osgi.service.url; version\="1.0.1", org.osgi.service.resolver; uses\:\="org.osgi.resource"; version\="1.1.1", org.osgi.util.tracker; uses\:\="org.osgi.framework"; version\="1.5.3", org.osgi.dto; version\="1.1.1", org.osgi.service.condition; version\="1.0.0", javax.security.auth.callback; version\="0.0.0.JavaSE_017", javax.security.auth.login; version\="0.0.0.JavaSE_017", javax.net.ssl; version\="0.0.0.JavaSE_017", java.lang.constant; version\="0.0.0.JavaSE_017", java.security.interfaces; version\="0.0.0.JavaSE_017", java.text.spi; version\="0.0.0.JavaSE_017", java.nio.channels.spi; version\="0.0.0.JavaSE_017", java.math; version\="0.0.0.JavaSE_017", java.nio.file; version\="0.0.0.JavaSE_017", java.util.concurrent.atomic; version\="0.0.0.JavaSE_017", java.security.cert; version\="0.0.0.JavaSE_017", java.security.spec; version\="0.0.0.JavaSE_017", java.nio.channels; version\="0.0.0.JavaSE_017", java.time.chrono; version\="0.0.0.JavaSE_017", javax.crypto; version\="0.0.0.JavaSE_017", java.time.zone; version\="0.0.0.JavaSE_017", java.nio.charset; version\="0.0.0.JavaSE_017", java.io; version\="0.0.0.JavaSE_017", java.util.spi; version\="0.0.0.JavaSE_017", java.net; version\="0.0.0.JavaSE_017", javax.security.cert; version\="0.0.0.JavaSE_017", java.lang.annotation; version\="0.0.0.JavaSE_017", javax.security.auth.spi; version\="0.0.0.JavaSE_017", java.util.concurrent; version\="0.0.0.JavaSE_017", java.nio.charset.spi; version\="0.0.0.JavaSE_017", javax.crypto.interfaces; version\="0.0.0.JavaSE_017", java.util; version\="0.0.0.JavaSE_017", java.security; version\="0.0.0.JavaSE_017", java.nio.file.spi; version\="0.0.0.JavaSE_017", java.nio; version\="0.0.0.JavaSE_017", java.util.jar; version\="0.0.0.JavaSE_017", javax.security.auth; version\="0.0.0.JavaSE_017", java.lang.ref; version\="0.0.0.JavaSE_017", java.util.regex; version\="0.0.0.JavaSE_017", java.net.spi; version\="0.0.0.JavaSE_017", java.lang.module; version\="0.0.0.JavaSE_017", java.lang.invoke; version\="0.0.0.JavaSE_017", java.time.format; version\="0.0.0.JavaSE_017", java.util.concurrent.locks; version\="0.0.0.JavaSE_017", java.time.temporal; version\="0.0.0.JavaSE_017", java.util.zip; version\="0.0.0.JavaSE_017", java.nio.file.attribute; version\="0.0.0.JavaSE_017", java.util.random; version\="0.0.0.JavaSE_017", java.text; version\="0.0.0.JavaSE_017", javax.crypto.spec; version\="0.0.0.JavaSE_017", java.util.stream; version\="0.0.0.JavaSE_017", java.time; version\="0.0.0.JavaSE_017", java.lang; version\="0.0.0.JavaSE_017", java.lang.runtime; version\="0.0.0.JavaSE_017", java.util.function; version\="0.0.0.JavaSE_017", javax.net; version\="0.0.0.JavaSE_017", javax.security.auth.x500; version\="0.0.0.JavaSE_017", java.lang.reflect; version\="0.0.0.JavaSE_017", javax.lang.model; version\="0.0.0.JavaSE_017", javax.annotation.processing; version\="0.0.0.JavaSE_017", javax.lang.model.element; version\="0.0.0.JavaSE_017", javax.tools; version\="0.0.0.JavaSE_017", javax.lang.model.type; version\="0.0.0.JavaSE_017", javax.lang.model.util; version\="0.0.0.JavaSE_017", java.awt.datatransfer; version\="0.0.0.JavaSE_017", java.awt.desktop; version\="0.0.0.JavaSE_017", javax.swing.plaf.synth; version\="0.0.0.JavaSE_017", java.beans; version\="0.0.0.JavaSE_017", javax.swing.text.html.parser; version\="0.0.0.JavaSE_017", javax.swing.text.rtf; version\="0.0.0.JavaSE_017", java.awt.font; version\="0.0.0.JavaSE_017", javax.imageio; version\="0.0.0.JavaSE_017", java.awt.im.spi; version\="0.0.0.JavaSE_017", java.applet; version\="0.0.0.JavaSE_017", javax.sound.midi; version\="0.0.0.JavaSE_017", java.awt.dnd; version\="0.0.0.JavaSE_017", javax.swing.text; version\="0.0.0.JavaSE_017", javax.swing.plaf.basic; version\="0.0.0.JavaSE_017", javax.swing.undo; version\="0.0.0.JavaSE_017", javax.swing.plaf; version\="0.0.0.JavaSE_017", javax.swing.filechooser; version\="0.0.0.JavaSE_017", javax.imageio.event; version\="0.0.0.JavaSE_017", javax.sound.sampled; version\="0.0.0.JavaSE_017", javax.print.attribute; version\="0.0.0.JavaSE_017", javax.print; version\="0.0.0.JavaSE_017", javax.swing.plaf.nimbus; version\="0.0.0.JavaSE_017", javax.accessibility; version\="0.0.0.JavaSE_017", java.awt.event; version\="0.0.0.JavaSE_017", javax.swing.text.html; version\="0.0.0.JavaSE_017", javax.imageio.spi; version\="0.0.0.JavaSE_017", javax.swing.border; version\="0.0.0.JavaSE_017", javax.sound.sampled.spi; version\="0.0.0.JavaSE_017", javax.imageio.plugins.bmp; version\="0.0.0.JavaSE_017", java.awt.color; version\="0.0.0.JavaSE_017", java.awt.geom; version\="0.0.0.JavaSE_017", javax.imageio.plugins.jpeg; version\="0.0.0.JavaSE_017", javax.swing.tree; version\="0.0.0.JavaSE_017", javax.imageio.plugins.tiff; version\="0.0.0.JavaSE_017", java.awt.print; version\="0.0.0.JavaSE_017", java.awt.image; version\="0.0.0.JavaSE_017", javax.imageio.metadata; version\="0.0.0.JavaSE_017", javax.swing.table; version\="0.0.0.JavaSE_017", javax.sound.midi.spi; version\="0.0.0.JavaSE_017", javax.print.attribute.standard; version\="0.0.0.JavaSE_017", javax.swing.colorchooser; version\="0.0.0.JavaSE_017", javax.swing; version\="0.0.0.JavaSE_017", java.awt.image.renderable; version\="0.0.0.JavaSE_017", javax.swing.plaf.multi; version\="0.0.0.JavaSE_017", java.awt.im; version\="0.0.0.JavaSE_017", javax.print.event; version\="0.0.0.JavaSE_017", javax.swing.plaf.metal; version\="0.0.0.JavaSE_017", java.beans.beancontext; version\="0.0.0.JavaSE_017", java.awt; version\="0.0.0.JavaSE_017", javax.imageio.stream; version\="0.0.0.JavaSE_017", javax.swing.event; version\="0.0.0.JavaSE_017", java.lang.instrument; version\="0.0.0.JavaSE_017", java.util.logging; version\="0.0.0.JavaSE_017", javax.management.timer; version\="0.0.0.JavaSE_017", javax.management; version\="0.0.0.JavaSE_017", javax.management.relation; version\="0.0.0.JavaSE_017", javax.management.loading; version\="0.0.0.JavaSE_017", javax.management.openmbean; version\="0.0.0.JavaSE_017", java.lang.management; version\="0.0.0.JavaSE_017", javax.management.remote; version\="0.0.0.JavaSE_017", javax.management.monitor; version\="0.0.0.JavaSE_017", javax.management.modelmbean; version\="0.0.0.JavaSE_017", javax.management.remote.rmi; version\="0.0.0.JavaSE_017", javax.naming.event; version\="0.0.0.JavaSE_017", javax.naming.ldap.spi; version\="0.0.0.JavaSE_017", javax.naming; version\="0.0.0.JavaSE_017", javax.naming.spi; version\="0.0.0.JavaSE_017", javax.naming.ldap; version\="0.0.0.JavaSE_017", javax.naming.directory; version\="0.0.0.JavaSE_017", java.net.http; version\="0.0.0.JavaSE_017", java.util.prefs; version\="0.0.0.JavaSE_017", java.rmi.registry; version\="0.0.0.JavaSE_017", javax.rmi.ssl; version\="0.0.0.JavaSE_017", java.rmi.dgc; version\="0.0.0.JavaSE_017", java.rmi; version\="0.0.0.JavaSE_017", java.rmi.server; version\="0.0.0.JavaSE_017", javax.script; version\="0.0.0.JavaSE_017", org.ietf.jgss; version\="0.0.0.JavaSE_017", javax.security.auth.kerberos; version\="0.0.0.JavaSE_017", javax.security.sasl; version\="0.0.0.JavaSE_017", javax.smartcardio; version\="0.0.0.JavaSE_017", java.sql; version\="0.0.0.JavaSE_017", javax.sql; version\="0.0.0.JavaSE_017", javax.sql.rowset.spi; version\="0.0.0.JavaSE_017", javax.sql.rowset.serial; version\="0.0.0.JavaSE_017", javax.sql.rowset; version\="0.0.0.JavaSE_017", javax.transaction.xa; version\="0.0.0.JavaSE_017", javax.xml; version\="0.0.0.JavaSE_017", javax.xml.transform.sax; version\="0.0.0.JavaSE_017", javax.xml.datatype; version\="0.0.0.JavaSE_017", javax.xml.catalog; version\="0.0.0.JavaSE_017", org.w3c.dom.traversal; version\="0.0.0.JavaSE_017", javax.xml.stream.events; version\="0.0.0.JavaSE_017", javax.xml.stream; version\="0.0.0.JavaSE_017", org.xml.sax; version\="0.0.0.JavaSE_017", javax.xml.transform; version\="0.0.0.JavaSE_017", javax.xml.xpath; version\="0.0.0.JavaSE_017", org.w3c.dom.events; version\="0.0.0.JavaSE_017", org.w3c.dom.ranges; version\="0.0.0.JavaSE_017", org.w3c.dom.ls; version\="0.0.0.JavaSE_017", javax.xml.stream.util; version\="0.0.0.JavaSE_017", javax.xml.namespace; version\="0.0.0.JavaSE_017", javax.xml.transform.stax; version\="0.0.0.JavaSE_017", org.xml.sax.helpers; version\="0.0.0.JavaSE_017", org.w3c.dom.views; version\="0.0.0.JavaSE_017", org.w3c.dom.bootstrap; version\="0.0.0.JavaSE_017", org.w3c.dom; version\="0.0.0.JavaSE_017", javax.xml.transform.stream; version\="0.0.0.JavaSE_017", javax.xml.transform.dom; version\="0.0.0.JavaSE_017", javax.xml.validation; version\="0.0.0.JavaSE_017", javax.xml.parsers; version\="0.0.0.JavaSE_017", org.xml.sax.ext; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.spec; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.keyinfo; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto.dom; version\="0.0.0.JavaSE_017", javax.xml.crypto; version\="0.0.0.JavaSE_017", javax.xml.crypto.dsig; version\="0.0.0.JavaSE_017", com.sun.java.accessibility.util; version\="0.0.0.JavaSE_017", com.sun.tools.attach.spi; version\="0.0.0.JavaSE_017", com.sun.tools.attach; version\="0.0.0.JavaSE_017", com.sun.source.doctree; version\="0.0.0.JavaSE_017", com.sun.source.tree; version\="0.0.0.JavaSE_017", com.sun.source.util; version\="0.0.0.JavaSE_017", com.sun.tools.javac; version\="0.0.0.JavaSE_017", jdk.dynalink.beans; version\="0.0.0.JavaSE_017", jdk.dynalink.linker.support; version\="0.0.0.JavaSE_017", jdk.dynalink.support; version\="0.0.0.JavaSE_017", jdk.dynalink; version\="0.0.0.JavaSE_017", jdk.dynalink.linker; version\="0.0.0.JavaSE_017", com.sun.net.httpserver; version\="0.0.0.JavaSE_017", com.sun.net.httpserver.spi; version\="0.0.0.JavaSE_017", com.sun.jarsigner; version\="0.0.0.JavaSE_017", jdk.security.jarsigner; version\="0.0.0.JavaSE_017", jdk.javadoc.doclet; version\="0.0.0.JavaSE_017", com.sun.tools.jconsole; version\="0.0.0.JavaSE_017", com.sun.jdi.connect; version\="0.0.0.JavaSE_017", com.sun.jdi.event; version\="0.0.0.JavaSE_017", com.sun.jdi.connect.spi; version\="0.0.0.JavaSE_017", com.sun.jdi; version\="0.0.0.JavaSE_017", com.sun.jdi.request; version\="0.0.0.JavaSE_017", jdk.jfr.consumer; version\="0.0.0.JavaSE_017", jdk.jfr; version\="0.0.0.JavaSE_017", jdk.jshell; version\="0.0.0.JavaSE_017", jdk.jshell.execution; version\="0.0.0.JavaSE_017", jdk.jshell.spi; version\="0.0.0.JavaSE_017", jdk.jshell.tool; version\="0.0.0.JavaSE_017", netscape.javascript; version\="0.0.0.JavaSE_017", com.sun.management; version\="0.0.0.JavaSE_017", jdk.management.jfr; version\="0.0.0.JavaSE_017", jdk.net; version\="0.0.0.JavaSE_017", jdk.nio; version\="0.0.0.JavaSE_017", jdk.nio.mapmode; version\="0.0.0.JavaSE_017", com.sun.nio.sctp; version\="0.0.0.JavaSE_017", com.sun.security.auth.module; version\="0.0.0.JavaSE_017", com.sun.security.auth.login; version\="0.0.0.JavaSE_017", com.sun.security.auth; version\="0.0.0.JavaSE_017", com.sun.security.auth.callback; version\="0.0.0.JavaSE_017", com.sun.security.jgss; version\="0.0.0.JavaSE_017", sun.reflect; version\="0.0.0.JavaSE_017", sun.misc; version\="0.0.0.JavaSE_017", com.sun.nio.file; version\="0.0.0.JavaSE_017", jdk.swing.interop; version\="0.0.0.JavaSE_017", org.w3c.dom.stylesheets; version\="0.0.0.JavaSE_017", org.w3c.dom.html; version\="0.0.0.JavaSE_017", org.w3c.dom.xpath; version\="0.0.0.JavaSE_017", org.w3c.dom.css; version\="0.0.0.JavaSE_017", com.yahoo.jdisc, com.yahoo.jdisc.application, com.yahoo.jdisc.handler, com.yahoo.jdisc.service, com.yahoo.jdisc.statistics, com.yahoo.jdisc.refcount, javax.inject;version\=1.0.0, org.aopalliance.intercept, org.aopalliance.aop, com.google.common.annotations;version\="27.1.0",com.google.common.base;version\="27.1.0",com.google.common.cache;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent",com.google.common.collect;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.escape;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.eventbus;version\="27.1.0",com.google.common.graph;version\="27.1.0";uses\:\="com.google.common.collect",com.google.common.hash;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.html;version\="27.1.0";uses\:\="com.google.common.escape",com.google.common.io;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.graph,com.google.common.hash",com.google.common.math;version\="27.1.0",com.google.common.net;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.escape",com.google.common.primitives;version\="27.1.0";uses\:\="com.google.common.base",com.google.common.reflect;version\="27.1.0";uses\:\="com.google.common.collect,com.google.common.io",com.google.common.util.concurrent;version\="27.1.0";uses\:\="com.google.common.base,com.google.common.collect,com.google.common.util.concurrent.internal",com.google.common.xml;version\="27.1.0";uses\:\="com.google.common.escape", com.google.inject;version\="1.4",com.google.inject.binder;version\="1.4",com.google.inject.matcher;version\="1.4",com.google.inject.multibindings;version\="1.4",com.google.inject.name;version\="1.4",com.google.inject.spi;version\="1.4",com.google.inject.util;version\="1.4", org.slf4j;version\=1.7.32, org.slf4j.spi;version\=1.7.32, org.slf4j.helpers;version\=1.7.32, org.slf4j.event;version\=1.7.32, org.slf4j.impl;version\=1.7.32, org.apache.commons.logging;version\=1.2, org.apache.commons.logging.impl;version\=1.2, org.apache.log4j;version\=1.2.17,org.apache.log4j.helpers;version\=1.2.17,org.apache.log4j.spi;version\=1.2.17,org.apache.log4j.xml;version\=1.2.17, com.yahoo.component.annotation;version\="1.0.0", com.yahoo.config;version\=1.0.0, com.yahoo.vespa.defaults;version\=1.0.0, ai.vespa.http;version\=1.0.0,ai.vespa.validation;version\=1.0.0,com.yahoo.binaryprefix;version\=1.0.0,com.yahoo.collections;version\=1.0.0,com.yahoo.compress;version\=1.0.0,com.yahoo.concurrent.classlock;version\=1.0.0,com.yahoo.concurrent.maintenance;version\=1.0.0,com.yahoo.concurrent;version\=1.0.0,com.yahoo.data.access.simple;version\=1.0.0,com.yahoo.data.access.slime;version\=1.0.0,com.yahoo.data.access;version\=1.0.0,com.yahoo.errorhandling;version\=1.0.0,com.yahoo.exception;version\=1.0.0,com.yahoo.geo;version\=1.0.0,com.yahoo.io.reader;version\=1.0.0,com.yahoo.io;version\=1.0.0,com.yahoo.javacc;version\=1.0.0,com.yahoo.lang;version\=1.0.0,com.yahoo.nativec;version\=1.0.0,com.yahoo.net;version\=1.0.0,com.yahoo.path;version\=1.0.0,com.yahoo.protect;version\=1.0.0,com.yahoo.reflection;version\=1.0.0,com.yahoo.slime;version\=1.0.0,com.yahoo.stream;version\=1.0.0,com.yahoo.system.execution;version\=1.0.0,com.yahoo.system;version\=1.0.0,com.yahoo.tensor.evaluation;version\=1.0.0,com.yahoo.tensor.functions;version\=1.0.0,com.yahoo.tensor.serialization;version\=1.0.0,com.yahoo.tensor;version\=1.0.0,com.yahoo.text.internal;version\=1.0.0,com.yahoo.text;version\=1.0.0,com.yahoo.time;version\=1.0.0,com.yahoo.transaction;version\=1.0.0,com.yahoo.vespa.objects;version\=1.0.0,com.yahoo.yolean.chain;version\=1.0.0,com.yahoo.yolean.concurrent;version\=1.0.0,com.yahoo.yolean.function;version\=1.0.0,com.yahoo.yolean.system;version\=1.0.0,com.yahoo.yolean.trace;version\=1.0.0,com.yahoo.yolean;version\=1.0.0, com.yahoo.log.event;version\=1.0.0,com.yahoo.log.impl;version\=1.0.0,com.yahoo.log;version\=1.0.0, javax.xml.bind;version\="2.3";uses\:\="javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.annotation;version\="2.3";uses\:\="javax.xml.bind,javax.xml.parsers,javax.xml.transform,javax.xml.transform.dom,org.w3c.dom",javax.xml.bind.annotation.adapters;version\="2.3",javax.xml.bind.attachment;version\="2.3";uses\:\="javax.activation",javax.xml.bind.helpers;version\="2.3";uses\:\="javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.stream,javax.xml.transform,javax.xml.validation,org.w3c.dom,org.xml.sax",javax.xml.bind.util;version\="2.3";uses\:\="javax.xml.bind,javax.xml.transform.sax", com.sun.istack;version\="3.0.5";uses\:\="javax.activation,javax.xml.stream,org.xml.sax,org.xml.sax.helpers",com.sun.istack.localization;version\="3.0.5",com.sun.istack.logging;version\="3.0.5",com.sun.xml.bind;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.annotation;version\="2.3.0",com.sun.xml.bind.api;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.api.impl;version\="2.3.0",com.sun.xml.bind.marshaller;uses\:\="javax.xml.parsers,org.w3c.dom,org.xml.sax,org.xml.sax.helpers";version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;version\="2.3.0",com.sun.xml.bind.v2;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.core;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.impl,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.namespace,javax.xml.transform";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.nav";version\="2.3.0",com.sun.xml.bind.v2.model.nav;uses\:\="com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.util;uses\:\="javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.xml.bind.v2.model.annotation,javax.activation,javax.xml.bind,javax.xml.bind.annotation.adapters";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="javax.xml.bind,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen.episode;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="javax.xml.parsers,javax.xml.transform,javax.xml.validation,javax.xml.xpath";version\="2.3.0",com.sun.xml.txw2;uses\:\="com.sun.xml.txw2.output,javax.xml.namespace";version\="2.3.0",com.sun.xml.txw2.annotation;version\="2.3.0",com.sun.xml.txw2.output;uses\:\="com.sun.xml.txw2,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.dom,javax.xml.transform.sax,javax.xml.transform.stream,org.w3c.dom,org.xml.sax,org.xml.sax.ext,org.xml.sax.helpers";version\="2.3.0", com.sun.xml.bind;uses\:\="com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.datatype,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.api;uses\:\="com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.xml.bind,javax.xml.bind.attachment,javax.xml.namespace,javax.xml.stream,javax.xml.transform,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.marshaller;version\="2.3.0",com.sun.xml.bind.unmarshaller;uses\:\="org.xml.sax";version\="2.3.0",com.sun.xml.bind.util;uses\:\="com.sun.xml.bind,javax.xml.bind.helpers,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,javax.xml.bind";version\="2.3.0",com.sun.xml.bind.v2.bytecode;version\="2.3.0",com.sun.xml.bind.v2.model.annotation;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.runtime";version\="2.3.0",com.sun.xml.bind.v2.model.impl;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,javax.activation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.model.runtime;uses\:\="com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.bind,javax.xml.namespace,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime;uses\:\="com.sun.istack,com.sun.xml.bind.api,com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.model.annotation,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.property,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.output;uses\:\="com.sun.xml.bind.marshaller,com.sun.xml.bind.v2.runtime,com.sun.xml.fastinfoset.stax,javax.xml.stream,org.jvnet.staxex,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.property;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,com.sun.xml.bind.v2.runtime.unmarshaller,com.sun.xml.bind.v2.util,javax.xml.namespace,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.xml.bind,javax.xml.bind.annotation.adapters,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.reflect.opt;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.runtime,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.reflect,javax.xml.stream,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.runtime.unmarshaller;uses\:\="com.sun.xml.bind,com.sun.xml.bind.api,com.sun.xml.bind.unmarshaller,com.sun.xml.bind.util,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.output,com.sun.xml.bind.v2.runtime.reflect,javax.activation,javax.xml.bind,javax.xml.bind.annotation,javax.xml.bind.annotation.adapters,javax.xml.bind.attachment,javax.xml.bind.helpers,javax.xml.namespace,javax.xml.stream,javax.xml.transform,javax.xml.transform.sax,javax.xml.validation,org.w3c.dom,org.xml.sax";version\="2.3.0",com.sun.xml.bind.v2.schemagen;uses\:\="com.sun.xml.bind.api,com.sun.xml.bind.v2.model.core,com.sun.xml.bind.v2.model.nav,com.sun.xml.txw2.output,javax.xml.bind,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.schemagen.xmlschema;uses\:\="com.sun.xml.txw2,com.sun.xml.txw2.annotation,javax.xml.namespace";version\="2.3.0",com.sun.xml.bind.v2.util;uses\:\="com.sun.xml.bind.v2.runtime,com.sun.xml.bind.v2.runtime.unmarshaller,javax.activation,javax.xml.namespace,javax.xml.transform.stream,org.xml.sax";version\="2.3.0", javax.activation;uses\:\="com.sun.activation.registries";version\="1.2",com.sun.activation.viewers;uses\:\="javax.activation";version\="1.2.0",com.sun.activation.registries;version\="1.2.0" diff --git a/jrt/src/com/yahoo/jrt/MandatoryMethods.java b/jrt/src/com/yahoo/jrt/MandatoryMethods.java index 19b2febf0de..a73e2bfc6dd 100644 --- a/jrt/src/com/yahoo/jrt/MandatoryMethods.java +++ b/jrt/src/com/yahoo/jrt/MandatoryMethods.java @@ -2,6 +2,8 @@ package com.yahoo.jrt; +import com.yahoo.security.tls.CapabilitySet; + import java.util.Collection; @@ -15,11 +17,13 @@ class MandatoryMethods { Method m; //--------------------------------------------------------------------- m = new Method("frt.rpc.ping", "", "", this::ping); + m.requireCapabilities(CapabilitySet.none()); m.methodDesc("Method that may be used to " + "check if the server is online"); parent.addMethod(m); //--------------------------------------------------------------------- m = new Method("frt.rpc.getMethodList", "", "SSS", this::getMethodList); + m.requireCapabilities(CapabilitySet.none()); m.methodDesc("Obtain a list of all available methods"); m.returnDesc(0, "names", "Method names"); m.returnDesc(1, "params", "Method parameter types"); @@ -27,6 +31,7 @@ class MandatoryMethods { parent.addMethod(m); //--------------------------------------------------------------------- m = new Method("frt.rpc.getMethodInfo", "s", "sssSSSS", this::getMethodInfo); + m.requireCapabilities(CapabilitySet.none()); m.methodDesc("Obtain detailed information about a single method"); m.paramDesc (0, "methodName", "The method we want information about"); m.returnDesc(0, "desc", "Description of what the method does"); diff --git a/jrt_test/CMakeLists.txt b/jrt_test/CMakeLists.txt index ea8c8b94faa..a678cbf112a 100644 --- a/jrt_test/CMakeLists.txt +++ b/jrt_test/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib fnet diff --git a/logd/CMakeLists.txt b/logd/CMakeLists.txt index d02b99a393a..5823ebf54a7 100644 --- a/logd/CMakeLists.txt +++ b/logd/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib config_cloudconfig diff --git a/logd/src/logd/watcher.cpp b/logd/src/logd/watcher.cpp index c3752fcc3e8..af1efc3077d 100644 --- a/logd/src/logd/watcher.cpp +++ b/logd/src/logd/watcher.cpp @@ -8,6 +8,7 @@ #include <vespa/vespalib/util/time.h> #include <vespa/vespalib/util/size_literals.h> #include <thread> +#include <cinttypes> #include <fcntl.h> #include <glob.h> #include <sys/stat.h> diff --git a/lowercasing_test/CMakeLists.txt b/lowercasing_test/CMakeLists.txt index b08a6ef350d..119209d4227 100644 --- a/lowercasing_test/CMakeLists.txt +++ b/lowercasing_test/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib searchlib diff --git a/maven-plugins/allowed-maven-dependencies.txt b/maven-plugins/allowed-maven-dependencies.txt index a3cc4cc7045..5272dd21a31 100644 --- a/maven-plugins/allowed-maven-dependencies.txt +++ b/maven-plugins/allowed-maven-dependencies.txt @@ -3,9 +3,9 @@ #[non-test] # Contains dependencies that are not used exclusively in 'test' scope aopalliance:aopalliance:1.0 -com.fasterxml.jackson.core:jackson-annotations:2.13.4 -com.fasterxml.jackson.core:jackson-core:2.13.4 -com.fasterxml.jackson.core:jackson-databind:2.13.4.2 +com.fasterxml.jackson.core:jackson-annotations:2.14.2 +com.fasterxml.jackson.core:jackson-core:2.14.2 +com.fasterxml.jackson.core:jackson-databind:2.14.2 com.google.errorprone:error_prone_annotations:2.18.0 com.google.guava:failureaccess:1.0.1 com.google.guava:guava:27.1-jre diff --git a/messagebus/CMakeLists.txt b/messagebus/CMakeLists.txt index 30e795cac1d..ab37173a5ea 100644 --- a/messagebus/CMakeLists.txt +++ b/messagebus/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog config_cloudconfig vespalib diff --git a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java index b4fa7d8f887..6afc2039c38 100644 --- a/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java +++ b/messagebus/src/main/java/com/yahoo/messagebus/network/rpc/RPCNetwork.java @@ -29,6 +29,7 @@ import com.yahoo.messagebus.network.NetworkOwner; import com.yahoo.messagebus.routing.Hop; import com.yahoo.messagebus.routing.Route; import com.yahoo.messagebus.routing.RoutingNode; +import com.yahoo.security.tls.CapabilitySet; import java.io.PrintWriter; import java.io.StringWriter; @@ -100,6 +101,7 @@ public class RPCNetwork implements Network, MethodHandler { servicePool = new RPCServicePool(this, 4096); Method method = new Method("mbus.getVersion", "", "s", this); + method.requireCapabilities(CapabilitySet.none()); method.methodDesc("Retrieves the message bus version."); method.returnDesc(0, "version", "The message bus version."); orb.addMethod(method); diff --git a/messagebus/src/tests/replygate/replygate.cpp b/messagebus/src/tests/replygate/replygate.cpp index 0acdc0a5611..3de48fce130 100644 --- a/messagebus/src/tests/replygate/replygate.cpp +++ b/messagebus/src/tests/replygate/replygate.cpp @@ -9,58 +9,59 @@ using namespace mbus; -struct MyGate : public ReplyGate -{ +namespace { + +struct MyGate : public ReplyGate { static int ctorCnt; static int dtorCnt; + MyGate(IMessageHandler &sender) : ReplyGate(sender) { ++ctorCnt; } - virtual ~MyGate() { + + ~MyGate() override { ++dtorCnt; } }; + int MyGate::ctorCnt = 0; int MyGate::dtorCnt = 0; -struct MyReply : public EmptyReply -{ +struct MyReply : public EmptyReply { static int ctorCnt; static int dtorCnt; + MyReply() : EmptyReply() { ++ctorCnt; } - virtual ~MyReply() { + + ~MyReply() override { ++dtorCnt; } }; + int MyReply::ctorCnt = 0; int MyReply::dtorCnt = 0; -struct MySender : public IMessageHandler -{ +struct MySender : public IMessageHandler { // giving a sync reply here is against the API contract, but it is // ok for testing. void handleMessage(Message::UP msg) override { - Reply::UP reply(new MyReply()); + auto reply = std::make_unique<MyReply>(); msg->swapState(*reply); IReplyHandler &handler = reply->getCallStack().pop(*reply); handler.handleReply(std::move(reply)); } }; +} -TEST_SETUP(Test); - -int -Test::Main() -{ - TEST_INIT("replygate_test"); +TEST("replygate_test") { { RoutableQueue q; MySender sender; MyGate *gate = new MyGate(sender); { - Message::UP msg(new SimpleMessage("test")); + auto msg = std::make_unique<SimpleMessage>("test"); msg->pushHandler(q); gate->handleMessage(std::move(msg)); } @@ -69,7 +70,7 @@ Test::Main() EXPECT_TRUE(MyReply::dtorCnt == 0); gate->close(); { - Message::UP msg(new SimpleMessage("test")); + auto msg = std::make_unique<SimpleMessage>("test"); msg->pushHandler(q); gate->handleMessage(std::move(msg)); } @@ -84,5 +85,6 @@ Test::Main() } EXPECT_TRUE(MyReply::ctorCnt == 2); EXPECT_TRUE(MyReply::dtorCnt == 2); - TEST_DONE(); } + +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/messagebus/src/tests/sequencer/sequencer.cpp b/messagebus/src/tests/sequencer/sequencer.cpp index 25d9934fe8d..7a3aaab9b1f 100644 --- a/messagebus/src/tests/sequencer/sequencer.cpp +++ b/messagebus/src/tests/sequencer/sequencer.cpp @@ -5,6 +5,7 @@ #include <vespa/messagebus/routablequeue.h> #include <vespa/messagebus/emptyreply.h> #include <vespa/vespalib/testkit/testapp.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP("sequencer_test"); diff --git a/messagebus/src/vespa/messagebus/callstack.h b/messagebus/src/vespa/messagebus/callstack.h index 937a9e939b6..8b2ab206c18 100644 --- a/messagebus/src/vespa/messagebus/callstack.h +++ b/messagebus/src/vespa/messagebus/callstack.h @@ -72,9 +72,12 @@ public: * @param ctx The context to store. * @param discardHandler The handler for discarded messages. **/ - void push(IReplyHandler &replyHandler, Context ctx, IDiscardHandler *discardHandler = nullptr) { + void push(IReplyHandler &replyHandler, Context ctx, IDiscardHandler *discardHandler) { _stack.emplace_back(&replyHandler, discardHandler, ctx); } + void push(IReplyHandler &replyHandler, Context ctx) { + _stack.emplace_back(&replyHandler, nullptr, ctx); + } /** * Pop a frame from this stack. The handler part of the frame will diff --git a/messagebus/src/vespa/messagebus/context.h b/messagebus/src/vespa/messagebus/context.h index edf875630e1..d6e7b4e82e5 100644 --- a/messagebus/src/vespa/messagebus/context.h +++ b/messagebus/src/vespa/messagebus/context.h @@ -17,11 +17,10 @@ struct Context { /** * This is a region of memory that can be interpreted as either an - * integer, a floating-point number or a pointer. + * integer or a pointer. **/ union { uint64_t UINT64; - double DOUBLE; void *PTR; } value; @@ -38,13 +37,6 @@ struct Context Context(uint64_t v) { value.UINT64 = v; } /** - * Create a context from a floating-point number - * - * @param v the value - **/ - Context(double v) { value.DOUBLE = v; } - - /** * Create a context from a pointer * * @param pt the pointer diff --git a/messagebus/src/vespa/messagebus/idiscardhandler.h b/messagebus/src/vespa/messagebus/idiscardhandler.h index 049b8dfb245..5618402f14d 100644 --- a/messagebus/src/vespa/messagebus/idiscardhandler.h +++ b/messagebus/src/vespa/messagebus/idiscardhandler.h @@ -15,7 +15,7 @@ public: /** * Virtual destructor required for inheritance. */ - virtual ~IDiscardHandler() { } + virtual ~IDiscardHandler() = default; /** * This method is invoked by message bus when a routable is being diff --git a/messagebus/src/vespa/messagebus/messenger.cpp b/messagebus/src/vespa/messagebus/messenger.cpp index 056be51609f..926d8b6ef22 100644 --- a/messagebus/src/vespa/messagebus/messenger.cpp +++ b/messagebus/src/vespa/messagebus/messenger.cpp @@ -18,70 +18,6 @@ struct DeleteFunctor } }; -class MessageTask : public mbus::Messenger::ITask { -private: - mbus::Message::UP _msg; - mbus::IMessageHandler &_handler; - -public: - MessageTask(mbus::Message::UP msg, mbus::IMessageHandler &handler) - : _msg(std::move(msg)), - _handler(handler) - { - // empty - } - - ~MessageTask() { - if (_msg) { - _msg->discard(); - } - } - - void run() override { - _handler.handleMessage(std::move(_msg)); - } - - uint8_t priority() const override { - if (_msg) { - return _msg->priority(); - } - - return 255; - } -}; - -class ReplyTask : public mbus::Messenger::ITask { -private: - mbus::Reply::UP _reply; - mbus::IReplyHandler &_handler; - -public: - ReplyTask(mbus::Reply::UP reply, mbus::IReplyHandler &handler) - : _reply(std::move(reply)), - _handler(handler) - { - // empty - } - - ~ReplyTask() { - if (_reply) { - _reply->discard(); - } - } - - void run() override { - _handler.handleReply(std::move(_reply)); - } - - uint8_t priority() const override { - if (_reply) { - return _reply->priority(); - } - - return 255; - } -}; - class SyncTask : public mbus::Messenger::ITask { private: vespalib::Gate &_gate; @@ -89,17 +25,13 @@ private: public: SyncTask(vespalib::Gate &gate) : _gate(gate) - { - // empty - } + { } ~SyncTask() { _gate.countDown(); } - void run() override { - // empty - } + void run() override { } uint8_t priority() const override { return 255; @@ -115,9 +47,7 @@ public: AddRecurrentTask(std::vector<ITask*> &tasks, mbus::Messenger::ITask::UP task) : _tasks(tasks), _task(std::move(task)) - { - // empty - } + { } void run() override { _tasks.push_back(_task.release()); @@ -136,9 +66,7 @@ public: DiscardRecurrentTasks(vespalib::Gate &gate, std::vector<ITask*> &tasks) : SyncTask(gate), _tasks(tasks) - { - // empty - } + { } void run() override { std::for_each(_tasks.begin(), _tasks.end(), DeleteFunctor<ITask>()); @@ -157,10 +85,11 @@ namespace mbus { Messenger::Messenger() : _lock(), - _pool(), + _cond(), _children(), _queue(), - _closed(false) + _closed(false), + _thread() {} Messenger::~Messenger() @@ -170,8 +99,9 @@ Messenger::~Messenger() _closed = true; } _cond.notify_all(); - - _pool.Close(); + if (_thread.joinable()) { + _thread.join(); + } std::for_each(_children.begin(), _children.end(), DeleteFunctor<ITask>()); if ( ! _queue.empty()) { LOG(warning, @@ -185,10 +115,8 @@ Messenger::~Messenger() } void -Messenger::Run(FastOS_ThreadInterface *thread, void *arg) +Messenger::run() { - (void)thread; - (void)arg; while (true) { ITask::UP task; { @@ -235,9 +163,7 @@ Messenger::discardRecurrentTasks() bool Messenger::start() { - if (_pool.NewThread(this) == nullptr) { - return false; - } + _thread = std::thread([this](){run();}); return true; } diff --git a/messagebus/src/vespa/messagebus/messenger.h b/messagebus/src/vespa/messagebus/messenger.h index 6ec42ed9c4c..86cd4bf3b7f 100644 --- a/messagebus/src/vespa/messagebus/messenger.h +++ b/messagebus/src/vespa/messagebus/messenger.h @@ -7,7 +7,8 @@ #include "reply.h" #include <vespa/vespalib/util/executor.h> #include <vespa/vespalib/util/arrayqueue.hpp> -#include <vespa/fastos/thread.h> +#include <condition_variable> +#include <mutex> namespace mbus { @@ -16,7 +17,7 @@ namespace mbus { * tasks. Tasks are enqueued using the synchronized {@link #enqueue(Task)} * method, and are run in the order they were enqueued. */ -class Messenger : public FastOS_Runnable { +class Messenger { public: /** * Defines the required interface for tasks to be posted to this worker. @@ -39,15 +40,15 @@ public: }; private: - mutable std::mutex _lock; - std::condition_variable _cond; - FastOS_ThreadPool _pool; - std::vector<ITask*> _children; - vespalib::ArrayQueue<ITask*> _queue; - bool _closed; - + mutable std::mutex _lock; + std::condition_variable _cond; + std::vector<ITask*> _children; + vespalib::ArrayQueue<ITask*> _queue; + bool _closed; + std::thread _thread; + protected: - void Run(FastOS_ThreadInterface *thread, void *arg) override; + void run(); public: Messenger(); @@ -55,7 +56,7 @@ public: /** * Frees any allocated resources. Also destroys all queued tasks. */ - ~Messenger() override; + ~Messenger(); /** * Adds a recurrent task to this that is to be run for every iteration of diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp index 030e3f956e1..8ab9aa13394 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp +++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.cpp @@ -18,7 +18,6 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/stringfmt.h> -#include <vespa/fastos/thread.h> #include <thread> #include <vespa/log/log.h> @@ -95,7 +94,6 @@ RPCNetwork::SendContext::handleVersion(const vespalib::Version *version) RPCNetwork::RPCNetwork(const RPCNetworkParams ¶ms) : _owner(nullptr), _ident(params.getIdentity()), - _threadPool(std::make_unique<FastOS_ThreadPool>()), _transport(std::make_unique<FNET_Transport>(toFNETConfig(params))), _orb(std::make_unique<FRT_Supervisor>(_transport.get())), _scheduler(*_transport->GetScheduler()), @@ -196,7 +194,7 @@ RPCNetwork::getSendAdapter(const vespalib::Version &version) bool RPCNetwork::start() { - if (!_transport->Start(_threadPool.get())) { + if (!_transport->Start()) { return false; } if (!_orb->Listen(_requestedPort)) { @@ -391,7 +389,6 @@ RPCNetwork::shutdown() // Unschedule any pending target pool flush task that may race with shutdown target flushing _scheduler.Kill(_targetPoolTask.get()); _transport->ShutDown(true); - _threadPool->Close(); } void diff --git a/messagebus/src/vespa/messagebus/network/rpcnetwork.h b/messagebus/src/vespa/messagebus/network/rpcnetwork.h index 0d2435e5dcd..8e296981458 100644 --- a/messagebus/src/vespa/messagebus/network/rpcnetwork.h +++ b/messagebus/src/vespa/messagebus/network/rpcnetwork.h @@ -16,7 +16,6 @@ #include <vespa/fnet/frt/invokable.h> class FNET_Transport; -class FastOS_ThreadPool; namespace slobrok { namespace api { class RegisterAPI; } @@ -56,7 +55,6 @@ private: INetworkOwner *_owner; Identity _ident; - std::unique_ptr<FastOS_ThreadPool> _threadPool; std::unique_ptr<FNET_Transport> _transport; std::unique_ptr<FRT_Supervisor> _orb; FNET_Scheduler &_scheduler; diff --git a/messagebus/src/vespa/messagebus/replygate.cpp b/messagebus/src/vespa/messagebus/replygate.cpp index d1bd6ef05c7..8094119f14c 100644 --- a/messagebus/src/vespa/messagebus/replygate.cpp +++ b/messagebus/src/vespa/messagebus/replygate.cpp @@ -38,9 +38,8 @@ ReplyGate::handleReply(Reply::UP reply) } void -ReplyGate::handleDiscard(Context ctx) +ReplyGate::handleDiscard(Context) { - (void)ctx; subRef(); } diff --git a/messagebus/src/vespa/messagebus/sendproxy.cpp b/messagebus/src/vespa/messagebus/sendproxy.cpp index dcb24b70eeb..fa2994a6b7e 100644 --- a/messagebus/src/vespa/messagebus/sendproxy.cpp +++ b/messagebus/src/vespa/messagebus/sendproxy.cpp @@ -13,9 +13,7 @@ SendProxy::SendProxy(MessageBus &mbus, INetwork &net, Resender *resender) : _msg(), _logTrace(false), _root() -{ - // empty -} +{ } void SendProxy::handleMessage(Message::UP msg) @@ -31,14 +29,13 @@ SendProxy::handleMessage(Message::UP msg) } } _msg = std::move(msg); - _root.reset(new RoutingNode(_mbus, _net, _resender, *this, *_msg, this)); + _root = std::make_unique<RoutingNode>(_mbus, _net, _resender, *this, *_msg, this); _root->send(); } void -SendProxy::handleDiscard(Context ctx) +SendProxy::handleDiscard(Context) { - (void)ctx; _msg->discard(); delete this; } diff --git a/messagebus/src/vespa/messagebus/sequencer.cpp b/messagebus/src/vespa/messagebus/sequencer.cpp index e17509033d6..59e3164e11d 100644 --- a/messagebus/src/vespa/messagebus/sequencer.cpp +++ b/messagebus/src/vespa/messagebus/sequencer.cpp @@ -18,8 +18,8 @@ Sequencer::Sequencer(IMessageHandler &sender) : Sequencer::~Sequencer() { - for (QueueMap::iterator it = _seqMap.begin(); it != _seqMap.end(); ++it) { - MessageQueue *queue = it->second; + for (auto & entry : _seqMap) { + MessageQueue *queue = entry.second; if (queue != nullptr) { while (queue->size() > 0) { Message *msg = queue->front(); @@ -39,7 +39,7 @@ Sequencer::filter(Message::UP msg) msg->setContext(Context(seqId)); { std::lock_guard guard(_lock); - QueueMap::iterator it = _seqMap.find(seqId); + auto it = _seqMap.find(seqId); if (it != _seqMap.end()) { if (it->second == nullptr) { it->second = new MessageQueue(); @@ -48,7 +48,7 @@ Sequencer::filter(Message::UP msg) make_string("Sequencer queued message with sequence id '%" PRIu64 "'.", seqId)); it->second->push(msg.get()); msg.release(); - return Message::UP(); + return {}; } _seqMap[seqId] = nullptr; // insert empty queue } @@ -87,7 +87,7 @@ Sequencer::handleReply(Reply::UP reply) Message::UP msg; { std::lock_guard guard(_lock); - QueueMap::iterator it = _seqMap.find(seq); + auto it = _seqMap.find(seq); MessageQueue *que = it->second; assert(it != _seqMap.end()); if (que == nullptr || que->size() == 0) { diff --git a/messagebus/src/vespa/messagebus/testlib/slobrok.cpp b/messagebus/src/vespa/messagebus/testlib/slobrok.cpp index 889daf538a3..b4cd2c846aa 100644 --- a/messagebus/src/vespa/messagebus/testlib/slobrok.cpp +++ b/messagebus/src/vespa/messagebus/testlib/slobrok.cpp @@ -9,80 +9,37 @@ #include <vespa/log/log.h> LOG_SETUP(".slobrok"); -namespace { -class WaitTask : public FNET_Task -{ -private: - bool _done; - std::mutex _mon; - std::condition_variable _cond; -public: - explicit WaitTask(FNET_Scheduler *s) : FNET_Task(s), _done(false), _mon() {} - ~WaitTask() override; - void wait() { - std::unique_lock guard(_mon); - while (!_done) { - _cond.wait(guard); - } - } - - void PerformTask() override { - std::lock_guard guard(_mon); - _done = true; - _cond.notify_one(); - } -}; - -WaitTask::~WaitTask() = default; -} // namespace <unnamed> - namespace mbus { void -Slobrok::Thread::setEnv(slobrok::SBEnv *env) -{ - _env = env; -} - -void -Slobrok::Thread::Run(FastOS_ThreadInterface *, void *) -{ - if (_env->MainLoop() != 0) { - LOG_ABORT("Slobrok main failed"); - } -} - -void Slobrok::init() { slobrok::ConfigShim shim(_port); _env = std::make_unique<slobrok::SBEnv>(shim); - _thread.setEnv(_env.get()); - WaitTask wt(_env->getTransport()->GetScheduler()); - wt.ScheduleNow(); - if (_pool.NewThread(&_thread, nullptr) == nullptr) { - LOG_ABORT("Could not spawn thread"); - } - wt.wait(); + _thread = std::thread([env = _env.get()]() + { + if (env->MainLoop() != 0) { + LOG_ABORT("Slobrok main failed"); + } + }); + _env->getTransport()->sync(); int p = _env->getSupervisor()->GetListenPort(); LOG_ASSERT(p != 0 && (p == _port || _port == 0)); _port = p; } Slobrok::Slobrok() - : _pool(), - _env(), - _port(0), - _thread() + : _env(), + _port(0), + _thread() { init(); } Slobrok::Slobrok(int p) - : _pool(), - _env(), - _port(p), - _thread() + : _env(), + _port(p), + _thread() { init(); } @@ -90,7 +47,7 @@ Slobrok::Slobrok(int p) Slobrok::~Slobrok() { _env->getTransport()->ShutDown(true); - _pool.Close(); + _thread.join(); } int diff --git a/messagebus/src/vespa/messagebus/testlib/slobrok.h b/messagebus/src/vespa/messagebus/testlib/slobrok.h index 7810222f07d..ff8e9cbb89a 100644 --- a/messagebus/src/vespa/messagebus/testlib/slobrok.h +++ b/messagebus/src/vespa/messagebus/testlib/slobrok.h @@ -4,7 +4,7 @@ #include <vespa/messagebus/common.h> #include <vespa/slobrok/cfg.h> -#include <vespa/fastos/thread.h> +#include <thread> namespace slobrok { class SBEnv; @@ -15,17 +15,9 @@ namespace mbus { class Slobrok { private: - class Thread : public FastOS_Runnable { - private: - slobrok::SBEnv *_env; - public: - void setEnv(slobrok::SBEnv *env); - void Run(FastOS_ThreadInterface *, void *) override; - }; - FastOS_ThreadPool _pool; std::unique_ptr<slobrok::SBEnv> _env; - int _port; - Thread _thread; + int _port; + std::thread _thread; Slobrok(const Slobrok &); Slobrok &operator=(const Slobrok &); @@ -42,4 +34,3 @@ public: }; } // namespace mbus - diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java index ab1c2e70735..ba51bed4407 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/core/MetricsManager.java @@ -97,7 +97,7 @@ public class MetricsManager { return metricsPackets; } - private MetricsPacket.Builder [] getMetricsBuildersAsArray(List<VespaService> services, Instant startTime, ConsumerId consumerId) { + private MetricsPacket.Builder[] getMetricsBuildersAsArray(List<VespaService> services, Instant startTime, ConsumerId consumerId) { List<MetricsPacket.Builder> builders = getMetricsAsBuilders(services, startTime, consumerId); return builders.toArray(new MetricsPacket.Builder[0]); } diff --git a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java index 821636336a8..21c7c78a224 100644 --- a/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java +++ b/metrics-proxy/src/main/java/ai/vespa/metricsproxy/http/application/Node.java @@ -25,14 +25,10 @@ public class Node { } public Node(String role, String hostname, int port, String path) { - Objects.requireNonNull(role, "Null role is not allowed"); - Objects.requireNonNull(hostname, "Null hostname is not allowed"); - Objects.requireNonNull(path, "Null path is not allowed"); - - this.role = role; - this.hostname = hostname; + this.role = Objects.requireNonNull(role, "Null role is not allowed"); + this.hostname = Objects.requireNonNull(hostname, "Null hostname is not allowed"); this.port = port; - this.path = path; + this.path = Objects.requireNonNull(path, "Null path is not allowed"); metricsUriBase = "http://" + hostname + ":" + port + path; } @@ -55,10 +51,10 @@ public class Node { public int hashCode() { return Objects.hash(role, hostname, port, path); } + @Override public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(role).append(":").append(metricsUriBase); - return sb.toString(); + return role + ":" + metricsUriBase; } + } diff --git a/metrics/CMakeLists.txt b/metrics/CMakeLists.txt index d75c17d4a00..5b95d8635d4 100644 --- a/metrics/CMakeLists.txt +++ b/metrics/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib config_cloudconfig diff --git a/metrics/src/tests/metricmanagertest.cpp b/metrics/src/tests/metricmanagertest.cpp index 604e9c46b80..6d6f21ea7b0 100644 --- a/metrics/src/tests/metricmanagertest.cpp +++ b/metrics/src/tests/metricmanagertest.cpp @@ -5,12 +5,10 @@ #include <vespa/metrics/metricmanager.h> #include <vespa/metrics/state_api_adapter.h> #include <vespa/metrics/textwriter.h> -#include <vespa/metrics/xmlwriter.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/size_literals.h> -#include <vespa/vespalib/util/xmlstream.h> #include <vespa/vespalib/util/time.h> #include <vespa/vespalib/data/simple_buffer.h> #include <vespa/vespalib/util/atomic.h> @@ -57,7 +55,7 @@ SubMetricSet::SubMetricSet(const Metric::String & name, MetricSet* owner) valsum.addMetricToSum(val1); valsum.addMetricToSum(val2); } -SubMetricSet::~SubMetricSet() { } +SubMetricSet::~SubMetricSet() = default; struct MultiSubMetricSet { @@ -152,11 +150,10 @@ namespace { std::pair<std::string, std::string> getMatchedMetrics(const vespalib::string& config) { - FastOS_ThreadPool pool; TestMetricSet mySet; MetricManager mm; mm.registerMetric(mm.getMetricLock(), mySet.set); - mm.init(ConfigUri(config), pool); + mm.init(ConfigUri(config)); MetricNameVisitor visitor; /** Take a copy to verify clone works. @@ -168,20 +165,15 @@ getMatchedMetrics(const vespalib::string& config) MetricLockGuard g(mm.getMetricLock()); mm.visit(g, mm.getActiveMetrics(g), visitor, "consumer"); - MetricManager::ConsumerSpec::SP consumerSpec( - mm.getConsumerSpec(g, "consumer")); - return std::pair<std::string, std::string>( - visitor.toString(), - consumerSpec.get() ? consumerSpec->toString() - : "Non-existing consumer"); + MetricManager::ConsumerSpec::SP consumerSpec(mm.getConsumerSpec(g, "consumer")); + return { visitor.toString(), consumerSpec ? consumerSpec->toString() : "Non-existing consumer" }; } } #define ASSERT_CONSUMER_MATCH(name, expected, config) \ { \ - std::pair<std::string, std::string> consumerMatch( \ - getMatchedMetrics(config)); \ + std::pair<std::string, std::string> consumerMatch(getMatchedMetrics(config)); \ EXPECT_EQ("\n" + expected, "\n" + consumerMatch.first) << (name + std::string(": ") + consumerMatch.second); \ } @@ -372,10 +364,10 @@ class FakeTimer : public MetricManager::Timer { std::atomic<time_t> _time; public: FakeTimer(time_t startTime = 0) : _time(startTime) {} - time_t getTime() const override { return load_relaxed(_time); } + time_point getTime() const override { return time_point(vespalib::from_s(load_relaxed(_time))); } void set_time(time_t t) noexcept { store_relaxed(_time, t); } // Not safe for multiple writers, only expected to be called by test. - void add_time(time_t t) noexcept { set_time(getTime() + t); } + void add_time(time_t t) noexcept { set_time(load_relaxed(_time) + t); } }; struct BriefValuePrinter : public MetricVisitor { @@ -392,8 +384,7 @@ struct BriefValuePrinter : public MetricVisitor { } }; -bool waitForTimeProcessed(const MetricManager& mm, - time_t processtime, uint32_t timeout = 120) +bool waitForTimeProcessed(const MetricManager& mm, time_t processtime, uint32_t timeout = 120) { uint32_t lastchance = time(0) + timeout; while (time(0) < lastchance) { @@ -404,23 +395,20 @@ bool waitForTimeProcessed(const MetricManager& mm, return false; } -std::string dumpAllSnapshots(const MetricManager& mm, - const std::string& consumer) +std::string dumpAllSnapshots(const MetricManager& mm, const std::string& consumer) { std::ostringstream ost; ost << "\n"; { MetricLockGuard metricLock(mm.getMetricLock()); BriefValuePrinter briefValuePrinter; - mm.visit(metricLock, mm.getActiveMetrics(metricLock), - briefValuePrinter, consumer); + mm.visit(metricLock, mm.getActiveMetrics(metricLock), briefValuePrinter, consumer); ost << "Current: " << briefValuePrinter.ost.str() << "\n"; } { MetricLockGuard metricLock(mm.getMetricLock()); BriefValuePrinter briefValuePrinter; - mm.visit(metricLock, mm.getTotalMetricSnapshot(metricLock), - briefValuePrinter, consumer); + mm.visit(metricLock, mm.getTotalMetricSnapshot(metricLock), briefValuePrinter, consumer); ost << "Total: " << briefValuePrinter.ost.str() << "\n"; } std::vector<uint32_t> periods; @@ -430,17 +418,15 @@ std::string dumpAllSnapshots(const MetricManager& mm, } for (uint32_t i=0; i<periods.size(); ++i) { MetricLockGuard metricLock(mm.getMetricLock()); - const MetricSnapshotSet& set(mm.getMetricSnapshotSet( - metricLock, periods[i])); + const MetricSnapshotSet& set(mm.getMetricSnapshotSet(metricLock, periods[i])); ost << set.getName() << "\n"; - uint32_t count = 0; - for (uint32_t j=0; j<2; ++j) { + for (uint32_t count=0,j=0; j<2; ++j) { if (set.getCount() == 1 && j == 1) continue; const MetricSnapshot& snap(set.getSnapshot(j == 1)); BriefValuePrinter briefValuePrinter; mm.visit(metricLock, snap, briefValuePrinter, consumer); ost << " " << count++ << " " << &snap.getMetrics() << ": " - << briefValuePrinter.ost.str() << "\n"; + << briefValuePrinter.ost.str() << "\n"; } } return ost.str(); @@ -453,14 +439,11 @@ std::string dumpAllSnapshots(const MetricManager& mm, MetricLockGuard lockGuard(mm.getMetricLock()); \ BriefValuePrinter briefValuePrinter; \ if (period == -1) { \ - mm.visit(lockGuard, mm.getActiveMetrics(lockGuard), \ - briefValuePrinter, "snapper"); \ + mm.visit(lockGuard, mm.getActiveMetrics(lockGuard), briefValuePrinter, "snapper"); \ } else if (period == 0) { \ - mm.visit(lockGuard, mm.getTotalMetricSnapshot(lockGuard), \ - briefValuePrinter, "snapper"); \ + mm.visit(lockGuard, mm.getTotalMetricSnapshot(lockGuard), briefValuePrinter, "snapper"); \ } else { \ - mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, period), \ - briefValuePrinter, "snapper"); \ + mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, period), briefValuePrinter, "snapper"); \ } \ EXPECT_EQ(std::string(expected), briefValuePrinter.ost.str()) << dumpAllSnapshots(mm, "snapper"); \ } @@ -475,9 +458,8 @@ std::string dumpAllSnapshots(const MetricManager& mm, TEST_F(MetricManagerTest, test_snapshots) { - FastOS_ThreadPool pool; - FakeTimer* timer = new FakeTimer(1000); - std::unique_ptr<MetricManager::Timer> timerImpl(timer); + auto timerImpl = std::make_unique<FakeTimer>(1000); + FakeTimer & timer = *timerImpl; TestMetricSet mySet; MetricManager mm(std::move(timerImpl)); { @@ -491,8 +473,7 @@ TEST_F(MetricManagerTest, test_snapshots) "consumer[0].tags[0] snaptest\n" "consumer[1].name log\n" "consumer[1].tags[1]\n" - "consumer[1].tags[0] snaptest\n"), - pool); + "consumer[1].tags[0] snaptest\n")); MetricNameVisitor visitor; { MetricLockGuard lockGuard(mm.getMetricLock()); @@ -525,7 +506,7 @@ TEST_F(MetricManagerTest, test_snapshots) mySet.val10.a.val1.addValue(7); mySet.val10.a.val2.addValue(2); mySet.val10.b.val1.addValue(1); - timer->add_time(5 * 60); + timer.add_time(5 * 60); ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60); ASSERT_VALUES(mm, 5 * 60, "2,4,4,1,7,9,1,1,8,2,10"); ASSERT_VALUES(mm, 60 * 60, ""); @@ -539,7 +520,7 @@ TEST_F(MetricManagerTest, test_snapshots) mySet.val10.a.val1.addValue(8); mySet.val10.a.val2.addValue(3); mySet.val10.b.val1.addValue(2); - timer->add_time(5 * 60); + timer.add_time(5 * 60); ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60 * 2); ASSERT_VALUES(mm, 5 * 60, "4,5,5,1,8,11,2,2,10,3,13"); ASSERT_VALUES(mm, 60 * 60, ""); @@ -547,7 +528,7 @@ TEST_F(MetricManagerTest, test_snapshots) // Adding another five minute period where nothing have happened. // Metric for last 5 minutes should be 0. - timer->add_time(5 * 60); + timer.add_time(5 * 60); ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60 * 3); ASSERT_VALUES(mm, 5 * 60, "0,0,0,0,0,0,0,0,0,0,0"); ASSERT_VALUES(mm, 60 * 60, ""); @@ -558,7 +539,7 @@ TEST_F(MetricManagerTest, test_snapshots) mySet.val6.addValue(6); for (uint32_t i=0; i<9; ++i) { // 9 x 5 minutes. Avoid snapshot bumping // due to taking snapshots in the past - timer->add_time(5 * 60); + timer.add_time(5 * 60); ASSERT_PROCESS_TIME(mm, 1000 + 5 * 60 * (4 + i)); } ASSERT_VALUES(mm, 5 * 60, "0,0,0,0,0,0,0,0,0,0,0"); @@ -573,89 +554,10 @@ TEST_F(MetricManagerTest, test_snapshots) ASSERT_VALUES(mm, 0 * 60, "0,0,0,0,0,0,0,0,0,0,0"); } -TEST_F(MetricManagerTest, test_xml_output) -{ - FastOS_ThreadPool pool; - FakeTimer* timer = new FakeTimer(1000); - std::unique_ptr<MetricManager::Timer> timerImpl(timer); - MetricManager mm(std::move(timerImpl)); - TestMetricSet mySet; - { - MetricLockGuard lockGuard(mm.getMetricLock()); - mm.registerMetric(lockGuard, mySet.set); - } - - // Initialize metric manager to get snapshots created. - mm.init(ConfigUri("raw:" - "consumer[2]\n" - "consumer[0].name snapper\n" - "consumer[0].tags[1]\n" - "consumer[0].tags[0] snaptest\n" - "consumer[1].name log\n" - "consumer[1].tags[1]\n" - "consumer[1].tags[0] snaptest\n"), - pool); - - takeSnapshots(mm, 1000); - - // Adding metrics to have some values in them - mySet.val6.addValue(2); - mySet.val9.val1.addValue(4); - mySet.val10.count.inc(); - mySet.val10.a.val1.addValue(7); - mySet.val10.a.val2.addValue(2); - mySet.val10.b.val1.addValue(1); - - timer->set_time(1300); - takeSnapshots(mm, 1300); - - std::string expected( - "'<snapshot name=\"5 minute\" from=\"1000\" to=\"1300\" period=\"300\">\n" - " <temp>\n" - " <val6 average=\"2\" last=\"2\" min=\"2\" max=\"2\" count=\"1\"/>\n" - " <sub>\n" - " <val1 average=\"4\" last=\"4\" min=\"4\" max=\"4\" count=\"1\"/>\n" - " <valsum average=\"4\" last=\"4\" min=\"4\" max=\"4\" count=\"1\"/>\n" - " </sub>\n" - " <multisub>\n" - " <count count=\"1\"/>\n" - " <a>\n" - " <val1 average=\"7\" last=\"7\" min=\"7\" max=\"7\" count=\"1\"/>\n" - " <valsum average=\"9\" last=\"9\"/>\n" - " </a>\n" - " <b>\n" - " <val1 average=\"1\" last=\"1\" min=\"1\" max=\"1\" count=\"1\"/>\n" - " <valsum average=\"1\" last=\"1\" min=\"1\" max=\"1\" count=\"1\"/>\n" - " </b>\n" - " <sum>\n" - " <val1 average=\"8\" last=\"8\"/>\n" - " <val2 average=\"2\" last=\"2\" min=\"2\" max=\"2\" count=\"1\"/>\n" - " <valsum average=\"10\" last=\"10\"/>\n" - " </sum>\n" - " </multisub>\n" - " </temp>\n" - "</snapshot>'"); - - std::ostringstream ost; - vespalib::XmlOutputStream xos(ost, " "); - XmlWriter writer(xos, 300, 0); - { - MetricLockGuard lockGuard(mm.getMetricLock()); - mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, 300, false), - writer, "snapper"); - } - std::string actual(ost.str()); - // Not bothering to match all the nitty gritty details as it will test - // more than it needs to. Just be here in order to check on XML output - // easily if needed. - EXPECT_EQ(expected, "'" + actual + "'"); -} - TEST_F(MetricManagerTest, test_json_output) { - FastOS_ThreadPool pool; - FakeTimer* timer = new FakeTimer(1000); - std::unique_ptr<MetricManager::Timer> timerImpl(timer); + auto timerImpl = std::make_unique<FakeTimer>(1000); + FakeTimer & timer = *timerImpl; MetricManager mm(std::move(timerImpl)); TestMetricSet mySet; { @@ -668,8 +570,7 @@ TEST_F(MetricManagerTest, test_json_output) "consumer[1]\n" "consumer[0].name snapper\n" "consumer[0].tags[1]\n" - "consumer[0].tags[0] snaptest\n"), - pool); + "consumer[0].tags[0] snaptest\n")); takeSnapshots(mm, 1000); @@ -681,7 +582,7 @@ TEST_F(MetricManagerTest, test_json_output) mySet.val10.a.val2.addValue(2); mySet.val10.b.val1.addValue(1); - timer->set_time(1300); + timer.set_time(1300); takeSnapshots(mm, 1300); // Create json output @@ -690,8 +591,7 @@ TEST_F(MetricManagerTest, test_json_output) JsonWriter writer(jsonStream); { MetricLockGuard lockGuard(mm.getMetricLock()); - mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, 300, false), - writer, "snapper"); + mm.visit(lockGuard, mm.getMetricSnapshot(lockGuard, 300, false), writer, "snapper"); } jsonStream.finalize(); std::string jsonData = as.str(); @@ -743,14 +643,12 @@ namespace { struct MetricSnapshotTestFixture { MetricManagerTest& test; - FastOS_ThreadPool pool; FakeTimer* timer; MetricManager manager; MetricSet& mset; MetricSnapshotTestFixture(MetricManagerTest& callerTest, MetricSet& metricSet) : test(callerTest), - pool(), timer(new FakeTimer(1000)), manager(std::unique_ptr<MetricManager::Timer>(timer)), mset(metricSet) @@ -765,8 +663,7 @@ struct MetricSnapshotTestFixture "consumer[1]\n" "consumer[0].name snapper\n" "consumer[0].addedmetrics[1]\n" - "consumer[0].addedmetrics[0] *\n"), - pool); + "consumer[0].addedmetrics[0] *\n")); test.takeSnapshots(manager, 1000); } @@ -783,22 +680,19 @@ struct MetricSnapshotTestFixture JsonWriter writer(jsonStream); { MetricLockGuard lockGuard(manager.getMetricLock()); - manager.visit(lockGuard, manager.getMetricSnapshot(lockGuard, 300, false), - writer, "snapper"); + manager.visit(lockGuard, manager.getMetricSnapshot(lockGuard, 300, false), writer, "snapper"); } jsonStream.finalize(); return as.str(); } - std::string renderLastSnapshotAsText( - const std::string& matchPattern = ".*") const + std::string renderLastSnapshotAsText(const std::string& matchPattern = ".*") const { std::ostringstream ss; TextWriter writer(ss, 300, matchPattern, true); { MetricLockGuard lockGuard(manager.getMetricLock()); - manager.visit(lockGuard, manager.getMetricSnapshot(lockGuard, 300, false), - writer, "snapper"); + manager.visit(lockGuard, manager.getMetricSnapshot(lockGuard, 300, false), writer, "snapper"); } return ss.str(); } @@ -829,8 +723,7 @@ public: } std::string nthMetricDimension(size_t metricIndex, const std::string& key) { - return nthMetric(metricIndex)["dimensions"][key] - .asString().make_string(); + return nthMetric(metricIndex)["dimensions"][key].asString().make_string(); } // Verify that the nth metric has the given name and the given set of @@ -853,7 +746,7 @@ JsonMetricWrapper::JsonMetricWrapper(const std::string& jsonText) { vespalib::slime::JsonFormat::decode(vespalib::Memory(jsonText), _tree); } -JsonMetricWrapper::~JsonMetricWrapper() { } +JsonMetricWrapper::~JsonMetricWrapper() = default; struct DimensionTestMetricSet : MetricSet { @@ -861,7 +754,7 @@ struct DimensionTestMetricSet : MetricSet LongCountMetric val2; DimensionTestMetricSet(MetricSet* owner = nullptr); - ~DimensionTestMetricSet(); + ~DimensionTestMetricSet() override; }; DimensionTestMetricSet::DimensionTestMetricSet(MetricSet* owner) @@ -869,7 +762,7 @@ DimensionTestMetricSet::DimensionTestMetricSet(MetricSet* owner) val1("val1", {{"tag1"}}, "val1 desc", this), val2("val2", {{"baz", "superbaz"}}, "val2 desc", this) { } -DimensionTestMetricSet::~DimensionTestMetricSet() { } +DimensionTestMetricSet::~DimensionTestMetricSet() = default; } @@ -986,10 +879,7 @@ TEST_F(MetricManagerTest, json_output_can_have_multiple_sets_with_same_name) TEST_F(MetricManagerTest, test_text_output) { - FastOS_ThreadPool pool; - FakeTimer* timer = new FakeTimer(1000); - std::unique_ptr<MetricManager::Timer> timerImpl(timer); - MetricManager mm(std::move(timerImpl)); + MetricManager mm(std::make_unique<FakeTimer>(1000)); TestMetricSet mySet; { MetricLockGuard lockGuard(mm.getMetricLock()); @@ -1010,8 +900,7 @@ TEST_F(MetricManagerTest, test_text_output) "consumer[0].tags[0] snaptest\n" "consumer[1].name log\n" "consumer[1].tags[1]\n" - "consumer[1].tags[0] snaptest\n"), - pool); + "consumer[1].tags[0] snaptest\n")); std::string expected( "snapshot \"Active metrics showing updates since last snapshot\" from 1000 to 0 period 0\n" "temp.val6 average=2 last=2 min=2 max=2 count=1 total=2\n" @@ -1063,10 +952,7 @@ namespace { std::mutex& _output_mutex; FakeTimer& _timer; - MyUpdateHook(std::ostringstream& output, - std::mutex& output_mutex, - const char* name, - FakeTimer& timer) + MyUpdateHook(std::ostringstream& output, std::mutex& output_mutex, const char* name, FakeTimer& timer) : UpdateHook(name), _output(output), _output_mutex(output_mutex), @@ -1076,7 +962,7 @@ namespace { void updateMetrics(const MetricLockGuard & ) override { std::lock_guard lock(_output_mutex); // updateMetrics() called from metric manager thread - _output << _timer.getTime() << ": " << getName() << " called\n"; + _output << vespalib::count_s(_timer.getTime().time_since_epoch()) << ": " << getName() << " called\n"; } }; } @@ -1085,9 +971,8 @@ TEST_F(MetricManagerTest, test_update_hooks) { std::mutex output_mutex; std::ostringstream output; - FastOS_ThreadPool pool; - FakeTimer* timer = new FakeTimer(1000); - std::unique_ptr<MetricManager::Timer> timerImpl(timer); + auto timerImpl = std::make_unique<FakeTimer>(1000); + FakeTimer & timer = *timerImpl; // Add a metric set just so one exist TestMetricSet mySet; MetricManager mm(std::move(timerImpl)); @@ -1096,9 +981,9 @@ TEST_F(MetricManagerTest, test_update_hooks) mm.registerMetric(lockGuard, mySet.set); } - MyUpdateHook preInitShort(output, output_mutex, "BIS", *timer); - MyUpdateHook preInitLong(output, output_mutex, "BIL", *timer); - MyUpdateHook preInitInfinite(output, output_mutex, "BII", *timer); + MyUpdateHook preInitShort(output, output_mutex, "BIS", timer); + MyUpdateHook preInitLong(output, output_mutex, "BIL", timer); + MyUpdateHook preInitInfinite(output, output_mutex, "BII", timer); mm.addMetricUpdateHook(preInitShort, 5); mm.addMetricUpdateHook(preInitLong, 300); mm.addMetricUpdateHook(preInitInfinite, 0); @@ -1114,59 +999,58 @@ TEST_F(MetricManagerTest, test_update_hooks) "consumer[0].tags[0] snaptest\n" "consumer[1].name log\n" "consumer[1].tags[1]\n" - "consumer[1].tags[0] snaptest\n"), - pool); + "consumer[1].tags[0] snaptest\n")); output << "Init done\n"; - MyUpdateHook postInitShort(output, output_mutex, "AIS", *timer); - MyUpdateHook postInitLong(output, output_mutex, "AIL", *timer); - MyUpdateHook postInitInfinite(output, output_mutex, "AII", *timer); + MyUpdateHook postInitShort(output, output_mutex, "AIS", timer); + MyUpdateHook postInitLong(output, output_mutex, "AIL", timer); + MyUpdateHook postInitInfinite(output, output_mutex, "AII", timer); mm.addMetricUpdateHook(postInitShort, 5); mm.addMetricUpdateHook(postInitLong, 400); mm.addMetricUpdateHook(postInitInfinite, 0); // After 5 seconds the short ones should get another. - timer->set_time(1006); + timer.set_time(1006); waitForTimeProcessed(mm, 1006); // After 4 more seconds the short ones should get another // since last update was a second late. (Stable periods, process time // should not affect how often they are updated) - timer->set_time(1010); + timer.set_time(1010); waitForTimeProcessed(mm, 1010); // Bumping considerably ahead, such that next update is in the past, // we should only get one update called in this period. - timer->set_time(1200); + timer.set_time(1200); waitForTimeProcessed(mm, 1200); // No updates at this time. - timer->set_time(1204); + timer.set_time(1204); waitForTimeProcessed(mm, 1204); // Give all hooks an update mm.updateMetrics(true); // Last update should not have interfered with periods - timer->set_time(1205); + timer.set_time(1205); waitForTimeProcessed(mm, 1205); // Time is just ahead of a snapshot. - timer->set_time(1299); + timer.set_time(1299); waitForTimeProcessed(mm, 1299); // At time 1300 we are at a 5 minute snapshot bump // All hooks should thus get an update. The one with matching period // should only get one - timer->set_time(1300); + timer.set_time(1300); waitForTimeProcessed(mm, 1300); // The snapshot time currently doesn't count for the metric at period // 400. It will get an event at this time. - timer->set_time(1450); + timer.set_time(1450); waitForTimeProcessed(mm, 1450); std::string expected( diff --git a/metrics/src/tests/snapshottest.cpp b/metrics/src/tests/snapshottest.cpp index 22eb3587eff..4d2ea96c36d 100644 --- a/metrics/src/tests/snapshottest.cpp +++ b/metrics/src/tests/snapshottest.cpp @@ -122,15 +122,15 @@ struct TestMetricSet : public MetricSet { SubMetricSet set2; SumMetric<SubMetricSet> setSum; - TestMetricSet(vespalib::stringref name, MetricSet* owner = 0); + TestMetricSet(vespalib::stringref name); ~TestMetricSet(); void incValues(); }; -TestMetricSet::TestMetricSet(vespalib::stringref name, MetricSet* owner) - : MetricSet(name, {}, "", owner), +TestMetricSet::TestMetricSet(vespalib::stringref name) + : MetricSet(name, {}, "", nullptr), set1("set1", this), set2("set2", this), setSum("setSum", {}, "", this) @@ -149,7 +149,7 @@ TestMetricSet::incValues() { struct FakeTimer : public MetricManager::Timer { uint32_t _timeInSecs; FakeTimer() : _timeInSecs(1) {} - time_t getTime() const override { return _timeInSecs; } + time_point getTime() const override { return time_point(vespalib::from_s(_timeInSecs)); } }; void ASSERT_VALUE(int32_t value, const MetricSnapshot & snapshot, const char *name) __attribute__((noinline)); @@ -176,16 +176,14 @@ TEST_F(SnapshotTest, test_snapshot_two_days) TestMetricSet set("test"); FakeTimer* timer; - FastOS_ThreadPool threadPool; - MetricManager mm( - std::unique_ptr<MetricManager::Timer>(timer = new FakeTimer)); + MetricManager mm(std::unique_ptr<MetricManager::Timer>(timer = new FakeTimer)); { MetricLockGuard lockGuard(mm.getMetricLock()); mm.registerMetric(lockGuard, set); } mm.init(config::ConfigUri("raw:consumer[1]\n" "consumer[0].name \"log\""), - threadPool, false); + false); tick(mm, timer->_timeInSecs * 1000); for (uint32_t days=0; days<2; ++days) { @@ -216,10 +214,9 @@ TEST_F(SnapshotTest, test_snapshot_two_days) << "\n"; */ - const MetricSnapshot* snap = 0; // active snapshot MetricLockGuard lockGuard(mm.getMetricLock()); - snap = &mm.getActiveMetrics(lockGuard); + const MetricSnapshot* snap = &mm.getActiveMetrics(lockGuard); ASSERT_VALUE(0, *snap, "test.set1.set1.count1"); ASSERT_VALUE(0, *snap, "test.set1.set1.countSum"); diff --git a/metrics/src/tests/stresstest.cpp b/metrics/src/tests/stresstest.cpp index e942d47b9de..afabf91d5c9 100644 --- a/metrics/src/tests/stresstest.cpp +++ b/metrics/src/tests/stresstest.cpp @@ -74,25 +74,29 @@ OuterMetricSet::OuterMetricSet(MetricSet* owner) OuterMetricSet::~OuterMetricSet() = default; -struct Hammer : public document::Runnable { +struct Hammer { using UP = std::unique_ptr<Hammer>; OuterMetricSet& _metrics; - - Hammer(OuterMetricSet& metrics,FastOS_ThreadPool& threadPool) - : _metrics(metrics) + std::atomic<bool> _stop_requested; + std::thread _thread; + + Hammer(OuterMetricSet& metrics) + : _metrics(metrics), + _stop_requested(false), + _thread() { - start(threadPool); + _thread = std::thread([this](){run();}); } ~Hammer() { - stop(); - join(); + _stop_requested = true; + _thread.join(); //std::cerr << "Loadgiver thread joined\n"; } - void run() override { + void run() { uint64_t i = 0; - while (running()) { + while (!_stop_requested.load(std::memory_order_relaxed)) { ++i; setMetrics(i, _metrics._inner1); setMetrics(i + 3, _metrics._inner2); @@ -114,10 +118,9 @@ TEST(StressTest, test_stress) OuterMetricSet metrics; LOG(info, "Starting load givers"); - FastOS_ThreadPool threadPool; std::vector<Hammer::UP> hammers; for (uint32_t i=0; i<10; ++i) { - hammers.push_back(std::make_unique<Hammer>(metrics, threadPool)); + hammers.push_back(std::make_unique<Hammer>(metrics)); } LOG(info, "Waiting to let loadgivers hammer a while"); std::this_thread::sleep_for(5s); diff --git a/metrics/src/vespa/metrics/CMakeLists.txt b/metrics/src/vespa/metrics/CMakeLists.txt index 2441bf95c0b..62254a8d620 100644 --- a/metrics/src/vespa/metrics/CMakeLists.txt +++ b/metrics/src/vespa/metrics/CMakeLists.txt @@ -18,7 +18,6 @@ vespa_add_library(metrics updatehook.cpp valuemetric.cpp valuemetricvalues.cpp - xmlwriter.cpp $<TARGET_OBJECTS:metrics_common> INSTALL lib64 diff --git a/metrics/src/vespa/metrics/metricmanager.cpp b/metrics/src/vespa/metrics/metricmanager.cpp index ae75968e605..df83001a4e2 100644 --- a/metrics/src/vespa/metrics/metricmanager.cpp +++ b/metrics/src/vespa/metrics/metricmanager.cpp @@ -22,25 +22,30 @@ LOG_SETUP(".metrics.manager"); namespace metrics { using Config = MetricsmanagerConfig; +using vespalib::IllegalStateException; +using vespalib::IllegalArgumentException; +using vespalib::make_string_short::fmt; +using vespalib::count_ms; +using vespalib::count_s; +using vespalib::from_s; MetricManager::ConsumerSpec::ConsumerSpec() = default; MetricManager::ConsumerSpec::~ConsumerSpec() = default; -time_t +time_point MetricManager::Timer::getTime() const { - return vespalib::count_s(vespalib::system_clock::now().time_since_epoch()); + return vespalib::system_clock::now(); } void MetricManager::assertMetricLockLocked(const MetricLockGuard& g) const { if ( ! g.owns(_waiter)) { - throw vespalib::IllegalArgumentException("Given lock does not lock the metric lock.", VESPA_STRLOC); + throw IllegalArgumentException("Given lock does not lock the metric lock.", VESPA_STRLOC); } } void -MetricManager::ConsumerSpec::print(std::ostream& out, bool verbose, - const std::string& indent) const +MetricManager::ConsumerSpec::print(std::ostream& out, bool verbose, const std::string& indent) const { (void) verbose; out << "ConsumerSpec("; @@ -63,6 +68,10 @@ MetricManager::ConsumerSpec::addMemoryUsage(MemoryConsumption& mc) const } } +MetricManager::MetricManager() + : MetricManager(std::make_unique<Timer>()) +{ } + MetricManager::MetricManager(std::unique_ptr<Timer> timer) : _activeMetrics("Active metrics showing updates since last snapshot"), _configSubscriber(), @@ -70,9 +79,7 @@ MetricManager::MetricManager(std::unique_ptr<Timer> timer) _config(), _consumerConfig(), _snapshots(), - _totalMetrics(std::make_shared<MetricSnapshot>( - "Empty metrics before init", 0, _activeMetrics.getMetrics(), - false)), + _totalMetrics(std::make_shared<MetricSnapshot>("Empty metrics before init", 0, _activeMetrics.getMetrics(), false)), _timer(std::move(timer)), _lastProcessedTime(0), _snapshotUnsetMetrics(false), @@ -82,7 +89,9 @@ MetricManager::MetricManager(std::unique_ptr<Timer> timer) _snapshotHookLatency("snapshothooklatency", {}, "Time in ms used to update a single snapshot hook", &_metricManagerMetrics), _resetLatency("resetlatency", {}, "Time in ms used to reset all metrics.", &_metricManagerMetrics), _snapshotLatency("snapshotlatency", {}, "Time in ms used to take a snapshot", &_metricManagerMetrics), - _sleepTimes("sleeptime", {}, "Time in ms worker thread is sleeping", &_metricManagerMetrics) + _sleepTimes("sleeptime", {}, "Time in ms worker thread is sleeping", &_metricManagerMetrics), + _stop_requested(false), + _thread() { registerMetric(getMetricLock(), _metricManagerMetrics); } @@ -95,15 +104,14 @@ MetricManager::~MetricManager() void MetricManager::stop() { - if (!running()) { - return; // Let stop() be idempotent. - } - Runnable::stop(); + request_stop(); { MetricLockGuard sync(_waiter); _cond.notify_all(); } - join(); + if (_thread.joinable()) { + _thread.join(); + } } void @@ -113,7 +121,7 @@ MetricManager::addMetricUpdateHook(UpdateHook& hook, uint32_t period) std::lock_guard sync(_waiter); // If we've already initialized manager, log period has been set. // In this case. Call first time after period - hook._nextCall = _timer->getTime() + period; + hook._nextCall = count_s(_timer->getTime().time_since_epoch()) + period; if (period == 0) { for (UpdateHook * sHook : _snapshotUpdateHooks) { if (sHook == &hook) { @@ -161,12 +169,11 @@ MetricManager::isInitialized() const { } void -MetricManager::init(const config::ConfigUri & uri, FastOS_ThreadPool& pool, bool startThread) +MetricManager::init(const config::ConfigUri & uri, bool startThread) { if (isInitialized()) { - throw vespalib::IllegalStateException( - "The metric manager have already been initialized. " - "It can only be initialized once.", VESPA_STRLOC); + throw IllegalStateException("The metric manager have already been initialized. " + "It can only be initialized once.", VESPA_STRLOC); } LOG(debug, "Initializing metric manager."); _configSubscriber = std::make_unique<config::ConfigSubscriber>(uri.getContext()); @@ -175,7 +182,7 @@ MetricManager::init(const config::ConfigUri & uri, FastOS_ThreadPool& pool, bool configure(getMetricLock(), _configHandle->getConfig()); LOG(debug, "Starting worker thread, waiting for first iteration to complete."); if (startThread) { - Runnable::start(pool); + _thread = std::thread([this](){run();}); // Wait for first iteration to have completed, such that it is safe // to access snapshots afterwards. MetricLockGuard sync(_waiter); @@ -232,8 +239,10 @@ struct ConsumerMetricBuilder : public MetricVisitor { bool nameRemoved; uint32_t metricCount; - Result() : tagAdded(false), tagRemoved(false), - nameAdded(false), nameRemoved(false), metricCount(0) {} + Result() + : tagAdded(false), tagRemoved(false), + nameAdded(false), nameRemoved(false), metricCount(0) + {} }; std::list<Result> result; @@ -352,9 +361,7 @@ ConsumerMetricBuilder::~ConsumerMetricBuilder() = default; void MetricManager::checkMetricsAltered(const MetricLockGuard & guard) { - if (_activeMetrics.getMetrics().isRegistrationAltered() - || _consumerConfigChanged) - { + if (_activeMetrics.getMetrics().isRegistrationAltered() || _consumerConfigChanged) { handleMetricsAltered(guard); } } @@ -384,8 +391,8 @@ MetricManager::handleMetricsAltered(const MetricLockGuard & guard) } LOG(debug, "Recreating snapshots to include altered metrics"); _totalMetrics->recreateSnapshot(_activeMetrics.getMetrics(), _snapshotUnsetMetrics); - for (uint32_t i=0; i<_snapshots.size(); ++i) { - _snapshots[i]->recreateSnapshot(_activeMetrics.getMetrics(), _snapshotUnsetMetrics); + for (const auto & snapshot: _snapshots) { + snapshot->recreateSnapshot(_activeMetrics.getMetrics(), _snapshotUnsetMetrics); } LOG(debug, "Setting new consumer config. Clearing dirty flag"); _consumerConfig.swap(configMap); @@ -393,24 +400,25 @@ MetricManager::handleMetricsAltered(const MetricLockGuard & guard) } namespace { - bool setSnapshotName(std::ostream& out, const char* name, uint32_t length, uint32_t period) - { - if (length % period != 0) return false; - out << (length / period) << ' ' << name; - if (length / period != 1) out << "s"; - return true; - } + +bool +setSnapshotName(std::ostream& out, const char* name, uint32_t length, uint32_t period) { + if (length % period != 0) return false; + out << (length / period) << ' ' << name; + if (length / period != 1) out << "s"; + return true; +} + } std::vector<MetricManager::SnapSpec> MetricManager::createSnapshotPeriods(const Config& config) { std::vector<SnapSpec> result; - try{ - for (uint32_t i=0; i<config.snapshot.periods.size(); ++i) { - uint32_t length = config.snapshot.periods[i]; + try { + for (auto length : config.snapshot.periods) { if (length < 1) - throw vespalib::IllegalStateException("Snapshot periods must be positive numbers", VESPA_STRLOC); + throw IllegalStateException("Snapshot periods must be positive numbers", VESPA_STRLOC); std::ostringstream name; if (setSnapshotName(name, "week", length, 60 * 60 * 24 * 7)) { } else if (setSnapshotName(name, "day", length, 60 * 60 * 24)) { @@ -424,16 +432,13 @@ MetricManager::createSnapshotPeriods(const Config& config) for (uint32_t i=1; i<result.size(); ++i) { if (result[i].first % result[i-1].first != 0) { std::ostringstream ost; - ost << "Period " << result[i].first - << " is not a multiplum of period " + ost << "Period " << result[i].first << " is not a multiplum of period " << result[i-1].first << " which is needs to be."; - throw vespalib::IllegalStateException( - ost.str(), VESPA_STRLOC); + throw IllegalStateException(ost.str(), VESPA_STRLOC); } } } catch (vespalib::Exception& e) { - LOG(warning, "Invalid snapshot periods specified. Using defaults: %s", - e.getMessage().c_str()); + LOG(warning, "Invalid snapshot periods specified. Using defaults: %s", e.getMessage().c_str()); result.clear(); } if (result.empty()) { @@ -453,8 +458,7 @@ MetricManager::configure(const MetricLockGuard & , std::unique_ptr<Config> confi std::ostringstream ost; config::OstreamConfigWriter w(ost); w.write(*config); - LOG(debug, "Received new config for metric manager: %s", - ost.str().c_str()); + LOG(debug, "Received new config for metric manager: %s", ost.str().c_str()); } if (_snapshots.empty()) { LOG(debug, "Initializing snapshots as this is first configure call"); @@ -462,21 +466,15 @@ MetricManager::configure(const MetricLockGuard & , std::unique_ptr<Config> confi // Set up snapshots only first time. We don't allow live reconfig // of snapshot periods. - time_t currentTime(_timer->getTime()); + time_t currentTime = count_s(_timer->getTime().time_since_epoch()); _activeMetrics.setFromTime(currentTime); uint32_t count = 1; - for (uint32_t i = 0; i< snapshotPeriods.size(); ++i) - { + for (uint32_t i = 0; i< snapshotPeriods.size(); ++i) { uint32_t nextCount = 1; if (i + 1 < snapshotPeriods.size()) { - nextCount = snapshotPeriods[i + 1].first - / snapshotPeriods[i].first; - if (snapshotPeriods[i + 1].first - % snapshotPeriods[i].first != 0) - { - throw vespalib::IllegalStateException( - "Snapshot periods must be multiplum of each other", - VESPA_STRLOC); + nextCount = snapshotPeriods[i + 1].first / snapshotPeriods[i].first; + if ((snapshotPeriods[i + 1].first % snapshotPeriods[i].first) != 0) { + throw IllegalStateException("Snapshot periods must be multiplum of each other",VESPA_STRLOC); } } _snapshots.push_back(std::make_shared<MetricSnapshotSet>( @@ -488,9 +486,7 @@ MetricManager::configure(const MetricLockGuard & , std::unique_ptr<Config> confi _totalMetrics = std::make_shared<MetricSnapshot>("All time snapshot", 0, _activeMetrics.getMetrics(), _snapshotUnsetMetrics); _totalMetrics->reset(currentTime); } - if (_config.get() == 0 - || _config->consumer.size() != config->consumer.size()) - { + if (_config.get() == 0 || (_config->consumer.size() != config->consumer.size())) { _consumerConfigChanged = true; } else { for (uint32_t i=0; i<_config->consumer.size(); ++i) { @@ -518,61 +514,55 @@ MetricManager::getConsumerSpec(const MetricLockGuard &, const Metric::String& co namespace { - struct ConsumerMetricVisitor : public MetricVisitor { - const MetricManager::ConsumerSpec& _metricsToMatch; - MetricVisitor& _client; +struct ConsumerMetricVisitor : public MetricVisitor { + const MetricManager::ConsumerSpec& _metricsToMatch; + MetricVisitor& _client; #ifdef VERIFY_ALL_METRICS_VISITED - std::set<Metric::String> _visitedMetrics; + std::set<Metric::String> _visitedMetrics; #endif - ConsumerMetricVisitor(const MetricManager::ConsumerSpec& spec, - MetricVisitor& clientVisitor) - : _metricsToMatch(spec), _client(clientVisitor) {} + ConsumerMetricVisitor(const MetricManager::ConsumerSpec& spec, MetricVisitor& clientVisitor) + : _metricsToMatch(spec), _client(clientVisitor) + {} - bool visitMetricSet(const MetricSet& metricSet, - bool autoGenerated) override - { - if (metricSet.isTopSet()) return true; - return (_metricsToMatch.contains(metricSet) - && _client.visitMetricSet(metricSet, autoGenerated)); - } - void doneVisitingMetricSet(const MetricSet& metricSet) override { - if (!metricSet.isTopSet()) { + bool visitMetricSet(const MetricSet& metricSet, bool autoGenerated) override { + if (metricSet.isTopSet()) return true; + return (_metricsToMatch.contains(metricSet) + && _client.visitMetricSet(metricSet, autoGenerated)); + } + void doneVisitingMetricSet(const MetricSet& metricSet) override { + if (!metricSet.isTopSet()) { #ifdef VERIFY_ALL_METRICS_VISITED - _visitedMetrics.insert(metricSet.getPath()); + _visitedMetrics.insert(metricSet.getPath()); #endif - _client.doneVisitingMetricSet(metricSet); - } + _client.doneVisitingMetricSet(metricSet); } - bool visitCountMetric(const AbstractCountMetric& metric, - bool autoGenerated) override - { - if (_metricsToMatch.contains(metric)) { + } + bool visitCountMetric(const AbstractCountMetric& metric, bool autoGenerated) override { + if (_metricsToMatch.contains(metric)) { #ifdef VERIFY_ALL_METRICS_VISITED - _visitedMetrics.insert(metric.getPath()); + _visitedMetrics.insert(metric.getPath()); #endif - return _client.visitCountMetric(metric, autoGenerated); - } - return true; + return _client.visitCountMetric(metric, autoGenerated); } - bool visitValueMetric(const AbstractValueMetric& metric, - bool autoGenerated) override - { - if (_metricsToMatch.contains(metric)) { + return true; + } + bool visitValueMetric(const AbstractValueMetric& metric, bool autoGenerated) override { + if (_metricsToMatch.contains(metric)) { #ifdef VERIFY_ALL_METRICS_VISITED - _visitedMetrics.insert(metric.getPath()); + _visitedMetrics.insert(metric.getPath()); #endif - return _client.visitValueMetric(metric, autoGenerated); - } - return true; + return _client.visitValueMetric(metric, autoGenerated); } - }; + return true; + } +}; } void -MetricManager::visit(const MetricLockGuard & guard, const MetricSnapshot& snapshot, MetricVisitor& visitor, - const std::string& consumer) const +MetricManager::visit(const MetricLockGuard & guard, const MetricSnapshot& snapshot, + MetricVisitor& visitor, const std::string& consumer) const { if (visitor.visitSnapshot(snapshot)) { if (consumer == "") { @@ -592,9 +582,7 @@ MetricManager::visit(const MetricLockGuard & guard, const MetricSnapshot& snapsh } #endif } else { - LOGBP(debug, - "Requested metrics for non-defined consumer '%s'.", - consumer.c_str()); + LOGBP(debug, "Requested metrics for non-defined consumer '%s'.", consumer.c_str()); } } visitor.doneVisitingSnapshot(snapshot); @@ -615,39 +603,31 @@ MetricManager::getSnapshotPeriods(const MetricLockGuard& l) const // Client should have grabbed metrics lock before doing this const MetricSnapshot& -MetricManager::getMetricSnapshot(const MetricLockGuard& l, - uint32_t period, bool getInProgressSet) const +MetricManager::getMetricSnapshot(const MetricLockGuard& l, uint32_t period, bool getInProgressSet) const { assertMetricLockLocked(l); - for (uint32_t i=0; i<_snapshots.size(); ++i) { - if (_snapshots[i]->getPeriod() == period) { - if (_snapshots[i]->getCount() == 1 && getInProgressSet) { - throw vespalib::IllegalStateException( - "No temporary snapshot for set " - + _snapshots[i]->getName(), VESPA_STRLOC); + for (const auto & snapshot : _snapshots) { + if (snapshot->getPeriod() == period) { + if (snapshot->getCount() == 1 && getInProgressSet) { + throw IllegalStateException("No temporary snapshot for set " + snapshot->getName(), VESPA_STRLOC); } - return _snapshots[i]->getSnapshot(getInProgressSet); + return snapshot->getSnapshot(getInProgressSet); } } - std::ostringstream ost; - ost << "No snapshot for period of length " << period << " exist."; - throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC); + throw IllegalArgumentException(fmt("No snapshot for period of length %u exist.", period), VESPA_STRLOC); } // Client should have grabbed metrics lock before doing this const MetricSnapshotSet& -MetricManager::getMetricSnapshotSet(const MetricLockGuard& l, - uint32_t period) const +MetricManager::getMetricSnapshotSet(const MetricLockGuard& l, uint32_t period) const { assertMetricLockLocked(l); - for (uint32_t i=0; i<_snapshots.size(); ++i) { - if (_snapshots[i]->getPeriod() == period) { - return *_snapshots[i]; + for (const auto & snapshot : _snapshots) { + if (snapshot->getPeriod() == period) { + return *snapshot; } } - std::ostringstream ost; - ost << "No snapshot set for period of length " << period << " exist."; - throw vespalib::IllegalArgumentException(ost.str(), VESPA_STRLOC); + throw IllegalArgumentException(fmt("No snapshot set for period of length %u exist.", period), VESPA_STRLOC); } void @@ -660,9 +640,8 @@ MetricManager::timeChangedNotification() const void MetricManager::updateMetrics(bool includeSnapshotOnlyHooks) { - LOG(debug, "Calling metric update hooks%s.", - includeSnapshotOnlyHooks ? ", including snapshot hooks" : ""); - // Ensure we're not in the way of the background thread + LOG(debug, "Calling metric update hooks%s.", includeSnapshotOnlyHooks ? ", including snapshot hooks" : ""); + // Ensure we're not in the way of the background thread MetricLockGuard sync(_waiter); LOG(debug, "Giving %zu periodic update hooks.", _periodicUpdateHooks.size()); updatePeriodicMetrics(sync, 0, true); @@ -676,30 +655,29 @@ MetricManager::updateMetrics(bool includeSnapshotOnlyHooks) time_t MetricManager::updatePeriodicMetrics(const MetricLockGuard & guard, time_t updateTime, bool outOfSchedule) { + assertMetricLockLocked(guard); time_t nextUpdateTime = std::numeric_limits<time_t>::max(); - time_t preTime = _timer->getTimeInMilliSecs(); + time_point preTime = _timer->getTimeInMilliSecs(); for (auto hook : _periodicUpdateHooks) { if (hook->_nextCall <= updateTime) { hook->updateMetrics(guard); if (hook->_nextCall + hook->_period < updateTime) { if (hook->_nextCall != 0) { - LOG(debug, "Updated hook %s at time %" PRIu64 ", but next " - "run in %u seconds have already passed as time" - " is %" PRIu64 ". Bumping next call to current " - "time + period.", + LOG(debug, "Updated hook %s at time %" PRIu64 ", but next run in %u seconds have already passed as " + "time is %" PRIu64 ". Bumping next call to current time + period.", hook->_name, static_cast<uint64_t>(hook->_nextCall), hook->_period, static_cast<uint64_t>(updateTime)); } hook->_nextCall = updateTime + hook->_period; } else { hook->_nextCall += hook->_period; } - time_t postTime = _timer->getTimeInMilliSecs(); - _periodicHookLatency.addValue(postTime - preTime); + time_point postTime = _timer->getTimeInMilliSecs(); + _periodicHookLatency.addValue(count_ms(postTime - preTime)); preTime = postTime; } else if (outOfSchedule) { hook->updateMetrics(guard); - time_t postTime = _timer->getTimeInMilliSecs(); - _periodicHookLatency.addValue(postTime - preTime); + time_point postTime = _timer->getTimeInMilliSecs(); + _periodicHookLatency.addValue(count_ms(postTime - preTime)); preTime = postTime; } nextUpdateTime = std::min(nextUpdateTime, hook->_nextCall); @@ -711,11 +689,12 @@ MetricManager::updatePeriodicMetrics(const MetricLockGuard & guard, time_t updat void MetricManager::updateSnapshotMetrics(const MetricLockGuard & guard) { - time_t preTime = _timer->getTimeInMilliSecs(); - for (auto it = _snapshotUpdateHooks.begin(); it != _snapshotUpdateHooks.end(); ++it) { - (**it).updateMetrics(guard); - time_t postTime = _timer->getTimeInMilliSecs(); - _snapshotHookLatency.addValue(postTime - preTime); + assertMetricLockLocked(guard); + time_point preTime = _timer->getTimeInMilliSecs(); + for (const auto & hook : _snapshotUpdateHooks) { + hook->updateMetrics(guard); + time_point postTime = _timer->getTimeInMilliSecs(); + _snapshotHookLatency.addValue(count_ms(postTime - preTime)); preTime = postTime; } } @@ -732,17 +711,17 @@ MetricManager::forceEventLogging() void MetricManager::reset(time_t currentTime) { - time_t preTime = _timer->getTimeInMilliSecs(); + time_point preTime = _timer->getTimeInMilliSecs(); // Resetting implies visiting metrics, which needs to grab metric lock // to avoid conflict with adding/removal of metrics std::lock_guard waiterLock(_waiter); _activeMetrics.reset(currentTime); - for (uint32_t i=0; i<_snapshots.size(); ++i) { - _snapshots[i]->reset(currentTime); + for (const auto & snapshot : _snapshots) { + snapshot->reset(currentTime); } _totalMetrics->reset(currentTime); - time_t postTime = _timer->getTimeInMilliSecs(); - _resetLatency.addValue(postTime - preTime); + time_point postTime = _timer->getTimeInMilliSecs(); + _resetLatency.addValue(count_ms(postTime - preTime)); } void @@ -754,7 +733,7 @@ MetricManager::run() // we constantly add next time to do something from the last timer. // For that to work, we need to initialize timers on first iteration // to set them to current time. - time_t currentTime = _timer->getTime(); + time_t currentTime = count_s(_timer->getTime().time_since_epoch()); for (auto & snapshot : _snapshots) { snapshot->setFromTime(currentTime); } @@ -763,13 +742,14 @@ MetricManager::run() } // Ensure correct time for first snapshot _snapshots[0]->getSnapshot().setToTime(currentTime); - while (!stopping()) { - currentTime = _timer->getTime(); + while (!stop_requested()) { + time_point now = _timer->getTime(); + currentTime = count_s(now.time_since_epoch()); time_t next = tick(sync, currentTime); if (currentTime < next) { - size_t ms = (next - currentTime) * 1000; - _cond.wait_for(sync, std::chrono::milliseconds(ms)); - _sleepTimes.addValue(ms); + vespalib::duration wait_time = from_s(next - currentTime); + _cond.wait_for(sync, wait_time); + _sleepTimes.addValue(count_ms(wait_time)); } else { _sleepTimes.addValue(0); } @@ -779,11 +759,10 @@ MetricManager::run() time_t MetricManager::tick(const MetricLockGuard & guard, time_t currentTime) { - LOG(spam, "Worker thread starting to process for time %" PRIu64 ".", - static_cast<uint64_t>(currentTime)); + LOG(spam, "Worker thread starting to process for time %" PRIu64 ".", static_cast<uint64_t>(currentTime)); // Check for new config and reconfigure - if (_configSubscriber.get() && _configSubscriber->nextConfigNow()) { + if (_configSubscriber && _configSubscriber->nextConfigNow()) { configure(guard, _configHandle->getConfig()); } @@ -810,8 +789,7 @@ MetricManager::tick(const MetricLockGuard & guard, time_t currentTime) // Do snapshotting if it is time if (nextWorkTime <= currentTime) takeSnapshots(guard, nextWorkTime); - _lastProcessedTime.store(nextWorkTime <= currentTime ? nextWorkTime : currentTime, - std::memory_order_relaxed); + _lastProcessedTime.store(nextWorkTime <= currentTime ? nextWorkTime : currentTime, std::memory_order_relaxed); LOG(spam, "Worker thread done with processing for time %" PRIu64 ".", static_cast<uint64_t>(_lastProcessedTime.load(std::memory_order_relaxed))); time_t next = _snapshots[0]->getPeriod() + _snapshots[0]->getToTime(); @@ -820,8 +798,9 @@ MetricManager::tick(const MetricLockGuard & guard, time_t currentTime) } void -MetricManager::takeSnapshots(const MetricLockGuard &, time_t timeToProcess) +MetricManager::takeSnapshots(const MetricLockGuard & guard, time_t timeToProcess) { + assertMetricLockLocked(guard); // If not time to do dump data from active snapshot yet, nothing to do if (!_snapshots[0]->timeForAnotherSnapshot(timeToProcess)) { LOG(spam, "Not time to process snapshot %s at time %" PRIu64 ". Current " @@ -831,7 +810,7 @@ MetricManager::takeSnapshots(const MetricLockGuard &, time_t timeToProcess) static_cast<uint64_t>(_snapshots[0]->getToTime())); return; } - time_t preTime = _timer->getTimeInMilliSecs(); + time_point preTime = _timer->getTimeInMilliSecs(); LOG(debug, "Updating %s snapshot and total metrics at time %" PRIu64 ".", _snapshots[0]->getName().c_str(), static_cast<uint64_t>(timeToProcess)); MetricSnapshot& firstTarget(_snapshots[0]->getNextTarget()); @@ -839,67 +818,54 @@ MetricManager::takeSnapshots(const MetricLockGuard &, time_t timeToProcess) _activeMetrics.addToSnapshot(firstTarget, false, timeToProcess); _activeMetrics.addToSnapshot(*_totalMetrics, false, timeToProcess); _activeMetrics.reset(timeToProcess); - LOG(debug, "After snapshotting, " - "active metrics goes from %" PRIu64 " to %" PRIu64", " + LOG(debug, "After snapshotting, active metrics goes from %" PRIu64 " to %" PRIu64", " "and 5 minute metrics goes from %" PRIu64 " to %" PRIu64".", static_cast<uint64_t>(_activeMetrics.getFromTime()), static_cast<uint64_t>(_activeMetrics.getToTime()), static_cast<uint64_t>(firstTarget.getFromTime()), static_cast<uint64_t>(firstTarget.getToTime())); // Update later snapshots if it is time for it for (uint32_t i=1; i<_snapshots.size(); ++i) { - LOG(debug, "Adding data from last snapshot to building snapshot of " - "next period snapshot %s.", + LOG(debug, "Adding data from last snapshot to building snapshot of next period snapshot %s.", _snapshots[i]->getName().c_str()); MetricSnapshot& target(_snapshots[i]->getNextTarget()); - _snapshots[i-1]->getSnapshot().addToSnapshot( - target, false, timeToProcess); + _snapshots[i-1]->getSnapshot().addToSnapshot(target, false, timeToProcess); target.setToTime(timeToProcess); if (!_snapshots[i]->haveCompletedNewPeriod(timeToProcess)) { - LOG(debug, "Not time to roll snapshot %s yet. %u of %u snapshot " - "taken at time %" PRIu64 ", and period of %u is not up " - "yet as we're currently processing for time %" PRIu64 ".", - _snapshots[i]->getName().c_str(), - _snapshots[i]->getBuilderCount(), - _snapshots[i]->getCount(), - static_cast<uint64_t> - (_snapshots[i]->getBuilderCount() * _snapshots[i]->getPeriod() - + _snapshots[i]->getFromTime()), - _snapshots[i]->getPeriod(), - static_cast<uint64_t>(timeToProcess)); + LOG(debug, "Not time to roll snapshot %s yet. %u of %u snapshot taken at time %" PRIu64 ", and period of %u " + "is not up yet as we're currently processing for time %" PRIu64 ".", + _snapshots[i]->getName().c_str(), _snapshots[i]->getBuilderCount(), _snapshots[i]->getCount(), + static_cast<uint64_t>(_snapshots[i]->getBuilderCount() * _snapshots[i]->getPeriod() + _snapshots[i]->getFromTime()), + _snapshots[i]->getPeriod(), static_cast<uint64_t>(timeToProcess)); break; } else { LOG(debug, "Rolled snapshot %s at time %" PRIu64 ".", - _snapshots[i]->getName().c_str(), - static_cast<uint64_t>(timeToProcess)); + _snapshots[i]->getName().c_str(), static_cast<uint64_t>(timeToProcess)); } } - time_t postTime = _timer->getTimeInMilliSecs(); - _snapshotLatency.addValue(postTime - preTime); + time_point postTime = _timer->getTimeInMilliSecs(); + _snapshotLatency.addValue(count_ms(postTime - preTime)); } MemoryConsumption::UP MetricManager::getMemoryConsumption(const MetricLockGuard & guard) const { - (void) guard; - MemoryConsumption::UP mc(new MemoryConsumption); + assertMetricLockLocked(guard); + auto mc = std::make_unique<MemoryConsumption>(); mc->_consumerCount += _consumerConfig.size(); - mc->_consumerMeta += (sizeof(ConsumerSpec::SP) + sizeof(ConsumerSpec)) - * _consumerConfig.size(); - for (auto it = _consumerConfig.begin(); it != _consumerConfig.end(); ++it) { - mc->_consumerId += mc->getStringMemoryUsage( - it->first, mc->_consumerIdUnique) - + sizeof(Metric::String); - it->second->addMemoryUsage(*mc); + mc->_consumerMeta += (sizeof(ConsumerSpec::SP) + sizeof(ConsumerSpec)) * _consumerConfig.size(); + for (const auto & consumer : _consumerConfig) { + mc->_consumerId += mc->getStringMemoryUsage(consumer.first, mc->_consumerIdUnique) + sizeof(Metric::String); + consumer.second->addMemoryUsage(*mc); } uint32_t preTotal = mc->getTotalMemoryUsage(); _activeMetrics.addMemoryUsage(*mc); uint32_t postTotal = mc->getTotalMemoryUsage(); mc->addSnapShotUsage("active", postTotal - preTotal); preTotal = postTotal; - for (uint32_t i=0; i<_snapshots.size(); ++i) { - _snapshots[i]->addMemoryUsage(*mc); + for (const auto & snapshot : _snapshots) { + snapshot->addMemoryUsage(*mc); postTotal = mc->getTotalMemoryUsage(); - mc->addSnapShotUsage(_snapshots[i]->getName(), postTotal - preTotal); + mc->addSnapShotUsage(snapshot->getName(), postTotal - preTotal); preTotal = postTotal; } _totalMetrics->addMemoryUsage(*mc); diff --git a/metrics/src/vespa/metrics/metricmanager.h b/metrics/src/vespa/metrics/metricmanager.h index 5f35c349f7f..c3ce37b451f 100644 --- a/metrics/src/vespa/metrics/metricmanager.h +++ b/metrics/src/vespa/metrics/metricmanager.h @@ -49,26 +49,26 @@ #include "valuemetric.h" #include "updatehook.h" #include <vespa/vespalib/stllike/hash_set.h> -#include <vespa/vespalib/util/document_runnable.h> #include <vespa/vespalib/util/jsonwriter.h> #include <vespa/metrics/config-metricsmanager.h> #include <vespa/config/subscription/configsubscriber.h> #include <vespa/config/subscription/configuri.h> #include <map> #include <list> +#include <thread> template class vespalib::hash_set<metrics::Metric::String>; namespace metrics { -class MetricManager : private document::Runnable +class MetricManager { public: struct Timer { virtual ~Timer() = default; - virtual time_t getTime() const; - virtual time_t getTimeInMilliSecs() const { return getTime() * 1000; } + virtual time_point getTime() const; + time_point getTimeInMilliSecs() const { return getTime(); } }; /** @@ -88,8 +88,7 @@ public: return (includedMetrics.find(m.getPath()) != includedMetrics.end()); } - void print(std::ostream& out, bool verbose, - const std::string& indent) const override; + void print(std::ostream& out, bool verbose, const std::string& indent) const override; void addMemoryUsage(MemoryConsumption&) const; }; @@ -119,10 +118,16 @@ private: LongAverageMetric _resetLatency; LongAverageMetric _snapshotLatency; LongAverageMetric _sleepTimes; + std::atomic<bool> _stop_requested; + std::thread _thread; + void request_stop() { _stop_requested.store(true, std::memory_order_relaxed); } + bool stop_requested() const { return _stop_requested.load(std::memory_order_relaxed); } + public: - MetricManager(std::unique_ptr<Timer> timer = std::make_unique<Timer>()); - ~MetricManager() override; + MetricManager(); + MetricManager(std::unique_ptr<Timer> timer); + ~MetricManager(); void stop(); @@ -194,7 +199,7 @@ public: * of consumers. readConfig() will start a config subscription. It should * not be called multiple times. */ - void init(const config::ConfigUri & uri, FastOS_ThreadPool&, bool startThread = true); + void init(const config::ConfigUri & uri, bool startThread = true); /** * Visit a given snapshot for a given consumer. (Empty consumer name means @@ -227,19 +232,16 @@ public: } /** While accessing the total metrics you should have the metric lock. */ - const MetricSnapshot& getTotalMetricSnapshot(const MetricLockGuard& l) const - { + const MetricSnapshot& getTotalMetricSnapshot(const MetricLockGuard& l) const { assertMetricLockLocked(l); return *_totalMetrics; } /** While accessing snapshots you should have the metric lock. */ - const MetricSnapshot& getMetricSnapshot( - const MetricLockGuard&, - uint32_t period, bool getInProgressSet = false) const; - const MetricSnapshotSet& getMetricSnapshotSet( - const MetricLockGuard&, uint32_t period) const; - bool hasTemporarySnapshot(const MetricLockGuard& l, uint32_t period) const - { return getMetricSnapshotSet(l, period).hasTemporarySnapshot(); } + const MetricSnapshot& getMetricSnapshot( const MetricLockGuard& guard, uint32_t period) const { + return getMetricSnapshot(guard, period, false); + } + const MetricSnapshot& getMetricSnapshot( const MetricLockGuard&, uint32_t period, bool getInProgressSet) const; + const MetricSnapshotSet& getMetricSnapshotSet(const MetricLockGuard&, uint32_t period) const; std::vector<uint32_t> getSnapshotPeriods(const MetricLockGuard& l) const; @@ -271,7 +273,7 @@ private: friend struct SnapshotTest; void configure(const MetricLockGuard & guard, std::unique_ptr<MetricsmanagerConfig> conf); - void run() override; + void run(); time_t tick(const MetricLockGuard & guard, time_t currentTime); /** * Utility function for updating periodic metrics. diff --git a/metrics/src/vespa/metrics/updatehook.h b/metrics/src/vespa/metrics/updatehook.h index 9fa0d52027e..58d9ef0d743 100644 --- a/metrics/src/vespa/metrics/updatehook.h +++ b/metrics/src/vespa/metrics/updatehook.h @@ -1,10 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include <vespa/vespalib/util/time.h> #include <mutex> namespace metrics { +using time_point = vespalib::system_time; + class MetricLockGuard { public: MetricLockGuard(std::mutex & mutex); diff --git a/metrics/src/vespa/metrics/xmlwriter.cpp b/metrics/src/vespa/metrics/xmlwriter.cpp deleted file mode 100644 index 11cb450e64d..00000000000 --- a/metrics/src/vespa/metrics/xmlwriter.cpp +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "xmlwriter.h" -#include "countmetric.h" -#include "metricset.h" -#include "metricsnapshot.h" -#include "valuemetric.h" -#include <vespa/vespalib/util/xmlstream.h> -#include <sstream> - -namespace metrics { - -XmlWriter::XmlWriter(vespalib::xml::XmlOutputStream& xos, - [[maybe_unused]] uint32_t period, int verbosity) - : _xos(xos), _verbosity(verbosity) {} - -bool -XmlWriter::visitSnapshot(const MetricSnapshot& snapshot) -{ - using namespace vespalib::xml; - _xos << XmlTag("snapshot") << XmlAttribute("name", snapshot.getName()) - << XmlAttribute("from", snapshot.getFromTime()) - << XmlAttribute("to", snapshot.getToTime()) - << XmlAttribute("period", snapshot.getPeriod()); - return true; -} - -void -XmlWriter::doneVisitingSnapshot(const MetricSnapshot&) -{ - using namespace vespalib::xml; - _xos << XmlEndTag(); -} - -bool -XmlWriter::visitMetricSet(const MetricSet& set, bool) -{ - using namespace vespalib::xml; - if (set.used() || _verbosity >= 2) { - _xos << XmlTag(set.getName(), XmlTagFlags::CONVERT_ILLEGAL_CHARACTERS); - printCommonXmlParts(set); - return true; - } - return false; -} -void -XmlWriter::doneVisitingMetricSet(const MetricSet&) { - using namespace vespalib::xml; - _xos << XmlEndTag(); -} - -bool -XmlWriter::visitCountMetric(const AbstractCountMetric& metric, bool) -{ - MetricValueClass::UP values(metric.getValues()); - if (!metric.inUse(*values) && _verbosity < 2) return true; - using namespace vespalib::xml; - std::ostringstream ost; - _xos << XmlTag(metric.getName(), XmlTagFlags::CONVERT_ILLEGAL_CHARACTERS) - << XmlAttribute(metric.sumOnAdd() - ? "count" : "value", values->toString("count")); - printCommonXmlParts(metric); - _xos << XmlEndTag(); - return true; -} - -bool -XmlWriter::visitValueMetric(const AbstractValueMetric& metric, bool) -{ - MetricValueClass::UP values(metric.getValues()); - if (!metric.inUse(*values) && _verbosity < 2) return true; - using namespace vespalib::xml; - _xos << XmlTag(metric.getName(), XmlTagFlags::CONVERT_ILLEGAL_CHARACTERS) - << XmlAttribute("average", values->getLongValue("count") == 0 - ? 0 : values->getDoubleValue("total") - / values->getDoubleValue("count")) - << XmlAttribute("last", values->toString("last")); - if (!metric.summedAverage()) { - if (values->getLongValue("count") > 0) { - _xos << XmlAttribute("min", values->toString("min")) - << XmlAttribute("max", values->toString("max")); - } - _xos << XmlAttribute("count", values->getLongValue("count")); - if (_verbosity >= 2) { - _xos << XmlAttribute("total", values->toString("total")); - } - } - printCommonXmlParts(metric); - _xos << XmlEndTag(); - return true; -} - -void -XmlWriter::printCommonXmlParts(const Metric& metric) const -{ - using namespace vespalib::xml; - const Metric::Tags& tags(metric.getTags()); - if (_verbosity >= 3 && tags.size() > 0) { - std::ostringstream ost; - // XXX print tag values as well - ost << tags[0].key(); - for (uint32_t i=1; i<tags.size(); ++i) { - ost << "," << tags[i].key(); - } - _xos << XmlAttribute("tags", ost.str()); - } - if (_verbosity >= 1 && !metric.getDescription().empty()) { - _xos << XmlAttribute("description", metric.getDescription()); - } -} - -} // metrics diff --git a/metrics/src/vespa/metrics/xmlwriter.h b/metrics/src/vespa/metrics/xmlwriter.h deleted file mode 100644 index a0e8a3efeea..00000000000 --- a/metrics/src/vespa/metrics/xmlwriter.h +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/metrics/metric.h> -#include <vespa/vespalib/util/xmlserializable.h> - -namespace metrics { - -class XmlWriter : public MetricVisitor { - vespalib::xml::XmlOutputStream& _xos; - int _verbosity; - -public: - XmlWriter(vespalib::xml::XmlOutputStream& xos, - uint32_t period, int verbosity); - - bool visitSnapshot(const MetricSnapshot&) override; - void doneVisitingSnapshot(const MetricSnapshot&) override; - bool visitMetricSet(const MetricSet& set, bool) override; - void doneVisitingMetricSet(const MetricSet&) override; - bool visitCountMetric(const AbstractCountMetric&, bool autoGenerated) override; - bool visitValueMetric(const AbstractValueMetric&, bool autoGenerated) override; - -private: - void printCommonXmlParts(const Metric& metric) const; -}; - -} - diff --git a/model-evaluation/abi-spec.json b/model-evaluation/abi-spec.json index a5bda6e1c21..667712d0daa 100644 --- a/model-evaluation/abi-spec.json +++ b/model-evaluation/abi-spec.json @@ -47,7 +47,9 @@ }, "ai.vespa.models.evaluation.Model" : { "superClass" : "java.lang.Object", - "interfaces" : [ ], + "interfaces" : [ + "java.lang.AutoCloseable" + ], "attributes" : [ "public" ], @@ -56,7 +58,8 @@ "public java.lang.String name()", "public java.util.List functions()", "public varargs ai.vespa.models.evaluation.FunctionEvaluator evaluatorOf(java.lang.String[])", - "public java.lang.String toString()" + "public java.lang.String toString()", + "public void close()" ], "fields" : [ ] }, @@ -67,12 +70,14 @@ "public" ], "methods" : [ + "public void <init>(com.yahoo.vespa.config.search.RankProfilesConfig, com.yahoo.vespa.config.search.core.RankingConstantsConfig, com.yahoo.vespa.config.search.core.RankingExpressionsConfig, com.yahoo.vespa.config.search.core.OnnxModelsConfig, com.yahoo.filedistribution.fileacquirer.FileAcquirer, ai.vespa.modelintegration.evaluator.OnnxRuntime)", "public void <init>(com.yahoo.vespa.config.search.RankProfilesConfig, com.yahoo.vespa.config.search.core.RankingConstantsConfig, com.yahoo.vespa.config.search.core.RankingExpressionsConfig, com.yahoo.vespa.config.search.core.OnnxModelsConfig, com.yahoo.filedistribution.fileacquirer.FileAcquirer)", "public void <init>(ai.vespa.models.evaluation.RankProfilesConfigImporter, com.yahoo.vespa.config.search.RankProfilesConfig, com.yahoo.vespa.config.search.core.RankingConstantsConfig, com.yahoo.vespa.config.search.core.RankingExpressionsConfig, com.yahoo.vespa.config.search.core.OnnxModelsConfig)", "public void <init>(java.util.Map)", "public java.util.Map models()", "public varargs ai.vespa.models.evaluation.FunctionEvaluator evaluatorOf(java.lang.String, java.lang.String[])", - "public ai.vespa.models.evaluation.Model requireModel(java.lang.String)" + "public ai.vespa.models.evaluation.Model requireModel(java.lang.String)", + "public void deconstruct()" ], "fields" : [ ] }, @@ -83,7 +88,7 @@ "public" ], "methods" : [ - "public void <init>(com.yahoo.filedistribution.fileacquirer.FileAcquirer)", + "public void <init>(com.yahoo.filedistribution.fileacquirer.FileAcquirer, ai.vespa.modelintegration.evaluator.OnnxRuntime)", "public java.util.Map importFrom(com.yahoo.vespa.config.search.RankProfilesConfig, com.yahoo.vespa.config.search.core.RankingConstantsConfig, com.yahoo.vespa.config.search.core.RankingExpressionsConfig, com.yahoo.vespa.config.search.core.OnnxModelsConfig)", "protected final java.lang.String readExpressionFromFile(java.io.File)", "protected com.yahoo.searchlib.rankingexpression.RankingExpression readExpressionFromFile(java.lang.String, com.yahoo.config.FileReference)", diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionReference.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionReference.java index 53592be7883..46134074137 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionReference.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/FunctionReference.java @@ -24,13 +24,13 @@ import java.util.regex.Pattern; class FunctionReference { private static final Pattern referencePattern = - Pattern.compile("rankingExpression\\(([a-zA-Z0-9_.]+)(@[a-f0-9]+\\.[a-f0-9]+)?\\)(\\.rankingScript)?"); + Pattern.compile("rankingExpression\\(([a-zA-Z0-9_.]+)(@[a-f0-9]+[.a-f0-9]*)?\\)(\\.rankingScript)?"); private static final Pattern externalReferencePattern = - Pattern.compile("rankingExpression\\(([a-zA-Z0-9_.]+)(@[a-f0-9]+\\.[a-f0-9]+)?\\)(\\.expressionName)?"); + Pattern.compile("rankingExpression\\(([a-zA-Z0-9_.]+)(@[a-f0-9]+[.a-f0-9]*)?\\)(\\.expressionName)?"); private static final Pattern argumentTypePattern = - Pattern.compile("rankingExpression\\(([a-zA-Z0-9_.]+)(@[a-f0-9]+\\.[a-f0-9]+)?\\)\\.([a-zA-Z0-9_]+)\\.type?"); + Pattern.compile("rankingExpression\\(([a-zA-Z0-9_.]+)(@[a-f0-9]+[.a-f0-9]*)?\\)\\.([a-zA-Z0-9_]+)\\.type"); private static final Pattern returnTypePattern = - Pattern.compile("rankingExpression\\(([a-zA-Z0-9_.]+)(@[a-f0-9]+\\.[a-f0-9]+)?\\)\\.type?"); + Pattern.compile("rankingExpression\\(([a-zA-Z0-9_.]+)(@[a-f0-9]+[.a-f0-9]*)?\\)\\.type"); /** The name of the function referenced */ private final String name; diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/Model.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/Model.java index d66d0330ea6..1da8121ba8e 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/Model.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/Model.java @@ -10,7 +10,6 @@ import com.yahoo.tensor.TensorType; import java.util.Arrays; import java.util.Collection; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -22,7 +21,7 @@ import java.util.stream.Collectors; * @author bratseth */ @Beta -public class Model { +public class Model implements AutoCloseable { /** The prefix generated by model-integration/../IntermediateOperation */ private final static String INTERMEDIATE_OPERATION_FUNCTION_PREFIX = "imported_ml_function_"; @@ -43,11 +42,14 @@ public class Model { private final ExpressionOptimizer expressionOptimizer = new ExpressionOptimizer(); + private final List<Runnable> closeActions; + /** Programmatically create a model containing functions without constant of function references only */ public Model(String name, Collection<ExpressionFunction> functions) { this(name, functions.stream().collect(Collectors.toMap(f -> FunctionReference.fromName(f.getName()), f -> f)), Map.of(), + Map.of(), List.of(), List.of()); } @@ -55,6 +57,7 @@ public class Model { Model(String name, Map<FunctionReference, ExpressionFunction> functions, Map<FunctionReference, ExpressionFunction> referencedFunctions, + Map<String, TensorType> declaredTypes, List<Constant> constants, List<OnnxModel> onnxModels) { this.name = name; @@ -84,8 +87,10 @@ public class Model { } else { // External functions have type info (when not scalar) - add argument types - if (function.getValue().getArgumentType(argument) == null) - functions.put(function.getKey(), function.getValue().withArgument(argument, TensorType.empty)); + if (function.getValue().getArgumentType(argument) == null) { + TensorType type = declaredTypes.getOrDefault(argument, TensorType.empty); + functions.put(function.getKey(), function.getValue().withArgument(argument, type)); + } } } } @@ -94,13 +99,18 @@ public class Model { } } this.contextPrototypes = Map.copyOf(contextBuilder); - this.functions = List.copyOf(functions.values()); + // Optimize free functions + this.functions = List.copyOf(functions.entrySet() + .stream() + .map(f -> optimize(f.getValue(), + contextPrototypes.get(f.getKey().functionName()))) + .collect(Collectors.toList())); + this.publicFunctions = functions.values().stream() .filter(f -> !f.getName().startsWith(INTERMEDIATE_OPERATION_FUNCTION_PREFIX)).toList(); - // Optimize functions - this.referencedFunctions = Map.copyOf(referencedFunctions.entrySet().stream() - .collect(CustomCollectors.toLinkedMap(f -> f.getKey(), f -> optimize(f.getValue(), contextPrototypes.get(f.getKey().functionName()))))); + this.referencedFunctions = Map.copyOf(referencedFunctions); + this.closeActions = onnxModels.stream().map(o -> (Runnable)o::close).toList(); } /** Returns an optimized version of the given function */ @@ -223,4 +233,5 @@ public class Model { @Override public String toString() { return "model '" + name + "'"; } + @Override public void close() { closeActions.forEach(Runnable::run); } } diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java index 28b613ca281..fd5306f9add 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/ModelsEvaluator.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.models.evaluation; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; import com.yahoo.api.annotations.Beta; import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; @@ -30,8 +31,17 @@ public class ModelsEvaluator extends AbstractComponent { RankingConstantsConfig constantsConfig, RankingExpressionsConfig expressionsConfig, OnnxModelsConfig onnxModelsConfig, + FileAcquirer fileAcquirer, + OnnxRuntime onnx) { + this(new RankProfilesConfigImporter(fileAcquirer, onnx), config, constantsConfig, expressionsConfig, onnxModelsConfig); + } + + public ModelsEvaluator(RankProfilesConfig config, + RankingConstantsConfig constantsConfig, + RankingExpressionsConfig expressionsConfig, + OnnxModelsConfig onnxModelsConfig, FileAcquirer fileAcquirer) { - this(new RankProfilesConfigImporter(fileAcquirer), config, constantsConfig, expressionsConfig, onnxModelsConfig); + this(config, constantsConfig, expressionsConfig, onnxModelsConfig, fileAcquirer, new OnnxRuntime()); } public ModelsEvaluator(RankProfilesConfigImporter importer, @@ -69,4 +79,5 @@ public class ModelsEvaluator extends AbstractComponent { return model; } + @Override public void deconstruct() { models.values().forEach(Model::close); } } diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/OnnxModel.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/OnnxModel.java index 19a9a1dccd5..cf97c20e881 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/OnnxModel.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/OnnxModel.java @@ -3,10 +3,14 @@ package ai.vespa.models.evaluation; import ai.vespa.modelintegration.evaluator.OnnxEvaluator; import ai.vespa.modelintegration.evaluator.OnnxEvaluatorOptions; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import java.io.File; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; import java.util.Map; /** @@ -14,18 +18,58 @@ import java.util.Map; * * @author lesters */ -class OnnxModel { +class OnnxModel implements AutoCloseable { + + static class InputSpec { + String onnxName; + String source; + TensorType wantedType; + InputSpec(String name, String source, TensorType tType) { + this.onnxName = name; + this.source = source; + this.wantedType = tType; + } + InputSpec(String name, String source) { this(name, source, null); } + } + + static class OutputSpec { + String onnxName; + String outputAs; + TensorType expectedType; + OutputSpec(String name, String as, TensorType tType) { + this.onnxName = name; + this.outputAs = as; + this.expectedType = tType; + } + OutputSpec(String name, String as) { this(name, as, null); } + } + + final List<InputSpec> inputSpecs = new ArrayList<>(); + final List<OutputSpec> outputSpecs = new ArrayList<>(); + + void addInputMapping(String onnxName, String source) { + if (evaluator != null) + throw new IllegalStateException("input mapping must be added before load()"); + inputSpecs.add(new InputSpec(onnxName, source)); + } + void addOutputMapping(String onnxName, String outputAs) { + if (evaluator != null) + throw new IllegalStateException("output mapping must be added before load()"); + outputSpecs.add(new OutputSpec(onnxName, outputAs)); + } private final String name; private final File modelFile; private final OnnxEvaluatorOptions options; + private final OnnxRuntime onnx; private OnnxEvaluator evaluator; - OnnxModel(String name, File modelFile, OnnxEvaluatorOptions options) { + OnnxModel(String name, File modelFile, OnnxEvaluatorOptions options, OnnxRuntime onnx) { this.name = name; this.modelFile = modelFile; this.options = options; + this.onnx = onnx; } public String name() { @@ -34,20 +78,103 @@ class OnnxModel { public void load() { if (evaluator == null) { - evaluator = new OnnxEvaluator(modelFile.getPath(), options); + evaluator = onnx.evaluatorOf(modelFile.getPath(), options); + fillInputTypes(evaluator().getInputs()); + fillOutputTypes(evaluator().getOutputs()); + } + } + + void fillInputTypes(Map<String, OnnxEvaluator.IdAndType> wantedTypes) { + if (inputSpecs.isEmpty()) { + for (var entry : wantedTypes.entrySet()) { + String name = entry.getKey(); + String source = entry.getValue().id(); + TensorType tType = entry.getValue().type(); + var spec = new InputSpec(name, source, tType); + inputSpecs.add(spec); + } + } else { + if (wantedTypes.size() != inputSpecs.size()) { + throw new IllegalArgumentException("Onnx model " + name() + + ": Mismatch between " + inputSpecs.size() + + " configured inputs and " + + wantedTypes.size() + " actual model inputs"); + } + for (var spec : inputSpecs) { + var entry = wantedTypes.get(spec.onnxName); + if (entry == null) { + throw new IllegalArgumentException("Onnx model " + name() + + ": No type in actual model for configured input " + + spec.onnxName); + } + spec.wantedType = entry.type(); + } + } + } + + void fillOutputTypes(Map<String, OnnxEvaluator.IdAndType> outputTypes) { + if (outputSpecs.isEmpty()) { + for (var entry : outputTypes.entrySet()) { + String name = entry.getKey(); + String as = entry.getValue().id(); + TensorType tType = entry.getValue().type(); + var spec = new OutputSpec(name, as, tType); + outputSpecs.add(spec); + } + } else { + if (outputTypes.size() != outputSpecs.size()) { + throw new IllegalArgumentException("Onnx model " + name() + + ": Mismatch between " + outputSpecs.size() + + " configured outputs and " + + outputTypes.size() + " actual model outputs"); + } + for (var spec : outputSpecs) { + var entry = outputTypes.get(spec.onnxName); + if (entry == null) { + throw new IllegalArgumentException("Onnx model " + name() + + ": No type in actual model for configured output " + + spec.onnxName); + } + spec.expectedType = entry.type(); + } } } public Map<String, TensorType> inputs() { - return evaluator().getInputInfo(); + var map = new HashMap<String, TensorType>(); + for (var spec : inputSpecs) { + map.put(spec.source, spec.wantedType); + } + return map; } public Map<String, TensorType> outputs() { - return evaluator().getOutputInfo(); + var map = new HashMap<String, TensorType>(); + for (var spec : outputSpecs) { + map.put(spec.outputAs, spec.expectedType); + } + return map; } public Tensor evaluate(Map<String, Tensor> inputs, String output) { - return evaluator().evaluate(inputs, output); + var mapped = new HashMap<String, Tensor>(); + for (var spec : inputSpecs) { + Tensor val = inputs.get(spec.source); + if (val == null) { + throw new IllegalArgumentException("evaluate ONNX model " + name() + ": missing input from source " + spec.source); + } + mapped.put(spec.onnxName, val); + } + String onnxName = null; + for (var spec : outputSpecs) { + if (spec.outputAs.equals(output)) { + onnxName = spec.onnxName; + } + } + if (onnxName == null) { + throw new IllegalArgumentException("evaluate ONNX model " + name() + ": no output available as: " + output); + } + return evaluator().evaluate(mapped, onnxName); } private OnnxEvaluator evaluator() { @@ -57,4 +184,5 @@ class OnnxModel { return evaluator; } + @Override public void close() { evaluator.close(); } } diff --git a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java index e8aae24ca9e..8c520e87001 100644 --- a/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java +++ b/model-evaluation/src/main/java/ai/vespa/models/evaluation/RankProfilesConfigImporter.java @@ -2,6 +2,7 @@ package ai.vespa.models.evaluation; import ai.vespa.modelintegration.evaluator.OnnxEvaluatorOptions; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; import com.yahoo.collections.Pair; import com.yahoo.config.FileReference; import com.yahoo.filedistribution.fileacquirer.FileAcquirer; @@ -46,9 +47,11 @@ import java.util.regex.Pattern; public class RankProfilesConfigImporter { private final FileAcquirer fileAcquirer; + private final OnnxRuntime onnx; - public RankProfilesConfigImporter(FileAcquirer fileAcquirer) { + public RankProfilesConfigImporter(FileAcquirer fileAcquirer, OnnxRuntime onnx) { this.fileAcquirer = fileAcquirer; + this.onnx = onnx; } /** @@ -87,11 +90,14 @@ public class RankProfilesConfigImporter { SmallConstantsInfo smallConstantsInfo = new SmallConstantsInfo(); ExpressionFunction firstPhase = null; ExpressionFunction secondPhase = null; + ExpressionFunction globalPhase = null; + Map<String, TensorType> declaredTypes = new LinkedHashMap<>(); for (RankProfilesConfig.Rankprofile.Fef.Property property : profile.fef().property()) { Optional<FunctionReference> reference = FunctionReference.fromSerial(property.name()); Optional<FunctionReference> externalReference = FunctionReference.fromExternalSerial(property.name()); Optional<Pair<FunctionReference, String>> argumentType = FunctionReference.fromTypeArgumentSerial(property.name()); Optional<FunctionReference> returnType = FunctionReference.fromReturnTypeSerial(property.name()); + Optional<String> typeDeclaredFeature = fromTypeDeclarationSerial(property.name()); if (externalReference.isPresent()) { RankingExpression expression = largeExpressions.get(property.value()); ExpressionFunction function = new ExpressionFunction(externalReference.get().functionName(), @@ -137,6 +143,13 @@ public class RankProfilesConfigImporter { secondPhase = new ExpressionFunction("secondphase", new ArrayList<>(), new RankingExpression("second-phase", property.value())); } + else if (property.name().equals("vespa.rank.globalphase")) { // Include in addition to functions + globalPhase = new ExpressionFunction("globalphase", new ArrayList<>(), + new RankingExpression("global-phase", property.value())); + } + else if (typeDeclaredFeature.isPresent()) { + declaredTypes.put(typeDeclaredFeature.get(), TensorType.fromSpec(property.value())); + } else { smallConstantsInfo.addIfSmallConstantInfo(property.name(), property.value()); } @@ -145,11 +158,13 @@ public class RankProfilesConfigImporter { functions.put(FunctionReference.fromName("firstphase"), firstPhase); if (functionByName("secondphase", functions.values()) == null && secondPhase != null) // may be already included, depending on body functions.put(FunctionReference.fromName("secondphase"), secondPhase); + if (functionByName("globalphase", functions.values()) == null && globalPhase != null) // may be already included, depending on body + functions.put(FunctionReference.fromName("globalphase"), globalPhase); constants.addAll(smallConstantsInfo.asConstants()); try { - return new Model(profile.name(), functions, referencedFunctions, constants, onnxModels); + return new Model(profile.name(), functions, referencedFunctions, declaredTypes, constants, onnxModels); } catch (RuntimeException e) { throw new IllegalArgumentException("Could not load model '" + profile.name() + "'", e); @@ -183,7 +198,14 @@ public class RankProfilesConfigImporter { options.setInterOpThreads(onnxModelConfig.stateless_interop_threads()); options.setIntraOpThreads(onnxModelConfig.stateless_intraop_threads()); options.setGpuDevice(onnxModelConfig.gpu_device(), onnxModelConfig.gpu_device_required()); - return new OnnxModel(name, file, options); + var m = new OnnxModel(name, file, options, onnx); + for (var spec : onnxModelConfig.input()) { + m.addInputMapping(spec.name(), spec.source()); + } + for (var spec : onnxModelConfig.output()) { + m.addOutputMapping(spec.name(), spec.as()); + } + return m; } catch (InterruptedException e) { throw new IllegalStateException("Gave up waiting for ONNX model " + onnxModelConfig.name()); } @@ -289,4 +311,15 @@ public class RankProfilesConfigImporter { } + private static final Pattern typeDeclarationPattern = + Pattern.compile("vespa[.]type[.]([a-zA-Z0-9]+)[.](.+)"); + + static Optional<String> fromTypeDeclarationSerial(String serialForm) { + Matcher expressionMatcher = typeDeclarationPattern.matcher(serialForm); + if ( ! expressionMatcher.matches()) return Optional.empty(); + String name = expressionMatcher.group(1); + String argument = expressionMatcher.group(2); + return Optional.of(name + "(" + argument + ")"); + } + } diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java index 992dae22aaf..0bee33be3cc 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/OnnxEvaluatorTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.models.evaluation; -import ai.vespa.modelintegration.evaluator.OnnxEvaluator; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.filedistribution.fileacquirer.FileAcquirer; import com.yahoo.filedistribution.fileacquirer.MockFileAcquirer; @@ -30,7 +30,7 @@ public class OnnxEvaluatorTest { @Test public void testOnnxEvaluation() { - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); + assumeTrue(OnnxRuntime.isRuntimeAvailable()); ModelsEvaluator models = createModels(); assertTrue(models.models().containsKey("add_mul")); diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfileImportingTest.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfileImportingTest.java index 3fdbb370a5c..1a6f6925caf 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfileImportingTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfileImportingTest.java @@ -31,4 +31,22 @@ public class RankProfileImportingTest { "4 * (match + rankBoost)", macros); } + @Test + public void testImportingSimpleGlobalPhase() { + ModelTester tester = new ModelTester("src/test/resources/config/dotproduct/"); + assertEquals(1, tester.models().size()); + Model m = tester.models().get("default"); + assertEquals("default", m.name()); + assertEquals(1, m.functions().size()); + tester.assertFunction("globalphase", "reduce(attribute(aa) * query(zz), sum)", m); + var f = m.functions().get(0); + assertEquals("globalphase", f.getName()); + assertEquals(2, f.arguments().size()); + assertEquals("tensor(d0[3])", f.getArgumentType("query(zz)").toString()); + assertEquals("tensor(d0[3])", f.getArgumentType("attribute(aa)").toString()); + var rt = f.returnType(); + assertEquals(true, rt.isPresent()); + assertEquals("tensor()", rt.get().toString()); + } + } diff --git a/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java b/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java index c11f4764678..0dd3bd29a2c 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java +++ b/model-evaluation/src/test/java/ai/vespa/models/evaluation/RankProfilesConfigImporterWithMockedConstants.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.models.evaluation; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; import com.yahoo.config.FileReference; import com.yahoo.filedistribution.fileacquirer.FileAcquirer; import com.yahoo.io.GrowableByteBuffer; @@ -24,7 +25,7 @@ public class RankProfilesConfigImporterWithMockedConstants extends RankProfilesC private final Path constantsPath; public RankProfilesConfigImporterWithMockedConstants(Path constantsPath, FileAcquirer fileAcquirer) { - super(fileAcquirer); + super(fileAcquirer, new OnnxRuntime()); this.constantsPath = constantsPath; } diff --git a/model-evaluation/src/test/java/ai/vespa/models/handler/HandlerTester.java b/model-evaluation/src/test/java/ai/vespa/models/handler/HandlerTester.java index 6c4dd886f4b..38215858366 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/handler/HandlerTester.java +++ b/model-evaluation/src/test/java/ai/vespa/models/handler/HandlerTester.java @@ -27,16 +27,22 @@ class HandlerTester { } private static Predicate<String> matchString(String expected) { return s -> { - //System.out.println("Expected: " + expected); - //System.out.println("Actual: " + s); - return expected.equals(s); + boolean result = expected.equals(s); + if (!result) { + System.out.println("Expected: " + expected); + System.out.println("Actual: " + s); + } + return result; }; } private static Predicate<String> matchJsonString(String expected) { return s -> { - //System.out.println("Expected: " + expected); - //System.out.println("Actual: " + s); - return JSON.canonical(expected).equals(JSON.canonical(s)); + boolean result = JSON.canonical(expected).equals(JSON.canonical(s)); + if (!result) { + System.out.println("Expected: " + expected); + System.out.println("Actual: " + s); + } + return result; }; } public static Predicate<String> matchJson(String... expectedJson) { diff --git a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java index 9b2b793212b..14da15f60d0 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/handler/ModelsEvaluationHandlerTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.models.handler; -import ai.vespa.modelintegration.evaluator.OnnxEvaluator; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; import ai.vespa.models.evaluation.ModelsEvaluator; import ai.vespa.models.evaluation.RankProfilesConfigImporterWithMockedConstants; import com.yahoo.config.subscription.ConfigGetter; @@ -323,7 +323,7 @@ public class ModelsEvaluationHandlerTest { @Test public void testMnistSavedEvaluateSpecificFunction() { - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); + assumeTrue(OnnxRuntime.isRuntimeAvailable()); Map<String, String> properties = new HashMap<>(); properties.put("input", inputTensor()); properties.put("format.tensors", "long"); diff --git a/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java b/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java index 86f56e14e2d..856031da72f 100644 --- a/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java +++ b/model-evaluation/src/test/java/ai/vespa/models/handler/OnnxEvaluationHandlerTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package ai.vespa.models.handler; -import ai.vespa.modelintegration.evaluator.OnnxEvaluator; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; import ai.vespa.models.evaluation.ModelsEvaluator; import com.yahoo.config.subscription.ConfigGetter; import com.yahoo.filedistribution.fileacquirer.FileAcquirer; @@ -27,7 +27,7 @@ public class OnnxEvaluationHandlerTest { @BeforeClass static public void setUp() { - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); + assumeTrue(OnnxRuntime.isRuntimeAvailable()); handler = new HandlerTester(createModels()); } diff --git a/model-evaluation/src/test/resources/config/dotproduct/onnx-models.cfg b/model-evaluation/src/test/resources/config/dotproduct/onnx-models.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/model-evaluation/src/test/resources/config/dotproduct/onnx-models.cfg diff --git a/model-evaluation/src/test/resources/config/dotproduct/rank-profiles.cfg b/model-evaluation/src/test/resources/config/dotproduct/rank-profiles.cfg new file mode 100644 index 00000000000..ae1e6791f3e --- /dev/null +++ b/model-evaluation/src/test/resources/config/dotproduct/rank-profiles.cfg @@ -0,0 +1,9 @@ +rankprofile[0].name "default" +rankprofile[0].fef.property[0].name "vespa.rank.globalphase" +rankprofile[0].fef.property[0].value "sum(attribute(aa) * query(zz))" +rankprofile[0].fef.property[1].name "vespa.match.feature" +rankprofile[0].fef.property[1].value "attribute(aa)" +rankprofile[0].fef.property[2].name "vespa.type.attribute.aa" +rankprofile[0].fef.property[2].value "tensor(d0[3])" +rankprofile[0].fef.property[3].name "vespa.type.query.zz" +rankprofile[0].fef.property[3].value "tensor(d0[3])" diff --git a/model-evaluation/src/test/resources/config/dotproduct/ranking-constants.cfg b/model-evaluation/src/test/resources/config/dotproduct/ranking-constants.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/model-evaluation/src/test/resources/config/dotproduct/ranking-constants.cfg diff --git a/model-evaluation/src/test/resources/config/dotproduct/ranking-expressions.cfg b/model-evaluation/src/test/resources/config/dotproduct/ranking-expressions.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/model-evaluation/src/test/resources/config/dotproduct/ranking-expressions.cfg diff --git a/model-evaluation/src/test/resources/config/expressions-as-arguments/onnx-models.cfg b/model-evaluation/src/test/resources/config/expressions-as-arguments/onnx-models.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/model-evaluation/src/test/resources/config/expressions-as-arguments/onnx-models.cfg diff --git a/model-evaluation/src/test/resources/config/expressions-as-arguments/rank-profiles.cfg b/model-evaluation/src/test/resources/config/expressions-as-arguments/rank-profiles.cfg new file mode 100644 index 00000000000..d8ccbeaf0fb --- /dev/null +++ b/model-evaluation/src/test/resources/config/expressions-as-arguments/rank-profiles.cfg @@ -0,0 +1,175 @@ +rankprofile[0].name "default" +rankprofile[0].fef.property[0].name "vespa.type.attribute.t1" +rankprofile[0].fef.property[0].value "tensor<float>(x{})" +rankprofile[0].fef.property[1].name "vespa.type.attribute.t2" +rankprofile[0].fef.property[1].value "tensor<float>(x{})" +rankprofile[1].name "unranked" +rankprofile[1].fef.property[0].name "vespa.rank.firstphase" +rankprofile[1].fef.property[0].value "value(0)" +rankprofile[1].fef.property[1].name "vespa.hitcollector.heapsize" +rankprofile[1].fef.property[1].value "0" +rankprofile[1].fef.property[2].name "vespa.hitcollector.arraysize" +rankprofile[1].fef.property[2].value "0" +rankprofile[1].fef.property[3].name "vespa.dump.ignoredefaultfeatures" +rankprofile[1].fef.property[3].value "true" +rankprofile[1].fef.property[4].name "vespa.type.attribute.t1" +rankprofile[1].fef.property[4].value "tensor<float>(x{})" +rankprofile[1].fef.property[5].name "vespa.type.attribute.t2" +rankprofile[1].fef.property[5].value "tensor<float>(x{})" +rankprofile[2].name "test" +rankprofile[2].fef.property[0].name "rankingExpression(my_square@31852fecfab75f29).rankingScript" +rankprofile[2].fef.property[0].value "2 * 2" +rankprofile[2].fef.property[1].name "rankingExpression(my_square@56bfa257b4b447a2).rankingScript" +rankprofile[2].fef.property[1].value "-3.14 * -3.14" +rankprofile[2].fef.property[2].name "rankingExpression(test_constant).rankingScript" +rankprofile[2].fef.property[2].value "rankingExpression(my_square@31852fecfab75f29) + rankingExpression(my_square@56bfa257b4b447a2)" +rankprofile[2].fef.property[3].name "rankingExpression(autogenerated_ranking_feature@c76aeb97d0b6610c).rankingScript" +rankprofile[2].fef.property[3].value "attribute(i1) * 2" +rankprofile[2].fef.property[4].name "rankingExpression(my_square@52d8ef83411bc8d0).rankingScript" +rankprofile[2].fef.property[4].value "rankingExpression(autogenerated_ranking_feature@c76aeb97d0b6610c) * rankingExpression(autogenerated_ranking_feature@c76aeb97d0b6610c)" +rankprofile[2].fef.property[5].name "rankingExpression(autogenerated_ranking_feature@828fd0618dc7fc06).rankingScript" +rankprofile[2].fef.property[5].value "attribute(i1) < 1" +rankprofile[2].fef.property[6].name "rankingExpression(my_square@6b77cba8e7358f11).rankingScript" +rankprofile[2].fef.property[6].value "rankingExpression(autogenerated_ranking_feature@828fd0618dc7fc06) * rankingExpression(autogenerated_ranking_feature@828fd0618dc7fc06)" +rankprofile[2].fef.property[7].name "rankingExpression(test_arithmetic).rankingScript" +rankprofile[2].fef.property[7].value "rankingExpression(my_square@52d8ef83411bc8d0) + rankingExpression(my_square@6b77cba8e7358f11)" +rankprofile[2].fef.property[8].name "rankingExpression(autogenerated_ranking_feature@675b0f8c6790c8bb).rankingScript" +rankprofile[2].fef.property[8].value "!attribute(i1)" +rankprofile[2].fef.property[9].name "rankingExpression(my_square@35879139f3786e2f).rankingScript" +rankprofile[2].fef.property[9].value "rankingExpression(autogenerated_ranking_feature@675b0f8c6790c8bb) * rankingExpression(autogenerated_ranking_feature@675b0f8c6790c8bb)" +rankprofile[2].fef.property[10].name "rankingExpression(autogenerated_ranking_feature@6084beaceb676bf2).rankingScript" +rankprofile[2].fef.property[10].value "-attribute(i1)" +rankprofile[2].fef.property[11].name "rankingExpression(my_square@819381f707f3ee78).rankingScript" +rankprofile[2].fef.property[11].value "rankingExpression(autogenerated_ranking_feature@6084beaceb676bf2) * rankingExpression(autogenerated_ranking_feature@6084beaceb676bf2)" +rankprofile[2].fef.property[12].name "rankingExpression(test_not_neg).rankingScript" +rankprofile[2].fef.property[12].value "rankingExpression(my_square@35879139f3786e2f) + rankingExpression(my_square@819381f707f3ee78)" +rankprofile[2].fef.property[13].name "rankingExpression(autogenerated_ranking_feature@b41d2fa3c2ee40a3).rankingScript" +rankprofile[2].fef.property[13].value "if (attribute(i1) in [0, 1, 2], 0, 1)" +rankprofile[2].fef.property[14].name "rankingExpression(my_square@643af1a7339a8b1b).rankingScript" +rankprofile[2].fef.property[14].value "rankingExpression(autogenerated_ranking_feature@b41d2fa3c2ee40a3) * rankingExpression(autogenerated_ranking_feature@b41d2fa3c2ee40a3)" +rankprofile[2].fef.property[15].name "rankingExpression(autogenerated_ranking_feature@335f7caa94692e97).rankingScript" +rankprofile[2].fef.property[15].value "attribute(i1) in [0, 1, 2]" +rankprofile[2].fef.property[16].name "rankingExpression(my_square@d48185ac029647a5).rankingScript" +rankprofile[2].fef.property[16].value "rankingExpression(autogenerated_ranking_feature@335f7caa94692e97) * rankingExpression(autogenerated_ranking_feature@335f7caa94692e97)" +rankprofile[2].fef.property[17].name "rankingExpression(test_if_in).rankingScript" +rankprofile[2].fef.property[17].value "rankingExpression(my_square@643af1a7339a8b1b) + rankingExpression(my_square@d48185ac029647a5)" +rankprofile[2].fef.property[18].name "rankingExpression(my_square@9a5117ae5a6d491b).rankingScript" +rankprofile[2].fef.property[18].value "cos(attribute(i1)) * cos(attribute(i1))" +rankprofile[2].fef.property[19].name "rankingExpression(test_function).rankingScript" +rankprofile[2].fef.property[19].value "rankingExpression(my_square@9a5117ae5a6d491b)" +rankprofile[2].fef.property[20].name "rankingExpression(autogenerated_ranking_feature@6c8232a0cd94322d).rankingScript" +rankprofile[2].fef.property[20].value "(attribute(i1) * 2)" +rankprofile[2].fef.property[21].name "rankingExpression(my_square@181aa0cc505c1788).rankingScript" +rankprofile[2].fef.property[21].value "rankingExpression(autogenerated_ranking_feature@6c8232a0cd94322d) * rankingExpression(autogenerated_ranking_feature@6c8232a0cd94322d)" +rankprofile[2].fef.property[22].name "rankingExpression(test_embraced).rankingScript" +rankprofile[2].fef.property[22].value "rankingExpression(my_square@181aa0cc505c1788)" +rankprofile[2].fef.property[23].name "rankingExpression(my_func@9bbaee2bad5a2fc0).rankingScript" +rankprofile[2].fef.property[23].value "reduce(attribute(t1), sum, x) + 1" +rankprofile[2].fef.property[24].name "rankingExpression(test_func).rankingScript" +rankprofile[2].fef.property[24].value "rankingExpression(my_func@9bbaee2bad5a2fc0)" +rankprofile[2].fef.property[25].name "rankingExpression(autogenerated_ranking_feature@43bc412603c00a4a).rankingScript" +rankprofile[2].fef.property[25].value "attribute(t1) * attribute(t2)" +rankprofile[2].fef.property[26].name "rankingExpression(my_func@7f288a910482845a).rankingScript" +rankprofile[2].fef.property[26].value "reduce(rankingExpression(autogenerated_ranking_feature@43bc412603c00a4a), sum, x) + 1" +rankprofile[2].fef.property[27].name "rankingExpression(test_tensor_func_with_expr).rankingScript" +rankprofile[2].fef.property[27].value "rankingExpression(my_func@7f288a910482845a)" +rankprofile[2].fef.property[28].name "rankingExpression(autogenerated_ranking_feature@c1057dea8228da3a).rankingScript" +rankprofile[2].fef.property[28].value "map(attribute(t1), f(x)(x * x))" +rankprofile[2].fef.property[29].name "rankingExpression(my_func@901c2cc6ceb37765).rankingScript" +rankprofile[2].fef.property[29].value "reduce(rankingExpression(autogenerated_ranking_feature@c1057dea8228da3a), sum, x) + 1" +rankprofile[2].fef.property[30].name "rankingExpression(test_func_with_tensor_func).rankingScript" +rankprofile[2].fef.property[30].value "rankingExpression(my_func@901c2cc6ceb37765)" +rankprofile[2].fef.property[31].name "rankingExpression(autogenerated_ranking_feature@fb1a4642f23d9d05).rankingScript" +rankprofile[2].fef.property[31].value "attribute(t1){x:0}" +rankprofile[2].fef.property[32].name "rankingExpression(my_square@1d27f1b495b50910).rankingScript" +rankprofile[2].fef.property[32].value "rankingExpression(autogenerated_ranking_feature@fb1a4642f23d9d05) * rankingExpression(autogenerated_ranking_feature@fb1a4642f23d9d05)" +rankprofile[2].fef.property[33].name "rankingExpression(test_func_with_slice).rankingScript" +rankprofile[2].fef.property[33].value "rankingExpression(my_square@1d27f1b495b50910)" +rankprofile[2].fef.property[34].name "rankingExpression(call_func_with_expr@640470df47a83000.c156faa8f98c0b0c).rankingScript" +rankprofile[2].fef.property[34].value "rankingExpression(my_func@7f288a910482845a)" +rankprofile[2].fef.property[35].name "rankingExpression(test_func_via_func_with_expr).rankingScript" +rankprofile[2].fef.property[35].value "rankingExpression(call_func_with_expr@640470df47a83000.c156faa8f98c0b0c)" +rankprofile[2].fef.property[36].name "rankingExpression(my_square).rankingScript" +rankprofile[2].fef.property[36].value "x * x" +rankprofile[2].fef.property[37].name "rankingExpression(my_func).rankingScript" +rankprofile[2].fef.property[37].value "reduce(t, sum, x) + 1" +rankprofile[2].fef.property[38].name "rankingExpression(autogenerated_ranking_feature@1044065d971a7507).rankingScript" +rankprofile[2].fef.property[38].value "a * b" +rankprofile[2].fef.property[39].name "rankingExpression(my_func@93366be10bade547).rankingScript" +rankprofile[2].fef.property[39].value "reduce(rankingExpression(autogenerated_ranking_feature@1044065d971a7507), sum, x) + 1" +rankprofile[2].fef.property[40].name "rankingExpression(call_func_with_expr).rankingScript" +rankprofile[2].fef.property[40].value "rankingExpression(my_func@93366be10bade547)" +rankprofile[2].fef.property[41].name "vespa.rank.firstphase" +rankprofile[2].fef.property[41].value "rankingExpression(firstphase)" +rankprofile[2].fef.property[42].name "rankingExpression(firstphase).rankingScript" +rankprofile[2].fef.property[42].value "42" +rankprofile[2].fef.property[43].name "vespa.summary.feature" +rankprofile[2].fef.property[43].value "rankingExpression(test_constant)" +rankprofile[2].fef.property[44].name "vespa.summary.feature" +rankprofile[2].fef.property[44].value "rankingExpression(test_arithmetic)" +rankprofile[2].fef.property[45].name "vespa.summary.feature" +rankprofile[2].fef.property[45].value "rankingExpression(test_not_neg)" +rankprofile[2].fef.property[46].name "vespa.summary.feature" +rankprofile[2].fef.property[46].value "rankingExpression(test_if_in)" +rankprofile[2].fef.property[47].name "vespa.summary.feature" +rankprofile[2].fef.property[47].value "rankingExpression(test_function)" +rankprofile[2].fef.property[48].name "vespa.summary.feature" +rankprofile[2].fef.property[48].value "rankingExpression(test_embraced)" +rankprofile[2].fef.property[49].name "vespa.summary.feature" +rankprofile[2].fef.property[49].value "rankingExpression(test_func)" +rankprofile[2].fef.property[50].name "vespa.summary.feature" +rankprofile[2].fef.property[50].value "rankingExpression(test_tensor_func_with_expr)" +rankprofile[2].fef.property[51].name "vespa.summary.feature" +rankprofile[2].fef.property[51].value "rankingExpression(test_func_with_tensor_func)" +rankprofile[2].fef.property[52].name "vespa.summary.feature" +rankprofile[2].fef.property[52].value "rankingExpression(test_func_with_slice)" +rankprofile[2].fef.property[53].name "vespa.summary.feature" +rankprofile[2].fef.property[53].value "rankingExpression(test_func_via_func_with_expr)" +rankprofile[2].fef.property[54].name "vespa.feature.rename" +rankprofile[2].fef.property[54].value "rankingExpression(test_constant)" +rankprofile[2].fef.property[55].name "vespa.feature.rename" +rankprofile[2].fef.property[55].value "test_constant" +rankprofile[2].fef.property[56].name "vespa.feature.rename" +rankprofile[2].fef.property[56].value "rankingExpression(test_arithmetic)" +rankprofile[2].fef.property[57].name "vespa.feature.rename" +rankprofile[2].fef.property[57].value "test_arithmetic" +rankprofile[2].fef.property[58].name "vespa.feature.rename" +rankprofile[2].fef.property[58].value "rankingExpression(test_not_neg)" +rankprofile[2].fef.property[59].name "vespa.feature.rename" +rankprofile[2].fef.property[59].value "test_not_neg" +rankprofile[2].fef.property[60].name "vespa.feature.rename" +rankprofile[2].fef.property[60].value "rankingExpression(test_if_in)" +rankprofile[2].fef.property[61].name "vespa.feature.rename" +rankprofile[2].fef.property[61].value "test_if_in" +rankprofile[2].fef.property[62].name "vespa.feature.rename" +rankprofile[2].fef.property[62].value "rankingExpression(test_function)" +rankprofile[2].fef.property[63].name "vespa.feature.rename" +rankprofile[2].fef.property[63].value "test_function" +rankprofile[2].fef.property[64].name "vespa.feature.rename" +rankprofile[2].fef.property[64].value "rankingExpression(test_embraced)" +rankprofile[2].fef.property[65].name "vespa.feature.rename" +rankprofile[2].fef.property[65].value "test_embraced" +rankprofile[2].fef.property[66].name "vespa.feature.rename" +rankprofile[2].fef.property[66].value "rankingExpression(test_func)" +rankprofile[2].fef.property[67].name "vespa.feature.rename" +rankprofile[2].fef.property[67].value "test_func" +rankprofile[2].fef.property[68].name "vespa.feature.rename" +rankprofile[2].fef.property[68].value "rankingExpression(test_tensor_func_with_expr)" +rankprofile[2].fef.property[69].name "vespa.feature.rename" +rankprofile[2].fef.property[69].value "test_tensor_func_with_expr" +rankprofile[2].fef.property[70].name "vespa.feature.rename" +rankprofile[2].fef.property[70].value "rankingExpression(test_func_with_tensor_func)" +rankprofile[2].fef.property[71].name "vespa.feature.rename" +rankprofile[2].fef.property[71].value "test_func_with_tensor_func" +rankprofile[2].fef.property[72].name "vespa.feature.rename" +rankprofile[2].fef.property[72].value "rankingExpression(test_func_with_slice)" +rankprofile[2].fef.property[73].name "vespa.feature.rename" +rankprofile[2].fef.property[73].value "test_func_with_slice" +rankprofile[2].fef.property[74].name "vespa.feature.rename" +rankprofile[2].fef.property[74].value "rankingExpression(test_func_via_func_with_expr)" +rankprofile[2].fef.property[75].name "vespa.feature.rename" +rankprofile[2].fef.property[75].value "test_func_via_func_with_expr" +rankprofile[2].fef.property[76].name "vespa.type.attribute.t1" +rankprofile[2].fef.property[76].value "tensor<float>(x{})" +rankprofile[2].fef.property[77].name "vespa.type.attribute.t2" +rankprofile[2].fef.property[77].value "tensor<float>(x{})" diff --git a/model-evaluation/src/test/resources/config/expressions-as-arguments/ranking-constants.cfg b/model-evaluation/src/test/resources/config/expressions-as-arguments/ranking-constants.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/model-evaluation/src/test/resources/config/expressions-as-arguments/ranking-constants.cfg diff --git a/model-evaluation/src/test/resources/config/expressions-as-arguments/ranking-expressions.cfg b/model-evaluation/src/test/resources/config/expressions-as-arguments/ranking-expressions.cfg new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/model-evaluation/src/test/resources/config/expressions-as-arguments/ranking-expressions.cfg diff --git a/model-integration/pom.xml b/model-integration/pom.xml index 1302984a314..9bb60827a68 100644 --- a/model-integration/pom.xml +++ b/model-integration/pom.xml @@ -69,6 +69,12 @@ <scope>provided</scope> </dependency> <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>component</artifactId> + <version>${project.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>net.java.dev.jna</groupId> <artifactId>jna</artifactId> <scope>provided</scope> @@ -105,6 +111,21 @@ <scope>test</scope> </dependency> <dependency> + <groupId>org.junit.vintage</groupId> + <artifactId>junit-vintage-engine</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter</artifactId> + <scope>test</scope> + </dependency> + <dependency> + <groupId>org.mockito</groupId> + <artifactId>mockito-core</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <scope>test</scope> diff --git a/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java b/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java index 002350ce3cf..b40e2b5be72 100644 --- a/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java +++ b/model-integration/src/main/java/ai/vespa/embedding/BertBaseEmbedder.java @@ -2,8 +2,10 @@ package ai.vespa.embedding; import ai.vespa.modelintegration.evaluator.OnnxEvaluator; import ai.vespa.modelintegration.evaluator.OnnxEvaluatorOptions; -import com.yahoo.embedding.BertBaseEmbedderConfig; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; +import com.yahoo.component.AbstractComponent; import com.yahoo.component.annotation.Inject; +import com.yahoo.embedding.BertBaseEmbedderConfig; import com.yahoo.language.process.Embedder; import com.yahoo.language.wordpiece.WordPieceEmbedder; import com.yahoo.tensor.IndexedTensor; @@ -28,7 +30,7 @@ import java.util.Map; * * @author lesters */ -public class BertBaseEmbedder implements Embedder { +public class BertBaseEmbedder extends AbstractComponent implements Embedder { private final static int TOKEN_CLS = 101; // [CLS] private final static int TOKEN_SEP = 102; // [SEP] @@ -44,7 +46,7 @@ public class BertBaseEmbedder implements Embedder { private final OnnxEvaluator evaluator; @Inject - public BertBaseEmbedder(BertBaseEmbedderConfig config) { + public BertBaseEmbedder(OnnxRuntime onnx, BertBaseEmbedderConfig config) { maxTokens = config.transformerMaxTokens(); inputIdsName = config.transformerInputIds(); attentionMaskName = config.transformerAttentionMask(); @@ -58,7 +60,7 @@ public class BertBaseEmbedder implements Embedder { options.setIntraOpThreads(modifyThreadCount(config.onnxIntraOpThreads())); tokenizer = new WordPieceEmbedder.Builder(config.tokenizerVocab().toString()).build(); - evaluator = new OnnxEvaluator(config.transformerModel().toString(), options); + this.evaluator = onnx.evaluatorOf(config.transformerModel().toString(), options); validateModel(); } @@ -100,6 +102,8 @@ public class BertBaseEmbedder implements Embedder { return embedTokens(tokens, type); } + @Override public void deconstruct() { evaluator.close(); } + Tensor embedTokens(List<Integer> tokens, TensorType type) { Tensor inputSequence = createTensorRepresentation(tokens, "d1"); Tensor attentionMask = createAttentionMask(inputSequence); diff --git a/model-integration/src/main/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedder.java b/model-integration/src/main/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedder.java index 81150fe99b0..9572cfcb0e4 100644 --- a/model-integration/src/main/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedder.java +++ b/model-integration/src/main/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedder.java @@ -3,22 +3,25 @@ package ai.vespa.embedding.huggingface; import ai.djl.huggingface.tokenizers.Encoding; import ai.djl.huggingface.tokenizers.HuggingFaceTokenizer; import ai.vespa.modelintegration.evaluator.OnnxEvaluator; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; +import com.yahoo.component.AbstractComponent; import com.yahoo.component.annotation.Inject; +import com.yahoo.embedding.huggingface.HuggingFaceEmbedderConfig; import com.yahoo.language.process.Embedder; import com.yahoo.tensor.IndexedTensor; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorAddress; import com.yahoo.tensor.TensorType; -import com.yahoo.embedding.huggingface.HuggingFaceEmbedderConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.io.*; +import java.io.IOException; import java.nio.file.Paths; -import java.util.*; +import java.util.Arrays; +import java.util.List; +import java.util.Map; -import org.slf4j.LoggerFactory; -import org.slf4j.Logger; - -public class HuggingFaceEmbedder implements Embedder { +public class HuggingFaceEmbedder extends AbstractComponent implements Embedder { private static final Logger LOG = LoggerFactory.getLogger(HuggingFaceEmbedder.class.getName()); @@ -30,7 +33,7 @@ public class HuggingFaceEmbedder implements Embedder { private final OnnxEvaluator evaluator; @Inject - public HuggingFaceEmbedder(HuggingFaceEmbedderConfig config) throws IOException { + public HuggingFaceEmbedder(OnnxRuntime onnx, HuggingFaceEmbedderConfig config) throws IOException { maxTokens = config.transformerMaxTokens(); inputIdsName = config.transformerInputIds(); attentionMaskName = config.transformerAttentionMask(); @@ -48,7 +51,7 @@ public class HuggingFaceEmbedder implements Embedder { LOG.info("Could not initialize the tokenizer"); throw new IOException("Could not initialize the tokenizer."); } - evaluator = new OnnxEvaluator(config.transformerModel().toString()); + evaluator = onnx.evaluatorOf(config.transformerModel().toString()); validateModel(); } @@ -83,6 +86,8 @@ public class HuggingFaceEmbedder implements Embedder { return tokenIds; } + @Override public void deconstruct() { evaluator.close(); } + public List<Integer> longToInteger(long[] values) { return Arrays.stream(values) .boxed().map(Long::intValue) diff --git a/model-integration/src/main/java/ai/vespa/llm/Generator.java b/model-integration/src/main/java/ai/vespa/llm/Generator.java new file mode 100644 index 00000000000..973b5ac2899 --- /dev/null +++ b/model-integration/src/main/java/ai/vespa/llm/Generator.java @@ -0,0 +1,230 @@ +package ai.vespa.llm; + +import ai.vespa.modelintegration.evaluator.OnnxEvaluator; +import ai.vespa.modelintegration.evaluator.OnnxEvaluatorOptions; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; +import com.yahoo.component.AbstractComponent; +import com.yahoo.component.annotation.Inject; +import com.yahoo.language.process.Embedder; +import com.yahoo.language.sentencepiece.SentencePieceEmbedder; +import com.yahoo.llm.GeneratorConfig; +import com.yahoo.tensor.DimensionSizes; +import com.yahoo.tensor.IndexedTensor; +import com.yahoo.tensor.PartialAddress; +import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.TensorType; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** +* A text generator based on language models (LLMs). By configuring a + * sentencepience tokenizer and models for encoding and decoding, this + * component generates text based on the given prompt. + * + * See llm.generator.def for configurable parameters. + * + * @author lesters + */ +public class Generator extends AbstractComponent { + + private final static int TOKEN_EOS = 1; // end of sequence + + private final static String BATCH_DIMENSION = "d0"; + private final static String SEQUENCE_DIMENSION = "d1"; + + private final int tokenizerMaxTokens; + private final String encoderInputIdsName; + private final String encoderAttentionMaskName; + private final String encoderOutputName; + private final String decoderInputIdsName; + private final String decoderAttentionMaskName; + private final String decoderEncoderHiddenStateName; + private final String decoderOutputName; + + private final SentencePieceEmbedder tokenizer; + private final OnnxEvaluator encoder; + private final OnnxEvaluator decoder; + + @Inject + public Generator(OnnxRuntime onnx, GeneratorConfig config) { + // Set up tokenizer + tokenizer = new SentencePieceEmbedder.Builder(config.tokenizerModel().toString()).build(); + tokenizerMaxTokens = config.tokenizerMaxTokens(); + + // Set up encoder + encoderInputIdsName = config.encoderModelInputIdsName(); + encoderAttentionMaskName = config.encoderModelAttentionMaskName(); + encoderOutputName = config.encoderModelOutputName(); + + OnnxEvaluatorOptions encoderOptions = new OnnxEvaluatorOptions(); + encoderOptions.setExecutionMode(config.encoderOnnxExecutionMode().toString()); + encoderOptions.setInterOpThreads(modifyThreadCount(config.encoderOnnxInterOpThreads())); + encoderOptions.setIntraOpThreads(modifyThreadCount(config.encoderOnnxIntraOpThreads())); + + encoder = onnx.evaluatorOf(config.encoderModel().toString(), encoderOptions); + + // Set up decoder + decoderInputIdsName = config.decoderModelInputIdsName(); + decoderAttentionMaskName = config.decoderModelAttentionMaskName(); + decoderEncoderHiddenStateName = config.decoderModelEncoderHiddenStateName(); + decoderOutputName = config.decoderModelOutputName(); + + OnnxEvaluatorOptions decoderOptions = new OnnxEvaluatorOptions(); + decoderOptions.setExecutionMode(config.decoderOnnxExecutionMode().toString()); + decoderOptions.setInterOpThreads(modifyThreadCount(config.decoderOnnxInterOpThreads())); + decoderOptions.setIntraOpThreads(modifyThreadCount(config.decoderOnnxIntraOpThreads())); + + decoder = onnx.evaluatorOf(config.decoderModel().toString(), decoderOptions); + + validateModels(); + } + + /** + * Generates text by evaluating an encoder model to encode the prompt, and + * repeatedly evaluating a decoding model to generate tokens until some + * stopping criteria has been met. + * + * @param prompt the prompt to generate text from + * @param options options for text generation + * @return a text generated from the prompt + */ + public String generate(String prompt, GeneratorOptions options) { + return switch (options.getSearchMethod()) { + case GREEDY -> generateGreedy(prompt, options); + default -> generateNotImplemented(options); + }; + } + + public String generate(String prompt) { + return generate(prompt, new GeneratorOptions()); + } + + @Override public void deconstruct() { encoder.close(); decoder.close(); } + + private String generateNotImplemented(GeneratorOptions options) { + throw new UnsupportedOperationException("Search method '" + options.getSearchMethod() + "' is currently not implemented"); + } + + private String generateGreedy(String prompt, GeneratorOptions options) { + var generatedTokens = new ArrayList<Integer>(); + generatedTokens.add(0); // Or target tokens + + // Tokenize + var inputTokens = tokenize(prompt); // Or source tokens + + // Evaluate encoder + var encoderInput = createTensorRepresentation(inputTokens, SEQUENCE_DIMENSION); + var encoderMask = createAttentionMask(encoderInput).expand(BATCH_DIMENSION); + var encoderOutput = evaluateEncoder(encoderInput.expand(BATCH_DIMENSION), encoderMask); + + // Greedy search just grabs the next most probable token + while (generatedTokens.size() < options.getMaxLength()) { // Todo: add stopping criteria + var decoderInput = createTensorRepresentation(generatedTokens, SEQUENCE_DIMENSION).expand(BATCH_DIMENSION); + var logits = evaluateDecoder(decoderInput, encoderMask, encoderOutput); + var nextToken = findMostProbableToken(logits, generatedTokens.size()-1, BATCH_DIMENSION, SEQUENCE_DIMENSION); + generatedTokens.add(nextToken); + } + + return detokenize(generatedTokens); + } + + private Tensor evaluateEncoder(Tensor input, Tensor mask) { + var encoderInputs = Map.of(encoderInputIdsName, input, + encoderAttentionMaskName, mask); + return encoder.evaluate(encoderInputs, encoderOutputName); + } + + private IndexedTensor evaluateDecoder(Tensor input, Tensor encoderMask, Tensor encoderOutput) { + var inputs = Map.of(decoderInputIdsName, input, + decoderAttentionMaskName, encoderMask, // yes, encoder's attention mask + decoderEncoderHiddenStateName, encoderOutput); + var output = decoder.evaluate(inputs, decoderOutputName); + if ( ! (output instanceof IndexedTensor indexedTensor)) { + throw new IllegalArgumentException("Output of decoder model is not an 'IndexedTensor'"); + } + return indexedTensor; + } + + /** + * Given a tensor 'logits' with 3 dimensions: batch, sequence, and vocabulary + * find the value in the vocabulary dimension with highest score for the given + * token in the sequence + */ + private static int findMostProbableToken(IndexedTensor logits, int seqIndex, String batchDim, String seqDim) { + if (logits.type().rank() != 3) { + throw new IllegalArgumentException("Expected a tensor with rank 3: batch, sequence, and vocabulary size. " + + "Got: " + logits.type()); + } + var iterator = logits.cellIterator(new PartialAddress.Builder(2). + add(batchDim, 0). + add(seqDim, seqIndex).build(), + DimensionSizes.of(logits.type())); + var maxVal = iterator.next().getValue(); + int maxIndex = 0; + for (int i = 1; iterator.hasNext(); ++i) { + var val = iterator.next().getValue(); + if (val >= maxVal && i != TOKEN_EOS) { + maxVal = val; + maxIndex = i; + } + } + return maxIndex; + } + + private List<Integer> tokenize(String text) { + var tokens = tokenizer.embed(text, new Embedder.Context("tokenizer")); + tokens = tokens.size() >= tokenizerMaxTokens ? tokens.subList(0,tokenizerMaxTokens-1): tokens; + tokens.add(TOKEN_EOS); + return tokens; + } + + private String detokenize(List<Integer> tokens) { + return tokenizer.decode(tokens, new Embedder.Context("tokenizer"), true); + } + + private static Tensor createTensorRepresentation(List<Integer> tokens, String dimension) { + var size = tokens.size(); + TensorType type = new TensorType.Builder(TensorType.Value.FLOAT).indexed(dimension, size).build(); + IndexedTensor.Builder builder = IndexedTensor.Builder.of(type); + for (int i = 0; i < size; ++i) { + builder.cell(tokens.get(i), i); + } + return builder.build(); + } + + private static Tensor createAttentionMask(Tensor d) { + return d.map((x) -> x > 0 ? 1:0); + } + + private void validateModels() { + Map<String, TensorType> inputs = encoder.getInputInfo(); + validateName(inputs, encoderInputIdsName, "input"); + validateName(inputs, encoderAttentionMaskName, "input"); + + Map<String, TensorType> outputs = encoder.getOutputInfo(); + validateName(outputs, encoderOutputName, "output"); + + inputs = decoder.getInputInfo(); + validateName(inputs, decoderInputIdsName, "input"); + validateName(inputs, decoderAttentionMaskName, "input"); + validateName(inputs, decoderEncoderHiddenStateName, "input"); + + outputs = decoder.getOutputInfo(); + validateName(outputs, decoderOutputName, "output"); + } + + private void validateName(Map<String, TensorType> types, String name, String type) { + if ( ! types.containsKey(name)) { + throw new IllegalArgumentException("Model does not contain required " + type + ": '" + name + "'. " + + "Model contains: " + String.join(",", types.keySet())); + } + } + + private int modifyThreadCount(int numThreads) { + if (numThreads >= 0) + return numThreads; + return Math.max(1, (int) Math.ceil(((double) Runtime.getRuntime().availableProcessors()) / (-1 * numThreads))); + } +} diff --git a/model-integration/src/main/java/ai/vespa/llm/GeneratorOptions.java b/model-integration/src/main/java/ai/vespa/llm/GeneratorOptions.java new file mode 100644 index 00000000000..743bb7c2f27 --- /dev/null +++ b/model-integration/src/main/java/ai/vespa/llm/GeneratorOptions.java @@ -0,0 +1,34 @@ +package ai.vespa.llm; + +public class GeneratorOptions { + + public enum SearchMethod { + GREEDY, + CONTRASTIVE, + BEAM, + SAMPLE, + } + + private SearchMethod searchMethod = SearchMethod.GREEDY; + private int maxLength = 20; + + public SearchMethod getSearchMethod() { + return searchMethod; + } + + public GeneratorOptions setSearchMethod(SearchMethod searchMethod) { + this.searchMethod = searchMethod; + return this; + } + + public int getMaxLength() { + return maxLength; + } + + public GeneratorOptions setMaxLength(int maxLength) { + this.maxLength = maxLength; + return this; + } + + +} diff --git a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java index 9961c24005c..7cdc27b6d63 100644 --- a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java +++ b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluator.java @@ -2,11 +2,12 @@ package ai.vespa.modelintegration.evaluator; +import ai.onnxruntime.NodeInfo; import ai.onnxruntime.OnnxTensor; import ai.onnxruntime.OnnxValue; -import ai.onnxruntime.OrtEnvironment; import ai.onnxruntime.OrtException; import ai.onnxruntime.OrtSession; +import ai.vespa.modelintegration.evaluator.OnnxRuntime.ReferencedOrtSession; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; @@ -14,32 +15,28 @@ import java.util.Collections; import java.util.HashMap; import java.util.Map; +import static ai.vespa.modelintegration.evaluator.OnnxRuntime.isCudaError; + /** * Evaluates an ONNX Model by deferring to ONNX Runtime. * * @author lesters */ -public class OnnxEvaluator { - - private final OrtEnvironment environment; - private final OrtSession session; +public class OnnxEvaluator implements AutoCloseable { - public OnnxEvaluator(String modelPath) { - this(modelPath, null); - } + private final ReferencedOrtSession session; - public OnnxEvaluator(String modelPath, OnnxEvaluatorOptions options) { - environment = OrtEnvironment.getEnvironment(); - session = createSession(modelPath, environment, options, true); + OnnxEvaluator(String modelPath, OnnxEvaluatorOptions options, OnnxRuntime runtime) { + session = createSession(modelPath, runtime, options, true); } public Tensor evaluate(Map<String, Tensor> inputs, String output) { Map<String, OnnxTensor> onnxInputs = null; try { output = mapToInternalName(output); - onnxInputs = TensorConverter.toOnnxTensors(inputs, environment, session); - try (OrtSession.Result result = session.run(onnxInputs, Collections.singleton(output))) { + onnxInputs = TensorConverter.toOnnxTensors(inputs, OnnxRuntime.ortEnvironment(), session.instance()); + try (OrtSession.Result result = session.instance().run(onnxInputs, Collections.singleton(output))) { return TensorConverter.toVespaTensor(result.get(0)); } } catch (OrtException e) { @@ -54,9 +51,9 @@ public class OnnxEvaluator { public Map<String, Tensor> evaluate(Map<String, Tensor> inputs) { Map<String, OnnxTensor> onnxInputs = null; try { - onnxInputs = TensorConverter.toOnnxTensors(inputs, environment, session); + onnxInputs = TensorConverter.toOnnxTensors(inputs, OnnxRuntime.ortEnvironment(), session.instance()); Map<String, Tensor> outputs = new HashMap<>(); - try (OrtSession.Result result = session.run(onnxInputs)) { + try (OrtSession.Result result = session.instance().run(onnxInputs)) { for (Map.Entry<String, OnnxValue> output : result) { String mapped = TensorConverter.asValidName(output.getKey()); outputs.put(mapped, TensorConverter.toVespaTensor(output.getValue())); @@ -72,9 +69,38 @@ public class OnnxEvaluator { } } + public record IdAndType(String id, TensorType type) { } + + private Map<String, IdAndType> toSpecMap(Map<String, NodeInfo> infoMap) { + Map<String, IdAndType> result = new HashMap<>(); + for (var info : infoMap.entrySet()) { + String name = info.getKey(); + String ident = TensorConverter.asValidName(name); + TensorType t = TensorConverter.toVespaType(info.getValue().getInfo()); + result.put(name, new IdAndType(ident, t)); + } + return result; + } + + public Map<String, IdAndType> getInputs() { + try { + return toSpecMap(session.instance().getInputInfo()); + } catch (OrtException e) { + throw new RuntimeException("ONNX Runtime exception", e); + } + } + + public Map<String, IdAndType> getOutputs() { + try { + return toSpecMap(session.instance().getOutputInfo()); + } catch (OrtException e) { + throw new RuntimeException("ONNX Runtime exception", e); + } + } + public Map<String, TensorType> getInputInfo() { try { - return TensorConverter.toVespaTypes(session.getInputInfo()); + return TensorConverter.toVespaTypes(session.instance().getInputInfo()); } catch (OrtException e) { throw new RuntimeException("ONNX Runtime exception", e); } @@ -82,25 +108,36 @@ public class OnnxEvaluator { public Map<String, TensorType> getOutputInfo() { try { - return TensorConverter.toVespaTypes(session.getOutputInfo()); + return TensorConverter.toVespaTypes(session.instance().getOutputInfo()); } catch (OrtException e) { throw new RuntimeException("ONNX Runtime exception", e); } } - private static OrtSession createSession(String modelPath, OrtEnvironment environment, OnnxEvaluatorOptions options, boolean tryCuda) { + @Override + public void close() throws IllegalStateException { + try { + session.close(); + } catch (UncheckedOrtException e) { + throw new IllegalStateException("Failed to close ONNX session", e); + } catch (IllegalStateException e) { + throw new IllegalStateException("Already closed", e); + } + } + + private static ReferencedOrtSession createSession(String modelPath, OnnxRuntime runtime, OnnxEvaluatorOptions options, boolean tryCuda) { if (options == null) { options = new OnnxEvaluatorOptions(); } try { - return environment.createSession(modelPath, options.getOptions(tryCuda && options.requestingGpu())); + return runtime.acquireSession(modelPath, options, tryCuda && options.requestingGpu()); } catch (OrtException e) { if (e.getCode() == OrtException.OrtErrorCode.ORT_NO_SUCHFILE) { throw new IllegalArgumentException("No such file: " + modelPath); } if (tryCuda && isCudaError(e) && !options.gpuDeviceRequired()) { // Failed in CUDA native code, but GPU device is optional, so we can proceed without it - return createSession(modelPath, environment, options, false); + return createSession(modelPath, runtime, options, false); } if (isCudaError(e)) { throw new IllegalArgumentException("GPU device is required, but CUDA initialization failed", e); @@ -109,34 +146,8 @@ public class OnnxEvaluator { } } - private static boolean isCudaError(OrtException e) { - return switch (e.getCode()) { - case ORT_FAIL -> e.getMessage().contains("cudaError"); - case ORT_EP_FAIL -> e.getMessage().contains("Failed to find CUDA"); - default -> false; - }; - } - - public static boolean isRuntimeAvailable() { - return isRuntimeAvailable(""); - } - - public static boolean isRuntimeAvailable(String modelPath) { - try { - new OnnxEvaluator(modelPath); - return true; - } catch (IllegalArgumentException e) { - if (e.getMessage().equals("No such file: ")) { - return true; - } - return false; - } catch (UnsatisfiedLinkError | RuntimeException | NoClassDefFoundError e) { - return false; - } - } - private String mapToInternalName(String outputName) throws OrtException { - var info = session.getOutputInfo(); + var info = session.instance().getOutputInfo(); var internalNames = info.keySet(); for (String name : internalNames) { if (name.equals(outputName)) { diff --git a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorOptions.java b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorOptions.java index b6de9698f1a..1ed219a8560 100644 --- a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorOptions.java +++ b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorOptions.java @@ -5,6 +5,8 @@ package ai.vespa.modelintegration.evaluator; import ai.onnxruntime.OrtException; import ai.onnxruntime.OrtSession; +import java.util.Objects; + /** * Session options for ONNX Runtime evaluation * @@ -12,7 +14,7 @@ import ai.onnxruntime.OrtSession; */ public class OnnxEvaluatorOptions { - private OrtSession.SessionOptions.OptLevel optimizationLevel; + private final OrtSession.SessionOptions.OptLevel optimizationLevel; private OrtSession.SessionOptions.ExecutionMode executionMode; private int interOpThreads; private int intraOpThreads; @@ -74,4 +76,18 @@ public class OnnxEvaluatorOptions { return gpuDeviceRequired; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + OnnxEvaluatorOptions that = (OnnxEvaluatorOptions) o; + return interOpThreads == that.interOpThreads && intraOpThreads == that.intraOpThreads + && gpuDeviceNumber == that.gpuDeviceNumber && gpuDeviceRequired == that.gpuDeviceRequired + && optimizationLevel == that.optimizationLevel && executionMode == that.executionMode; + } + + @Override + public int hashCode() { + return Objects.hash(optimizationLevel, executionMode, interOpThreads, intraOpThreads, gpuDeviceNumber, gpuDeviceRequired); + } } diff --git a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxRuntime.java b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxRuntime.java new file mode 100644 index 00000000000..42830041c02 --- /dev/null +++ b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/OnnxRuntime.java @@ -0,0 +1,170 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package ai.vespa.modelintegration.evaluator; + +import ai.onnxruntime.OrtEnvironment; +import ai.onnxruntime.OrtException; +import ai.onnxruntime.OrtSession; +import com.yahoo.component.AbstractComponent; +import com.yahoo.component.annotation.Inject; +import com.yahoo.jdisc.ResourceReference; +import com.yahoo.jdisc.refcount.DebugReferencesWithStack; +import com.yahoo.jdisc.refcount.References; + +import java.util.HashMap; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +import static com.yahoo.yolean.Exceptions.throwUnchecked; + +/** + * Provides ONNX runtime environment with session management. + * + * @author bjorncs + */ +public class OnnxRuntime extends AbstractComponent { + + // For unit testing + @FunctionalInterface interface OrtSessionFactory { + OrtSession create(String path, OrtSession.SessionOptions opts) throws OrtException; + } + + private static final Logger log = Logger.getLogger(OnnxRuntime.class.getName()); + + private static final OrtEnvironmentResult ortEnvironment = getOrtEnvironment(); + private static final OrtSessionFactory defaultFactory = (path, opts) -> ortEnvironment().createSession(path, opts); + + private final Object monitor = new Object(); + private final Map<OrtSessionId, SharedOrtSession> sessions = new HashMap<>(); + private final OrtSessionFactory factory; + + @Inject public OnnxRuntime() { this(defaultFactory); } + + OnnxRuntime(OrtSessionFactory factory) { this.factory = factory; } + + public OnnxEvaluator evaluatorOf(String modelPath) { + return new OnnxEvaluator(modelPath, null, this); + } + + public OnnxEvaluator evaluatorOf(String modelPath, OnnxEvaluatorOptions options) { + return new OnnxEvaluator(modelPath, options, this); + } + + public static OrtEnvironment ortEnvironment() { + if (ortEnvironment.env() != null) return ortEnvironment.env(); + throw throwUnchecked(ortEnvironment.failure()); + } + + @Override + public void deconstruct() { + synchronized (monitor) { + sessions.forEach((id, sharedSession) -> { + int hash = System.identityHashCode(sharedSession.session()); + var refs = sharedSession.references(); + log.warning("Closing leaked session %s (%s) with %d outstanding references:\n%s" + .formatted(id, hash, refs.referenceCount(), refs.currentState())); + try { + sharedSession.session().close(); + } catch (Exception e) { + log.log(Level.WARNING, "Failed to close session %s (%s)".formatted(id, hash), e); + } + }); + sessions.clear(); + } + } + + private static OrtEnvironmentResult getOrtEnvironment() { + try { + return new OrtEnvironmentResult(OrtEnvironment.getEnvironment(), null); + } catch (UnsatisfiedLinkError | RuntimeException | NoClassDefFoundError e) { + log.log(Level.FINE, e, () -> "Failed to load ONNX runtime"); + return new OrtEnvironmentResult(null, e); + } + } + + public static boolean isRuntimeAvailable() { return ortEnvironment.env() != null; } + public static boolean isRuntimeAvailable(String modelPath) { + if (!isRuntimeAvailable()) return false; + try { + // Expensive way of checking if runtime is available as it incurs the cost of loading the model if successful + defaultFactory.create(modelPath, new OnnxEvaluatorOptions().getOptions(false)); + return true; + } catch (OrtException e) { + return e.getCode() == OrtException.OrtErrorCode.ORT_NO_SUCHFILE; + } catch (UnsatisfiedLinkError | RuntimeException | NoClassDefFoundError e) { + return false; + } + } + + static boolean isCudaError(OrtException e) { + return switch (e.getCode()) { + case ORT_FAIL -> e.getMessage().contains("cudaError"); + case ORT_EP_FAIL -> e.getMessage().contains("Failed to find CUDA"); + default -> false; + }; + } + + ReferencedOrtSession acquireSession(String modelPath, OnnxEvaluatorOptions options, boolean loadCuda) throws OrtException { + var sessionId = new OrtSessionId(modelPath, options, loadCuda); + synchronized (monitor) { + var sharedSession = sessions.get(sessionId); + if (sharedSession != null) { + return sharedSession.newReference(); + } + } + + // Note: identical models loaded simultaneously will result in duplicate session instances + var session = factory.create(modelPath, options.getOptions(loadCuda)); + log.fine(() -> "Created new session (%s)".formatted(System.identityHashCode(session))); + + var sharedSession = new SharedOrtSession(sessionId, session); + var referencedSession = sharedSession.newReference(); + synchronized (monitor) { sessions.put(sessionId, sharedSession); } + sharedSession.references().release(); // Release initial reference + return referencedSession; + } + + int sessionsCached() { synchronized(monitor) { return sessions.size(); } } + + public static class ReferencedOrtSession implements AutoCloseable { + private final OrtSession instance; + private final ResourceReference ref; + + public ReferencedOrtSession(OrtSession instance, ResourceReference ref) { + this.instance = instance; + this.ref = ref; + } + + public OrtSession instance() { return instance; } + @Override public void close() { ref.close(); } + } + + // Assumes options are never modified after being stored in `onnxSessions` + record OrtSessionId(String modelPath, OnnxEvaluatorOptions options, boolean loadCuda) {} + + record OrtEnvironmentResult(OrtEnvironment env, Throwable failure) {} + + private class SharedOrtSession { + private final OrtSessionId id; + private final OrtSession session; + private final References refs = new DebugReferencesWithStack(this::close); + + SharedOrtSession(OrtSessionId id, OrtSession session) { + this.id = id; + this.session = session; + } + + ReferencedOrtSession newReference() { return new ReferencedOrtSession(session, refs.refer(id)); } + References references() { return refs; } + OrtSession session() { return session; } + + void close() { + try { + synchronized (OnnxRuntime.this.monitor) { sessions.remove(id); } + log.fine(() -> "Closing session (%s)".formatted(System.identityHashCode(session))); + session.close(); + } catch (OrtException e) { throw new UncheckedOrtException(e);} + } + } +} diff --git a/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/UncheckedOrtException.java b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/UncheckedOrtException.java new file mode 100644 index 00000000000..1f2c8ba2cf7 --- /dev/null +++ b/model-integration/src/main/java/ai/vespa/modelintegration/evaluator/UncheckedOrtException.java @@ -0,0 +1,15 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package ai.vespa.modelintegration.evaluator; + +import ai.onnxruntime.OrtException; + +/** + * @author bjorncs + */ +public class UncheckedOrtException extends RuntimeException { + + public UncheckedOrtException(Throwable e) { super(e.getMessage(), e); } + + @Override public synchronized OrtException getCause() { return (OrtException) super.getCause(); } +} diff --git a/model-integration/src/main/resources/configdefinitions/llm.generator.def b/model-integration/src/main/resources/configdefinitions/llm.generator.def new file mode 100644 index 00000000000..478daad6ede --- /dev/null +++ b/model-integration/src/main/resources/configdefinitions/llm.generator.def @@ -0,0 +1,32 @@ +namespace=llm + +# SentencePiece tokenizer +tokenizerModel model +tokenizerMaxTokens int default=1000 + +# +# The encoder model +# +encoderModel model +encoderModelInputIdsName string default=input_ids +encoderModelAttentionMaskName string default=attention_mask +encoderModelOutputName string default=last_hidden_state + +encoderOnnxExecutionMode enum { parallel, sequential } default=sequential +encoderOnnxInterOpThreads int default=1 +encoderOnnxIntraOpThreads int default=-4 # n=number of threads -> n<0: CPUs/(-n), n==0: CPUs, n>0: n +# enable GPU? + +# +# The decoder model +# +decoderModel model +decoderModelInputIdsName string default=input_ids +decoderModelAttentionMaskName string default=encoder_attention_mask +decoderModelEncoderHiddenStateName string default=encoder_hidden_states +decoderModelOutputName string default=logits + +decoderOnnxExecutionMode enum { parallel, sequential } default=sequential +decoderOnnxInterOpThreads int default=1 +decoderOnnxIntraOpThreads int default=-4 # n=number of threads -> n<0: CPUs/(-n), n==0: CPUs, n>0: n +# enable GPU? diff --git a/model-integration/src/test/java/ai/vespa/embedding/BertBaseEmbedderTest.java b/model-integration/src/test/java/ai/vespa/embedding/BertBaseEmbedderTest.java index 73359736536..329b87cacd1 100644 --- a/model-integration/src/test/java/ai/vespa/embedding/BertBaseEmbedderTest.java +++ b/model-integration/src/test/java/ai/vespa/embedding/BertBaseEmbedderTest.java @@ -1,15 +1,12 @@ package ai.vespa.embedding; -import ai.vespa.modelintegration.evaluator.OnnxEvaluator; -import com.yahoo.config.FileReference; +import ai.vespa.modelintegration.evaluator.OnnxRuntime; import com.yahoo.config.ModelReference; -import com.yahoo.config.UrlReference; import com.yahoo.embedding.BertBaseEmbedderConfig; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import org.junit.Test; -import java.lang.IllegalArgumentException; import java.util.List; import static org.junit.Assert.assertEquals; @@ -22,12 +19,12 @@ public class BertBaseEmbedderTest { public void testEmbedder() { String vocabPath = "src/test/models/onnx/transformer/dummy_vocab.txt"; String modelPath = "src/test/models/onnx/transformer/dummy_transformer.onnx"; - assumeTrue(OnnxEvaluator.isRuntimeAvailable(modelPath)); + assumeTrue(OnnxRuntime.isRuntimeAvailable(modelPath)); BertBaseEmbedderConfig.Builder builder = new BertBaseEmbedderConfig.Builder(); builder.tokenizerVocab(ModelReference.valueOf(vocabPath)); builder.transformerModel(ModelReference.valueOf(modelPath)); - BertBaseEmbedder embedder = new BertBaseEmbedder(builder.build()); + BertBaseEmbedder embedder = newBertBaseEmbedder(builder.build()); TensorType destType = TensorType.fromSpec("tensor<float>(x[7])"); List<Integer> tokens = List.of(1,2,3,4,5); // use random tokens instead of invoking the tokenizer @@ -41,13 +38,13 @@ public class BertBaseEmbedderTest { public void testEmbedderWithoutTokenTypeIdsName() { String vocabPath = "src/test/models/onnx/transformer/dummy_vocab.txt"; String modelPath = "src/test/models/onnx/transformer/dummy_transformer_without_type_ids.onnx"; - assumeTrue(OnnxEvaluator.isRuntimeAvailable(modelPath)); + assumeTrue(OnnxRuntime.isRuntimeAvailable(modelPath)); BertBaseEmbedderConfig.Builder builder = new BertBaseEmbedderConfig.Builder(); builder.tokenizerVocab(ModelReference.valueOf(vocabPath)); builder.transformerModel(ModelReference.valueOf(modelPath)); builder.transformerTokenTypeIds(""); - BertBaseEmbedder embedder = new BertBaseEmbedder(builder.build()); + BertBaseEmbedder embedder = newBertBaseEmbedder(builder.build()); TensorType destType = TensorType.fromSpec("tensor<float>(x[7])"); List<Integer> tokens = List.of(1,2,3,4,5); // use random tokens instead of invoking the tokenizer @@ -61,14 +58,18 @@ public class BertBaseEmbedderTest { public void testEmbedderWithoutTokenTypeIdsNameButWithConfig() { String vocabPath = "src/test/models/onnx/transformer/dummy_vocab.txt"; String modelPath = "src/test/models/onnx/transformer/dummy_transformer_without_type_ids.onnx"; - assumeTrue(OnnxEvaluator.isRuntimeAvailable(modelPath)); + assumeTrue(OnnxRuntime.isRuntimeAvailable(modelPath)); BertBaseEmbedderConfig.Builder builder = new BertBaseEmbedderConfig.Builder(); builder.tokenizerVocab(ModelReference.valueOf(vocabPath)); builder.transformerModel(ModelReference.valueOf(modelPath)); // we did not configured BertBaseEmbedder to accept missing token type ids // so we expect ctor to throw - assertThrows(IllegalArgumentException.class, () -> { new BertBaseEmbedder(builder.build()); }); + assertThrows(IllegalArgumentException.class, () -> { newBertBaseEmbedder(builder.build()); }); + } + + private static BertBaseEmbedder newBertBaseEmbedder(BertBaseEmbedderConfig cfg) { + return new BertBaseEmbedder(new OnnxRuntime(), cfg); } } diff --git a/model-integration/src/test/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedderTest.java b/model-integration/src/test/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedderTest.java index c67b6b0dcab..0ff9acc9a69 100644 --- a/model-integration/src/test/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedderTest.java +++ b/model-integration/src/test/java/ai/vespa/embedding/huggingface/HuggingFaceEmbedderTest.java @@ -1,19 +1,5 @@ package ai.vespa.embedding.huggingface; -import ai.vespa.modelintegration.evaluator.OnnxEvaluator; -import com.yahoo.config.ModelReference; -import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.TensorType; -import org.junit.Test; - -import com.yahoo.embedding.huggingface.HuggingFaceEmbedderConfig; - -import java.io.IOException; -import java.util.List; - -import static org.junit.Assume.assumeTrue; -import static org.junit.Assert.assertEquals; - public class HuggingFaceEmbedderTest { /* @Test @@ -21,7 +7,7 @@ public class HuggingFaceEmbedderTest { String modelPath = "src/test/models/hf/model.onnx"; String tokenizerPath = "src/test/models/hf/tokenizer.json"; - assumeTrue(OnnxEvaluator.isRuntimeAvailable(modelPath)); + assumeTrue(OnnxRuntime.isRuntimeAvailable(modelPath)); HuggingFaceEmbedderConfig.Builder builder = new HuggingFaceEmbedderConfig.Builder(); builder.tokenizerPath(ModelReference.valueOf(tokenizerPath)); diff --git a/model-integration/src/test/java/ai/vespa/llm/GeneratorTest.java b/model-integration/src/test/java/ai/vespa/llm/GeneratorTest.java new file mode 100644 index 00000000000..c22902b344f --- /dev/null +++ b/model-integration/src/test/java/ai/vespa/llm/GeneratorTest.java @@ -0,0 +1,40 @@ +package ai.vespa.llm; + +import ai.vespa.modelintegration.evaluator.OnnxRuntime; +import com.yahoo.config.ModelReference; +import com.yahoo.llm.GeneratorConfig; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assume.assumeTrue; + +public class GeneratorTest { + + @Test + public void testGenerator() { + String vocabPath = "src/test/models/onnx/llm/en.wiki.bpe.vs10000.model"; + String encoderModelPath = "src/test/models/onnx/llm/random_encoder.onnx"; + String decoderModelPath = "src/test/models/onnx/llm/random_decoder.onnx"; + assumeTrue(OnnxRuntime.isRuntimeAvailable(encoderModelPath)); + + GeneratorConfig.Builder builder = new GeneratorConfig.Builder(); + builder.tokenizerModel(ModelReference.valueOf(vocabPath)); + builder.encoderModel(ModelReference.valueOf(encoderModelPath)); + builder.decoderModel(ModelReference.valueOf(decoderModelPath)); + Generator generator = newGenerator(builder.build()); + + GeneratorOptions options = new GeneratorOptions(); + options.setSearchMethod(GeneratorOptions.SearchMethod.GREEDY); + options.setMaxLength(10); + + String prompt = "generate some random text"; + String result = generator.generate(prompt, options); + + assertEquals("<unk> linear recruit latest sack annually institutions cert solid references", result); + } + + private static Generator newGenerator(GeneratorConfig cfg) { + return new Generator(new OnnxRuntime(), cfg); + } + +} diff --git a/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java b/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java index 83f355821e5..5aba54de11b 100644 --- a/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java +++ b/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxEvaluatorTest.java @@ -5,23 +5,31 @@ package ai.vespa.modelintegration.evaluator; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import org.junit.Test; +import org.junit.jupiter.api.BeforeAll; import java.util.HashMap; import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -import static org.junit.Assume.assumeTrue; +import static org.junit.Assume.assumeNotNull; /** * @author lesters */ public class OnnxEvaluatorTest { + private static OnnxRuntime runtime; + + @BeforeAll + public static void beforeAll() { + if (OnnxRuntime.isRuntimeAvailable()) runtime = new OnnxRuntime(); + } + @Test public void testSimpleModel() { - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); - OnnxEvaluator evaluator = new OnnxEvaluator("src/test/models/onnx/simple/simple.onnx"); + assumeNotNull(runtime); + OnnxEvaluator evaluator = runtime.evaluatorOf("src/test/models/onnx/simple/simple.onnx"); // Input types Map<String, TensorType> inputTypes = evaluator.getInputInfo(); @@ -45,8 +53,8 @@ public class OnnxEvaluatorTest { @Test public void testBatchDimension() { - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); - OnnxEvaluator evaluator = new OnnxEvaluator("src/test/models/onnx/pytorch/one_layer.onnx"); + assumeNotNull(runtime); + OnnxEvaluator evaluator = runtime.evaluatorOf("src/test/models/onnx/pytorch/one_layer.onnx"); // Input types Map<String, TensorType> inputTypes = evaluator.getInputInfo(); @@ -64,7 +72,7 @@ public class OnnxEvaluatorTest { @Test public void testMatMul() { - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); + assumeNotNull(runtime); String expected = "tensor<float>(d0[2],d1[4]):[38,44,50,56,83,98,113,128]"; String input1 = "tensor<float>(d0[2],d1[3]):[1,2,3,4,5,6]"; String input2 = "tensor<float>(d0[3],d1[4]):[1,2,3,4,5,6,7,8,9,10,11,12]"; @@ -73,7 +81,7 @@ public class OnnxEvaluatorTest { @Test public void testTypes() { - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); + assumeNotNull(runtime); assertEvaluate("add_double.onnx", "tensor(d0[1]):[3]", "tensor(d0[1]):[1]", "tensor(d0[1]):[2]"); assertEvaluate("add_float.onnx", "tensor<float>(d0[1]):[3]", "tensor<float>(d0[1]):[1]", "tensor<float>(d0[1]):[2]"); assertEvaluate("add_int64.onnx", "tensor<double>(d0[1]):[3]", "tensor<double>(d0[1]):[1]", "tensor<double>(d0[1]):[2]"); @@ -86,8 +94,8 @@ public class OnnxEvaluatorTest { @Test public void testNotIdentifiers() { - assumeTrue(OnnxEvaluator.isRuntimeAvailable()); - OnnxEvaluator evaluator = new OnnxEvaluator("src/test/models/onnx/badnames.onnx"); + assumeNotNull(runtime); + OnnxEvaluator evaluator = runtime.evaluatorOf("src/test/models/onnx/badnames.onnx"); var inputInfo = evaluator.getInputInfo(); var outputInfo = evaluator.getOutputInfo(); for (var entry : inputInfo.entrySet()) { @@ -152,7 +160,7 @@ public class OnnxEvaluatorTest { } private void assertEvaluate(String model, String output, String... input) { - OnnxEvaluator evaluator = new OnnxEvaluator("src/test/models/onnx/" + model); + OnnxEvaluator evaluator = runtime.evaluatorOf("src/test/models/onnx/" + model); Map<String, Tensor> inputs = new HashMap<>(); for (int i = 0; i < input.length; ++i) { inputs.put("input" + (i+1), Tensor.from(input[i])); diff --git a/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxRuntimeTest.java b/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxRuntimeTest.java new file mode 100644 index 00000000000..81b1237e770 --- /dev/null +++ b/model-integration/src/test/java/ai/vespa/modelintegration/evaluator/OnnxRuntimeTest.java @@ -0,0 +1,48 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package ai.vespa.modelintegration.evaluator; + +import ai.onnxruntime.OrtException; +import ai.onnxruntime.OrtSession; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotSame; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +/** + * @author bjorncs + */ +class OnnxRuntimeTest { + + @Test + void reuses_sessions_while_active() throws OrtException { + var runtime = new OnnxRuntime((__, ___) -> mock(OrtSession.class)); + var session1 = runtime.acquireSession("model1", new OnnxEvaluatorOptions(), false); + var session2 = runtime.acquireSession("model1", new OnnxEvaluatorOptions(), false); + var session3 = runtime.acquireSession("model2", new OnnxEvaluatorOptions(), false); + assertSame(session1.instance(), session2.instance()); + assertNotSame(session1.instance(), session3.instance()); + assertEquals(2, runtime.sessionsCached()); + + session1.close(); + session2.close(); + assertEquals(1, runtime.sessionsCached()); + verify(session1.instance()).close(); + verify(session3.instance(), never()).close(); + + session3.close(); + assertEquals(0, runtime.sessionsCached()); + verify(session3.instance()).close(); + + var session4 = runtime.acquireSession("model1", new OnnxEvaluatorOptions(), false); + assertNotSame(session1.instance(), session4.instance()); + assertEquals(1, runtime.sessionsCached()); + session4.close(); + assertEquals(0, runtime.sessionsCached()); + verify(session4.instance()).close(); + } +}
\ No newline at end of file diff --git a/model-integration/src/test/models/onnx/llm/en.wiki.bpe.vs10000.model b/model-integration/src/test/models/onnx/llm/en.wiki.bpe.vs10000.model Binary files differnew file mode 100644 index 00000000000..89f93ef3517 --- /dev/null +++ b/model-integration/src/test/models/onnx/llm/en.wiki.bpe.vs10000.model diff --git a/model-integration/src/test/models/onnx/llm/random_decoder.onnx b/model-integration/src/test/models/onnx/llm/random_decoder.onnx Binary files differnew file mode 100644 index 00000000000..a8c5f18ddf2 --- /dev/null +++ b/model-integration/src/test/models/onnx/llm/random_decoder.onnx diff --git a/model-integration/src/test/models/onnx/llm/random_encoder.onnx b/model-integration/src/test/models/onnx/llm/random_encoder.onnx Binary files differnew file mode 100644 index 00000000000..a9100fcd6af --- /dev/null +++ b/model-integration/src/test/models/onnx/llm/random_encoder.onnx diff --git a/model-integration/src/test/models/onnx/llm/random_llm.py b/model-integration/src/test/models/onnx/llm/random_llm.py new file mode 100644 index 00000000000..722906fc48b --- /dev/null +++ b/model-integration/src/test/models/onnx/llm/random_llm.py @@ -0,0 +1,82 @@ +import torch +import torch.onnx +import torch.nn as nn +from torch.nn import TransformerEncoderLayer, TransformerEncoder, TransformerDecoder, TransformerDecoderLayer + + +class EncoderModel(nn.Module): + def __init__(self, vocab_size, emb_size, hidden_dim_size, num_heads, num_layers, dropout=0.2, batch_first=True): + super(EncoderModel, self).__init__() + self.embedding = nn.Embedding(vocab_size, emb_size) + encoder_layers = TransformerEncoderLayer(emb_size, num_heads, hidden_dim_size, dropout, batch_first=batch_first) + self.transformer_encoder = TransformerEncoder(encoder_layers, num_layers) + + def forward(self, tokens, attention_mask): + src = self.embedding(tokens * attention_mask) # N, S, E + output = self.transformer_encoder(src) + return output + + +class DecoderModel(nn.Module): + def __init__(self, vocab_size, emb_size, hidden_dim_size, num_heads, num_layers, dropout=0.2, batch_first=True): + super(DecoderModel, self).__init__() + self.embedding = nn.Embedding(vocab_size, emb_size) + decoder_layers = nn.TransformerDecoderLayer(emb_size, num_heads, hidden_dim_size, batch_first=batch_first) + self.transformer_decoder = nn.TransformerDecoder(decoder_layers, num_layers) + self.linear = nn.Linear(emb_size, vocab_size) + + def forward(self, tokens, attention_mask, encoder_hidden_state): + tgt = self.embedding(tokens) # N, T, E + out = self.transformer_decoder(tgt, encoder_hidden_state, memory_mask=attention_mask) + logits = self.linear(out) + return logits + + +def main(): + vocabulary_size = 10000 + embedding_size = 8 + hidden_dim_size = 16 + num_heads = 1 + num_layers = 1 + + encoder = EncoderModel(vocabulary_size, embedding_size, hidden_dim_size, num_heads, num_layers) + decoder = DecoderModel(vocabulary_size, embedding_size, hidden_dim_size, num_heads, num_layers) + + # Omit training - just export randomly initialized network + + tokens = torch.LongTensor([[1, 2, 3, 4, 5]]) + attention_mask = torch.LongTensor([[1, 1, 1, 1, 1]]) + + torch.onnx.export(encoder, + (tokens, attention_mask), + "random_encoder.onnx", + input_names=["input_ids", "attention_mask"], + output_names=["last_hidden_state"], + dynamic_axes={ + "input_ids": {0: "batch", 1: "tokens"}, + "attention_mask": {0: "batch", 1: "tokens"}, + "last_hidden_state": {0: "batch", 1: "tokens"}, + }, + opset_version=12) + + last_hidden_state = encoder.forward(tokens, attention_mask) + tokens = torch.LongTensor([[0]]) #1, 2]]) + + torch.onnx.export(decoder, + (tokens, attention_mask.float(), last_hidden_state), + "random_decoder.onnx", + input_names=["input_ids", "encoder_attention_mask", "encoder_hidden_states"], + output_names=["logits"], + dynamic_axes={ + "input_ids": {0: "batch", 1: "target_tokens"}, + "encoder_attention_mask": {0: "batch", 1: "source_tokens"}, + "encoder_hidden_states": {0: "batch", 1: "source_tokens"}, + "logits": {0: "batch", 1: "target_tokens"}, + }, + opset_version=12) + + +if __name__ == "__main__": + main() + + diff --git a/node-admin/pom.xml b/node-admin/pom.xml index 90bb72efbcd..0f153663e3d 100644 --- a/node-admin/pom.xml +++ b/node-admin/pom.xml @@ -86,6 +86,11 @@ <!-- Test --> <dependency> + <groupId>com.google.guava</groupId> + <artifactId>guava-testlib</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> <scope>test</scope> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java index 61ee612e3de..6e6a6dec9d3 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/ConfigServerApiImpl.java @@ -249,6 +249,7 @@ public class ConfigServerApiImpl implements ConfigServerApi { return HttpClientBuilder.create() .setDefaultRequestConfig(DEFAULT_REQUEST_CONFIG) .disableAutomaticRetries() + .disableConnectionState() // Share connections between subsequent requests. .setUserAgent("node-admin") .setConnectionManager(cm) .build(); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepository.java index b423eb5dbdf..f543416115b 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/NodeRepository.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.configserver.noderepository; -import com.yahoo.vespa.hosted.node.admin.wireguard.ConfigserverPeer; +import com.yahoo.vespa.hosted.node.admin.wireguard.WireguardPeer; import java.util.List; import java.util.Map; @@ -24,7 +24,9 @@ public interface NodeRepository { Map<String, Acl> getAcls(String hostname); - List<ConfigserverPeer> getConfigserverPeers(); + List<WireguardPeer> getExclavePeers(); + + List<WireguardPeer> getConfigserverPeers(); void updateNodeAttributes(String hostName, NodeAttributes nodeAttributes); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java index e092cc15145..3a7e12f5661 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepository.java @@ -18,7 +18,7 @@ import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.Ge import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.GetWireguardResponse; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.bindings.NodeRepositoryNode; import com.yahoo.vespa.hosted.node.admin.task.util.network.VersionedIpAddress; -import com.yahoo.vespa.hosted.node.admin.wireguard.ConfigserverPeer; +import com.yahoo.vespa.hosted.node.admin.wireguard.WireguardPeer; import java.net.URI; import java.time.Instant; @@ -130,12 +130,23 @@ public class RealNodeRepository implements NodeRepository { } @Override - public List<ConfigserverPeer> getConfigserverPeers() { - GetWireguardResponse nodeResponse = configServerApi.get("/nodes/v2/wireguard", - GetWireguardResponse.class); - return nodeResponse.configservers.stream() + public List<WireguardPeer> getExclavePeers() { + String path = "/nodes/v2/node/?recursive=true&enclave=true"; + final GetNodesResponse response = configServerApi.get(path, GetNodesResponse.class); + + return response.nodes.stream() + .filter(node -> node.wireguardPubkey != null && ! node.wireguardPubkey.isEmpty()) + .map(RealNodeRepository::createTenantPeer) + .sorted() + .toList(); + } + + @Override + public List<WireguardPeer> getConfigserverPeers() { + GetWireguardResponse response = configServerApi.get("/nodes/v2/wireguard", GetWireguardResponse.class); + return response.configservers.stream() .map(RealNodeRepository::createConfigserverPeer) - .sorted(Comparator.comparing(ConfigserverPeer::hostname)) + .sorted(Comparator.comparing(WireguardPeer::hostname)) .toList(); } @@ -340,10 +351,16 @@ public class RealNodeRepository implements NodeRepository { return node; } - private static ConfigserverPeer createConfigserverPeer(GetWireguardResponse.Configserver configServer) { - return new ConfigserverPeer(HostName.of(configServer.hostname), - configServer.ipAddresses.stream().map(VersionedIpAddress::from).toList(), - configServer.wireguardKey()); + private static WireguardPeer createTenantPeer(NodeRepositoryNode node) { + return new WireguardPeer(HostName.of(node.hostname), + node.ipAddresses.stream().map(VersionedIpAddress::from).toList(), + WireguardKey.from(node.wireguardPubkey)); + } + + private static WireguardPeer createConfigserverPeer(GetWireguardResponse.Configserver configServer) { + return new WireguardPeer(HostName.of(configServer.hostname), + configServer.ipAddresses.stream().map(VersionedIpAddress::from).toList(), + WireguardKey.from(configServer.wireguardPubkey)); } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetWireguardResponse.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetWireguardResponse.java index 9f30f8e0fb5..a71b2a74b31 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetWireguardResponse.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/bindings/GetWireguardResponse.java @@ -4,10 +4,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import com.yahoo.config.provision.WireguardKey; import java.util.List; -import java.util.Optional; /** * A response from the /nodes/v2/wireguard api. @@ -45,10 +43,6 @@ public class GetWireguardResponse { this.ipAddresses = ipAddresses; this.wireguardPubkey = wireguardPubkey; } - - public Optional<WireguardKey> wireguardKey() { - return (wireguardPubkey == null || wireguardPubkey.isEmpty()) ? Optional.empty() : Optional.of(new WireguardKey(wireguardPubkey)); - } } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java index d884c608ead..e2a7f9167ac 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContext.java @@ -61,4 +61,6 @@ public interface NodeAgentContext extends TaskContext { double vcpuOnThisHost(); Optional<ApplicationId> hostExclusiveTo(); + + boolean exclave(); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java index ed2de691eb0..210bdf2fcb3 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextImpl.java @@ -45,11 +45,12 @@ public class NodeAgentContextImpl implements NodeAgentContext { private final double cpuSpeedup; private final Set<NodeAgentTask> disabledNodeAgentTasks; private final Optional<ApplicationId> hostExclusiveTo; + private final boolean exclave; public NodeAgentContextImpl(NodeSpec node, Acl acl, AthenzIdentity identity, ContainerNetworkMode containerNetworkMode, ZoneApi zone, FlagSource flagSource, UserScope userScope, PathScope pathScope, - double cpuSpeedup, Optional<ApplicationId> hostExclusiveTo) { + double cpuSpeedup, Optional<ApplicationId> hostExclusiveTo, boolean exclave) { if (cpuSpeedup <= 0) throw new IllegalArgumentException("cpuSpeedUp must be positive, was: " + cpuSpeedup); @@ -68,6 +69,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { .with(FetchVector.Dimension.HOSTNAME, node.hostname()) .with(FetchVector.Dimension.NODE_TYPE, node.type().name()).value()); this.hostExclusiveTo = hostExclusiveTo; + this.exclave = exclave; } @Override @@ -140,6 +142,11 @@ public class NodeAgentContextImpl implements NodeAgentContext { logger.log(level, logPrefix + message, throwable); } + @Override + public boolean exclave() { + return exclave; + } + public static NodeAgentContextImpl.Builder builder(NodeSpec node) { return new Builder(new NodeSpec.Builder(node)); } @@ -168,6 +175,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { private FlagSource flagSource; private double cpuSpeedUp = 1; private Optional<ApplicationId> hostExclusiveTo = Optional.empty(); + private boolean exclave = false; private Builder(NodeSpec.Builder nodeSpecBuilder) { this.nodeSpecBuilder = nodeSpecBuilder; @@ -234,6 +242,11 @@ public class NodeAgentContextImpl implements NodeAgentContext { return this; } + public Builder exclave(boolean exclave) { + this.exclave = exclave; + return this; + } + public NodeAgentContextImpl build() { Objects.requireNonNull(containerStorage, "Must set one of containerStorage or fileSystem"); @@ -273,7 +286,7 @@ public class NodeAgentContextImpl implements NodeAgentContext { Optional.ofNullable(flagSource).orElseGet(InMemoryFlagSource::new), userScope, new PathScope(containerFs, "/opt/vespa"), - cpuSpeedUp, hostExclusiveTo); + cpuSpeedUp, hostExclusiveTo, exclave); } } } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/VersionedIpAddress.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/VersionedIpAddress.java index 987a8909142..03dd4d026ec 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/VersionedIpAddress.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/network/VersionedIpAddress.java @@ -5,20 +5,32 @@ import com.google.common.net.InetAddresses; import java.net.Inet4Address; import java.net.Inet6Address; import java.net.InetAddress; +import java.util.Objects; /** + * Encapsulates an IP address and its version along with some convenience methods. + * Default sorting is by version (IPv6 first), then by address. + * * @author gjoranv */ -public class VersionedIpAddress { +public class VersionedIpAddress implements Comparable<VersionedIpAddress> { private final InetAddress address; private final IPVersion version; private VersionedIpAddress(InetAddress address) { - this.address = address; + this.address = Objects.requireNonNull(address); version = getVersionOrThrow(address); } + public static VersionedIpAddress from(InetAddress address) { + return new VersionedIpAddress(address); + } + + public static VersionedIpAddress from(String address) { + return from(InetAddresses.forString(address)); + } + public IPVersion version() { return version; } @@ -27,13 +39,36 @@ public class VersionedIpAddress { return InetAddresses.toAddrString(address); } - public static VersionedIpAddress from(InetAddress address) { - return new VersionedIpAddress(address); + public String asEndpoint(int port) { + var format = (version == IPVersion.IPv6) ? "[%s]:%d" : "%s:%d"; + return String.format(format, asString(), port); } - // TODO: remove? - public static VersionedIpAddress from(String address) { - return from(InetAddresses.forString(address)); + @Override + public int compareTo(VersionedIpAddress o) { + int version = version().compareTo(o.version()); + return (version != 0) ? version : asString().compareTo(o.asString()); + } + + @Override + public String toString() { + return "VersionedIpAddress{" + + "address=" + address + + ", version=" + version + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + VersionedIpAddress that = (VersionedIpAddress) o; + return address.equals(that.address) && version == that.version; + } + + @Override + public int hashCode() { + return Objects.hash(address, version); } private static IPVersion getVersionOrThrow(InetAddress address) { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/ConfigserverParameters.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/ConfigserverParameters.java deleted file mode 100644 index c74ba2b7d6c..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/ConfigserverParameters.java +++ /dev/null @@ -1,36 +0,0 @@ -package com.yahoo.vespa.hosted.node.admin.wireguard; - -import com.yahoo.config.provision.WireguardKey; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; - -/** - * Wireguard parameters for a configserver. - * - * @author gjoranv - */ -public record ConfigserverParameters(String hostname, String endpoint, WireguardKey publicKey) { - - public static ConfigserverParameters fromJson(String json) { - Slime slime = SlimeUtils.jsonToSlime(json); - Cursor root = slime.get(); - return new ConfigserverParameters( - root.field("hostname").asString(), - root.field("endpoint").asString(), - WireguardKey.from(root.field("publicKey").asString()) - ); - } - - public String toJson() { - Slime slime = new Slime(); - Cursor cursor = slime.setObject(); - cursor.setString("hostname", hostname); - cursor.setString("endpoint", endpoint); - cursor.setString("publicKey", publicKey.value()); - return SlimeUtils.toJson(slime); - } - -} - - diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/ConfigserverPeer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/ConfigserverPeer.java deleted file mode 100644 index 63ddc2f3dd2..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/ConfigserverPeer.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.yahoo.vespa.hosted.node.admin.wireguard; - -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.WireguardKey; -import com.yahoo.vespa.hosted.node.admin.task.util.network.VersionedIpAddress; - -import java.util.List; -import java.util.Optional; - -/** - * @author gjoranv - */ -public record ConfigserverPeer(HostName hostname, - List<VersionedIpAddress> ipAddresses, - Optional<WireguardKey> publicKey) { - - public ConfigserverPeer { - if (ipAddresses.isEmpty()) throw new IllegalArgumentException("No IP addresses for configserver " + hostname.value()); - ipAddresses = List.copyOf(ipAddresses); - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/ParameterStore.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/ParameterStore.java deleted file mode 100644 index 4c7ddb23ecc..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/ParameterStore.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.yahoo.vespa.hosted.node.admin.wireguard; - -import com.yahoo.config.provision.zone.ZoneApi; - -import java.util.List; - -/** - * A cloud-agnostic store of parameters for Wireguard. - * - * @author gjoranv - */ -public interface ParameterStore { - - /** Returns the configservers for the given zone. */ - List<ConfigserverParameters> getConfigservers(ZoneApi zoneApi); - - /** Returns the tenant nodes for the given zone. */ - List<TenantParameters> getTenantNodes(ZoneApi zoneApi); - - void addConfigserver(ConfigserverParameters configserver); - - void addTenantNode(TenantParameters tenant); - - void removeConfigserver(String hostname); - - void removeTenantNode(String hostname); - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/TenantParameters.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/TenantParameters.java deleted file mode 100644 index e06ffacdf3b..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/TenantParameters.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.yahoo.vespa.hosted.node.admin.wireguard; - -import com.yahoo.config.provision.WireguardKey; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; - -/** - * Wireguard parameters for a tenant host/node. - * - * @author gjoranv - */ -public record TenantParameters(String hostname, WireguardKey publicKey) { - - public static TenantParameters fromJson(String json) { - Slime slime = SlimeUtils.jsonToSlime(json); - Cursor root = slime.get(); - return new TenantParameters( - root.field("hostname").asString(), - WireguardKey.from(root.field("publicKey").asString()) - ); - } - - public String toJson() { - Slime slime = new Slime(); - Cursor cursor = slime.setObject(); - cursor.setString("hostname", hostname); - cursor.setString("publicKey", publicKey.value()); - return SlimeUtils.toJson(slime); - } - -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeer.java new file mode 100644 index 00000000000..0f4d2d5d8e0 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeer.java @@ -0,0 +1,29 @@ +package com.yahoo.vespa.hosted.node.admin.wireguard; + +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.WireguardKey; +import com.yahoo.vespa.hosted.node.admin.task.util.network.VersionedIpAddress; + +import java.util.List; + +/** + * A wireguard peer. Sorted by hostname. IP addresses are sorted by version, IPv6 first. + * The public key should always be non-null. + * + * @author gjoranv + */ +public record WireguardPeer(HostName hostname, + List<VersionedIpAddress> ipAddresses, + WireguardKey publicKey) implements Comparable<WireguardPeer> { + + public WireguardPeer { + if (ipAddresses.isEmpty()) throw new IllegalArgumentException("No IP addresses for peer node " + hostname.value()); + ipAddresses = ipAddresses.stream().sorted().toList(); + } + + @Override + public int compareTo(WireguardPeer o) { + return hostname.value().compareTo(o.hostname.value()); + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java index 9c33db0355f..00a805790f1 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java @@ -7,12 +7,13 @@ import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.WireguardKey; import com.yahoo.config.provision.host.FlavorOverrides; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApi; import com.yahoo.vespa.hosted.node.admin.configserver.ConfigServerApiImpl; import com.yahoo.vespa.hosted.node.admin.task.util.network.VersionedIpAddress; -import com.yahoo.vespa.hosted.node.admin.wireguard.ConfigserverPeer; +import com.yahoo.vespa.hosted.node.admin.wireguard.WireguardPeer; import com.yahoo.vespa.hosted.provision.restapi.NodesV2ApiHandler; import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; import org.junit.jupiter.api.AfterEach; @@ -67,7 +68,7 @@ public class RealNodeRepositoryTest { for (int i = 0; i < 3; i++) { try { int port = findRandomOpenPort(); - container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(port, CloudAccount.empty), Networking.enable); + container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(port, SystemName.main, CloudAccount.empty), Networking.enable); ConfigServerApi configServerApi = ConfigServerApiImpl.createForTesting( List.of(URI.create("http://127.0.0.1:" + port))); waitForJdiscContainerToServe(configServerApi); @@ -199,24 +200,37 @@ public class RealNodeRepositoryTest { } @Test - void wireguard_peer_config_for_configservers_can_be_retrieved() { - List<ConfigserverPeer> cfgPeers = nodeRepositoryApi.getConfigserverPeers(); - assertEquals(2, cfgPeers.size()); - - var cfg1 = cfgPeers.get(0); - assertEquals("cfg1.yahoo.com", cfg1.hostname().value()); - assertEquals(2, cfg1.ipAddresses().size()); - assertIp(cfg1.ipAddresses().get(0), "127.0.201.1", 4); - assertIp(cfg1.ipAddresses().get(1), "::201:1", 6); - assertEquals("lololololololololololololololololololololoo=", cfg1.publicKey().get().value()); - - var cfg2 = cfgPeers.get(1); - assertEquals("cfg2.yahoo.com", cfg2.hostname().value()); - assertEquals(2, cfg1.ipAddresses().size()); - assertIp(cfg2.ipAddresses().get(0), "127.0.202.1", 4); - assertIp(cfg2.ipAddresses().get(1), "::202:1", 6); - assertEquals("olololololololololololololololololololololo=", cfg2.publicKey().get().value()); + void wireguard_peer_config_can_be_retrieved_for_configservers_and_exclave_nodes() { + //// Configservers //// + + List<WireguardPeer> cfgPeers = nodeRepositoryApi.getConfigserverPeers(); + + // cfg2 does not have a wg public key, so should not be included + assertEquals(1, cfgPeers.size()); + + assertWireguardPeer(cfgPeers.get(0), "cfg1.yahoo.com", + "::201:1", "127.0.201.1", + "lololololololololololololololololololololoo="); + + //// Exclave nodes //// + + List<WireguardPeer> exclavePeers = nodeRepositoryApi.getExclavePeers(); + + // host3 does not have a wg public key, so should not be included + assertEquals(1, exclavePeers.size()); + + assertWireguardPeer(exclavePeers.get(0), "dockerhost2.yahoo.com", + "::101:1", "127.0.101.1", + "000011112222333344445555666677778888999900c="); + } + + private void assertWireguardPeer(WireguardPeer peer, String hostname, String ipv6, String ipv4, String publicKey) { + assertEquals(hostname, peer.hostname().value()); + assertEquals(2, peer.ipAddresses().size()); + assertIp(peer.ipAddresses().get(0), ipv6, 6); + assertIp(peer.ipAddresses().get(1), ipv4, 4); + assertEquals(publicKey, peer.publicKey().value()); } private void assertIp(VersionedIpAddress ip, String expectedIp, int expectedVersion) { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/NodeRepoMock.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/NodeRepoMock.java index 06729083494..3dab2f1e776 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/NodeRepoMock.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/NodeRepoMock.java @@ -8,14 +8,13 @@ import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttribu import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; -import com.yahoo.vespa.hosted.node.admin.wireguard.ConfigserverPeer; +import com.yahoo.vespa.hosted.node.admin.wireguard.WireguardPeer; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.ConcurrentHashMap; import java.util.function.Function; -import java.util.stream.Collectors; /** * Mock with some simple logic @@ -48,7 +47,12 @@ public class NodeRepoMock implements NodeRepository { } @Override - public List<ConfigserverPeer> getConfigserverPeers() { + public List<WireguardPeer> getExclavePeers() { + throw new UnsupportedOperationException(); + } + + @Override + public List<WireguardPeer> getConfigserverPeers() { throw new UnsupportedOperationException(); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/VersionedIpAddressTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/VersionedIpAddressTest.java new file mode 100644 index 00000000000..32fbcf9f6a4 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/network/VersionedIpAddressTest.java @@ -0,0 +1,62 @@ +package com.yahoo.vespa.hosted.node.admin.task.util.network; + +import com.google.common.testing.EqualsTester; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author gjoranv + */ +public class VersionedIpAddressTest { + + @Test + void ip4_address_can_be_generated_from_string() { + var ip4 = VersionedIpAddress.from("10.0.0.1"); + assertEquals(IPVersion.IPv4, ip4.version()); + assertEquals("10.0.0.1", ip4.asString()); + } + + @Test + void ip6_address_can_be_generated_from_string() { + var ip6 = VersionedIpAddress.from("::1"); + assertEquals(IPVersion.IPv6, ip6.version()); + assertEquals("::1", ip6.asString()); + } + + @Test + void they_are_sorted_by_version_then_by_address() { + var ip4 = VersionedIpAddress.from("10.0.0.1"); + var ip4_2 = VersionedIpAddress.from("127.0.0.1"); + var ip6 = VersionedIpAddress.from("::1"); + var ip6_2 = VersionedIpAddress.from("::2"); + + var sorted = Stream.of(ip4_2, ip6, ip4, ip6_2) + .sorted() + .toList(); + assertEquals(List.of(ip6, ip6_2, ip4, ip4_2), sorted); + } + + @Test + void endpoint_with_port_is_generated_correctly_for_both_versions() { + var ip4 = VersionedIpAddress.from("10.0.0.1"); + var ip6 = VersionedIpAddress.from("::1"); + + assertEquals("10.0.0.1:8080", ip4.asEndpoint(8080)); + assertEquals("[::1]:8080", ip6.asEndpoint(8080)); + } + + @Test + void equals_and_hashCode_are_implemented() { + new EqualsTester() + .addEqualityGroup(VersionedIpAddress.from("::1"), VersionedIpAddress.from("::1")) + .addEqualityGroup(VersionedIpAddress.from("::2")) + .addEqualityGroup(VersionedIpAddress.from("127.0.0.1"), VersionedIpAddress.from("127.0.0.1")) + .addEqualityGroup(VersionedIpAddress.from("10.0.0.1")) + .testEquals(); + } + +} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/ConfigserverParametersTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/ConfigserverParametersTest.java deleted file mode 100644 index 616590d4993..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/ConfigserverParametersTest.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.yahoo.vespa.hosted.node.admin.wireguard; - -import com.yahoo.config.provision.WireguardKey; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author gjoranv - */ -public class ConfigserverParametersTest { - - private static final String dummyKey = "qT+1Kdx7qZZpbqBxHupj7XgmVXSfcXol1RccaSd40XA="; - - @Test - public void parameters_can_be_converted_to_json_and_back() { - ConfigserverParameters params = new ConfigserverParameters("host", "endpoint", - WireguardKey.from(dummyKey)); - ConfigserverParameters params2 = ConfigserverParameters.fromJson(params.toJson()); - assertEquals(params, params2); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/TenantParametersTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/TenantParametersTest.java deleted file mode 100644 index b1109ea351c..00000000000 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/TenantParametersTest.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.yahoo.vespa.hosted.node.admin.wireguard; - -import com.yahoo.config.provision.WireguardKey; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * @author gjoranv - */ -public class TenantParametersTest { - - private static final String dummyKey = "qT+1Kdx7qZZpbqBxHupj7XgmVXSfcXol1RccaSd40XA="; - - @Test - public void parameters_can_be_converted_to_json_and_back() { - TenantParameters params = new TenantParameters("host", WireguardKey.from(dummyKey)); - TenantParameters params2 = TenantParameters.fromJson(params.toJson()); - assertEquals(params, params2); - } - -} diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeerTest.java new file mode 100644 index 00000000000..00aca5c5e4d --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/wireguard/WireguardPeerTest.java @@ -0,0 +1,35 @@ +package com.yahoo.vespa.hosted.node.admin.wireguard; + +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.WireguardKey; +import com.yahoo.vespa.hosted.node.admin.task.util.network.VersionedIpAddress; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * @author gjoranv + */ +public class WireguardPeerTest { + + @Test + void peers_are_sorted_by_hostname_ascending() { + List<WireguardPeer> peers = Stream.of( + peer("b"), + peer("a"), + peer("c") + ).sorted().toList(); + + assertEquals("a", peers.get(0).hostname().value()); + assertEquals("b", peers.get(1).hostname().value()); + assertEquals("c", peers.get(2).hostname().value()); + } + + private static WireguardPeer peer(String hostname) { + return new WireguardPeer(HostName.of(hostname), List.of(VersionedIpAddress.from("::1:1")), + WireguardKey.generateRandomForTesting()); + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 31283998702..ba99219638b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -116,11 +116,11 @@ public final class Node implements Nodelike { if (state == State.ready && type.isHost()) { requireNonEmpty(ipConfig.primary(), "A " + type + " must have at least one primary IP address in state " + state); - requireNonEmpty(ipConfig.pool().asSet(), "A " + type + " must have a non-empty IP address pool in state " + state); + requireNonEmpty(ipConfig.pool().ipSet(), "A " + type + " must have a non-empty IP address pool in state " + state); } if (parentHostname.isPresent()) { - if (!ipConfig.pool().asSet().isEmpty()) throw new IllegalArgumentException("A child node cannot have an IP address pool"); + if (!ipConfig.pool().ipSet().isEmpty()) throw new IllegalArgumentException("A child node cannot have an IP address pool"); if (modelName.isPresent()) throw new IllegalArgumentException("A child node cannot have model name set"); if (switchHostname.isPresent()) throw new IllegalArgumentException("A child node cannot have switch hostname set"); if (status.wantToRebuild()) throw new IllegalArgumentException("A child node cannot be rebuilt"); @@ -388,7 +388,7 @@ public final class Node implements Nodelike { return with(history.with(new History.Event(History.Event.Type.up, agent, instant))); } - /** Returns whether this node is down, according to its recorded 'down' and 'up' events */ + /** Returns true if we have positive evidence that this node is down. */ public boolean isDown() { Optional<Instant> downAt = history().event(History.Event.Type.down).map(History.Event::at); if (downAt.isEmpty()) return false; @@ -396,6 +396,14 @@ public final class Node implements Nodelike { return !history().hasEventAfter(History.Event.Type.up, downAt.get()); } + /** Returns true if we have positive evidence that this node is up. */ + public boolean isUp() { + Optional<Instant> upAt = history().event(History.Event.Type.up).map(History.Event::at); + if (upAt.isEmpty()) return false; + + return !history().hasEventAfter(History.Event.Type.down, upAt.get()); + } + /** Returns a copy of this with allocation set as specified. <code>node.state</code> is *not* changed. */ public Node allocate(ApplicationId owner, ClusterMembership membership, NodeResources requestedResources, Instant at) { return this.with(new Allocation(owner, membership, requestedResources, new Generation(0, 0), false)) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java index 1cf7bcfa4f2..800cf2150e9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.node.ClusterId; +import com.yahoo.vespa.hosted.provision.node.IP; import java.util.ArrayList; import java.util.Comparator; @@ -329,22 +330,22 @@ public class NodeList extends AbstractFilteringList<Node, NodeList> { /** * Returns the number of unused IP addresses in the pool, assuming any and all unaccounted for hostnames - * in the pool are resolved to exactly 1 IP address (or 2 if dual-stack). + * in the pool are resolved to exactly 1 IP address (or 2 with {@link IP.IpAddresses.Protocol#dualStack}). */ public int eventuallyUnusedIpAddressCount(Node host) { // The count in this method relies on the size of the IP address pool if that's non-empty, // otherwise fall back to the address/hostname pool. - if (host.ipConfig().pool().asSet().isEmpty()) { + if (host.ipConfig().pool().ipSet().isEmpty()) { Set<String> allHostnames = cache().keySet(); - return (int) host.ipConfig().pool().hostnames().stream() - .filter(hostname -> !allHostnames.contains(hostname.value())) + return (int) host.ipConfig().pool().getAddressList().stream() + .filter(address -> !allHostnames.contains(address.hostname())) .count(); } Set<String> allIps = ipCache.updateAndGet(old -> old != null ? old : stream().flatMap(node -> node.ipConfig().primary().stream()) .collect(Collectors.toUnmodifiableSet()) ); - return (int) host.ipConfig().pool().asSet().stream() + return (int) host.ipConfig().pool().ipSet().stream() .filter(address -> !allIps.contains(address)) .count(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index ea22c768e90..510c4041efb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision; -import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; +import com.yahoo.component.annotation.Inject; import com.yahoo.concurrent.maintenance.JobControl; import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.ClusterSpec; @@ -28,7 +28,7 @@ import com.yahoo.vespa.hosted.provision.persistence.CuratorDb; import com.yahoo.vespa.hosted.provision.persistence.DnsNameResolver; import com.yahoo.vespa.hosted.provision.persistence.JobControlFlags; import com.yahoo.vespa.hosted.provision.persistence.NameResolver; -import com.yahoo.vespa.hosted.provision.provisioning.ArchiveUris; +import com.yahoo.vespa.hosted.provision.archive.ArchiveUriManager; import com.yahoo.vespa.hosted.provision.provisioning.ContainerImages; import com.yahoo.vespa.hosted.provision.provisioning.FirmwareChecks; import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; @@ -57,7 +57,7 @@ public class NodeRepository extends AbstractComponent { private final InfrastructureVersions infrastructureVersions; private final FirmwareChecks firmwareChecks; private final ContainerImages containerImages; - private final ArchiveUris archiveUris; + private final ArchiveUriManager archiveUriManager; private final JobControl jobControl; private final Applications applications; private final LoadBalancers loadBalancers; @@ -134,7 +134,7 @@ public class NodeRepository extends AbstractComponent { this.infrastructureVersions = new InfrastructureVersions(db); this.firmwareChecks = new FirmwareChecks(db, clock); this.containerImages = new ContainerImages(containerImage, tenantContainerImage, tenantGpuContainerImage); - this.archiveUris = new ArchiveUris(db); + this.archiveUriManager = new ArchiveUriManager(db, zone); this.jobControl = new JobControl(new JobControlFlags(db, flagSource)); this.loadBalancers = new LoadBalancers(db); this.metricsDb = metricsDb; @@ -166,7 +166,7 @@ public class NodeRepository extends AbstractComponent { public ContainerImages containerImages() { return containerImages; } /** Returns the archive URIs to use for nodes in this. */ - public ArchiveUris archiveUris() { return archiveUris; } + public ArchiveUriManager archiveUriManager() { return archiveUriManager; } /** Returns the status of maintenance jobs managed by this. */ public JobControl jobControl() { return jobControl; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java index ea4944c2bd5..16016815b7c 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.applications; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; @@ -33,6 +34,7 @@ public class Cluster { private final boolean required; private final Autoscaling suggested; private final Autoscaling target; + private final ClusterInfo clusterInfo; private final BcpGroupInfo bcpGroupInfo; /** The maxScalingEvents last scaling events of this, sorted by increasing time (newest last) */ @@ -46,6 +48,7 @@ public class Cluster { boolean required, Autoscaling suggested, Autoscaling target, + ClusterInfo clusterInfo, BcpGroupInfo bcpGroupInfo, List<ScalingEvent> scalingEvents) { this.id = Objects.requireNonNull(id); @@ -57,9 +60,10 @@ public class Cluster { this.suggested = Objects.requireNonNull(suggested); Objects.requireNonNull(target); if (target.resources().isPresent() && ! target.resources().get().isWithin(minResources, maxResources)) - this.target = Autoscaling.empty(); + this.target = target.withResources(Optional.empty()); // Delete illegal target else this.target = target; + this.clusterInfo = clusterInfo; this.bcpGroupInfo = Objects.requireNonNull(bcpGroupInfo); this.scalingEvents = List.copyOf(scalingEvents); } @@ -105,6 +109,8 @@ public class Cluster { return true; } + public ClusterInfo clusterInfo() { return clusterInfo; } + /** Returns info about the BCP group of clusters this belongs to. */ public BcpGroupInfo bcpGroupInfo() { return bcpGroupInfo; } @@ -119,19 +125,19 @@ public class Cluster { public Cluster withConfiguration(boolean exclusive, Capacity capacity) { return new Cluster(id, exclusive, capacity.minResources(), capacity.maxResources(), capacity.groupSize(), capacity.isRequired(), - suggested, target, bcpGroupInfo, scalingEvents); + suggested, target, capacity.clusterInfo(), bcpGroupInfo, scalingEvents); } public Cluster withSuggested(Autoscaling suggested) { - return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, bcpGroupInfo, scalingEvents); + return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, clusterInfo, bcpGroupInfo, scalingEvents); } public Cluster withTarget(Autoscaling target) { - return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, bcpGroupInfo, scalingEvents); + return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, clusterInfo, bcpGroupInfo, scalingEvents); } public Cluster with(BcpGroupInfo bcpGroupInfo) { - return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, bcpGroupInfo, scalingEvents); + return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, clusterInfo, bcpGroupInfo, scalingEvents); } /** Add or update (based on "at" time) a scaling event */ @@ -145,7 +151,7 @@ public class Cluster { scalingEvents.add(scalingEvent); prune(scalingEvents); - return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, bcpGroupInfo, scalingEvents); + return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, clusterInfo, bcpGroupInfo, scalingEvents); } @Override @@ -177,7 +183,7 @@ public class Cluster { public static Cluster create(ClusterSpec.Id id, boolean exclusive, Capacity requested) { return new Cluster(id, exclusive, requested.minResources(), requested.maxResources(), requested.groupSize(), requested.isRequired(), - Autoscaling.empty(), Autoscaling.empty(), BcpGroupInfo.empty(), List.of()); + Autoscaling.empty(), Autoscaling.empty(), requested.clusterInfo(), BcpGroupInfo.empty(), List.of()); } /** The predicted time it will take to rescale this cluster. */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java new file mode 100644 index 00000000000..faa360bbcb1 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManager.java @@ -0,0 +1,93 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.archive; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.Zone; +import com.yahoo.lang.CachedSupplier; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.persistence.CuratorDb; + +import java.time.Duration; +import java.util.Optional; +import java.util.function.Function; + +/** + * Thread safe class to get and set archive URI for given account and tenants. + * + * @author freva + */ +public class ArchiveUriManager { + + private static final Duration cacheTtl = Duration.ofMinutes(1); + + private final CuratorDb db; + private final Zone zone; + private final CachedSupplier<ArchiveUris> archiveUris; + + public ArchiveUriManager(CuratorDb db, Zone zone) { + this.db = db; + this.zone = zone; + this.archiveUris = new CachedSupplier<>(db::readArchiveUris, cacheTtl); + } + + public ArchiveUris archiveUris() { + return archiveUris.get(); + } + + /** Returns the archive URI to use for given node */ + public Optional<String> archiveUriFor(Node node) { + if (node.allocation().isEmpty()) return Optional.empty(); + ApplicationId app = node.allocation().get().owner(); + + return Optional.ofNullable(node.cloudAccount().isEnclave(zone) ? + archiveUris.get().accountArchiveUris().get(node.cloudAccount()) : + archiveUris.get().tenantArchiveUris().get(app.tenant())) + .map(uri -> { + // TODO (freva): Remove when all URIs dont have tenant name in them anymore + String tenantSuffix = "/" + app.tenant().value() + "/"; + if (uri.endsWith(tenantSuffix)) return uri.substring(0, uri.length() - tenantSuffix.length() + 1); + return uri; + }) + .map(uri -> { + StringBuilder sb = new StringBuilder(100).append(uri) + .append(app.tenant().value()).append('/') + .append(app.application().value()).append('/') + .append(app.instance().value()).append('/') + .append(node.allocation().get().membership().cluster().id().value()).append('/'); + + for (char c: node.hostname().toCharArray()) { + if (c == '.') break; + sb.append(c); + } + + return sb.append('/').toString(); + }); + } + + /** Set (or remove, if archiveURI is empty) archive URI to use for given tenant */ + public void setArchiveUri(TenantName tenant, Optional<String> archiveUri) { + setArchiveUri(au -> au.with(tenant, archiveUri)); + } + + /** Set (or remove, if archiveURI is empty) archive URI to use for given account */ + public void setArchiveUri(CloudAccount account, Optional<String> archiveUri) { + if (!account.isEnclave(zone) || account.isUnspecified()) + throw new IllegalArgumentException("Cannot set archive URI for non-enclave account: " + account); + setArchiveUri(au -> au.with(account, archiveUri)); + } + + private void setArchiveUri(Function<ArchiveUris, ArchiveUris> mapper) { + try (Lock lock = db.lockArchiveUris()) { + ArchiveUris archiveUris = db.readArchiveUris(); + ArchiveUris updated = mapper.apply(archiveUris); + if (archiveUris.equals(updated)) return; // No change + + db.writeArchiveUris(updated); + this.archiveUris.invalidate(); // Throw away current cache + } + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUris.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUris.java new file mode 100644 index 00000000000..7ec9bd36744 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUris.java @@ -0,0 +1,45 @@ +package com.yahoo.vespa.hosted.provision.archive; + +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.TenantName; + +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.regex.Pattern; + +public record ArchiveUris(Map<TenantName, String> tenantArchiveUris, Map<CloudAccount, String> accountArchiveUris) { + private static final Pattern validUriPattern = Pattern.compile("[a-z0-9]+://(?:(?:[a-z0-9]+(?:[-_][a-z0-9.]+)*)+/)+"); + + public ArchiveUris(Map<TenantName, String> tenantArchiveUris, Map<CloudAccount, String> accountArchiveUris) { + this.tenantArchiveUris = Map.copyOf(tenantArchiveUris); + this.accountArchiveUris = Map.copyOf(accountArchiveUris); + } + + public ArchiveUris with(TenantName tenant, Optional<String> archiveUri) { + Map<TenantName, String> updated = updateIfChanged(tenantArchiveUris, tenant, archiveUri); + if (updated == tenantArchiveUris) return this; + return new ArchiveUris(updated, accountArchiveUris); + } + + public ArchiveUris with(CloudAccount account, Optional<String> archiveUri) { + Map<CloudAccount, String> updated = updateIfChanged(accountArchiveUris, account, archiveUri); + if (updated == accountArchiveUris) return this; + return new ArchiveUris(tenantArchiveUris, updated); + } + + private static <T> Map<T, String> updateIfChanged(Map<T, String> current, T key, Optional<String> archiveUri) { + archiveUri = archiveUri.map(ArchiveUris::normalizeUri); + if (Optional.ofNullable(current.get(key)).equals(archiveUri)) return current; + Map<T, String> updated = new HashMap<>(current); + archiveUri.ifPresentOrElse(uri -> updated.put(key, uri), () -> updated.remove(key)); + return updated; + } + + static String normalizeUri(String uri) { + if (!uri.endsWith("/")) uri = uri + "/"; + if (!validUriPattern.matcher(uri).matches()) + throw new IllegalArgumentException("Invalid archive URI: " + uri); + return uri; + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java index 1406aaecb71..a2ef76e84d0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocatableClusterResources.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; @@ -37,7 +38,7 @@ public class AllocatableClusterResources { NodeRepository nodeRepository) { this.nodes = requested.nodes(); this.groups = requested.groups(); - this.realResources = nodeRepository.resourcesCalculator().requestToReal(requested.nodeResources(), nodeRepository.exclusiveAllocation(clusterSpec)); + this.realResources = nodeRepository.resourcesCalculator().requestToReal(requested.nodeResources(), nodeRepository.exclusiveAllocation(clusterSpec), false); this.advertisedResources = requested.nodeResources(); this.clusterSpec = clusterSpec; this.fulfilment = 1; @@ -155,6 +156,7 @@ public class AllocatableClusterResources { } public static Optional<AllocatableClusterResources> from(ClusterResources wantedResources, + ApplicationId applicationId, ClusterSpec clusterSpec, Limits applicationLimits, List<NodeResources> availableRealHostResources, @@ -163,24 +165,32 @@ public class AllocatableClusterResources { boolean exclusive = nodeRepository.exclusiveAllocation(clusterSpec); if (! exclusive) { // We decide resources: Add overhead to what we'll request (advertised) to make sure real becomes (at least) cappedNodeResources - var advertisedResources = nodeRepository.resourcesCalculator().realToRequest(wantedResources.nodeResources(), exclusive); - advertisedResources = systemLimits.enlargeToLegal(advertisedResources, clusterSpec, exclusive); // Ask for something legal - advertisedResources = applicationLimits.cap(advertisedResources); // Overrides other conditions, even if it will then fail - var realResources = nodeRepository.resourcesCalculator().requestToReal(advertisedResources, exclusive); // What we'll really get - if ( ! systemLimits.isWithinRealLimits(realResources, clusterSpec) && advertisedResources.storageType() == NodeResources.StorageType.any) { - // Since local disk resreves some of the storage, try to constrain to remote disk - advertisedResources = advertisedResources.with(NodeResources.StorageType.remote); - realResources = nodeRepository.resourcesCalculator().requestToReal(advertisedResources, exclusive); + var allocatableResources = calculateAllocatableResources(wantedResources, + nodeRepository, + applicationId, + clusterSpec, + applicationLimits, + exclusive, + true); + + var worstCaseRealResources = nodeRepository.resourcesCalculator().requestToReal(allocatableResources.advertisedResources, + exclusive, + false); + if ( ! systemLimits.isWithinRealLimits(worstCaseRealResources, applicationId, clusterSpec)) { + allocatableResources = calculateAllocatableResources(wantedResources, + nodeRepository, + applicationId, + clusterSpec, + applicationLimits, + exclusive, + false); } - if ( ! systemLimits.isWithinRealLimits(realResources, clusterSpec)) + + if ( ! systemLimits.isWithinRealLimits(allocatableResources.realResources, applicationId, clusterSpec)) return Optional.empty(); - if (anySatisfies(realResources, availableRealHostResources)) - return Optional.of(new AllocatableClusterResources(wantedResources.with(realResources), - advertisedResources, - wantedResources, - clusterSpec)); - else + if ( ! anySatisfies(allocatableResources.realResources, availableRealHostResources)) return Optional.empty(); + return Optional.of(allocatableResources); } else { // Return the cheapest flavor satisfying the requested resources, if any NodeResources cappedWantedResources = applicationLimits.cap(wantedResources.nodeResources()); @@ -188,11 +198,11 @@ public class AllocatableClusterResources { for (Flavor flavor : nodeRepository.flavors().getFlavors()) { // Flavor decide resources: Real resources are the worst case real resources we'll get if we ask for these advertised resources NodeResources advertisedResources = nodeRepository.resourcesCalculator().advertisedResourcesOf(flavor); - NodeResources realResources = nodeRepository.resourcesCalculator().requestToReal(advertisedResources, exclusive); + NodeResources realResources = nodeRepository.resourcesCalculator().requestToReal(advertisedResources, exclusive, false); // Adjust where we don't need exact match to the flavor if (flavor.resources().storageType() == NodeResources.StorageType.remote) { - double diskGb = systemLimits.enlargeToLegal(cappedWantedResources, clusterSpec, exclusive).diskGb(); + double diskGb = systemLimits.enlargeToLegal(cappedWantedResources, applicationId, clusterSpec, exclusive).diskGb(); advertisedResources = advertisedResources.withDiskGb(diskGb); realResources = realResources.withDiskGb(diskGb); } @@ -202,7 +212,7 @@ public class AllocatableClusterResources { } if ( ! between(applicationLimits.min().nodeResources(), applicationLimits.max().nodeResources(), advertisedResources)) continue; - if ( ! systemLimits.isWithinRealLimits(realResources, clusterSpec)) continue; + if ( ! systemLimits.isWithinRealLimits(realResources, applicationId, clusterSpec)) continue; var candidate = new AllocatableClusterResources(wantedResources.with(realResources), advertisedResources, wantedResources, @@ -215,6 +225,30 @@ public class AllocatableClusterResources { } } + private static AllocatableClusterResources calculateAllocatableResources(ClusterResources wantedResources, + NodeRepository nodeRepository, + ApplicationId applicationId, + ClusterSpec clusterSpec, + Limits applicationLimits, + boolean exclusive, + boolean bestCase) { + var systemLimits = new NodeResourceLimits(nodeRepository); + var advertisedResources = nodeRepository.resourcesCalculator().realToRequest(wantedResources.nodeResources(), exclusive, bestCase); + advertisedResources = systemLimits.enlargeToLegal(advertisedResources, applicationId, clusterSpec, exclusive); // Ask for something legal + advertisedResources = applicationLimits.cap(advertisedResources); // Overrides other conditions, even if it will then fail + var realResources = nodeRepository.resourcesCalculator().requestToReal(advertisedResources, exclusive, bestCase); // What we'll really get + if ( ! systemLimits.isWithinRealLimits(realResources, applicationId, clusterSpec) + && advertisedResources.storageType() == NodeResources.StorageType.any) { + // Since local disk reserves some of the storage, try to constrain to remote disk + advertisedResources = advertisedResources.with(NodeResources.StorageType.remote); + realResources = nodeRepository.resourcesCalculator().requestToReal(advertisedResources, exclusive, bestCase); + } + return new AllocatableClusterResources(wantedResources.with(realResources), + advertisedResources, + wantedResources, + clusterSpec); + } + /** Returns true if the given resources could be allocated on any of the given host flavors */ private static boolean anySatisfies(NodeResources realResources, List<NodeResources> availableRealHostResources) { return availableRealHostResources.stream().anyMatch(realHostResources -> realHostResources.satisfies(realResources)); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java index e7278699ade..b56e8d1b247 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.autoscale; -import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.ClusterResources; +import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -32,11 +32,10 @@ public class AllocationOptimizer { * @return the best allocation, if there are any possible legal allocations, fulfilling the target * fully or partially, within the limits */ - public Optional<AllocatableClusterResources> findBestAllocation(Load targetLoad, + public Optional<AllocatableClusterResources> findBestAllocation(Load loadAdjustment, AllocatableClusterResources current, ClusterModel clusterModel, Limits limits) { - int minimumNodes = AllocationOptimizer.minimumNodes; if (limits.isEmpty()) limits = Limits.of(new ClusterResources(minimumNodes, 1, NodeResources.unspecified()), new ClusterResources(maximumNodes, maximumNodes, NodeResources.unspecified()), @@ -53,13 +52,16 @@ public class AllocationOptimizer { for (int nodes = limits.min().nodes(); nodes <= limits.max().nodes(); nodes++) { if (nodes % groups != 0) continue; if ( ! limits.groupSize().includes(nodes / groups)) continue; - var resources = new ClusterResources(nodes, groups, nodeResourcesWith(nodes, groups, - limits, targetLoad, current, clusterModel)); - var allocatableResources = AllocatableClusterResources.from(resources, current.clusterSpec(), limits, - availableRealHostResources, nodeRepository); + limits, loadAdjustment, current, clusterModel)); + var allocatableResources = AllocatableClusterResources.from(resources, + clusterModel.application().id(), + current.clusterSpec(), + limits, + availableRealHostResources, + nodeRepository); if (allocatableResources.isEmpty()) continue; if (bestAllocation.isEmpty() || allocatableResources.get().preferableTo(bestAllocation.get())) { bestAllocation = allocatableResources; @@ -83,10 +85,10 @@ public class AllocationOptimizer { private NodeResources nodeResourcesWith(int nodes, int groups, Limits limits, - Load targetLoad, + Load loadAdjustment, AllocatableClusterResources current, ClusterModel clusterModel) { - var scaled = targetLoad // redundancy aware target relative to current load + var scaled = loadAdjustment // redundancy aware target relative to current load .multiply(clusterModel.loadWith(nodes, groups)) // redundancy aware adjustment with these counts .divide(clusterModel.redundancyAdjustment()) // correct for double redundancy adjustment .scaled(current.realResources().nodeResources()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java index 410a4fcb773..f52c4cc85f7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaler.java @@ -57,12 +57,12 @@ public class Autoscaler { private Autoscaling autoscale(Application application, Cluster cluster, NodeList clusterNodes, Limits limits) { ClusterModel clusterModel = new ClusterModel(nodeRepository.zone(), application, - clusterNodes.clusterSpec(), + clusterNodes.not().retired().clusterSpec(), cluster, clusterNodes, nodeRepository.metricsDb(), nodeRepository.clock()); - if (clusterModel.isEmpty()) return Autoscaling.empty(); + if (clusterModel.isEmpty()) return Autoscaling.empty(clusterModel.description()); if (! limits.isEmpty() && cluster.minResources().equals(cluster.maxResources())) return Autoscaling.dontScale(Autoscaling.Status.unavailable, "Autoscaling is not enabled", clusterModel); @@ -70,7 +70,7 @@ public class Autoscaler { if ( ! clusterModel.isStable(nodeRepository)) return Autoscaling.dontScale(Status.waiting, "Cluster change in progress", clusterModel); - var currentAllocation = new AllocatableClusterResources(clusterNodes, nodeRepository); + var currentAllocation = new AllocatableClusterResources(clusterNodes.not().retired(), nodeRepository); Optional<AllocatableClusterResources> bestAllocation = allocationOptimizer.findBestAllocation(clusterModel.loadAdjustment(), currentAllocation, clusterModel, limits); if (bestAllocation.isEmpty()) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java index fd769ec913d..2cc43a1eb33 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Autoscaling.java @@ -57,6 +57,10 @@ public class Autoscaling { return new Autoscaling(status, description, resources, at, peak, ideal, metrics); } + public Autoscaling withResources(Optional<ClusterResources> resources) { + return new Autoscaling(status, description, resources, at, peak, ideal, metrics); + } + /** Converts this autoscaling into an ideal one at the completion of it. */ public Autoscaling asIdeal(Instant at) { return new Autoscaling(Status.ideal, @@ -68,7 +72,14 @@ public class Autoscaling { metrics); } - public boolean isEmpty() { return this.equals(empty()); } + /** Returns true if no measurements was used in creating this instance. */ + public boolean isEmpty() { + return peak.equals(Load.zero()); + } + + public boolean isPresent() { + return ! isEmpty(); + } @Override public boolean equals(Object o) { @@ -95,8 +106,12 @@ public class Autoscaling { } public static Autoscaling empty() { + return empty(""); + } + + public static Autoscaling empty(String description) { return new Autoscaling(Status.unavailable, - "", + description, Optional.empty(), Instant.EPOCH, Load.zero(), diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java index d122f719eff..4e1523834e4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java @@ -113,13 +113,17 @@ public class ClusterModel { public ClusterSpec clusterSpec() { return clusterSpec; } public Cluster cluster() { return cluster; } + public String description() { + return nodeTimeseries.description(); + } + public boolean isEmpty() { return nodeTimeseries().isEmpty(); } /** Returns the relative load adjustment that should be made to this cluster given available measurements. */ public Load loadAdjustment() { - if (nodeTimeseries().isEmpty()) return Load.one(); + if (nodeTimeseries().measurementsPerNode() < 0.5) return Load.one(); // Don't change based on very little data Load adjustment = peakLoad().divide(idealLoad()); if (! safeToScaleDown()) adjustment = adjustment.map(v -> v < 1 ? 1 : v); @@ -127,10 +131,6 @@ public class ClusterModel { } public boolean isStable(NodeRepository nodeRepository) { - // An autoscaling decision was recently made - if (hasScaledIn(Duration.ofMinutes(5))) - return false; - // The cluster is processing recent changes if (nodes.stream().anyMatch(node -> node.status().wantToRetire() || node.allocation().get().membership().retired() || @@ -193,21 +193,28 @@ public class ClusterModel { /** * Returns the ideal load across the nodes of this such that each node will be at ideal load - * if one of the nodes go down. + * if one of the nodes go down. */ public Load idealLoad() { var ideal = new Load(idealCpuLoad(), idealMemoryLoad(), idealDiskLoad()).divide(redundancyAdjustment()); - if (! cluster.bcpGroupInfo().isEmpty()) { + if ( !cluster.bcpGroupInfo().isEmpty() && cluster.bcpGroupInfo().queryRate() > 0) { + // Since we have little local information, use information about query cost in other groups + + Load bcpGroupIdeal = adjustQueryDependentIdealLoadByBcpGroupInfo(ideal); + // Do a weighted sum of the ideal "vote" based on local and bcp group info. // This avoids any discontinuities with a near-zero local query rate. double localInformationWeight = Math.min(1, averageQueryRate().orElse(0) / Math.min(queryRateGivingFullConfidence, cluster.bcpGroupInfo().queryRate())); - Load bcpGroupIdeal = adjustQueryDependentIdealLoadByBcpGroupInfo(ideal); ideal = ideal.multiply(localInformationWeight).add(bcpGroupIdeal.multiply(1 - localInformationWeight)); } return ideal; } + private boolean canRescaleWithinBcpDeadline() { + return scalingDuration().minus(cluster.clusterInfo().bcpDeadline()).isNegative(); + } + public Autoscaling.Metrics metrics() { return new Autoscaling.Metrics(averageQueryRate().orElse(0), growthRateHeadroom(), @@ -218,7 +225,7 @@ public class ClusterModel { public Instant at() { return at;} private OptionalDouble cpuCostPerQuery() { - if (averageQueryRate().isEmpty()) return OptionalDouble.empty(); + if (averageQueryRate().isEmpty() || averageQueryRate().getAsDouble() == 0.0) return OptionalDouble.empty(); // TODO: Query rate should generally be sampled at the time where we see the peak resource usage int fanOut = clusterSpec.type().isContainer() ? 1 : groupSize(); return OptionalDouble.of(peakLoad().cpu() * queryCpuFraction() * fanOut * nodes.not().retired().first().get().resources().vcpu() @@ -228,7 +235,9 @@ public class ClusterModel { private Load adjustQueryDependentIdealLoadByBcpGroupInfo(Load ideal) { double currentClusterTotalVcpuPerGroup = nodes.not().retired().first().get().resources().vcpu() * groupSize(); - double targetQueryRateToHandle = cluster.bcpGroupInfo().queryRate() * cluster.bcpGroupInfo().growthRateHeadroom() * trafficShiftHeadroom(); + double targetQueryRateToHandle = ( canRescaleWithinBcpDeadline() ? averageQueryRate().orElse(0) + : cluster.bcpGroupInfo().queryRate() ) + * cluster.bcpGroupInfo().growthRateHeadroom() * trafficShiftHeadroom(); double neededTotalVcpPerGroup = cluster.bcpGroupInfo().cpuCostPerQuery() * targetQueryRateToHandle / groupCount() + ( 1 - queryCpuFraction()) * idealCpuLoad() * (clusterSpec.type().isContainer() ? 1 : groupSize()); @@ -270,14 +279,14 @@ public class ClusterModel { /** The number of nodes this cluster has, or will have if not deployed yet. */ // TODO: Make this the deployed, not current count private int nodeCount() { - if ( ! nodes.isEmpty()) return (int)nodes.stream().count(); + if ( ! nodes.isEmpty()) return (int)nodes.not().retired().stream().count(); return cluster.minResources().nodes(); } /** The number of groups this cluster has, or will have if not deployed yet. */ // TODO: Make this the deployed, not current count private int groupCount() { - if ( ! nodes.isEmpty()) return (int)nodes.stream().mapToInt(node -> node.allocation().get().membership().cluster().group().get().index()).distinct().count(); + if ( ! nodes.isEmpty()) return (int)nodes.not().retired().stream().mapToInt(node -> node.allocation().get().membership().cluster().group().get().index()).distinct().count(); return cluster.minResources().groups(); } @@ -323,6 +332,7 @@ public class ClusterModel { */ private double trafficShiftHeadroom() { if ( ! zone.environment().isProduction()) return 1; + if (canRescaleWithinBcpDeadline()) return 1; double trafficShiftHeadroom; if (application.status().maxReadShare() == 0) // No traffic fraction data trafficShiftHeadroom = 2.0; // assume we currently get half of the max possible share of traffic diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java index b86a24af5c9..e94ddd8e543 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterNodesTimeseries.java @@ -6,10 +6,8 @@ import com.yahoo.vespa.hosted.provision.applications.Cluster; import java.time.Duration; import java.util.List; -import java.util.Optional; import java.util.OptionalDouble; import java.util.function.Predicate; -import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.provision.autoscale.ClusterModel.warmupDuration; @@ -25,20 +23,26 @@ public class ClusterNodesTimeseries { /** The measurements for all nodes in this snapshot */ private final List<NodeTimeseries> timeseries; + private int initialCount, afterWarmupCount, afterStableCount, finalCount; + public ClusterNodesTimeseries(Duration period, Cluster cluster, NodeList clusterNodes, MetricsDb db) { this.clusterNodes = clusterNodes; - // See warmupSeconds*4 into the past to see any generation change in it + // See warmupDuration*4 into the past to see any generation change in it. // If none can be detected we assume the node is new/was down. // If either this is the case, or there is a generation change, we ignore - // the first warmupWindow metrics + // the first warmupWindow metrics. var timeseries = db.getNodeTimeseries(period.plus(warmupDuration.multipliedBy(4)), clusterNodes); + initialCount = totalMeasurementsIn(timeseries); if (cluster.lastScalingEvent().isPresent()) { long currentGeneration = cluster.lastScalingEvent().get().generation(); timeseries = keepGenerationAfterWarmup(timeseries, currentGeneration); + afterWarmupCount = totalMeasurementsIn(timeseries); } timeseries = keep(timeseries, snapshot -> snapshot.inService() && snapshot.stable()); + afterStableCount = totalMeasurementsIn(timeseries); timeseries = keep(timeseries, snapshot -> ! snapshot.at().isBefore(db.clock().instant().minus(period))); + finalCount = totalMeasurementsIn(timeseries); this.timeseries = timeseries; } @@ -47,46 +51,29 @@ public class ClusterNodesTimeseries { this.timeseries = timeseries; } + public String description() { + return "initial measurements: " + initialCount + + ", after warmup: " + afterWarmupCount + + ", after stable: " + afterStableCount + + ", final measurements: " + finalCount; + } + public boolean isEmpty() { return measurementsPerNode() == 0; } /** Returns the average number of measurements per node */ - public int measurementsPerNode() { + public double measurementsPerNode() { if (clusterNodes.size() == 0) return 0; - int measurementCount = timeseries.stream().mapToInt(m -> m.size()).sum(); - return measurementCount / clusterNodes.size(); + return (double) totalMeasurementsIn(timeseries) / clusterNodes.size(); } - /** Returns the number of nodes measured in this */ - public int nodesMeasured() { return timeseries.size(); } - - /** Returns the average load after the given instant */ - public Load averageLoad() { - Load total = Load.zero(); - int count = 0; - for (var nodeTimeseries : timeseries) { - for (var snapshot : nodeTimeseries.asList()) { - total = total.add(snapshot.load()); - count++; - } - } - return total.divide(count); + private int totalMeasurementsIn(List<NodeTimeseries> timeseries) { + return timeseries.stream().mapToInt(m -> m.size()).sum(); } - /** Returns average of the latest load reading from each node */ - public Load currentLoad() { - Load total = Load.zero(); - int count = 0; - for (var nodeTimeseries : timeseries) { - Optional<NodeMetricSnapshot> last = nodeTimeseries.last(); - if (last.isEmpty()) continue; - - total = total.add(last.get().load()); - count++; - } - return total.divide(count); - } + /** Returns the number of nodes measured in this */ + public int nodesMeasured() { return timeseries.size(); } /** * Returns the "peak load" in this: Which is for each load dimension, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java index dc0327c9537..428ea784115 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java @@ -162,7 +162,8 @@ public class MetricsResponse { generation { // application config generation active on the node @Override - public List<String> metricResponseNames() { return List.of("application_generation"); } + public List<String> metricResponseNames() { return List.of("application_generation", + "content.proton.config.generation"); } @Override double computeFinal(ListMap<String, Double> values) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java index c25b0684f5a..17444ef9d2e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/NodeTimeseries.java @@ -11,7 +11,6 @@ import java.util.List; import java.util.Optional; import java.util.OptionalDouble; import java.util.function.Predicate; -import java.util.stream.Collectors; import static com.yahoo.vespa.hosted.provision.autoscale.ClusterModel.warmupDuration; @@ -98,7 +97,6 @@ public class NodeTimeseries { } private boolean onAtLeastGeneration(long generation, NodeMetricSnapshot snapshot) { - if (snapshot.generation() < 0) return true; // Content nodes do not yet send generation return snapshot.generation() >= generation; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java index c4b30f2f5ec..f166e73da77 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java @@ -40,7 +40,7 @@ public class LoadBalancerInstance { this.networks = ImmutableSortedSet.copyOf(Objects.requireNonNull(networks, "networks must be non-null")); this.reals = ImmutableSortedSet.copyOf(Objects.requireNonNull(reals, "targets must be non-null")); this.settings = Objects.requireNonNull(settings, "settings must be non-null"); - this.serviceIds = Objects.requireNonNull(serviceIds, "private service id must be non-null"); + this.serviceIds = List.copyOf(Objects.requireNonNull(serviceIds, "private service id must be non-null")); this.cloudAccount = Objects.requireNonNull(cloudAccount, "cloudAccount must be non-null"); if (hostname.isEmpty() == ipAddress.isEmpty()) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java index f792c511adb..69c03dbf6dc 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainer.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Deployer; @@ -47,49 +48,63 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { @Override protected double maintain() { if ( ! nodeRepository().nodes().isWorking()) return 0.0; - - if ( ! nodeRepository().zone().environment().isAnyOf(Environment.dev, Environment.perf, Environment.prod)) return 1.0; - - activeNodesByApplication().forEach(this::autoscale); - return 1.0; - } - - private void autoscale(ApplicationId application, NodeList applicationNodes) { - try { - nodesByCluster(applicationNodes).forEach((clusterId, clusterNodes) -> autoscale(application, clusterId)); - } - catch (IllegalArgumentException e) { - throw new IllegalArgumentException("Illegal arguments for " + application, e); + if (nodeRepository().zone().environment().isTest()) return 1.0; + + int attempts = 0; + int failures = 0; + for (var applicationNodes : activeNodesByApplication().entrySet()) { + for (var clusterNodes : nodesByCluster(applicationNodes.getValue()).entrySet()) { + attempts++; + if ( ! autoscale(applicationNodes.getKey(), clusterNodes.getKey())) + failures++; + } } + return asSuccessFactor(attempts, failures); } - private void autoscale(ApplicationId applicationId, ClusterSpec.Id clusterId) { + /** + * Autoscales the given cluster. + * + * @return true if an autoscaling decision was made or nothing should be done, false if there was an error + */ + private boolean autoscale(ApplicationId applicationId, ClusterSpec.Id clusterId) { try (var lock = nodeRepository().applications().lock(applicationId)) { Optional<Application> application = nodeRepository().applications().get(applicationId); - if (application.isEmpty()) return; - Optional<Cluster> cluster = application.get().cluster(clusterId); - if (cluster.isEmpty()) return; + if (application.isEmpty()) return true; + if (application.get().cluster(clusterId).isEmpty()) return true; + Cluster cluster = application.get().cluster(clusterId).get(); NodeList clusterNodes = nodeRepository().nodes().list(Node.State.active).owner(applicationId).cluster(clusterId); - Cluster updatedCluster = updateCompletion(cluster.get(), clusterNodes); - var autoscaling = autoscaler.autoscale(application.get(), updatedCluster, clusterNodes); - - // 1. Update cluster info - updatedCluster = updateCompletion(cluster.get(), clusterNodes); - if ( ! autoscaling.isEmpty()) // Ignore empties we'll get from servers recently started - updatedCluster = updatedCluster.withTarget(autoscaling); - applications().put(application.get().with(updatedCluster), lock); - - var current = new AllocatableClusterResources(clusterNodes, nodeRepository()).advertisedResources(); - if (autoscaling.resources().isPresent() && !current.equals(autoscaling.resources().get())) { - // 2. Also autoscale + cluster = updateCompletion(cluster, clusterNodes); + + var current = new AllocatableClusterResources(clusterNodes.not().retired(), nodeRepository()).advertisedResources(); + + // Autoscale unless an autoscaling is already in progress + Autoscaling autoscaling = null; + if (cluster.target().resources().isEmpty() || current.equals(cluster.target().resources().get())) { + autoscaling = autoscaler.autoscale(application.get(), cluster, clusterNodes); + if ( autoscaling.isPresent() || cluster.target().isEmpty()) // Ignore empty from recently started servers + cluster = cluster.withTarget(autoscaling); + } + + // Always store updates + applications().put(application.get().with(cluster), lock); + + // Attempt to perform the autoscaling immediately, and log it regardless + if (autoscaling != null && autoscaling.resources().isPresent() && !current.equals(autoscaling.resources().get())) { try (MaintenanceDeployment deployment = new MaintenanceDeployment(applicationId, deployer, metric, nodeRepository())) { - if (deployment.isValid()) { + if (deployment.isValid()) deployment.activate(); - logAutoscaling(current, autoscaling.resources().get(), applicationId, clusterNodes); - } + logAutoscaling(current, autoscaling.resources().get(), applicationId, clusterNodes.not().retired()); } } + return true; + } + catch (ApplicationLockException e) { + return false; + } + catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Illegal arguments for " + applicationId + " cluster " + clusterId, e); } } @@ -123,7 +138,7 @@ public class AutoscalingMaintainer extends NodeRepositoryMaintainer { } private void logAutoscaling(ClusterResources from, ClusterResources to, ApplicationId application, NodeList clusterNodes) { - log.info("Autoscaled " + application + " " + clusterNodes.clusterSpec() + ":" + + log.info("Autoscaling " + application + " " + clusterNodes.clusterSpec() + ":" + "\nfrom " + toString(from) + "\nto " + toString(to)); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java index c8b736cb25b..603056856e2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityChecker.java @@ -127,12 +127,12 @@ public class CapacityChecker { for (var host : hosts) { NodeResources hostResources = host.flavor().resources(); int occupiedIps = 0; - Set<String> ipPool = host.ipConfig().pool().asSet(); + Set<String> ipPool = host.ipConfig().pool().ipSet(); for (var child : nodeChildren.get(host)) { hostResources = hostResources.subtract(child.resources().justNumbers()); occupiedIps += child.ipConfig().primary().stream().filter(ipPool::contains).count(); } - availableResources.put(host, new AllocationResources(hostResources, host.ipConfig().pool().asSet().size() - occupiedIps)); + availableResources.put(host, new AllocationResources(hostResources, host.ipConfig().pool().ipSet().size() - occupiedIps)); } return availableResources; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java index 86c5a926900..c606ede05d1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisioner.java @@ -9,14 +9,17 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException; -import com.yahoo.vespa.hosted.provision.provisioning.HostIpConfig; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; import com.yahoo.yolean.Exceptions; import javax.naming.NamingException; import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; /** * Resumes provisioning (requests additional IP addresses, updates DNS when IPs are ready) of hosts in state provisioned @@ -38,13 +41,23 @@ public class HostResumeProvisioner extends NodeRepositoryMaintainer { @Override protected double maintain() { NodeList allNodes = nodeRepository().nodes().list(); + Map<String, Set<Node>> nodesByProvisionedParentHostname = + allNodes.nodeType(NodeType.tenant, NodeType.config, NodeType.controller) + .asList() + .stream() + .filter(node -> node.parentHostname().isPresent()) + .collect(Collectors.groupingBy(node -> node.parentHostname().get(), Collectors.toSet())); + NodeList hosts = allNodes.state(Node.State.provisioned).nodeType(NodeType.host, NodeType.confighost, NodeType.controllerhost); int failures = 0; for (Node host : hosts) { - NodeList children = allNodes.childrenOf(host); - try { - HostIpConfig hostIpConfig = hostProvisioner.provision(host, children.asSet()); - setIpConfig(host, children, hostIpConfig); + Set<Node> children = nodesByProvisionedParentHostname.getOrDefault(host.hostname(), Set.of()); + // This doesn't actually require unallocated lock, but that is much easier than simultaneously holding + // the application locks of the host and all it's children. + try (var lock = nodeRepository().nodes().lockUnallocated()) { + List<Node> updatedNodes = hostProvisioner.provision(host, children); + verifyDns(updatedNodes); + nodeRepository().nodes().write(updatedNodes, lock); } catch (IllegalArgumentException | IllegalStateException e) { log.log(Level.INFO, "Could not provision " + host.hostname() + " with " + children.size() + " children, will retry in " + interval() + ": " + Exceptions.toMessageString(e)); @@ -66,21 +79,13 @@ public class HostResumeProvisioner extends NodeRepositoryMaintainer { return asSuccessFactor(hosts.size(), failures); } - private void setIpConfig(Node host, NodeList children, HostIpConfig hostIpConfig) { - if (hostIpConfig.isEmpty()) return; - NodeList nodes = NodeList.of(host).and(children); + /** Verify DNS configuration of given nodes */ + private void verifyDns(List<Node> nodes) { for (var node : nodes) { - verifyDns(node, hostIpConfig.require(node.hostname())); - } - nodeRepository().nodes().setIpConfig(hostIpConfig); - } - - /** Verify DNS configuration of given node */ - private void verifyDns(Node node, IP.Config ipConfig) { - boolean enclave = node.cloudAccount().isEnclave(nodeRepository().zone()); - for (var ipAddress : ipConfig.primary()) { - IP.verifyDns(node.hostname(), ipAddress, nodeRepository().nameResolver(), !enclave); + boolean enclave = node.cloudAccount().isEnclave(nodeRepository().zone()); + for (var ipAddress : node.ipConfig().primary()) { + IP.verifyDns(node.hostname(), ipAddress, nodeRepository().nameResolver(), !enclave); + } } } - } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index 67c1c7359f7..5af74214648 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -267,13 +267,20 @@ public class MetricsReporter extends NodeRepositoryMaintainer { } private void updateNodeCountMetrics(NodeList nodes) { - Map<State, List<Node>> nodesByState = nodes.nodeType(NodeType.tenant).asList().stream() - .collect(Collectors.groupingBy(Node::state)); + var nodesByState = nodes.nodeType(NodeType.tenant) + .asList().stream() + .collect(Collectors.groupingBy(Node::state)); + + var hostsByState = nodes.nodeType(NodeType.host) + .asList().stream() + .collect(Collectors.groupingBy(Node::state)); // Count per state for (State state : State.values()) { - List<Node> nodesInState = nodesByState.getOrDefault(state, List.of()); - metric.set("hostedVespa." + state.name() + "Hosts", nodesInState.size(), null); + var nodesInState = nodesByState.getOrDefault(state, List.of()); + var hostsInState = hostsByState.getOrDefault(state, List.of()); + metric.set("hostedVespa." + state.name() + "Nodes", nodesInState.size(), null); + metric.set("hostedVespa." + state.name() + "Hosts", hostsInState.size(), null); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java index 94683d463af..781debe26a0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeHealthTracker.java @@ -42,14 +42,13 @@ public class NodeHealthTracker extends NodeRepositoryMaintainer { @Override protected double maintain() { - return updateActiveNodeDownState(); + return updateNodeHealth(); } /** - * If the node is down (see {@link #allDown}), and there is no "down" history record, we add it. - * Otherwise we remove any "down" history record. + * Update UP and DOWN node records for each node as they change. */ - private double updateActiveNodeDownState() { + private double updateNodeHealth() { var attempts = new MutableInteger(0); var failures = new MutableInteger(0); NodeList activeNodes = nodeRepository().nodes().list(Node.State.active); @@ -59,7 +58,7 @@ public class NodeHealthTracker extends NodeRepositoryMaintainer { // Already correct record, nothing to do boolean isDown = allDown(serviceInstances); - if (isDown == node.get().isDown()) return; + if (isDown == node.get().isDown() && isDown != node.get().isUp()) return; // Lock and update status ApplicationId owner = node.get().allocation().get().owner(); @@ -82,7 +81,7 @@ public class NodeHealthTracker extends NodeRepositoryMaintainer { } /** - * Returns true if the node is considered bad: All monitored services services are down. + * Returns true if the node is considered bad: All monitored services are down. * If a node remains bad for a long time, the NodeFailer will try to fail the node. */ static boolean allDown(List<ServiceInstance> services) { @@ -110,7 +109,7 @@ public class NodeHealthTracker extends NodeRepositoryMaintainer { /** Clear down record for node, if any */ private void recordAsUp(Node node, Mutex lock) { - if (!node.isDown()) return; // already up: Don't change down timestamp + if (node.isUp()) return; // already up: Don't change up timestamp nodeRepository().nodes().write(node.upAt(clock().instant(), Agent.NodeHealthTracker), lock); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Address.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Address.java new file mode 100644 index 00000000000..d7ef2228960 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Address.java @@ -0,0 +1,17 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.node; + +import java.util.Objects; + +/** + * Address info about a container that might run on a host. + * + * @author hakon + */ +public record Address(String hostname) { + public Address { + Objects.requireNonNull(hostname, "hostname cannot be null"); + if (hostname.isEmpty()) + throw new IllegalArgumentException("hostname cannot be empty"); + } +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java index 708d84ba655..b2becc7ecfd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/IP.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.node; +import com.google.common.collect.ImmutableSet; import com.google.common.net.InetAddresses; import com.google.common.primitives.UnsignedBytes; -import com.yahoo.config.provision.HostName; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; @@ -31,7 +31,7 @@ import static com.yahoo.config.provision.NodeType.proxyhost; * * @author mpolden */ -public record IP() { +public class IP { /** Comparator for sorting IP addresses by their natural order */ public static final Comparator<InetAddress> NATURAL_ORDER = (ip1, ip2) -> { @@ -59,26 +59,31 @@ public record IP() { }; /** IP configuration of a node */ - public record Config(Set<String> primary, Pool pool) { + public static class Config { public static final Config EMPTY = Config.ofEmptyPool(Set.of()); + private final Set<String> primary; + private final Pool pool; + public static Config ofEmptyPool(Set<String> primary) { - return new Config(primary, Pool.EMPTY); + return new Config(primary, Set.of(), List.of()); } - public static Config of(Set<String> primary, Set<String> ipPool, List<HostName> hostnames) { - return new Config(primary, new Pool(IpAddresses.of(ipPool), hostnames)); + public static Config of(Set<String> primary, Set<String> ipPool, List<Address> addressPool) { + return new Config(primary, ipPool, addressPool); } - public static Config of(Set<String> primary, Set<String> pool) { - return of(primary, pool, List.of()); + /** LEGACY TEST CONSTRUCTOR - use of() variants and/or the with- methods. */ + public Config(Set<String> primary, Set<String> pool) { + this(primary, pool, List.of()); } /** DO NOT USE: Public for NodeSerializer. */ - public Config(Set<String> primary, Pool pool) { - this.primary = Collections.unmodifiableSet(new LinkedHashSet<>(Objects.requireNonNull(primary, "primary must be non-null"))); - this.pool = Objects.requireNonNull(pool, "pool must be non-null"); + public Config(Set<String> primary, Set<String> pool, List<Address> addresses) { + this.primary = ImmutableSet.copyOf(Objects.requireNonNull(primary, "primary must be non-null")); + this.pool = Pool.of(Objects.requireNonNull(pool, "pool must be non-null"), + Objects.requireNonNull(addresses, "addresses must be non-null")); } /** The primary addresses of this. These addresses are used when communicating with the node itself */ @@ -93,12 +98,31 @@ public record IP() { /** Returns a copy of this with pool set to given value */ public Config withPool(Pool pool) { - return new Config(primary, pool); + return new Config(primary, pool.ipSet(), pool.getAddressList()); } /** Returns a copy of this with pool set to given value */ public Config withPrimary(Set<String> primary) { - return new Config(primary, pool); + return new Config(primary, pool.ipSet(), pool.getAddressList()); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Config config = (Config) o; + return primary.equals(config.primary) && + pool.equals(config.pool); + } + + @Override + public int hashCode() { + return Objects.hash(primary, pool); + } + + @Override + public String toString() { + return String.format("ip config primary=%s pool=%s", primary, pool.ipSet()); } /** @@ -116,8 +140,8 @@ public record IP() { var addresses = new HashSet<>(node.ipConfig().primary()); var otherAddresses = new HashSet<>(other.ipConfig().primary()); if (node.type().isHost()) { // Addresses of a host can never overlap with any other nodes - addresses.addAll(node.ipConfig().pool().asSet()); - otherAddresses.addAll(other.ipConfig().pool().asSet()); + addresses.addAll(node.ipConfig().pool().ipSet()); + otherAddresses.addAll(other.ipConfig().pool().ipSet()); } otherAddresses.retainAll(addresses); if (!otherAddresses.isEmpty()) @@ -134,12 +158,12 @@ public record IP() { if (node.parentHostname().isPresent() == existingNode.parentHostname().isPresent()) return false; // Not a parent-child node if (node.parentHostname().isEmpty()) return canAssignIpOf(node, existingNode); if (!node.parentHostname().get().equals(existingNode.hostname())) return false; // Wrong host - return switch (node.type()) { - case proxy -> existingNode.type() == proxyhost; - case config -> existingNode.type() == confighost; - case controller -> existingNode.type() == controllerhost; - default -> false; - }; + switch (node.type()) { + case proxy: return existingNode.type() == proxyhost; + case config: return existingNode.type() == confighost; + case controller: return existingNode.type() == controllerhost; + } + return false; } public static Node verify(Node node, LockedNodeList allNodes) { @@ -149,13 +173,25 @@ public record IP() { } /** A list of IP addresses and their protocol */ - record IpAddresses(Set<String> addresses, Protocol protocol) { + public static class IpAddresses { - public IpAddresses(Set<String> addresses, Protocol protocol) { - this.addresses = Collections.unmodifiableSet(new LinkedHashSet<>(Objects.requireNonNull(addresses, "addresses must be non-null"))); + private final Set<String> ipAddresses; + private final Protocol protocol; + + private IpAddresses(Set<String> ipAddresses, Protocol protocol) { + this.ipAddresses = ImmutableSet.copyOf(Objects.requireNonNull(ipAddresses, "addresses must be non-null")); this.protocol = Objects.requireNonNull(protocol, "type must be non-null"); } + public Set<String> asSet() { + return ipAddresses; + } + + /** The protocol of addresses in this */ + public Protocol protocol() { + return protocol; + } + /** Create addresses of the given set */ private static IpAddresses of(Set<String> addresses) { long ipv6AddrCount = addresses.stream().filter(IP::isV6).count(); @@ -180,7 +216,6 @@ public record IP() { } public enum Protocol { - dualStack("dual-stack"), ipv4("IPv4-only"), ipv6("IPv6-only"); @@ -189,8 +224,21 @@ public record IP() { Protocol(String description) { this.description = description; } + public String getDescription() { return description; } } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + IpAddresses that = (IpAddresses) o; + return ipAddresses.equals(that.ipAddresses) && protocol == that.protocol; + } + + @Override + public int hashCode() { + return Objects.hash(ipAddresses, protocol); + } } /** @@ -198,23 +246,25 @@ public record IP() { * * Addresses in this are available for use by Linux containers. */ - public record Pool(IpAddresses ipAddresses, List<HostName> hostnames) { + public static class Pool { - public static final Pool EMPTY = new Pool(IpAddresses.of(Set.of()), List.of()); + private final IpAddresses ipAddresses; + private final List<Address> addresses; + + /** Creates an empty pool. */ + public static Pool of() { + return of(Set.of(), List.of()); + } /** Create a new pool containing given ipAddresses */ - public static Pool of(Set<String> ipAddresses, List<HostName> hostnames) { + public static Pool of(Set<String> ipAddresses, List<Address> addresses) { IpAddresses ips = IpAddresses.of(ipAddresses); - return new Pool(ips, hostnames); + return new Pool(ips, addresses); } - public Pool(IpAddresses ipAddresses, List<HostName> hostnames) { + private Pool(IpAddresses ipAddresses, List<Address> addresses) { this.ipAddresses = Objects.requireNonNull(ipAddresses, "ipAddresses must be non-null"); - this.hostnames = List.copyOf(Objects.requireNonNull(hostnames, "hostnames must be non-null")); - } - - public Set<String> asSet() { - return ipAddresses.addresses; + this.addresses = Objects.requireNonNull(addresses, "addresses must be non-null"); } /** @@ -224,15 +274,16 @@ public record IP() { * @return an allocation from the pool, if any can be made */ public Optional<Allocation> findAllocation(LockedNodeList nodes, NameResolver resolver, boolean hasPtr) { - if (ipAddresses.addresses.isEmpty()) { + if (ipAddresses.asSet().isEmpty()) { // IP addresses have not yet been resolved and should be done later. - return findUnusedHostnames(nodes).map(Allocation::ofHostname) - .findFirst(); + return findUnusedAddressStream(nodes) + .map(Allocation::ofAddress) + .findFirst(); } if (!hasPtr) { // Without PTR records (reverse IP mapping): Ensure only forward resolving from hostnames. - return findUnusedHostnames(nodes).findFirst().map(address -> Allocation.fromHostname(address, resolver, ipAddresses.protocol)); + return findUnusedAddressStream(nodes).findFirst().map(address -> Allocation.fromAddress(address, resolver, ipAddresses.protocol)); } if (ipAddresses.protocol == IpAddresses.Protocol.ipv4) { @@ -262,34 +313,64 @@ public record IP() { * @param nodes a list of all nodes in the repository */ public Set<String> findUnusedIpAddresses(NodeList nodes) { - Set<String> unusedAddresses = new LinkedHashSet<>(asSet()); - nodes.matching(node -> node.ipConfig().primary().stream().anyMatch(ip -> asSet().contains(ip))) + var unusedAddresses = new LinkedHashSet<>(ipSet()); + nodes.matching(node -> node.ipConfig().primary().stream().anyMatch(ip -> ipSet().contains(ip))) .forEach(node -> unusedAddresses.removeAll(node.ipConfig().primary())); return Collections.unmodifiableSet(unusedAddresses); } - private Stream<HostName> findUnusedHostnames(NodeList nodes) { - Set<String> usedHostnames = nodes.stream().map(Node::hostname).collect(Collectors.toSet()); - return hostnames.stream().filter(hostname -> !usedHostnames.contains(hostname.value())); + private Stream<Address> findUnusedAddressStream(NodeList nodes) { + Set<String> hostnames = nodes.stream().map(Node::hostname).collect(Collectors.toSet()); + return addresses.stream().filter(address -> !hostnames.contains(address.hostname())); + } + + public IpAddresses.Protocol getProtocol() { + return ipAddresses.protocol; + } + + /** Returns the IP addresses in this pool as a set */ + public Set<String> ipSet() { + return ipAddresses.asSet(); + } + + public List<Address> getAddressList() { + return addresses; } public Pool withIpAddresses(Set<String> ipAddresses) { - return Pool.of(ipAddresses, hostnames); + return Pool.of(ipAddresses, addresses); } - public Pool withHostnames(List<HostName> hostnames) { - return Pool.of(ipAddresses.addresses, hostnames); + public Pool withAddresses(List<Address> addresses) { + return Pool.of(ipAddresses.ipAddresses, addresses); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Pool pool = (Pool) o; + return ipAddresses.equals(pool.ipAddresses) && addresses.equals(pool.addresses); + } + + @Override + public int hashCode() { + return Objects.hash(ipAddresses, addresses); } } /** An address allocation from a pool */ - public record Allocation(String hostname, Optional<String> ipv4Address, Optional<String> ipv6Address) { + public static class Allocation { + + private final String hostname; + private final Optional<String> ipv4Address; + private final Optional<String> ipv6Address; - public Allocation { - Objects.requireNonNull(hostname, "hostname must be non-null"); - Objects.requireNonNull(ipv4Address, "ipv4Address must be non-null"); - Objects.requireNonNull(ipv6Address, "ipv6Address must be non-null"); + private Allocation(String hostname, Optional<String> ipv4Address, Optional<String> ipv6Address) { + this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null"); + this.ipv4Address = Objects.requireNonNull(ipv4Address, "ipv4Address must be non-null"); + this.ipv6Address = Objects.requireNonNull(ipv6Address, "ipv6Address must be non-null"); } /** @@ -346,22 +427,22 @@ public record IP() { return new Allocation(hostname4, Optional.of(addresses.get(0)), Optional.empty()); } - private static Allocation fromHostname(HostName hostname, NameResolver resolver, IpAddresses.Protocol protocol) { + private static Allocation fromAddress(Address address, NameResolver resolver, IpAddresses.Protocol protocol) { // Resolve both A and AAAA to verify they match the protocol and to avoid surprises later on. - Optional<String> ipv4Address = resolveOptional(hostname.value(), resolver, RecordType.A); + Optional<String> ipv4Address = resolveOptional(address.hostname(), resolver, RecordType.A); if (protocol != IpAddresses.Protocol.ipv6 && ipv4Address.isEmpty()) - throw new IllegalArgumentException(protocol.description + " hostname " + hostname.value() + " did not resolve to an IPv4 address"); + throw new IllegalArgumentException(protocol.description + " hostname " + address.hostname() + " did not resolve to an IPv4 address"); if (protocol == IpAddresses.Protocol.ipv6 && ipv4Address.isPresent()) - throw new IllegalArgumentException(protocol.description + " hostname " + hostname.value() + " has an IPv4 address: " + ipv4Address.get()); + throw new IllegalArgumentException(protocol.description + " hostname " + address.hostname() + " has an IPv4 address: " + ipv4Address.get()); - Optional<String> ipv6Address = resolveOptional(hostname.value(), resolver, RecordType.AAAA); + Optional<String> ipv6Address = resolveOptional(address.hostname(), resolver, RecordType.AAAA); if (protocol != IpAddresses.Protocol.ipv4 && ipv6Address.isEmpty()) - throw new IllegalArgumentException(protocol.description + " hostname " + hostname.value() + " did not resolve to an IPv6 address"); + throw new IllegalArgumentException(protocol.description + " hostname " + address.hostname() + " did not resolve to an IPv6 address"); if (protocol == IpAddresses.Protocol.ipv4 && ipv6Address.isPresent()) - throw new IllegalArgumentException(protocol.description + " hostname " + hostname.value() + " has an IPv6 address: " + ipv6Address.get()); + throw new IllegalArgumentException(protocol.description + " hostname " + address.hostname() + " has an IPv6 address: " + ipv6Address.get()); - return new Allocation(hostname.value(), ipv4Address, ipv6Address); + return new Allocation(address.hostname(), ipv4Address, ipv6Address); } private static Optional<String> resolveOptional(String hostname, NameResolver resolver, RecordType recordType) { @@ -373,14 +454,37 @@ public record IP() { }; } - private static Allocation ofHostname(HostName hostName) { - return new Allocation(hostName.value(), Optional.empty(), Optional.empty()); + private static Allocation ofAddress(Address address) { + return new Allocation(address.hostname(), Optional.empty(), Optional.empty()); + } + + /** Hostname pointing to the IP addresses in this */ + public String hostname() { + return hostname; + } + + /** IPv4 address of this allocation */ + public Optional<String> ipv4Address() { + return ipv4Address; + } + + /** IPv6 address of this allocation */ + public Optional<String> ipv6Address() { + return ipv6Address; } /** All IP addresses in this */ public Set<String> addresses() { - return Stream.concat(ipv4Address.stream(), ipv6Address.stream()) - .collect(Collectors.toUnmodifiableSet()); + ImmutableSet.Builder<String> builder = ImmutableSet.builder(); + ipv4Address.ifPresent(builder::add); + ipv6Address.ifPresent(builder::add); + return builder.build(); + } + + @Override + public String toString() { + return String.format("Address allocation [hostname=%s, IPv4=%s, IPv6=%s]", + hostname, ipv4Address.orElse("<none>"), ipv6Address.orElse("<none>")); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java index 9134c376f38..bb3d288e555 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java @@ -21,7 +21,6 @@ import com.yahoo.vespa.hosted.provision.applications.Applications; import com.yahoo.vespa.hosted.provision.maintenance.NodeFailer; import com.yahoo.vespa.hosted.provision.node.filter.NodeFilter; import com.yahoo.vespa.hosted.provision.persistence.CuratorDb; -import com.yahoo.vespa.hosted.provision.provisioning.HostIpConfig; import com.yahoo.vespa.orchestrator.HostNameNotFoundException; import com.yahoo.vespa.orchestrator.Orchestrator; @@ -110,7 +109,7 @@ public class Nodes { */ public boolean isWorking() { NodeList activeNodes = list(Node.State.active); - if (activeNodes.size() <= 5) return true; // Not enough data to decide + if (activeNodes.size() < 20) return true; // Not enough data to decide NodeList downNodes = activeNodes.down(); return ! ( (double)downNodes.size() / (double)activeNodes.size() > 0.2 ); } @@ -126,7 +125,7 @@ public class Nodes { illegal("Cannot add " + node + ": Child nodes need to be allocated"); Optional<Node> existing = node(node.hostname()); if (existing.isPresent()) - illegal("Cannot add " + node + ": A node with this name already exists"); + throw new IllegalStateException("Cannot add " + node + ": A node with this name already exists"); } return db.addNodesInState(nodes.asList(), Node.State.reserved, Agent.system); } @@ -152,7 +151,7 @@ public class Nodes { Optional<Node> existing = node(node.hostname()); if (existing.isPresent()) { if (existing.get().state() != Node.State.deprovisioned) - illegal("Cannot add " + node + ": A node with this name already exists"); + throw new IllegalStateException("Cannot add " + node + ": A node with this name already exists"); node = node.with(existing.get().history()); node = node.with(existing.get().reports()); node = node.with(node.status().withFailCount(existing.get().status().failCount())); @@ -355,15 +354,6 @@ public class Nodes { } } - /** Update IP config for nodes in given config */ - public void setIpConfig(HostIpConfig hostIpConfig) { - Predicate<Node> nodeInConfig = (node) -> hostIpConfig.contains(node.hostname()); - performOn(nodeInConfig, (node, lock) -> { - IP.Config ipConfig = hostIpConfig.require(node.hostname()); - return write(node.with(ipConfig), lock); - }); - } - /** * Parks this node and returns it in its new state. * diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java index 38675ba3758..1e378c80f90 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterResources; @@ -20,6 +21,7 @@ import com.yahoo.vespa.hosted.provision.autoscale.Load; import java.io.IOException; import java.io.UncheckedIOException; +import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.Collection; @@ -54,6 +56,8 @@ public class ApplicationSerializer { private static final String groupSizeKey = "groupSize"; private static final String requiredKey = "required"; private static final String suggestedKey = "suggested"; + private static final String clusterInfoKey = "clusterInfo"; + private static final String bcpDeadlineKey = "bcpDeadline"; private static final String bcpGroupInfoKey = "bcpGroupInfo"; private static final String queryRateKey = "queryRateKey"; private static final String growthRateHeadroomKey = "growthRateHeadroomKey"; @@ -134,6 +138,8 @@ public class ApplicationSerializer { clusterObject.setBool(requiredKey, cluster.required()); toSlime(cluster.suggested(), clusterObject.setObject(suggestedKey)); toSlime(cluster.target(), clusterObject.setObject(targetKey)); + if (! cluster.clusterInfo().isEmpty()) + toSlime(cluster.clusterInfo(), clusterObject.setObject(clusterInfoKey)); if (! cluster.bcpGroupInfo().isEmpty()) toSlime(cluster.bcpGroupInfo(), clusterObject.setObject(bcpGroupInfoKey)); scalingEventsToSlime(cluster.scalingEvents(), clusterObject.setArray(scalingEventsKey)); @@ -148,6 +154,7 @@ public class ApplicationSerializer { clusterObject.field(requiredKey).asBool(), autoscalingFromSlime(clusterObject.field(suggestedKey)), autoscalingFromSlime(clusterObject.field(targetKey)), + clusterInfoFromSlime(clusterObject.field(clusterInfoKey)), bcpGroupInfoFromSlime(clusterObject.field(bcpGroupInfoKey)), scalingEventsFromSlime(clusterObject.field(scalingEventsKey))); } @@ -225,8 +232,18 @@ public class ApplicationSerializer { metricsFromSlime(autoscalingObject.field(metricsKey))); } + private static void toSlime(ClusterInfo clusterInfo, Cursor clusterInfoObject) { + clusterInfoObject.setLong(bcpDeadlineKey, clusterInfo.bcpDeadline().toMinutes()); + } + + private static ClusterInfo clusterInfoFromSlime(Inspector clusterInfoObject) { + if ( ! clusterInfoObject.valid()) return ClusterInfo.empty(); + ClusterInfo.Builder builder = new ClusterInfo.Builder(); + builder.bcpDeadline(Duration.ofMinutes(clusterInfoObject.field(bcpDeadlineKey).asLong())); + return builder.build(); + } + private static void toSlime(BcpGroupInfo bcpGroupInfo, Cursor bcpGroupInfoObject) { - if (bcpGroupInfo.isEmpty()) return; bcpGroupInfoObject.setDouble(queryRateKey, bcpGroupInfo.queryRate()); bcpGroupInfoObject.setDouble(growthRateHeadroomKey, bcpGroupInfo.growthRateHeadroom()); bcpGroupInfoObject.setDouble(cpuCostPerQueryKey, bcpGroupInfo.cpuCostPerQuery()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ArchiveUriSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ArchiveUriSerializer.java new file mode 100644 index 00000000000..3bf37816dd4 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ArchiveUriSerializer.java @@ -0,0 +1,56 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.TenantName; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.ObjectTraverser; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; +import com.yahoo.vespa.hosted.provision.archive.ArchiveUris; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.HashMap; +import java.util.Map; + +/** + * Serializer for archive URIs that are set per tenant and per account. + * + * @author freva + */ +public class ArchiveUriSerializer { + + private ArchiveUriSerializer() {} + + public static byte[] toJson(ArchiveUris archiveUris) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + + Cursor tenantObject = root.setObject("tenant"); + archiveUris.tenantArchiveUris().forEach((tenant, uri) -> tenantObject.setString(tenant.value(), uri)); + + Cursor accountObject = root.setObject("account"); + archiveUris.accountArchiveUris().forEach((account, uri) -> accountObject.setString(account.value(), uri)); + + try { + return SlimeUtils.toJsonBytes(slime); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public static ArchiveUris fromJson(byte[] data) { + Inspector inspector = SlimeUtils.jsonToSlime(data).get(); + + Map<TenantName, String> tenantArchiveUris = new HashMap<>(); + inspector.field("tenant").traverse((ObjectTraverser) (key, value) -> tenantArchiveUris.put(TenantName.from(key), value.asString())); + + Map<CloudAccount, String> accountArchiveUris = new HashMap<>(); + inspector.field("account").traverse((ObjectTraverser) (key, value) -> accountArchiveUris.put(CloudAccount.from(key), value.asString())); + + return new ArchiveUris(tenantArchiveUris, accountArchiveUris); + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java index c25dfe9f1e2..5aea6858f60 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDb.java @@ -10,7 +10,6 @@ import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.TenantName; import com.yahoo.path.Path; import com.yahoo.transaction.NestedTransaction; import com.yahoo.transaction.Transaction; @@ -21,6 +20,7 @@ import com.yahoo.vespa.curator.transaction.CuratorOperations; import com.yahoo.vespa.curator.transaction.CuratorTransaction; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.applications.Application; +import com.yahoo.vespa.hosted.provision.archive.ArchiveUris; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerId; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -70,7 +70,7 @@ public class CuratorDb { private static final Path infrastructureVersionsPath = root.append("infrastructureVersions"); private static final Path osVersionsPath = root.append("osVersions"); private static final Path firmwareCheckPath = root.append("firmwareCheck"); - private static final Path archiveUrisPath = root.append("archiveUris"); + private static final Path archiveUrisPath = root.append("archiveUri"); private static final Duration defaultLockTimeout = Duration.ofMinutes(1); @@ -102,6 +102,7 @@ public class CuratorDb { db.create(archiveUrisPath); db.create(loadBalancersPath); provisionIndexCounter.initialize(100); + CuratorOperations.delete(root.append("archiveUris").toString()); // TODO (freva): March 2023 } /** Adds a set of nodes. Rollbacks/fails transaction if any node is not in the expected state. */ @@ -375,16 +376,16 @@ public class CuratorDb { // Archive URIs ----------------------------------------------------------- - public void writeArchiveUris(Map<TenantName, String> archiveUris) { - byte[] data = TenantArchiveUriSerializer.toJson(archiveUris); + public void writeArchiveUris(ArchiveUris archiveUris) { + byte[] data = ArchiveUriSerializer.toJson(archiveUris); NestedTransaction transaction = new NestedTransaction(); CuratorTransaction curatorTransaction = db.newCuratorTransactionIn(transaction); curatorTransaction.add(CuratorOperations.setData(archiveUrisPath.getAbsolute(), data)); transaction.commit(); } - public Map<TenantName, String> readArchiveUris() { - return read(archiveUrisPath, TenantArchiveUriSerializer::fromJson).orElseGet(Map::of); + public ArchiveUris readArchiveUris() { + return read(archiveUrisPath, ArchiveUriSerializer::fromJson).orElseGet(() -> new ArchiveUris(Map.of(), Map.of())); } public Lock lockArchiveUris() { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 23ea14da4cc..39cccafb8ef 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -14,7 +14,6 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; @@ -29,6 +28,7 @@ import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.Generation; @@ -167,8 +167,8 @@ public class NodeSerializer { object.setString(hostnameKey, node.hostname()); object.setString(stateKey, toString(node.state())); toSlime(node.ipConfig().primary(), object.setArray(ipAddressesKey)); - toSlime(node.ipConfig().pool().asSet(), object.setArray(ipAddressPoolKey)); - toSlime(node.ipConfig().pool().hostnames(), object); + toSlime(node.ipConfig().pool().ipSet(), object.setArray(ipAddressPoolKey)); + toSlime(node.ipConfig().pool().getAddressList(), object); object.setString(idKey, node.id()); node.parentHostname().ifPresent(hostname -> object.setString(parentHostnameKey, hostname)); toSlime(node.flavor(), object); @@ -247,11 +247,11 @@ public class NodeSerializer { ipAddresses.stream().map(IP::parse).sorted(IP.NATURAL_ORDER).map(IP::asString).forEach(array::addString); } - private void toSlime(List<HostName> hostnames, Cursor object) { - if (hostnames.isEmpty()) return; - Cursor containersArray = object.setArray(containersKey); - hostnames.forEach(hostname -> { - containersArray.addObject().setString(containerHostnameKey, hostname.value()); + private void toSlime(List<Address> addresses, Cursor object) { + if (addresses.isEmpty()) return; + Cursor addressCursor = object.setArray(containersKey); + addresses.forEach(address -> { + addressCursor.addObject().setString(containerHostnameKey, address.hostname()); }); } @@ -277,9 +277,9 @@ public class NodeSerializer { private Node nodeFromSlime(Inspector object) { Flavor flavor = flavorFromSlime(object); return new Node(object.field(idKey).asString(), - IP.Config.of(ipAddressesFromSlime(object, ipAddressesKey), - ipAddressesFromSlime(object, ipAddressPoolKey), - hostnamesFromSlime(object)), + new IP.Config(ipAddressesFromSlime(object, ipAddressesKey), + ipAddressesFromSlime(object, ipAddressPoolKey), + addressesFromSlime(object)), object.field(hostnameKey).asString(), SlimeUtils.optionalString(object.field(parentHostnameKey)), flavor, @@ -394,9 +394,9 @@ public class NodeSerializer { return ipAddresses.build(); } - private List<HostName> hostnamesFromSlime(Inspector object) { + private List<Address> addressesFromSlime(Inspector object) { return SlimeUtils.entriesStream(object.field(containersKey)) - .map(elem -> HostName.of(elem.field(containerHostnameKey).asString())) + .map(elem -> new Address(elem.field(containerHostnameKey).asString())) .toList(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/TenantArchiveUriSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/TenantArchiveUriSerializer.java deleted file mode 100644 index d381d25704a..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/TenantArchiveUriSerializer.java +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.persistence; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Inspector; -import com.yahoo.slime.ObjectTraverser; -import com.yahoo.slime.Slime; -import com.yahoo.slime.SlimeUtils; - -import java.io.IOException; -import java.io.UncheckedIOException; -import java.util.Map; -import java.util.TreeMap; - -/** - * Serializer for archive URIs that are set per tenant. - * - * @author freva - */ -public class TenantArchiveUriSerializer { - - private TenantArchiveUriSerializer() {} - - public static byte[] toJson(Map<TenantName, String> archiveUrisByTenantName) { - Slime slime = new Slime(); - Cursor object = slime.setObject(); - archiveUrisByTenantName.forEach((tenantName, archiveUri) -> - object.setString(tenantName.value(), archiveUri)); - try { - return SlimeUtils.toJsonBytes(slime); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - public static Map<TenantName, String> fromJson(byte[] data) { - Map<TenantName, String> archiveUrisByTenantName = new TreeMap<>(); // Use TreeMap to sort by tenant name - Inspector inspector = SlimeUtils.jsonToSlime(data).get(); - inspector.traverse((ObjectTraverser) (key, value) -> - archiveUrisByTenantName.put(TenantName.from(key), value.asString())); - return archiveUrisByTenantName; - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ArchiveUris.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ArchiveUris.java deleted file mode 100644 index 8b8801d5b53..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ArchiveUris.java +++ /dev/null @@ -1,89 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.provisioning; - -import com.yahoo.config.provision.TenantName; -import com.yahoo.lang.CachedSupplier; -import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.node.Allocation; -import com.yahoo.vespa.hosted.provision.persistence.CuratorDb; - -import java.time.Duration; -import java.util.Map; -import java.util.Optional; -import java.util.TreeMap; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.regex.Pattern; - -/** - * Thread safe class to get and set archive URI for given tenants. Archive URIs are stored in ZooKeeper so that - * nodes within the same tenant have the same archive URI from all the config servers. - * - * @author freva - */ -public class ArchiveUris { - - private static final Logger log = Logger.getLogger(ArchiveUris.class.getName()); - private static final Pattern validUriPattern = Pattern.compile("[a-z0-9]+://(?:(?:[a-z0-9]+(?:[-_][a-z0-9.]+)*)+/)+"); - private static final Duration cacheTtl = Duration.ofMinutes(1); - - private final CuratorDb db; - private final CachedSupplier<Map<TenantName, String>> archiveUris; - - public ArchiveUris(CuratorDb db) { - this.db = db; - this.archiveUris = new CachedSupplier<>(db::readArchiveUris, cacheTtl); - } - - /** Returns the current archive URI for each tenant */ - public Map<TenantName, String> getArchiveUris() { - return archiveUris.get(); - } - - /** Returns the archive URI to use for given tenant */ - public Optional<String> archiveUriFor(TenantName tenant) { - return Optional.ofNullable(archiveUris.get().get(tenant)); - } - - /** Returns the archive URI to use for given node */ - public Optional<String> archiveUriFor(Node node) { - return node.allocation().map(Allocation::owner) - .flatMap(app -> archiveUriFor(app.tenant()) - .map(uri -> { - StringBuilder sb = new StringBuilder(100).append(uri) - .append(app.application().value()).append('/') - .append(app.instance().value()).append('/') - .append(node.allocation().get().membership().cluster().id().value()).append('/'); - - for (char c: node.hostname().toCharArray()) { - if (c == '.') break; - sb.append(c); - } - - return sb.append('/').toString(); - })); - } - - /** Set (or remove, if archiveURI is empty) archive URI to use for given tenant */ - public void setArchiveUri(TenantName tenant, Optional<String> archiveUri) { - try (Lock lock = db.lockArchiveUris()) { - Map<TenantName, String> archiveUris = new TreeMap<>(db.readArchiveUris()); - if (Optional.ofNullable(archiveUris.get(tenant)).equals(archiveUri)) return; // No change - - archiveUri.map(ArchiveUris::normalizeUri).ifPresentOrElse(uri -> archiveUris.put(tenant, uri), - () -> archiveUris.remove(tenant)); - db.writeArchiveUris(archiveUris); - this.archiveUris.invalidate(); // Throw away current cache - log.log(Level.FINE, () -> archiveUri.map(s -> "Set archive URI for " + tenant + " to " + s) - .orElseGet(() -> "Remove archive URI for " + tenant)); - } - } - - static String normalizeUri(String uri) { - if (!uri.endsWith("/")) uri = uri + "/"; - if (!validUriPattern.matcher(uri).matches()) - throw new IllegalArgumentException("Invalid archive URI: " + uri); - return uri; - } -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index 8af1df93bde..5732e94956a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -115,10 +115,12 @@ public class CapacityPolicies { if (nodeRepository.exclusiveAllocation(clusterSpec)) { return versioned(clusterSpec, Map.of(new Version(0), smallestExclusiveResources())); } - // TODO (hmusum): Go back to 1.14 Gb memory when bug in resource limits for admin nodes - // has been fixed + + // 1.32 fits floor(8/1.32) = 6 cluster controllers on each 8Gb host, and each will have + // 1.32-(0.7+0.6)*(1.32/8) = 1.1 Gb real memory given current taxes. return versioned(clusterSpec, Map.of(new Version(0), new NodeResources(0.25, 1.14, 10, 0.3), - new Version(8, 127, 11), new NodeResources(0.25, 1.5, 10, 0.3))); + new Version(8, 127, 11), new NodeResources(0.25, 1.5, 10, 0.3), + new Version(8, 129, 4), new NodeResources(0.25, 1.32, 10, 0.3))); } private Architecture adminClusterArchitecture(ApplicationId instance) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java index e7332f6474d..b7e7ac7ee4b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/EmptyProvisionServiceProvider.java @@ -41,10 +41,10 @@ public class EmptyProvisionServiceProvider implements ProvisionServiceProvider { public NodeResources advertisedResourcesOf(Flavor flavor) { return flavor.resources(); } @Override - public NodeResources requestToReal(NodeResources resources, boolean exclusive) { return resources; } + public NodeResources requestToReal(NodeResources resources, boolean exclusive, boolean bestCase) { return resources; } @Override - public NodeResources realToRequest(NodeResources resources, boolean exclusive) { return resources; } + public NodeResources realToRequest(NodeResources resources, boolean exclusive, boolean bestCase) { return resources; } @Override public long reservedDiskSpaceInBase2Gb(NodeType nodeType, boolean sharedHost) { return 0; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostIpConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostIpConfig.java deleted file mode 100644 index 891251fc892..00000000000 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostIpConfig.java +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.provisioning; - -import com.yahoo.vespa.hosted.provision.node.IP; - -import java.util.Map; -import java.util.Objects; - -/** - * IP config of a host and its children. - * - * @author mpolden - */ -public record HostIpConfig(Map<String, IP.Config> ipConfigByHostname) { - - public static final HostIpConfig EMPTY = new HostIpConfig(Map.of()); - - public HostIpConfig(Map<String, IP.Config> ipConfigByHostname) { - this.ipConfigByHostname = Map.copyOf(Objects.requireNonNull(ipConfigByHostname)); - } - - public Map<String, IP.Config> asMap() { - return ipConfigByHostname; - } - - public boolean contains(String hostname) { - return ipConfigByHostname.containsKey(hostname); - } - - public IP.Config require(String hostname) { - IP.Config ipConfig = this.ipConfigByHostname.get(hostname); - if (ipConfig == null) throw new IllegalArgumentException("No IP config exists for node '" + hostname + "'"); - return ipConfig; - } - - public boolean isEmpty() { - return this.equals(EMPTY); - } - -} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java index 3144f42a92c..38fa1abf8e2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java @@ -10,6 +10,7 @@ import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeRepository; import java.util.List; import java.util.Optional; @@ -71,11 +72,12 @@ public interface HostProvisioner { * * @param host the host to provision * @param children list of all the nodes that run on the given host - * @return IP config for the provisioned host and its children + * @return a subset of {@code host} and {@code children} where the values have been modified and should + * be written back to node-repository. * @throws FatalProvisioningException if the provisioning has irrecoverably failed and the input nodes * should be deleted from node-repo. */ - HostIpConfig provision(Node host, Set<Node> children) throws FatalProvisioningException; + List<Node> provision(Node host, Set<Node> children) throws FatalProvisioningException; /** * Deprovisions a given host and resources associated with it and its children (such as DNS entries). diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java index 0f186337b6d..90cdf932f17 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostResourcesCalculator.java @@ -28,13 +28,13 @@ public interface HostResourcesCalculator { * Used with exclusive hosts: * Returns the lowest possible real resources we'll get if requesting the given advertised resources */ - NodeResources requestToReal(NodeResources advertisedResources, boolean exclusiveAllocation); + NodeResources requestToReal(NodeResources advertisedResources, boolean exclusiveAllocation, boolean bestCase); /** * Used with shared hosts: * Returns the advertised resources we need to request to be sure to get at least the given real resources. */ - NodeResources realToRequest(NodeResources realResources, boolean exclusiveAllocation); + NodeResources realToRequest(NodeResources realResources, boolean exclusiveAllocation, boolean bestCase); /** * Returns the disk space to reserve in base2 GB. This space is reserved for use by the host, e.g. for storing diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index a6d292b1a17..ab5cd577ea1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -131,7 +131,7 @@ class NodeAllocation { } } else if (! saturated() && hasCompatibleResources(candidate)) { - if (! nodeResourceLimits.isWithinRealLimits(candidate, cluster)) { + if (! nodeResourceLimits.isWithinRealLimits(candidate, application, cluster)) { ++rejectedDueToInsufficientRealResources; continue; } @@ -163,7 +163,7 @@ class NodeAllocation { boolean alreadyRetired = candidate.allocation().map(a -> a.membership().retired()).orElse(false); return alreadyRetired ? Retirement.alreadyRetired : Retirement.none; } - if ( ! nodeResourceLimits.isWithinRealLimits(candidate, cluster)) return Retirement.outsideRealLimits; + if ( ! nodeResourceLimits.isWithinRealLimits(candidate, application, cluster)) return Retirement.outsideRealLimits; if (violatesParentHostPolicy(candidate)) return Retirement.violatesParentHostPolicy; if ( ! hasCompatibleResources(candidate)) return Retirement.incompatibleResources; if (candidate.wantToRetire()) return Retirement.hardRequest; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java index b194730727f..fa07782057b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java @@ -181,11 +181,11 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat if ( ! lessThanHalfTheHost(this) && lessThanHalfTheHost(other)) return 1; } - // Prefer host with the least skew + // Prefer host with least skew int hostPriority = hostPriority(other); if (hostPriority != 0) return hostPriority; - // Prefer node with the cheapest flavor + // Prefer node with cheapest flavor if (this.flavor().cost() < other.flavor().cost()) return -1; if (other.flavor().cost() < this.flavor().cost()) return 1; @@ -199,7 +199,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat return Integer.compare(this.allocation().get().membership().index(), other.allocation().get().membership().index()); - // Prefer host with the latest OS version + // Prefer host with latest OS version Version thisHostOsVersion = this.parent.flatMap(host -> host.status().osVersion().current()).orElse(Version.emptyVersion); Version otherHostOsVersion = other.parent.flatMap(host -> host.status().osVersion().current()).orElse(Version.emptyVersion); if (thisHostOsVersion.isAfter(otherHostOsVersion)) return -1; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 9f2ff373b28..526e6ed5a4e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -88,8 +88,8 @@ public class NodeRepositoryProvisioner implements Provisioner { if (cluster.group().isPresent()) throw new IllegalArgumentException("Node requests cannot specify a group"); - nodeResourceLimits.ensureWithinAdvertisedLimits("Min", requested.minResources().nodeResources(), cluster); - nodeResourceLimits.ensureWithinAdvertisedLimits("Max", requested.maxResources().nodeResources(), cluster); + nodeResourceLimits.ensureWithinAdvertisedLimits("Min", requested.minResources().nodeResources(), application, cluster); + nodeResourceLimits.ensureWithinAdvertisedLimits("Max", requested.maxResources().nodeResources(), application, cluster); int groups; NodeResources resources; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java index a944bf62534..4d33e1c7bad 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeResourceLimits.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeResources; @@ -25,11 +26,11 @@ public class NodeResourceLimits { } /** Validates the resources applications ask for (which are in "advertised" resource space) */ - public void ensureWithinAdvertisedLimits(String type, NodeResources requested, ClusterSpec cluster) { + public void ensureWithinAdvertisedLimits(String type, NodeResources requested, ApplicationId applicationId, ClusterSpec cluster) { if (requested.isUnspecified()) return; - if (requested.vcpu() < minAdvertisedVcpu(cluster)) - illegal(type, "vcpu", "", cluster, requested.vcpu(), minAdvertisedVcpu(cluster)); + if (requested.vcpu() < minAdvertisedVcpu(applicationId, cluster)) + illegal(type, "vcpu", "", cluster, requested.vcpu(), minAdvertisedVcpu(applicationId, cluster)); if (requested.memoryGb() < minAdvertisedMemoryGb(cluster)) illegal(type, "memoryGb", "Gb", cluster, requested.memoryGb(), minAdvertisedMemoryGb(cluster)); if (requested.diskGb() < minAdvertisedDiskGb(requested, cluster.isExclusive())) @@ -37,33 +38,34 @@ public class NodeResourceLimits { } /** Returns whether the real resources we'll end up with on a given tenant node are within limits */ - public boolean isWithinRealLimits(NodeCandidate candidateNode, ClusterSpec cluster) { + public boolean isWithinRealLimits(NodeCandidate candidateNode, ApplicationId applicationId, ClusterSpec cluster) { if (candidateNode.type() != NodeType.tenant) return true; // Resource limits only apply to tenant nodes return isWithinRealLimits(nodeRepository.resourcesCalculator().realResourcesOf(candidateNode, nodeRepository), - cluster); + applicationId, cluster); } /** Returns whether the real resources we'll end up with on a given tenant node are within limits */ - public boolean isWithinRealLimits(NodeResources realResources, ClusterSpec cluster) { + public boolean isWithinRealLimits(NodeResources realResources, ApplicationId applicationId, ClusterSpec cluster) { if (realResources.isUnspecified()) return true; - if (realResources.vcpu() < minRealVcpu(cluster)) return false; + if (realResources.vcpu() < minRealVcpu(applicationId, cluster)) return false; if (realResources.memoryGb() < minRealMemoryGb(cluster)) return false; if (realResources.diskGb() < minRealDiskGb()) return false; return true; } - public NodeResources enlargeToLegal(NodeResources requested, ClusterSpec cluster, boolean exclusive) { + public NodeResources enlargeToLegal(NodeResources requested, ApplicationId applicationId, ClusterSpec cluster, boolean exclusive) { if (requested.isUnspecified()) return requested; - return requested.withVcpu(Math.max(minAdvertisedVcpu(cluster), requested.vcpu())) + return requested.withVcpu(Math.max(minAdvertisedVcpu(applicationId, cluster), requested.vcpu())) .withMemoryGb(Math.max(minAdvertisedMemoryGb(cluster), requested.memoryGb())) .withDiskGb(Math.max(minAdvertisedDiskGb(requested, exclusive), requested.diskGb())); } - private double minAdvertisedVcpu(ClusterSpec cluster) { + private double minAdvertisedVcpu(ApplicationId applicationId, ClusterSpec cluster) { if (cluster.type() == ClusterSpec.Type.admin) return 0.1; - if (zone().environment().isProduction() && ! zone().system().isCd() && nodeRepository.exclusiveAllocation(cluster)) return 2; + if (zone().environment().isProduction() && ! zone().system().isCd() && + nodeRepository.exclusiveAllocation(cluster) && ! applicationId.instance().isTester()) return 2; if (zone().environment().isProduction() && cluster.type().isContent()) return 1.0; if (zone().environment() == Environment.dev && ! nodeRepository.exclusiveAllocation(cluster)) return 0.1; return 0.5; @@ -86,10 +88,13 @@ public class NodeResourceLimits { return 4; } - private double minRealVcpu(ClusterSpec cluster) { return minAdvertisedVcpu(cluster); } + private double minRealVcpu(ApplicationId applicationId, ClusterSpec cluster) { + return minAdvertisedVcpu(applicationId, cluster); + } private double minRealMemoryGb(ClusterSpec cluster) { - return minAdvertisedMemoryGb(cluster) - 1.7; + if (cluster.type() == ClusterSpec.Type.admin) return 0.95; // TODO: Increase to 1.05 after March 2023 + return 2.3; } private double minRealDiskGb() { return 6; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java index d083d81c196..15a6b6ba523 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java @@ -6,10 +6,10 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.node.OsVersion; import com.yahoo.vespa.hosted.provision.node.Status; @@ -32,38 +32,38 @@ public class ProvisionedHost { private final NodeType hostType; private final Optional<ApplicationId> exclusiveToApplicationId; private final Optional<ClusterSpec.Type> exclusiveToClusterType; - private final List<HostName> nodeHostnames; + private final List<Address> nodeAddresses; private final NodeResources nodeResources; private final Version osVersion; private final CloudAccount cloudAccount; public ProvisionedHost(String id, String hostHostname, Flavor hostFlavor, NodeType hostType, Optional<ApplicationId> exclusiveToApplicationId, Optional<ClusterSpec.Type> exclusiveToClusterType, - List<HostName> nodeHostnames, NodeResources nodeResources, Version osVersion, CloudAccount cloudAccount) { + List<Address> nodeAddresses, NodeResources nodeResources, Version osVersion, CloudAccount cloudAccount) { this.id = Objects.requireNonNull(id, "Host id must be set"); this.hostHostname = Objects.requireNonNull(hostHostname, "Host hostname must be set"); this.hostFlavor = Objects.requireNonNull(hostFlavor, "Host flavor must be set"); this.hostType = Objects.requireNonNull(hostType, "Host type must be set"); this.exclusiveToApplicationId = Objects.requireNonNull(exclusiveToApplicationId, "exclusiveToApplicationId must be set"); this.exclusiveToClusterType = Objects.requireNonNull(exclusiveToClusterType, "exclusiveToClusterType must be set"); - this.nodeHostnames = validateNodeAddresses(nodeHostnames); + this.nodeAddresses = validateNodeAddresses(nodeAddresses); this.nodeResources = Objects.requireNonNull(nodeResources, "Node resources must be set"); this.osVersion = Objects.requireNonNull(osVersion, "OS version must be set"); this.cloudAccount = Objects.requireNonNull(cloudAccount, "Cloud account must be set"); if (!hostType.isHost()) throw new IllegalArgumentException(hostType + " is not a host"); } - private static List<HostName> validateNodeAddresses(List<HostName> nodeHostnames) { - Objects.requireNonNull(nodeHostnames, "Node hostnames must be set"); - if (nodeHostnames.isEmpty()) { - throw new IllegalArgumentException("There must be at least one node hostname"); + private static List<Address> validateNodeAddresses(List<Address> nodeAddresses) { + Objects.requireNonNull(nodeAddresses, "Node addresses must be set"); + if (nodeAddresses.isEmpty()) { + throw new IllegalArgumentException("There must be at least one node address"); } - return nodeHostnames; + return nodeAddresses; } /** Generate {@link Node} instance representing the provisioned physical host */ public Node generateHost() { - Node.Builder builder = Node.create(id, IP.Config.of(Set.of(), Set.of(), nodeHostnames), hostHostname, hostFlavor, + Node.Builder builder = Node.create(id, IP.Config.of(Set.of(), Set.of(), nodeAddresses), hostHostname, hostFlavor, hostType) .status(Status.initial().withOsVersion(OsVersion.EMPTY.withCurrent(Optional.of(osVersion)))) .cloudAccount(cloudAccount); @@ -85,12 +85,12 @@ public class ProvisionedHost { public NodeType hostType() { return hostType; } public Optional<ApplicationId> exclusiveToApplicationId() { return exclusiveToApplicationId; } public Optional<ClusterSpec.Type> exclusiveToClusterType() { return exclusiveToClusterType; } - public List<HostName> nodeHostnames() { return nodeHostnames; } + public List<Address> nodeAddresses() { return nodeAddresses; } public NodeResources nodeResources() { return nodeResources; } public Version osVersion() { return osVersion; } public CloudAccount cloudAccount() { return cloudAccount; } - public String nodeHostname() { return nodeHostnames.get(0).value(); } + public String nodeHostname() { return nodeAddresses.get(0).hostname(); } @Override public boolean equals(Object o) { @@ -103,7 +103,7 @@ public class ProvisionedHost { hostType == that.hostType && exclusiveToApplicationId.equals(that.exclusiveToApplicationId) && exclusiveToClusterType.equals(that.exclusiveToClusterType) && - nodeHostnames.equals(that.nodeHostnames) && + nodeAddresses.equals(that.nodeAddresses) && nodeResources.equals(that.nodeResources) && osVersion.equals(that.osVersion) && cloudAccount.equals(that.cloudAccount); @@ -111,7 +111,7 @@ public class ProvisionedHost { @Override public int hashCode() { - return Objects.hash(id, hostHostname, hostFlavor, hostType, exclusiveToApplicationId, exclusiveToClusterType, nodeHostnames, nodeResources, osVersion, cloudAccount); + return Objects.hash(id, hostHostname, hostFlavor, hostType, exclusiveToApplicationId, exclusiveToClusterType, nodeAddresses, nodeResources, osVersion, cloudAccount); } @Override @@ -123,7 +123,7 @@ public class ProvisionedHost { ", hostType=" + hostType + ", exclusiveToApplicationId=" + exclusiveToApplicationId + ", exclusiveToClusterType=" + exclusiveToClusterType + - ", nodeAddresses=" + nodeHostnames + + ", nodeAddresses=" + nodeAddresses + ", nodeResources=" + nodeResources + ", osVersion=" + osVersion + ", cloudAccount=" + cloudAccount + diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveResponse.java index 3cff3c5e05f..84c82d314c9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveResponse.java @@ -4,6 +4,9 @@ package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.archive.ArchiveUris; + +import java.util.Map; /** * Returns tenant archive URIs. @@ -13,11 +16,22 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; public class ArchiveResponse extends SlimeJsonResponse { public ArchiveResponse(NodeRepository nodeRepository) { + ArchiveUris archiveUris = nodeRepository.archiveUriManager().archiveUris(); Cursor archivesArray = slime.setObject().setArray("archives"); - nodeRepository.archiveUris().getArchiveUris().forEach((tenant, uri) -> { + + archiveUris.tenantArchiveUris().entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .forEach(entry -> { + Cursor archiveObject = archivesArray.addObject(); + archiveObject.setString("tenant", entry.getKey().value()); + archiveObject.setString("uri", entry.getValue()); + }); + archiveUris.accountArchiveUris().entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .forEach(entry -> { Cursor archiveObject = archivesArray.addObject(); - archiveObject.setString("tenant", tenant.value()); - archiveObject.setString("uri", uri); + archiveObject.setString("account", entry.getKey().value()); + archiveObject.setString("uri", entry.getValue()); }); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java index 8c74409d771..582d5963cfd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java @@ -6,7 +6,6 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.TenantName; @@ -20,6 +19,7 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeMutex; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.IP; @@ -100,7 +100,8 @@ public class NodePatcher { "currentRestartGeneration", "reports", "trustStore", - "vespaVersion")); + "vespaVersion", + "wireguardPubkey")); if (!disallowedFields.isEmpty()) { throw new IllegalArgumentException("Patching fields not supported: " + disallowedFields); } @@ -244,15 +245,12 @@ public class NodePatcher { private Node applyIpconfigField(Node node, String name, Inspector value, LockedNodeList nodes) { switch (name) { - case "ipAddresses" -> { + case "ipAddresses": return IP.Config.verify(node.with(node.ipConfig().withPrimary(asStringSet(value))), nodes); - } - case "additionalIpAddresses" -> { + case "additionalIpAddresses": return IP.Config.verify(node.with(node.ipConfig().withPool(node.ipConfig().pool().withIpAddresses(asStringSet(value)))), nodes); - } - case "additionalHostnames" -> { - return IP.Config.verify(node.with(node.ipConfig().withPool(node.ipConfig().pool().withHostnames(asHostnames(value)))), nodes); - } + case "additionalHostnames": + return IP.Config.verify(node.with(node.ipConfig().withPool(node.ipConfig().pool().withAddresses(asAddressList(value)))), nodes); } throw new IllegalArgumentException("Could not apply field '" + name + "' on a node: No such modifiable field"); } @@ -319,19 +317,20 @@ public class NodePatcher { return strings; } - private List<HostName> asHostnames(Inspector field) { + private List<Address> asAddressList(Inspector field) { if ( ! field.type().equals(Type.ARRAY)) throw new IllegalArgumentException("Expected an ARRAY value, got a " + field.type()); - List<HostName> hostnames = new ArrayList<>(field.entries()); + List<Address> addresses = new ArrayList<>(field.entries()); for (int i = 0; i < field.entries(); i++) { Inspector entry = field.entry(i); if ( ! entry.type().equals(Type.STRING)) throw new IllegalArgumentException("Expected a STRING value, got a " + entry.type()); - hostnames.add(HostName.of(entry.asString())); + Address address = new Address(entry.asString()); + addresses.add(address); } - return hostnames; + return addresses; } private Node patchRequiredDiskSpeed(Node node, String value) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index f98c4ba1199..6bf07077c81 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -6,17 +6,18 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.serialization.NetworkPortsSerializer; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; +import com.yahoo.vespa.applicationmodel.HostName; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.History; import com.yahoo.vespa.hosted.provision.node.TrustStoreItem; @@ -51,7 +52,7 @@ class NodesResponse extends SlimeJsonResponse { private final NodeFilter filter; private final boolean recursive; - private final Function<com.yahoo.vespa.applicationmodel.HostName, Optional<HostInfo>> orchestrator; + private final Function<HostName, Optional<HostInfo>> orchestrator; private final NodeRepository nodeRepository; private final StringFlag wantedDockerTagFlag; @@ -150,7 +151,7 @@ class NodesResponse extends SlimeJsonResponse { object.setString("wantedVespaVersion", allocation.membership().cluster().vespaVersion().toFullString()); NodeResourcesSerializer.toSlime(allocation.requestedResources(), object.setObject("requestedResources")); allocation.networkPorts().ifPresent(ports -> NetworkPortsSerializer.toSlime(ports, object.setArray("networkPorts"))); - orchestrator.apply(new com.yahoo.vespa.applicationmodel.HostName(node.hostname())) + orchestrator.apply(new HostName(node.hostname())) .ifPresent(info -> { if (info.status() != HostStatus.NO_REMARKS) { object.setString("orchestratorStatus", info.status().asString()); @@ -179,12 +180,12 @@ class NodesResponse extends SlimeJsonResponse { toSlime(node.history().events(), object.setArray("history")); toSlime(node.history().log(), object.setArray("log")); ipAddressesToSlime(node.ipConfig().primary(), object.setArray("ipAddresses")); - ipAddressesToSlime(node.ipConfig().pool().asSet(), object.setArray("additionalIpAddresses")); - hostnamesToSlime(node.ipConfig().pool().hostnames(), object); + ipAddressesToSlime(node.ipConfig().pool().ipSet(), object.setArray("additionalIpAddresses")); + addressesToSlime(node.ipConfig().pool().getAddressList(), object); node.reports().toSlime(object, "reports"); node.modelName().ifPresent(modelName -> object.setString("modelName", modelName)); node.switchHostname().ifPresent(switchHostname -> object.setString("switchHostname", switchHostname)); - nodeRepository.archiveUris().archiveUriFor(node).ifPresent(uri -> object.setString("archiveUri", uri)); + nodeRepository.archiveUriManager().archiveUriFor(node).ifPresent(uri -> object.setString("archiveUri", uri)); trustedCertsToSlime(node.trustedCertificates(), object); if (!node.cloudAccount().isUnspecified()) { object.setString("cloudAccount", node.cloudAccount().value()); @@ -248,11 +249,11 @@ class NodesResponse extends SlimeJsonResponse { ipAddresses.forEach(array::addString); } - private void hostnamesToSlime(List<HostName> hostnames, Cursor object) { - if (hostnames.isEmpty()) return; + private void addressesToSlime(List<Address> addresses, Cursor object) { + if (addresses.isEmpty()) return; // When/if Address becomes richer: add another field (e.g. "addresses") and expand to array of objects Cursor addressesArray = object.setArray("additionalHostnames"); - hostnames.forEach(hostname -> addressesArray.addString(hostname.value())); + addresses.forEach(address -> addressesArray.addString(address.hostname())); } private void trustedCertsToSlime(List<TrustStoreItem> trustStoreItems, Cursor object) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java index dadef5ce243..0fbe912812e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiHandler.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostFilter; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; @@ -34,6 +33,7 @@ import com.yahoo.vespa.hosted.provision.NodeMutex; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.autoscale.Load; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; @@ -186,9 +186,9 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { return new MessageResponse("Updated " + patcher.application()); } } - else if (path.matches("/nodes/v2/archive/{tenant}")) { + else if (path.matches("/nodes/v2/archive/account/{key}") || path.matches("/nodes/v2/archive/tenant/{key}") || path.matches("/nodes/v2/archive/{key}") /* TODO (freva): Remove March 2023 */) { String uri = requiredField(toSlime(request), "uri", Inspector::asString); - return setTenantArchiveUri(path.get("tenant"), Optional.of(uri)); + return setArchiveUri(path.get("key"), Optional.of(uri), !path.getPath().segments().get(3).equals("account")); } else if (path.matches("/nodes/v2/upgrade/{nodeType}")) { return setTargetVersions(path.get("nodeType"), toSlime(request)); @@ -229,7 +229,8 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { private HttpResponse handleDELETE(HttpRequest request) { Path path = new Path(request.getUri()); if (path.matches("/nodes/v2/node/{hostname}")) return deleteNode(path.get("hostname")); - if (path.matches("/nodes/v2/archive/{tenant}")) return setTenantArchiveUri(path.get("tenant"), Optional.empty()); + if (path.matches("/nodes/v2/archive/account/{key}") || path.matches("/nodes/v2/archive/tenant/{key}") || path.matches("/nodes/v2/archive/{key}") /* TODO (freva): Remove March 2023) */) + return setArchiveUri(path.get("key"), Optional.empty(), !path.getPath().segments().get(3).equals("account")); if (path.matches("/nodes/v2/upgrade/firmware")) return cancelFirmwareCheckResponse(); throw new NotFoundException("Nothing at path '" + request.getUri().getPath() + "'"); @@ -280,12 +281,12 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { Set<String> ipAddressPool = new HashSet<>(); inspector.field("additionalIpAddresses").traverse((ArrayTraverser) (i, item) -> ipAddressPool.add(item.asString())); - List<HostName> hostnames = new ArrayList<>(); + List<Address> addressPool = new ArrayList<>(); inspector.field("additionalHostnames").traverse((ArrayTraverser) (i, item) -> - hostnames.add(HostName.of(item.asString()))); + addressPool.add(new Address(item.asString()))); Node.Builder builder = Node.create(inspector.field("id").asString(), - IP.Config.of(ipAddresses, ipAddressPool, hostnames), + IP.Config.of(ipAddresses, ipAddressPool, addressPool), inspector.field("hostname").asString(), flavorFromSlime(inspector), nodeTypeFromSlime(inspector.field("type"))) @@ -422,9 +423,10 @@ public class NodesV2ApiHandler extends ThreadedHttpRequestHandler { return new MessageResponse("Will request firmware checks on all hosts."); } - private HttpResponse setTenantArchiveUri(String tenant, Optional<String> archiveUri) { - nodeRepository.archiveUris().setArchiveUri(TenantName.from(tenant), archiveUri); - return new MessageResponse(archiveUri.map(a -> "Updated").orElse("Removed") + " archive URI for " + tenant); + private HttpResponse setArchiveUri(String key, Optional<String> archiveUri, boolean isTenant) { + if (isTenant) nodeRepository.archiveUriManager().setArchiveUri(TenantName.from(key), archiveUri); + else nodeRepository.archiveUriManager().setArchiveUri(CloudAccount.from(key), archiveUri); + return new MessageResponse(archiveUri.map(a -> "Updated").orElse("Removed") + " archive URI for " + key); } private static String hostnamesAsString(List<Node> nodes) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/WireguardResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/WireguardResponse.java index 0bac6f09029..11be80de990 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/WireguardResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/WireguardResponse.java @@ -1,13 +1,18 @@ package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.WireguardKey; import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; +import java.util.Set; + /** + * A response containing the wireguard peer config for each configserver that has a public key. + * * @author gjoranv */ public class WireguardResponse extends SlimeJsonResponse { @@ -20,17 +25,18 @@ public class WireguardResponse extends SlimeJsonResponse { .list(Node.State.active) .nodeType(NodeType.config); - configservers.forEach( - configserver -> addConfigserver(cfgArray.addObject(), configserver)); + configservers.stream() + .filter(node -> node.wireguardPubKey().isPresent()) + .forEach(configserver -> addConfigserver(cfgArray.addObject(), + configserver.hostname(), + configserver.wireguardPubKey().get(), + configserver.ipConfig().primary())); } - private void addConfigserver(Cursor cfgEntry, Node configserver) { - cfgEntry.setString("hostname", configserver.hostname()); - - configserver.wireguardPubKey().ifPresent( - key -> cfgEntry.setString("wireguardPubkey", key.value())); - - NodesResponse.ipAddressesToSlime(configserver.ipConfig().primary(), cfgEntry.setArray("ipAddresses")); + private void addConfigserver(Cursor cfgEntry, String hostname, WireguardKey key, Set<String> ipAddresses) { + cfgEntry.setString("hostname", hostname); + cfgEntry.setString("wireguardPubkey", key.value()); + NodesResponse.ipAddressesToSlime(ipAddresses, cfgEntry.setArray("ipAddresses")); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java index f0f85b6523f..4f88a10dff0 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/ContainerConfig.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.testutils; import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.SystemName; /** * For running NodeRepository API with some mocked data. @@ -11,12 +12,15 @@ import com.yahoo.config.provision.CloudAccount; */ public class ContainerConfig { - public static String servicesXmlV2(int port, CloudAccount cloudAccount) { + public static String servicesXmlV2(int port, SystemName systemName, CloudAccount cloudAccount) { return """ <container version='1.0'> <config name="container.handler.threadpool"> <maxthreads>20</maxthreads> </config> + <config name="cloud.config.configserver"> + <system>%s</system> + </config> <config name="config.provisioning.cloud"> <account>%s</account> </config> @@ -47,7 +51,7 @@ public class ContainerConfig { <server id='myServer' port='%s'/> </http> </container> - """.formatted(cloudAccount.value(), port); + """.formatted(systemName.value(), cloudAccount.value(), port); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java index 84856ab310b..5f3cb873e10 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java @@ -7,16 +7,16 @@ import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostEvent; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.FatalProvisioningException; -import com.yahoo.vespa.hosted.provision.provisioning.HostIpConfig; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; +import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import com.yahoo.vespa.hosted.provision.provisioning.ProvisionedHost; import java.time.Instant; @@ -46,7 +46,7 @@ public class MockHostProvisioner implements HostProvisioner { private int deprovisionedHosts = 0; private EnumSet<Behaviour> behaviours = EnumSet.noneOf(Behaviour.class); - private Optional<Flavor> hostFlavor = Optional.empty(); + private Map<ClusterSpec.Type, Flavor> hostFlavors = new HashMap<>(); public MockHostProvisioner(List<Flavor> flavors, MockNameResolver nameResolver, int memoryTaxGb) { this.flavors = List.copyOf(flavors); @@ -67,11 +67,14 @@ public class MockHostProvisioner implements HostProvisioner { ApplicationId applicationId, Version osVersion, HostSharing sharing, Optional<ClusterSpec.Type> clusterType, CloudAccount cloudAccount, Consumer<List<ProvisionedHost>> provisionedHostsConsumer) { - Flavor hostFlavor = this.hostFlavor.orElseGet(() -> flavors.stream() - .filter(f -> sharing == HostSharing.exclusive ? compatible(f, resources) - : f.resources().satisfies(resources)) - .findFirst() - .orElseThrow(() -> new NodeAllocationException("No host flavor matches " + resources, true))); + Flavor hostFlavor = hostFlavors.get(clusterType.orElse(ClusterSpec.Type.content)); + if (hostFlavor == null) + hostFlavor = flavors.stream() + .filter(f -> sharing == HostSharing.exclusive ? compatible(f, resources) + : f.resources().satisfies(resources)) + .findFirst() + .orElseThrow(() -> new NodeAllocationException("No host flavor matches " + resources, true)); + List<ProvisionedHost> hosts = new ArrayList<>(); for (int index : provisionIndices) { String hostHostname = hostType == NodeType.host ? "host" + index : hostType.name() + index; @@ -81,7 +84,7 @@ public class MockHostProvisioner implements HostProvisioner { hostType, sharing == HostSharing.exclusive ? Optional.of(applicationId) : Optional.empty(), Optional.empty(), - createHostnames(hostType, hostFlavor, index), + createAddressesForHost(hostType, hostFlavor, index), resources, osVersion, cloudAccount)); @@ -91,16 +94,16 @@ public class MockHostProvisioner implements HostProvisioner { } @Override - public HostIpConfig provision(Node host, Set<Node> children) throws FatalProvisioningException { + public List<Node> provision(Node host, Set<Node> children) throws FatalProvisioningException { if (behaviours.contains(Behaviour.failProvisioning)) throw new FatalProvisioningException("Failed to provision node(s)"); if (host.state() != Node.State.provisioned) throw new IllegalStateException("Host to provision must be in " + Node.State.provisioned); - Map<String, IP.Config> result = new HashMap<>(); - result.put(host.hostname(), createIpConfig(host)); + List<Node> result = new ArrayList<>(); + result.add(withIpAssigned(host)); for (var child : children) { if (child.state() != Node.State.reserved) throw new IllegalStateException("Child to provisioned must be in " + Node.State.reserved); - result.put(child.hostname(), createIpConfig(child)); + result.add(withIpAssigned(child)); } - return new HostIpConfig(result); + return result; } @Override @@ -152,14 +155,30 @@ public class MockHostProvisioner implements HostProvisioner { return this; } - public MockHostProvisioner overrideHostFlavor(String flavorName) { + public MockHostProvisioner setHostFlavor(String flavorName, ClusterSpec.Type ... types) { Flavor flavor = flavors.stream().filter(f -> f.name().equals(flavorName)) .findFirst() .orElseThrow(() -> new IllegalArgumentException("No such flavor '" + flavorName + "'")); - hostFlavor = Optional.of(flavor); + if (types.length == 0) + types = ClusterSpec.Type.values(); + for (var type : types) + hostFlavors.put(type, flavor); return this; } + /** Sets the host flavor to use to the flavor matching these resources exactly, if any. */ + public MockHostProvisioner setHostFlavorIfAvailable(NodeResources flavorAdvertisedResources, HostResourcesCalculator calculator, ClusterSpec.Type ... types) { + Optional<Flavor> hostFlavor = flavors.stream().filter(f -> calculator.advertisedResourcesOf(f).compatibleWith(flavorAdvertisedResources)) + .findFirst(); + if (types.length == 0) + types = ClusterSpec.Type.values(); + for (var type : types) + hostFlavor.ifPresent(f -> hostFlavors.put(type, f)); + return this; + } + + public Optional<Flavor> getHostFlavor(ClusterSpec.Type type) { return Optional.ofNullable(hostFlavors.get(type)); } + public MockHostProvisioner addEvent(HostEvent event) { hostEvents.add(event); return this; @@ -176,21 +195,21 @@ public class MockHostProvisioner implements HostProvisioner { return flavor.resources().compatibleWith(resourcesToVerify); } - private List<HostName> createHostnames(NodeType hostType, Flavor flavor, int hostIndex) { + private List<Address> createAddressesForHost(NodeType hostType, Flavor flavor, int hostIndex) { long numAddresses = Math.max(2, Math.round(flavor.resources().bandwidthGbps())); return IntStream.range(1, (int) numAddresses) .mapToObj(i -> { String hostname = hostType == NodeType.host ? "host" + hostIndex + "-" + i : hostType.childNodeType().name() + i; - return HostName.of(hostname); + return new Address(hostname); }) .toList(); } - public IP.Config createIpConfig(Node node) { + public Node withIpAssigned(Node node) { if (!node.type().isHost()) { - return node.ipConfig().withPrimary(nameResolver.resolveAll(node.hostname())); + return node.with(node.ipConfig().withPrimary(nameResolver.resolveAll(node.hostname()))); } int hostIndex = Integer.parseInt(node.hostname().replaceAll("^[a-z]+|-\\d+$", "")); Set<String> addresses = Set.of("::" + hostIndex + ":0"); @@ -204,7 +223,7 @@ public class MockHostProvisioner implements HostProvisioner { } } IP.Pool pool = node.ipConfig().pool().withIpAddresses(ipAddressPool); - return node.ipConfig().withPrimary(addresses).withPool(pool); + return node.with(node.ipConfig().withPrimary(addresses).withPool(pool)); } public enum Behaviour { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index d27bd3aea4a..0a614cc9b2b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.component.Version; import com.yahoo.config.provision.ActivationContext; @@ -110,7 +111,7 @@ public class MockNodeRepository extends NodeRepository { .cloudAccount(defaultCloudAccount).build()); // Emulate node in tenant account nodes.add(Node.create("node3", ipConfig(3), "host3.yahoo.com", resources(0.5, 48, 500, 1, fast, local), NodeType.tenant) - .cloudAccount(tenantAccount).build()); + .cloudAccount(tenantAccount).build()); Node node4 = Node.create("node4", ipConfig(4), "host4.yahoo.com", resources(1, 4, 100, 1, fast, local), NodeType.tenant) .parentHostname("dockerhost1.yahoo.com") .status(Status.initial() @@ -156,7 +157,9 @@ public class MockNodeRepository extends NodeRepository { flavors.getFlavorOrThrow("large"), NodeType.host).cloudAccount(defaultCloudAccount).build()); // Emulate host in tenant account nodes.add(Node.create("dockerhost2", ipConfig(101, 1, 3), "dockerhost2.yahoo.com", - flavors.getFlavorOrThrow("large"), NodeType.host).cloudAccount(tenantAccount).build()); + flavors.getFlavorOrThrow("large"), NodeType.host) + .wireguardPubKey(WireguardKey.from("000011112222333344445555666677778888999900c=")) + .cloudAccount(tenantAccount).build()); nodes.add(Node.create("dockerhost3", ipConfig(102, 1, 3), "dockerhost3.yahoo.com", flavors.getFlavorOrThrow("large"), NodeType.host).cloudAccount(defaultCloudAccount).build()); nodes.add(Node.create("dockerhost4", ipConfig(103, 1, 3), "dockerhost4.yahoo.com", @@ -170,7 +173,7 @@ public class MockNodeRepository extends NodeRepository { nodes.add(Node.create("cfg1", ipConfig(201), "cfg1.yahoo.com", flavors.getFlavorOrThrow("default"), NodeType.config) .wireguardPubKey(WireguardKey.from("lololololololololololololololololololololoo=")).build()); nodes.add(Node.create("cfg2", ipConfig(202), "cfg2.yahoo.com", flavors.getFlavorOrThrow("default"), NodeType.config) - .wireguardPubKey(WireguardKey.from("olololololololololololololololololololololo=")).build()); + .build()); // Ready all nodes, except 7 and 55 nodes = nodes().addNodes(nodes, Agent.system); @@ -206,7 +209,8 @@ public class MockNodeRepository extends NodeRepository { IntRange.empty(), false, true, - Optional.empty()), + Optional.empty(), + ClusterInfo.empty()), null), app1Id, provisioner); Application app1 = applications().get(app1Id).get(); Cluster cluster1 = app1.cluster(cluster1Id.id()).get(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java index 4d0b3e75740..b964bf871c1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/NodeRepositoryTester.java @@ -74,7 +74,7 @@ public class NodeRepositoryTester { private Node addNode(String id, String hostname, String parentHostname, Flavor flavor, NodeType type) { Set<String> ips = nodeRepository.nameResolver().resolveAll(hostname); - IP.Config ipConfig = IP.Config.of(ips, type.isHost() ? ips : Set.of()); + IP.Config ipConfig = new IP.Config(ips, type.isHost() ? ips : Set.of()); Node node = Node.create(id, ipConfig, hostname, flavor, type).parentHostname(parentHostname).build(); return nodeRepository.nodes().addNodes(List.of(node), Agent.system).get(0); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java new file mode 100644 index 00000000000..44c1c976355 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUriManagerTest.java @@ -0,0 +1,87 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.archive; + +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.Cloud; +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Allocation; +import com.yahoo.vespa.hosted.provision.node.Generation; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; +import org.junit.Test; + +import java.util.Optional; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertThrows; + +/** + * @author freva + */ +public class ArchiveUriManagerTest { + + @Test + public void archive_uri() { + ApplicationId app1 = ApplicationId.from("vespa", "music", "main"); + ApplicationId app2 = ApplicationId.from("yahoo", "music", "main"); + CloudAccount account1 = CloudAccount.from("123456789012"); + CloudAccount account2 = CloudAccount.from("210987654321"); + CloudAccount accountSystem = CloudAccount.from("555444333222"); + ArchiveUriManager archiveUriManager = new ProvisioningTester.Builder() + .zone(new Zone(Cloud.builder().account(accountSystem).build(), SystemName.Public, Environment.prod, RegionName.defaultName())) + .build().nodeRepository().archiveUriManager(); + + // Initially no uris are set + assertFalse(archiveUriManager.archiveUriFor(createNode(null, null)).isPresent()); + assertFalse(archiveUriManager.archiveUriFor(createNode(app1, account1)).isPresent()); + + archiveUriManager.setArchiveUri(app1.tenant(), Optional.of("scheme://tenant-bucket/dir")); + archiveUriManager.setArchiveUri(account1, Optional.of("scheme://account-bucket/dir")); + assertThrows(IllegalArgumentException.class, () -> archiveUriManager.setArchiveUri(accountSystem, Optional.of("scheme://something"))); + assertThrows(IllegalArgumentException.class, () -> archiveUriManager.setArchiveUri(CloudAccount.empty, Optional.of("scheme://something"))); + + assertFalse(archiveUriManager.archiveUriFor(createNode(null, null)).isPresent()); // Not allocated + assertFalse(archiveUriManager.archiveUriFor(createNode(null, account1)).isPresent()); // URI set for this account, but not allocated + assertFalse(archiveUriManager.archiveUriFor(createNode(null, account2)).isPresent()); // Not allocated + assertFalse(archiveUriManager.archiveUriFor(createNode(app2, null)).isPresent()); // No URI set for this tenant or account + assertEquals("scheme://tenant-bucket/dir/vespa/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, null)).get()); + assertEquals("scheme://account-bucket/dir/vespa/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, account1)).get()); // Account has precedence + assertFalse(archiveUriManager.archiveUriFor(createNode(app1, account2)).isPresent()); // URI set for this tenant, but is ignored because enclave account + assertEquals("scheme://tenant-bucket/dir/vespa/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, accountSystem)).get()); // URI for tenant because non-enclave acocunt + } + + @Test + public void handles_uri_with_tenant_name() { + ApplicationId app1 = ApplicationId.from("vespa", "music", "main"); + ArchiveUriManager archiveUriManager = new ProvisioningTester.Builder().build().nodeRepository().archiveUriManager(); + archiveUriManager.setArchiveUri(app1.tenant(), Optional.of("scheme://tenant-bucket/vespa")); + assertEquals("scheme://tenant-bucket/vespa/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, null)).get()); + + // Archive URI ends with the tenant name + archiveUriManager.setArchiveUri(app1.tenant(), Optional.of("scheme://tenant-vespa/")); + assertEquals("scheme://tenant-vespa/vespa/music/main/default/h432a/", archiveUriManager.archiveUriFor(createNode(app1, null)).get()); + } + + private Node createNode(ApplicationId appId, CloudAccount account) { + Node.Builder nodeBuilder = Node.create("id", "h432a.prod.us-south-1.vespa.domain.tld", new Flavor(NodeResources.unspecified()), Node.State.parked, NodeType.tenant); + Optional.ofNullable(appId) + .map(app -> new Allocation(app, + ClusterMembership.from("container/default/0/0", Version.fromString("1.2.3"), Optional.empty()), + NodeResources.unspecified(), + Generation.initial(), + false)) + .ifPresent(nodeBuilder::allocation); + Optional.ofNullable(account).ifPresent(nodeBuilder::cloudAccount); + return nodeBuilder.build(); + } +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUrisTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUrisTest.java new file mode 100644 index 00000000000..9770d60b27a --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/archive/ArchiveUrisTest.java @@ -0,0 +1,49 @@ +package com.yahoo.vespa.hosted.provision.archive; + + +import com.yahoo.config.provision.TenantName; +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.Optional; + +import static com.yahoo.vespa.hosted.provision.archive.ArchiveUris.normalizeUri; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class ArchiveUrisTest { + + @Test + void normalize_test() { + assertEquals("ftp://domain/legal-dir123/", normalizeUri("ftp://domain/legal-dir123")); + assertEquals("ftp://domain/legal-dir123/", normalizeUri("ftp://domain/legal-dir123/")); + assertEquals("s3://my-bucket-prod.region/my-tenant-123/", normalizeUri("s3://my-bucket-prod.region/my-tenant-123/")); + assertEquals("s3://my-bucket-prod.region/my-tenant_123/", normalizeUri("s3://my-bucket-prod.region/my-tenant_123/")); + assertThrows(IllegalArgumentException.class, () -> normalizeUri("domain/dir/")); + assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp:/domain/dir/")); + assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp:/domain//dir/")); + assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/illegal:dir/")); + assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/-illegal-dir/")); + assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/_illegal-dir/")); + assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/illegal-dir-/")); + assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/illegal-dir_/")); + } + + @Test + void updates_in_place_if_possible() { + TenantName t1 = TenantName.from("t1"); + + ArchiveUris uris0 = new ArchiveUris(Map.of(), Map.of()); + ArchiveUris uris1 = uris0.with(t1, Optional.empty()); + assertSame(uris0, uris1); + + ArchiveUris uris2 = uris0.with(t1, Optional.of("scheme://test123")); + assertEquals(Map.of(t1, "scheme://test123/"), uris2.tenantArchiveUris()); + + assertSame(uris2, uris2.with(t1, Optional.of("scheme://test123"))); + assertSame(uris2, uris2.with(t1, Optional.of("scheme://test123/"))); + assertEquals(uris0, uris2.with(t1, Optional.empty())); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java index 158c5116e19..3ee72c18318 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingIntegrationTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.NodeResources; import com.yahoo.test.ManualClock; +import com.yahoo.vespa.hosted.provision.provisioning.DynamicProvisioningTester; import com.yahoo.vespa.hosted.provision.testutils.OrchestratorMock; import org.junit.Test; @@ -22,12 +23,12 @@ public class AutoscalingIntegrationTest { @Test public void testComponentIntegration() { - var fixture = AutoscalingTester.fixture() - .hostCount(20) - .hostFlavors(new NodeResources(3, 20, 200, 1)) - .initialResources(Optional.of(new ClusterResources(2, 1, + var fixture = DynamicProvisioningTester.fixture() + .hostCount(20) + .hostFlavors(new NodeResources(3, 20, 200, 1)) + .initialResources(Optional.of(new ClusterResources(2, 1, new NodeResources(1, 10, 100, 1)))) - .build(); + .build(); MetricsV2MetricsFetcher fetcher = new MetricsV2MetricsFetcher(fixture.tester().nodeRepository(), new OrchestratorMock(), new MockHttpClient(fixture.tester().clock())); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index a5cfc04afd4..d69d9267cfd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; @@ -12,6 +13,7 @@ import com.yahoo.config.provision.NodeResources.StorageType; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.hosted.provision.provisioning.CapacityPolicies; +import com.yahoo.vespa.hosted.provision.provisioning.DynamicProvisioningTester; import org.junit.Test; import java.time.Duration; @@ -20,6 +22,7 @@ import java.util.Optional; import static com.yahoo.config.provision.NodeResources.DiskSpeed.fast; import static com.yahoo.config.provision.NodeResources.DiskSpeed.slow; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; /** @@ -28,12 +31,37 @@ import static org.junit.Assert.assertTrue; public class AutoscalingTest { @Test + public void test_autoscaling_nodes_only() { + var resources = new NodeResources(16, 32, 200, 0.1); + var min = new ClusterResources( 8, 1, resources); + var now = new ClusterResources(12, 1, resources.with(StorageType.remote)); + var max = new ClusterResources(12, 1, resources); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .clusterType(ClusterSpec.Type.content) + .initialResources(Optional.of(now)) + .capacity(Capacity.from(min, max)) + .build(); + fixture.tester.clock().advance(Duration.ofDays(2)); + fixture.loader().applyLoad(new Load(0.17f, 0.17, 0.12), 1, true, true, 100); + var result = fixture.autoscale(); + assertTrue(result.resources().isEmpty()); + assertNotEquals(Autoscaling.Status.insufficient, result.status()); + + fixture.tester.clock().advance(Duration.ofDays(2)); + fixture.loader().applyLoad(new Load(0.08f, 0.17, 0.12), 1, true, true, 100); + fixture.tester().assertResources("Scaling down", + 8, 1, 16, 32, 200, + fixture.autoscale()); + } + + @Test public void test_autoscaling_single_content_group() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); fixture.loader().applyCpuLoad(0.7f, 10); var scaledResources = fixture.tester().assertResources("Scaling up since resource usage is too high", - 7, 1, 4.6, 11.1, 55.1, + 9, 1, 3.6, 7.7, 31.7, fixture.autoscale()); fixture.deploy(Capacity.from(scaledResources)); @@ -50,31 +78,31 @@ public class AutoscalingTest { fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyCpuLoad(0.1f, 10); fixture.tester().assertResources("Scaling cpu down since usage has gone down significantly", - 6, 1, 1.3, 11.8, 78.6, + 8, 1, 1.0, 7.3, 22.1, fixture.autoscale()); } /** Using too many resources for a short period is proof we should scale up regardless of the time that takes. */ @Test public void test_no_autoscaling_with_no_measurements() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); assertTrue(fixture.autoscale().resources().isEmpty()); } @Test public void test_no_autoscaling_with_no_measurements_exclusive() { - var fixture = AutoscalingTester.fixture().awsProdSetup(false).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(false).build(); assertTrue(fixture.autoscale().resources().isEmpty()); } /** Using too many resources for a short period is proof we should scale up regardless of the time that takes. */ @Test public void test_autoscaling_up_is_fast() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); fixture.loader().applyLoad(new Load(0.1, 0.1, 0.1), 3); fixture.loader().applyLoad(new Load(1.0, 1.0, 1.0), 1); fixture.tester().assertResources("Scaling up since resource usage is too high", - 8, 1, 5.3, 17.7, 93.6, + 9, 1, 4.7, 14.8, 66.0, fixture.autoscale()); } @@ -83,12 +111,12 @@ public class AutoscalingTest { var min = new ClusterResources(2, 1, new NodeResources(4, 8, 50, 0.1)); var now = new ClusterResources(8, 1, new NodeResources(4, 8, 50, 0.1)); var max = new ClusterResources(8, 1, new NodeResources(4, 8, 50, 0.1)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(false) - .clusterType(ClusterSpec.Type.container) - .initialResources(Optional.of(now)) - .capacity(Capacity.from(min, max)) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(false) + .clusterType(ClusterSpec.Type.container) + .initialResources(Optional.of(now)) + .capacity(Capacity.from(min, max)) + .build(); fixture.tester().setScalingDuration(fixture.applicationId(), fixture.clusterSpec.id(), Duration.ofMinutes(5)); fixture.loader().applyLoad(new Load(0.01, 0.38, 0), 5); @@ -101,12 +129,12 @@ public class AutoscalingTest { public void initial_deployment_with_host_sharing_flag() { var min = new ClusterResources(7, 1, new NodeResources(2.0, 10.0, 384.0, 0.1)); var max = new ClusterResources(7, 1, new NodeResources(2.4, 32.0, 768.0, 0.1)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(false) - .capacity(Capacity.from(min, max)) - .initialResources(Optional.empty()) - .hostSharingFlag() - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(false) + .capacity(Capacity.from(min, max)) + .initialResources(Optional.empty()) + .hostSharingFlag() + .build(); fixture.tester().assertResources("Initial resources at min, since flag turns on host sharing", 7, 1, 2.0, 10.0, 384.0, fixture.currentResources().advertisedResources()); @@ -116,13 +144,13 @@ public class AutoscalingTest { public void initial_deployment_with_host_sharing_flag_and_too_small_min() { var min = new ClusterResources(1, 1, new NodeResources(0.5, 4.0, 10, 0.1)); var max = new ClusterResources(1, 1, new NodeResources(2.0, 8.0, 50, 0.1)); - var fixture = AutoscalingTester.fixture() - .awsSetup(false, Environment.test) - .clusterType(ClusterSpec.Type.container) - .capacity(Capacity.from(min, max)) - .initialResources(Optional.empty()) - .hostSharingFlag() - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsSetup(false, Environment.test) + .clusterType(ClusterSpec.Type.container) + .capacity(Capacity.from(min, max)) + .initialResources(Optional.empty()) + .hostSharingFlag() + .build(); fixture.tester().assertResources("Initial resources at min, since flag turns on host sharing", 1, 1, 0.5, 4.0, 10.0, fixture.currentResources().advertisedResources()); @@ -131,27 +159,27 @@ public class AutoscalingTest { /** When scaling up, disregard underutilized dimensions (memory here) */ @Test public void test_only_autoscaling_up_quickly() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); fixture.loader().applyLoad(new Load(1.0, 0.1, 1.0), 10); fixture.tester().assertResources("Scaling up (only) since resource usage is too high", - 7, 1, 8.2, 10.7, 99.5, + 8, 1, 7.1, 8.8, 75.4, fixture.autoscale()); } /** When ok to scale down, scale in both directions simultaneously (compare to test_only_autoscaling_up_quickly) */ @Test public void test_scale_in_both_directions_when_ok_to_scale_down() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); fixture.tester.clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(1.0, 0.1, 1.0), 10); fixture.tester().assertResources("Scaling cpu and disk up and memory down", - 7, 1, 8.2, 4.0, 99.5, + 7, 1, 8.2, 4.0, 88.0, fixture.autoscale()); } @Test public void test_scale_in_both_directions_when_ok_to_scale_down_exclusive() { - var fixture = AutoscalingTester.fixture().awsProdSetup(false).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(false).build(); fixture.tester.clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(1.0, 0.1, 1.0), 10); fixture.tester().assertResources("Scaling cpu and disk up, memory follows", @@ -161,18 +189,18 @@ public class AutoscalingTest { @Test public void test_autoscaling_uses_peak() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); fixture.loader().applyCpuLoad(0.01, 100); fixture.loader().applyCpuLoad(0.70, 1); fixture.loader().applyCpuLoad(0.01, 100); fixture.tester().assertResources("Scaling up since peak resource usage is too high", - 8, 1, 4.3, 9.5, 47.2, + 9, 1, 3.8, 7.7, 31.7, fixture.autoscale()); } @Test public void test_autoscaling_uses_peak_exclusive() { - var fixture = AutoscalingTester.fixture().awsProdSetup(false).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(false).build(); fixture.loader().applyCpuLoad(0.01, 100); fixture.loader().applyCpuLoad(0.70, 1); fixture.loader().applyCpuLoad(0.01, 100); @@ -183,7 +211,7 @@ public class AutoscalingTest { @Test public void test_autoscaling_uses_peak_preprovisioned() { - var fixture = AutoscalingTester.fixture().hostCount(15).build(); + var fixture = DynamicProvisioningTester.fixture().hostCount(15).build(); fixture.loader().applyCpuLoad(0.01, 100); fixture.loader().applyCpuLoad(0.70, 1); fixture.loader().applyCpuLoad(0.01, 100); @@ -197,10 +225,10 @@ public class AutoscalingTest { var min = new ClusterResources(1, 1, new NodeResources(0.5, 4, 10, 0.3)); var now = new ClusterResources(4, 1, new NodeResources(8, 16, 10, 0.3)); var max = new ClusterResources(4, 1, new NodeResources(16, 32, 50, 0.3)); - var fixture = AutoscalingTester.fixture(min, now, max) - .clusterType(ClusterSpec.Type.container) - .awsProdSetup(false) - .build(); + var fixture = DynamicProvisioningTester.fixture(min, now, max) + .clusterType(ClusterSpec.Type.container) + .awsProdSetup(false) + .build(); var duration = fixture.loader().addMeasurements(new Load(0.04, 0.39, 0.01), 20); fixture.tester().clock().advance(duration.negated()); fixture.loader().zeroTraffic(20, 1); @@ -212,7 +240,7 @@ public class AutoscalingTest { /** We prefer fewer nodes for container clusters as (we assume) they all use the same disk and memory */ @Test public void test_autoscaling_single_container_group() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).clusterType(ClusterSpec.Type.container).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).clusterType(ClusterSpec.Type.container).build(); fixture.loader().applyCpuLoad(0.25f, 120); ClusterResources scaledResources = fixture.tester().assertResources("Scaling cpu up", @@ -229,12 +257,12 @@ public class AutoscalingTest { @Test public void autoscaling_handles_disk_setting_changes_exclusive_preprovisioned() { var resources = new NodeResources(3, 100, 100, 1, slow); - var fixture = AutoscalingTester.fixture() - .hostCount(20) - .hostFlavors(resources) - .initialResources(Optional.of(new ClusterResources(5, 1, resources))) - .capacity(Capacity.from(new ClusterResources(5, 1, resources))) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .hostCount(20) + .hostFlavors(resources) + .initialResources(Optional.of(new ClusterResources(5, 1, resources))) + .capacity(Capacity.from(new ClusterResources(5, 1, resources))) + .build(); assertTrue(fixture.tester().nodeRepository().nodes().list().owner(fixture.applicationId).stream() .allMatch(n -> n.allocation().get().requestedResources().diskSpeed() == slow)); @@ -263,11 +291,11 @@ public class AutoscalingTest { NodeResources resources = new NodeResources(1, 100, 100, 1); var capacity = Capacity.from(new ClusterResources( 2, 1, resources.with(DiskSpeed.any)), new ClusterResources( 10, 1, resources.with(DiskSpeed.any))); - var fixture = AutoscalingTester.fixture() - .capacity(capacity) - .awsProdSetup(true) - .initialResources(Optional.empty()) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .capacity(capacity) + .awsProdSetup(true) + .initialResources(Optional.empty()) + .build(); // Redeployment without target: Uses current resource numbers with *requested* non-numbers (i.e disk-speed any) assertTrue(fixture.tester().nodeRepository().applications().get(fixture.applicationId).get().cluster(fixture.clusterSpec.id()).get().target().resources().isEmpty()); @@ -278,6 +306,7 @@ public class AutoscalingTest { fixture.deactivateRetired(capacity); fixture.tester().clock().advance(Duration.ofDays(1)); fixture.loader().applyCpuLoad(0.8, 120); + System.out.println("Autoscaling ----------"); assertEquals(DiskSpeed.any, fixture.autoscale(capacity).resources().get().nodeResources().diskSpeed()); } @@ -286,10 +315,10 @@ public class AutoscalingTest { var min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); var now = new ClusterResources(5, 1, new NodeResources(1.9, 70, 70, 1)); var max = new ClusterResources( 6, 1, new NodeResources(2.4, 78, 79, 1)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(true) - .initialResources(Optional.of(now)) - .capacity(Capacity.from(min, max)).build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .initialResources(Optional.of(now)) + .capacity(Capacity.from(min, max)).build(); fixture.tester().clock().advance(Duration.ofDays(1)); fixture.loader().applyLoad(new Load(0.25, 0.95, 0.95), 120); @@ -302,7 +331,7 @@ public class AutoscalingTest { public void autoscaling_respects_lower_limit() { var min = new ClusterResources( 4, 1, new NodeResources(1.8, 7.4, 8.5, 1)); var max = new ClusterResources( 6, 1, new NodeResources(2.4, 78, 79, 1)); - var fixture = AutoscalingTester.fixture().awsProdSetup(true).capacity(Capacity.from(min, max)).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).capacity(Capacity.from(min, max)).build(); // deploy fixture.tester().clock().advance(Duration.ofDays(2)); @@ -316,11 +345,11 @@ public class AutoscalingTest { public void autoscaling_with_unspecified_resources_use_defaults_exclusive() { var min = new ClusterResources( 2, 1, NodeResources.unspecified()); var max = new ClusterResources( 6, 1, NodeResources.unspecified()); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(false) - .initialResources(Optional.empty()) - .capacity(Capacity.from(min, max)) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(false) + .initialResources(Optional.empty()) + .capacity(Capacity.from(min, max)) + .build(); NodeResources defaultResources = new CapacityPolicies(fixture.tester().nodeRepository()).defaultNodeResources(fixture.clusterSpec, fixture.applicationId); @@ -341,15 +370,15 @@ public class AutoscalingTest { var min = new ClusterResources( 2, 2, new NodeResources(1, 1, 1, 1)); var now = new ClusterResources(5, 5, new NodeResources(3.0, 10, 10, 1)); var max = new ClusterResources(18, 6, new NodeResources(100, 1000, 1000, 1)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(true) - .initialResources(Optional.of(now)) - .capacity(Capacity.from(min, max)) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .initialResources(Optional.of(now)) + .capacity(Capacity.from(min, max)) + .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyCpuLoad(0.4, 240); fixture.tester().assertResources("Scaling cpu up", - 6, 6, 5.0, 8.1, 10.0, + 6, 6, 5.0, 7.4, 10.0, fixture.autoscale()); } @@ -358,22 +387,22 @@ public class AutoscalingTest { var min = new ClusterResources( 2, 2, new NodeResources(1, 1, 1, 1)); var now = new ClusterResources(5, 5, new NodeResources(3.0, 10, 10, 1)); var max = new ClusterResources(18, 6, new NodeResources(100, 1000, 1000, 1)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(true) - .initialResources(Optional.of(now)) - .capacity(Capacity.from(min, max, IntRange.of(2, 3), false, true, Optional.empty())) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .initialResources(Optional.of(now)) + .capacity(Capacity.from(min, max, IntRange.of(2, 3), false, true, Optional.empty(), ClusterInfo.empty())) + .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyCpuLoad(0.4, 240); fixture.tester().assertResources("Scaling cpu up", - 12, 6, 2.8, 4.3, 10.0, + 8, 4, 4.6, 4.0, 10.0, fixture.autoscale()); } @Test public void test_autoscaling_limits_when_min_equals_max() { ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); - var fixture = AutoscalingTester.fixture().awsProdSetup(true).capacity(Capacity.from(min, min)).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).capacity(Capacity.from(min, min)).build(); fixture.tester().clock().advance(Duration.ofDays(1)); fixture.loader().applyCpuLoad(0.25, 120); @@ -385,14 +414,14 @@ public class AutoscalingTest { var resources = new ClusterResources( 2, 1, new NodeResources(3, 100, 50, 1)); var local = new NodeResources(3, 100, 75, 1, fast, StorageType.local); var remote = new NodeResources(3, 100, 50, 1, fast, StorageType.remote); - var fixture = AutoscalingTester.fixture() - .dynamicProvisioning(true) - .allowHostSharing(false) - .clusterType(ClusterSpec.Type.container) - .hostFlavors(local, remote) - .capacity(Capacity.from(resources)) - .initialResources(Optional.of(new ClusterResources(3, 1, resources.nodeResources()))) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .dynamicProvisioning(true) + .allowHostSharing(false) + .clusterType(ClusterSpec.Type.container) + .hostFlavors(local, remote) + .capacity(Capacity.from(resources)) + .initialResources(Optional.of(new ClusterResources(3, 1, resources.nodeResources()))) + .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(0.01, 0.01, 0.01), 120); @@ -407,18 +436,18 @@ public class AutoscalingTest { @Test public void suggestions_ignores_limits() { ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); - var fixture = AutoscalingTester.fixture().awsProdSetup(true).capacity(Capacity.from(min, min)).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).capacity(Capacity.from(min, min)).build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyCpuLoad(1.0, 120); fixture.tester().assertResources("Suggesting above capacity limit", - 8, 1, 6.2, 7.6, 37.8, + 8, 1, 6.2, 7.0, 29.0, fixture.tester().suggest(fixture.applicationId, fixture.clusterSpec.id(), min, min)); } @Test public void suggestions_ignores_limits_exclusive() { ClusterResources min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); - var fixture = AutoscalingTester.fixture().awsProdSetup(false).capacity(Capacity.from(min, min)).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(false).capacity(Capacity.from(min, min)).build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyCpuLoad(1.0, 120); fixture.tester().assertResources("Suggesting above capacity limit", @@ -428,7 +457,7 @@ public class AutoscalingTest { @Test public void not_using_out_of_service_measurements() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(0.9, 0.6, 0.7), 1, false, true, 120); assertTrue("Not scaling up since nodes were measured while cluster was out of service", @@ -437,7 +466,7 @@ public class AutoscalingTest { @Test public void not_using_unstable_measurements() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(0.9, 0.6, 0.7), 1, true, false, 120); assertTrue("Not scaling up since nodes were measured while cluster was unstable", @@ -449,15 +478,15 @@ public class AutoscalingTest { var min = new ClusterResources( 2, 2, new NodeResources(1, 1, 1, 1)); var now = new ClusterResources(5, 5, new NodeResources(3, 100, 100, 1)); var max = new ClusterResources(20, 20, new NodeResources(10, 1000, 1000, 1)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(true) - .initialResources(Optional.of(now)) - .capacity(Capacity.from(min, max)) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .initialResources(Optional.of(now)) + .capacity(Capacity.from(min, max)) + .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyCpuLoad(0.9, 120); fixture.tester().assertResources("Scaling up to 2 nodes, scaling memory and disk down at the same time", - 10, 5, 7.7, 40.6, 47.8, + 10, 5, 7.7, 39.3, 38.5, fixture.autoscale()); } @@ -466,15 +495,15 @@ public class AutoscalingTest { var min = new ClusterResources( 2, 2, new NodeResources(1, 1, 1, 1)); var now = new ClusterResources(5, 5, new NodeResources(3, 100, 100, 1)); var max = new ClusterResources(20, 20, new NodeResources(10, 1000, 1000, 1)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(true) - .initialResources(Optional.of(now)) - .capacity(Capacity.from(min, max, IntRange.of(1), false, true, Optional.empty())) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .initialResources(Optional.of(now)) + .capacity(Capacity.from(min, max, IntRange.of(1), false, true, Optional.empty(), ClusterInfo.empty())) + .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyCpuLoad(0.9, 120); fixture.tester().assertResources("Scaling up to 2 nodes, scaling memory and disk down at the same time", - 7, 7, 9.4, 80.8, 85.2, + 7, 7, 9.4, 78.6, 77.0, fixture.autoscale()); } @@ -483,17 +512,17 @@ public class AutoscalingTest { var min = new ClusterResources( 3, 1, new NodeResources(1, 1, 1, 1)); var now = new ClusterResources(6, 2, new NodeResources(3, 100, 100, 1)); var max = new ClusterResources(21, 7, new NodeResources(100, 1000, 1000, 1)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(true) - .initialResources(Optional.of(now)) - .capacity(Capacity.from(min, max)) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .initialResources(Optional.of(now)) + .capacity(Capacity.from(min, max)) + .build(); fixture.tester().clock().advance(Duration.ofDays(2)); Duration timePassed = fixture.loader().addCpuMeasurements(0.25, 120); fixture.tester().clock().advance(timePassed.negated()); fixture.loader().addLoadMeasurements(10, t -> t == 0 ? 200.0 : 100.0, t -> 10.0); fixture.tester().assertResources("Scaling up cpu, others down, changing to 1 group is cheaper", - 8, 1, 2.8, 36.2, 56.4, + 9, 1, 2.5, 30.7, 30.1, fixture.autoscale()); } @@ -503,17 +532,17 @@ public class AutoscalingTest { var min = new ClusterResources( 3, 1, new NodeResources(1, 1, 1, 1)); var now = new ClusterResources(6, 2, new NodeResources(3, 100, 100, 1)); var max = new ClusterResources(21, 7, new NodeResources(100, 1000, 1000, 1)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(true) - .initialResources(Optional.of(now)) - .capacity(Capacity.from(min, max)) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .initialResources(Optional.of(now)) + .capacity(Capacity.from(min, max)) + .build(); fixture.tester().clock().advance(Duration.ofDays(2)); Duration timePassed = fixture.loader().addCpuMeasurements(0.25, 120); fixture.tester().clock().advance(timePassed.negated()); fixture.loader().addLoadMeasurements(10, t -> t == 0 ? 20.0 : 10.0, t -> 100.0); fixture.tester().assertResources("Scaling down since resource usage is too high, changing to 1 group is cheaper", - 6, 1, 1.0, 50.7, 79.0, + 6, 1, 1.0, 49.1, 48.1, fixture.autoscale()); } @@ -522,15 +551,15 @@ public class AutoscalingTest { var min = new ClusterResources( 2, 2, new NodeResources(1, 1, 1, 1)); var now = new ClusterResources(6, 2, new NodeResources(10, 100, 100, 1)); var max = new ClusterResources(30, 30, new NodeResources(100, 100, 1000, 1)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(true) - .initialResources(Optional.of(now)) - .capacity(Capacity.from(min, max)) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .initialResources(Optional.of(now)) + .capacity(Capacity.from(min, max)) + .build(); fixture.tester().clock().advance(Duration.ofDays(1)); fixture.loader().applyMemLoad(1.0, 1000); fixture.tester().assertResources("Increase group size to reduce memory load", - 8, 2, 13.9, 97.1, 66.6, + 8, 2, 13.9, 94.5, 60.1, fixture.autoscale()); } @@ -539,27 +568,27 @@ public class AutoscalingTest { var min = new ClusterResources( 2, 1, new NodeResources(1, 1, 1, 1)); var now = new ClusterResources(6, 1, new NodeResources(3, 100, 100, 1)); var max = new ClusterResources(20, 1, new NodeResources(100, 1000, 1000, 1)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(true) - .initialResources(Optional.of(now)) - .capacity(Capacity.from(min, max)) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .initialResources(Optional.of(now)) + .capacity(Capacity.from(min, max)) + .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(0.16, 0.02, 0.5), 120); fixture.tester().assertResources("Scaling down memory", - 6, 1, 3.0, 4.2, 139.9, + 6, 1, 3.0, 4.0, 96.2, fixture.autoscale()); } @Test public void scaling_down_only_after_delay() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); fixture.loader().applyCpuLoad(0.02, 120); assertTrue("Too soon after initial deployment", fixture.autoscale().resources().isEmpty()); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyCpuLoad(0.02, 120); fixture.tester().assertResources("Scaling down since enough time has passed", - 3, 1, 1.0, 25.8, 147.4, + 3, 1, 1.0, 24.6, 101.4, fixture.autoscale()); } @@ -567,35 +596,35 @@ public class AutoscalingTest { public void test_autoscaling_considers_read_share() { var min = new ClusterResources( 1, 1, new NodeResources(3, 100, 100, 1)); var max = new ClusterResources(10, 1, new NodeResources(3, 100, 100, 1)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(true) - .capacity(Capacity.from(min, max)) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .capacity(Capacity.from(min, max)) + .build(); fixture.tester.clock().advance(Duration.ofDays(1)); fixture.loader().applyCpuLoad(0.25, 120); // (no read share stored) fixture.tester().assertResources("Advice to scale up since we set aside for bcp by default", - 6, 1, 3, 100, 100, + 5, 1, 3, 100, 100, fixture.autoscale()); fixture.loader().applyCpuLoad(0.25, 120); fixture.storeReadShare(0.25, 0.5); fixture.tester().assertResources("Half of global share is the same as the default assumption used above", - 6, 1, 3, 100, 100, + 5, 1, 3, 100, 100, fixture.autoscale()); fixture.tester.clock().advance(Duration.ofDays(1)); fixture.loader().applyCpuLoad(0.25, 120); fixture.storeReadShare(0.5, 0.5); fixture.tester().assertResources("Advice to scale down since we don't need room for bcp", - 5, 1, 3, 100, 100, + 4, 1, 3, 100, 100, fixture.autoscale()); } @Test public void test_autoscaling_considers_growth_rate() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); fixture.tester().clock().advance(Duration.ofDays(2)); Duration timeAdded = fixture.loader().addLoadMeasurements(100, t -> t == 0 ? 200.0 : 100.0, t -> 0.0); @@ -603,7 +632,7 @@ public class AutoscalingTest { fixture.loader().addCpuMeasurements(0.25, 200); fixture.tester().assertResources("Scale up since we assume we need 2x cpu for growth when no data scaling time data", - 6, 1, 2.1, 10.6, 66.5, + 10, 1, 1.2, 5.5, 22.5, fixture.autoscale()); fixture.setScalingDuration(Duration.ofMinutes(5)); @@ -612,7 +641,7 @@ public class AutoscalingTest { fixture.tester.clock().advance(timeAdded.negated()); fixture.loader().addCpuMeasurements(0.25, 200); fixture.tester().assertResources("Scale down since observed growth is slower than scaling time", - 5, 1, 2.1, 13.3, 83.2, + 10, 1, 1.0, 5.5, 22.5, fixture.autoscale()); fixture.setScalingDuration(Duration.ofMinutes(60)); @@ -623,13 +652,13 @@ public class AutoscalingTest { fixture.tester.clock().advance(timeAdded.negated()); fixture.loader().addCpuMeasurements(0.25, 200); fixture.tester().assertResources("Scale up since observed growth is faster than scaling time", - 6, 1, 2.1, 10.6, 66.5, + 9, 1, 1.4, 6.1, 25.3, fixture.autoscale()); } @Test public void test_autoscaling_weights_growth_rate_by_confidence() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); double scalingFactor = 1.0/6000; // To make the average query rate low fixture.setScalingDuration(Duration.ofMinutes(60)); @@ -640,13 +669,13 @@ public class AutoscalingTest { fixture.tester.clock().advance(timeAdded.negated()); fixture.loader().addCpuMeasurements(0.7, 200); fixture.tester().assertResources("Scale up slightly since observed growth is faster than scaling time, but we are not confident", - 5, 1, 2.1, 13.3, 83.2, + 10, 1, 1.0, 5.5, 22.5, fixture.autoscale()); } @Test public void test_autoscaling_considers_query_vs_write_rate() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); fixture.loader().addCpuMeasurements(0.4, 220); @@ -658,7 +687,7 @@ public class AutoscalingTest { fixture.tester.clock().advance(timeAdded.negated()); fixture.loader().addCpuMeasurements(0.4, 200); fixture.tester.assertResources("Query and write load is equal -> scale up somewhat", - 7, 1, 2, 8.9, 55.5, + 10, 1, 1.4, 5.5, 22.5, fixture.autoscale()); fixture.tester().clock().advance(Duration.ofDays(2)); @@ -667,7 +696,7 @@ public class AutoscalingTest { fixture.loader().addCpuMeasurements(0.4, 200); // TODO: Ackhually, we scale down here - why? fixture.tester().assertResources("Query load is 4x write load -> scale up more", - 6, 1, 2.1, 10.6, 66.5, + 10, 1, 1.3, 5.5, 22.5, fixture.autoscale()); fixture.tester().clock().advance(Duration.ofDays(2)); @@ -675,7 +704,7 @@ public class AutoscalingTest { fixture.tester.clock().advance(timeAdded.negated()); fixture.loader().addCpuMeasurements(0.4, 200); fixture.tester().assertResources("Write load is 10x query load -> scale down", - 5, 1, 1.4, 13.3, 83.2, + 6, 1, 1.1, 9.8, 40.5, fixture.autoscale()); fixture.tester().clock().advance(Duration.ofDays(2)); @@ -683,7 +712,7 @@ public class AutoscalingTest { fixture.tester.clock().advance(timeAdded.negated()); fixture.loader().addCpuMeasurements(0.4, 200); fixture.tester().assertResources("Query only -> largest possible", - 7, 1, 3.5, 8.9, 55.5, + 9, 1, 2.7, 6.1, 25.3, fixture.autoscale()); fixture.tester().clock().advance(Duration.ofDays(2)); @@ -691,16 +720,16 @@ public class AutoscalingTest { fixture.tester.clock().advance(timeAdded.negated()); fixture.loader().addCpuMeasurements(0.4, 200); fixture.tester().assertResources("Write only -> smallest possible", - 4, 1, 1.1, 17.2, 110.9, + 4, 1, 1.1, 16.4, 67.6, fixture.autoscale()); } @Test public void test_autoscaling_in_dev_preprovisioned() { - var fixture = AutoscalingTester.fixture() - .hostCount(5) - .zone(new Zone(Environment.dev, RegionName.from("us-east"))) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .hostCount(5) + .zone(new Zone(Environment.dev, RegionName.from("us-east"))) + .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(1.0, 1.0, 1.0), 200); assertTrue("Not attempting to scale up because policies dictate we'll only get one node", @@ -713,14 +742,14 @@ public class AutoscalingTest { new NodeResources(1, 4, 10, 1, NodeResources.DiskSpeed.any)); var max = new ClusterResources(20, 20, new NodeResources(100, 1000, 1000, 1, NodeResources.DiskSpeed.any)); - var fixture = AutoscalingTester.fixture() - .awsSetup(true, Environment.dev) - .capacity(Capacity.from(min, max, IntRange.of(3, 5), false, true, Optional.empty())) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsSetup(true, Environment.dev) + .capacity(Capacity.from(min, max, IntRange.of(3, 5), false, true, Optional.empty(), ClusterInfo.empty())) + .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(1.0, 1.0, 1.0), 200); fixture.tester().assertResources("Scale only to a single node and group since this is dev", - 1, 1, 0.1, 24.8, 131.1, + 1, 1, 0.1, 23.6, 105.6, fixture.autoscale()); } @@ -735,13 +764,14 @@ public class AutoscalingTest { IntRange.empty(), true, true, - Optional.empty()); - - var fixture = AutoscalingTester.fixture() - .hostCount(5) - .capacity(requiredCapacity) - .zone(new Zone(Environment.dev, RegionName.from("us-east"))) - .build(); + Optional.empty(), + ClusterInfo.empty()); + + var fixture = DynamicProvisioningTester.fixture() + .hostCount(5) + .capacity(requiredCapacity) + .zone(new Zone(Environment.dev, RegionName.from("us-east"))) + .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(1.0, 1.0, 1.0), 200); fixture.tester().assertResources("We scale even in dev because resources are 'required'", @@ -757,13 +787,14 @@ public class AutoscalingTest { IntRange.empty(), true, true, - Optional.empty()); - - var fixture = AutoscalingTester.fixture() - .hostCount(5) - .capacity(requiredCapacity) - .zone(new Zone(Environment.dev, RegionName.from("us-east"))) - .build(); + Optional.empty(), + ClusterInfo.empty()); + + var fixture = DynamicProvisioningTester.fixture() + .hostCount(5) + .capacity(requiredCapacity) + .zone(new Zone(Environment.dev, RegionName.from("us-east"))) + .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(1.0, 1.0, 1.0), 200); fixture.tester().assertResources("We scale even in dev because resources are required", @@ -775,21 +806,20 @@ public class AutoscalingTest { public void test_changing_exclusivity() { var min = new ClusterResources( 2, 1, new NodeResources( 3, 4, 100, 1)); var max = new ClusterResources(20, 1, new NodeResources(100, 1000, 1000, 1)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(true) - .cluster(clusterSpec(true)) - .capacity(Capacity.from(min, max)) - .initialResources(Optional.empty()) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .cluster(clusterSpec(true)) + .capacity(Capacity.from(min, max)) + .initialResources(Optional.empty()) + .build(); fixture.tester().assertResources("Initial deployment at minimum", 2, 1, 4, 8, 100, fixture.currentResources().advertisedResources()); fixture.tester().deploy(fixture.applicationId(), clusterSpec(false), fixture.capacity()); fixture.loader().applyLoad(new Load(0.1, 0.1, 0.1), 100); - fixture.tester().assertResources("With non-exclusive nodes, a better solution is " + - "50% more nodes with less cpu and memory", - 3, 1, 3, 4, 100.0, + fixture.tester().assertResources("Exclusive nodes makes no difference here", + 2, 1, 4, 8, 100.0, fixture.autoscale()); fixture.tester().deploy(fixture.applicationId(), clusterSpec(true), fixture.capacity()); @@ -805,17 +835,17 @@ public class AutoscalingTest { var min = new ClusterResources(7, 1, new NodeResources( 2, 10, 384, 1)); var now = new ClusterResources(7, 1, new NodeResources( 3.4, 16.2, 450.1, 1)); var max = new ClusterResources(7, 1, new NodeResources( 4, 32, 768, 1)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(true) - .capacity(Capacity.from(min, max)) - .initialResources(Optional.of(now)) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .capacity(Capacity.from(min, max)) + .initialResources(Optional.of(now)) + .build(); var initialNodes = fixture.nodes().asList(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.loader().applyLoad(new Load(0.06, 0.52, 0.27), 100); var autoscaling = fixture.autoscale(); fixture.tester().assertResources("Scaling down", - 7, 1, 2, 15.9, 384.0, + 7, 1, 2, 14.7, 384.0, autoscaling); fixture.deploy(Capacity.from(autoscaling.resources().get())); assertEquals("Initial nodes are kept", initialNodes, fixture.nodes().asList()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java index 0bd94872557..704491ed44f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java @@ -1,15 +1,20 @@ package com.yahoo.vespa.hosted.provision.autoscale; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.applications.BcpGroupInfo; +import com.yahoo.vespa.hosted.provision.provisioning.DynamicProvisioningTester; import org.junit.Test; import java.time.Duration; import java.util.Optional; +import static org.junit.Assert.assertEquals; + /** * Tests autoscaling using information from the BCP group this cluster deployment * is part of to supplement local data when the local deployment lacks sufficient traffic. @@ -21,21 +26,21 @@ public class AutoscalingUsingBcpGroupInfoTest { /** Tests with varying BCP group info parameters. */ @Test public void test_autoscaling_single_content_group() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.store(new BcpGroupInfo(100, 1.1, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 4.0, 7.6, 37.8, + 9, 1, 3.6, 6.1, 25.3, fixture.autoscale()); - // Higher query rate (mem and disk changes are due to being assigned larger hosts where we get less overhead share + // Higher query rate fixture.tester().clock().advance(Duration.ofDays(2)); fixture.store(new BcpGroupInfo(200, 1.1, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 8.0, 7.4, 32.8, + 9, 1, 7.1, 6.1, 25.3, fixture.autoscale()); // Higher headroom @@ -43,7 +48,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.3, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 9, 1, 4.2, 6.6, 33.1, + 9, 1, 4.2, 6.1, 25.3, fixture.autoscale()); // Higher per query cost @@ -51,7 +56,15 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.1, 0.45)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 9, 1, 5.4, 6.6, 33.1, + 9, 1, 5.4, 6.1, 25.3, + fixture.autoscale()); + + // Bcp elsewhere is 0 - use local only + fixture.tester().clock().advance(Duration.ofDays(2)); + fixture.store(new BcpGroupInfo(0, 1.1, 0.45)); + fixture.loader().addCpuMeasurements(0.7f, 10); + fixture.tester().assertResources("Scaling using local info", + 8, 1, 1, 7.0, 29.0, fixture.autoscale()); } @@ -62,25 +75,25 @@ public class AutoscalingUsingBcpGroupInfoTest { new NodeResources(1, 4, 10, 1, NodeResources.DiskSpeed.any)); var max = new ClusterResources(21, 3, new NodeResources(100, 1000, 1000, 1, NodeResources.DiskSpeed.any)); - var fixture = AutoscalingTester.fixture() - .awsProdSetup(true) - .initialResources(Optional.of(new ClusterResources(9, 3, new NodeResources(2, 16, 75, 1)))) - .capacity(Capacity.from(min, max)) - .build(); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .initialResources(Optional.of(new ClusterResources(9, 3, new NodeResources(2, 16, 75, 1)))) + .capacity(Capacity.from(min, max)) + .build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.store(new BcpGroupInfo(100, 1.1, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 3, 3, 10.5, 42.3, 187.0, + 3, 3, 10.5, 41.0, 168.9, fixture.autoscale()); - // Higher query rate (mem and disk changes are due to being assigned larger hosts where we get less overhead share + // Higher query rate fixture.tester().clock().advance(Duration.ofDays(2)); fixture.store(new BcpGroupInfo(200, 1.1, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 3, 3, 20.9, 42.3, 178.0, + 3, 3, 20.9, 41.0, 168.9, fixture.autoscale()); // Higher headroom @@ -88,7 +101,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.3, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 3, 3, 12.4, 42.3, 187.0, + 3, 3, 12.4, 41.0, 168.9, fixture.autoscale()); // Higher per query cost @@ -96,7 +109,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.1, 0.45)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 3, 3, 15.7, 42.3, 187.0, + 3, 3, 15.7, 41.0, 168.9, fixture.autoscale()); } @@ -108,13 +121,13 @@ public class AutoscalingUsingBcpGroupInfoTest { */ @Test public void test_autoscaling_container() { - var fixture = AutoscalingTester.fixture().clusterType(ClusterSpec.Type.container).awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().clusterType(ClusterSpec.Type.container).awsProdSetup(true).build(); fixture.tester().clock().advance(Duration.ofDays(2)); fixture.store(new BcpGroupInfo(100, 1.1, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 4.0, 16.0, 40.8, + 8, 1, 4.0, 16.0, 40.8, fixture.autoscale()); // Higher query rate (mem and disk changes are due to being assigned larger hosts where we get less overhead share @@ -122,7 +135,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(200, 1.1, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 8.0, 16.0, 40.8, + 8, 1, 8.0, 16.0, 40.8, fixture.autoscale()); // Higher headroom @@ -130,7 +143,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.3, 0.3)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 5, 1, 8.0, 16.0, 40.8, + 5, 1, 8.0, 16.0, 40.8, fixture.autoscale()); // Higher per query cost @@ -138,20 +151,42 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.store(new BcpGroupInfo(100, 1.1, 0.45)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 6, 1, 8.0, 16.0, 40.8, + 6, 1, 8.0, 16.0, 40.8, + fixture.autoscale()); + } + + @Test + public void test_autoscaling_with_bcp_deadline() { + var capacity = Capacity.from(new ClusterResources(2, 1, + new NodeResources(1, 4, 10, 1, NodeResources.DiskSpeed.any)), + new ClusterResources(20, 1, + new NodeResources(100, 1000, 1000, 1, NodeResources.DiskSpeed.any)), + IntRange.empty(), false, true, Optional.empty(), + new ClusterInfo.Builder().bcpDeadline(Duration.ofMinutes(60)).build()); + + var fixture = DynamicProvisioningTester.fixture() + .capacity(capacity) + .clusterType(ClusterSpec.Type.container).awsProdSetup(true).build(); + + // We can rescale within deadline - do not take BCP info into account + fixture.tester().clock().advance(Duration.ofDays(2)); + fixture.store(new BcpGroupInfo(100, 1.1, 0.45)); + fixture.loader().addCpuMeasurements(0.7f, 10); + fixture.tester().assertResources("No need for traffic shift headroom", + 2, 1, 2.0, 16.0, 40.8, fixture.autoscale()); } @Test public void test_autoscaling_single_content_group_with_some_local_traffic() { - var fixture = AutoscalingTester.fixture().awsProdSetup(true).build(); + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); // Baseline: No local traffic, group traffic indicates much higher cpu usage than local fixture.tester().clock().advance(Duration.ofDays(2)); fixture.store(new BcpGroupInfo(200, 1.3, 0.45)); fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 14.2, 7.4, 32.8, + 8, 1, 14.2, 7.0, 29.0, fixture.autoscale()); // Some local traffic @@ -161,7 +196,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.tester().clock().advance(duration1.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 10.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 8, 1, 6.9, 7.6, 37.8, + 8, 1, 6.9, 7.0, 29.0, fixture.autoscale()); // Enough local traffic to get half the votes @@ -171,7 +206,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.tester().clock().advance(duration2.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 50.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 7, 1, 3.5, 8.9, 55.5, + 9, 1, 2.7, 6.1, 25.3, fixture.autoscale()); // Mostly local @@ -181,7 +216,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.tester().clock().advance(duration3.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 90.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 7, 1, 2.7, 8.9, 55.5, + 9, 1, 2.1, 6.1, 25.3, fixture.autoscale()); // Local only @@ -191,7 +226,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.tester().clock().advance(duration4.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 100.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 7, 1, 2.6, 8.9, 55.5, + 9, 1, 2.0, 6.1, 25.3, fixture.autoscale()); // No group info, should be the same as the above @@ -201,7 +236,7 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.tester().clock().advance(duration5.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 100.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 7, 1, 2.6, 8.9, 55.5, + 9, 1, 2.0, 6.1, 25.3, fixture.autoscale()); // 40 query rate, no group info (for reference to the below) @@ -211,28 +246,65 @@ public class AutoscalingUsingBcpGroupInfoTest { fixture.tester().clock().advance(duration6.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 40.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 6, 1, 2.2, 10.6, 66.5, + 9, 1, 1.4, 6.1, 25.3, fixture.autoscale()); // Local query rate is too low but global is even lower so disregard it, giving the same as above fixture.tester().clock().advance(Duration.ofDays(2)); - fixture.store(new BcpGroupInfo(200/40.0, 1.3, 0.45*40.0)); + fixture.store(new BcpGroupInfo(200 / 40.0, 1.3, 0.45 * 40.0)); Duration duration7 = fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().clock().advance(duration7.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 40.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 6, 1, 2.2, 10.6, 66.5, + 9, 1, 1.4, 6.1, 25.3, fixture.autoscale()); // Local query rate is too low to be fully confident, and so is global but as it is slightly larger, incorporate it slightly fixture.tester().clock().advance(Duration.ofDays(2)); - fixture.store(new BcpGroupInfo(200/4.0, 1.3, 0.45*4.0)); + fixture.store(new BcpGroupInfo(200 / 4.0, 1.3, 0.45 * 4.0)); Duration duration8 = fixture.loader().addCpuMeasurements(0.7f, 10); fixture.tester().clock().advance(duration8.negated()); fixture.loader().addQueryRateMeasurements(10, __ -> 40.0); fixture.tester().assertResources("Scaling up cpu using bcp group cpu info", - 7, 1, 2.2, 8.9, 55.5, + 9, 1, 1.8, 6.1, 25.3, fixture.autoscale()); } + /** Tests with varying BCP group info parameters. */ + @Test + public void test_autoscaling_metrics() { + var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build(); + + // Empty has metrics at zero + assertEquals(new Autoscaling.Metrics(0, 0, 0), + fixture.autoscale().metrics()); + + + // No external load mesurements -> 0 + fixture.tester().clock().advance(Duration.ofDays(2)); + fixture.loader().addCpuMeasurements(0.7f, 10); + assertEquals(new Autoscaling.Metrics(0, 1.0, 0), + fixture.autoscale().metrics()); + + // External load is measured to zero -> 0 + fixture.tester().clock().advance(Duration.ofDays(2)); + fixture.loader().addCpuMeasurements(0.7f, 10); + fixture.loader().addQueryRateMeasurements(10, i -> 0.0); + assertEquals(new Autoscaling.Metrics(0, 1.0, 0), + fixture.autoscale().metrics()); + + // External load + fixture.tester().clock().advance(Duration.ofDays(2)); + fixture.loader().addCpuMeasurements(0.7f, 10); + fixture.loader().addQueryRateMeasurements(10, i -> 110.0); + assertEquals(new Autoscaling.Metrics(110, 1.1, 0.05), + round(fixture.autoscale().metrics())); + } + + private Autoscaling.Metrics round(Autoscaling.Metrics metrics) { + return new Autoscaling.Metrics(Math.round(metrics.queryRate() * 100) / 100.0, + Math.round(metrics.growthRateHeadroom() * 100) / 100.0, + Math.round(metrics.cpuCostPerQuery() * 100) / 100.0); + } + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java index 5caf50a4e83..48db3fade95 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java @@ -5,6 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.Cloud; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; @@ -24,6 +25,7 @@ import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.applications.BcpGroupInfo; import com.yahoo.vespa.hosted.provision.autoscale.awsnodes.AwsHostResourcesCalculatorImpl; import com.yahoo.vespa.hosted.provision.autoscale.awsnodes.AwsNodeTypes; +import com.yahoo.vespa.hosted.provision.provisioning.DynamicProvisioningTester; import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; import java.time.Duration; import java.util.Arrays; @@ -37,7 +39,7 @@ import java.util.Optional; */ public class Fixture { - final AutoscalingTester tester; + final DynamicProvisioningTester tester; final Zone zone; final ApplicationId applicationId; final ClusterSpec clusterSpec; @@ -49,13 +51,13 @@ public class Fixture { applicationId = builder.application; clusterSpec = builder.cluster; capacity = builder.capacity; - tester = new AutoscalingTester(builder.zone, builder.resourceCalculator, builder.hostFlavors, builder.flagSource, hostCount); + tester = new DynamicProvisioningTester(builder.zone, builder.resourceCalculator, builder.hostFlavors, builder.flagSource, hostCount); var deployCapacity = initialResources.isPresent() ? Capacity.from(initialResources.get()) : capacity; tester.deploy(builder.application, builder.cluster, deployCapacity); this.loader = new Loader(this); } - public AutoscalingTester tester() { return tester; } + public DynamicProvisioningTester tester() { return tester; } public ApplicationId applicationId() { return applicationId; } @@ -141,7 +143,7 @@ public class Fixture { public static class Builder { - ApplicationId application = AutoscalingTester.applicationId("application1"); + ApplicationId application = DynamicProvisioningTester.applicationId("application1"); ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("cluster1")).vespaVersion("7").build(); Zone zone = new Zone(Environment.prod, RegionName.from("us-east")); List<Flavor> hostFlavors = List.of(new Flavor(new NodeResources(100, 100, 100, 1))); @@ -150,7 +152,7 @@ public class Fixture { new NodeResources(1, 4, 10, 1, NodeResources.DiskSpeed.any)), new ClusterResources(20, 1, new NodeResources(100, 1000, 1000, 1, NodeResources.DiskSpeed.any))); - HostResourcesCalculator resourceCalculator = new AutoscalingTester.MockHostResourcesCalculator(zone); + HostResourcesCalculator resourceCalculator = new DynamicProvisioningTester.MockHostResourcesCalculator(zone); final InMemoryFlagSource flagSource = new InMemoryFlagSource(); int hostCount = 0; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsHostResourcesCalculatorImpl.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsHostResourcesCalculatorImpl.java index 2ae1fe18714..b4db8c36d18 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsHostResourcesCalculatorImpl.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsHostResourcesCalculatorImpl.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.provision.autoscale.awsnodes; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.Zone; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.Nodelike; @@ -44,45 +43,35 @@ public class AwsHostResourcesCalculatorImpl implements HostResourcesCalculator { } @Override - public NodeResources requestToReal(NodeResources advertisedResources, boolean exclusive) { - // Only consider exactly matched flavors if any to avoid concluding we have slightly too little resources - // on an exactly matched flavor if we move from exclusive to shared hosts - List<VespaFlavor> consideredFlavors = flavorsCompatibleWithAdvertised(advertisedResources, true); - if (consideredFlavors.isEmpty()) - consideredFlavors = flavorsCompatibleWithAdvertised(advertisedResources, false); - + public NodeResources requestToReal(NodeResources advertisedResources, boolean exclusive, boolean bestCase) { + var consideredFlavors = consideredFlavorsGivenAdvertised(advertisedResources); double memoryOverhead = consideredFlavors.stream() .mapToDouble(flavor -> resourcesCalculator.memoryOverhead(flavor, advertisedResources, false)) - .max().orElse(0); + .reduce(bestCase ? Double::min : Double::max).orElse(0); double diskOverhead = consideredFlavors.stream() .mapToDouble(flavor -> resourcesCalculator.diskOverhead(flavor, advertisedResources, false, exclusive)) - .max().orElse(0); + .reduce(bestCase ? Double::min : Double::max).orElse(0); return advertisedResources.withMemoryGb(advertisedResources.memoryGb() - memoryOverhead) .withDiskGb(advertisedResources.diskGb() - diskOverhead); } @Override - public NodeResources realToRequest(NodeResources realResources, boolean exclusive) { - // Only consider exactly matched flavors if any to avoid concluding we have slightly too little resources - // on an exactly matched flavor if we move from exclusive to shared hosts - List<VespaFlavor> consideredFlavors = flavorsCompatibleWithReal(realResources, true); - if (consideredFlavors.isEmpty()) - consideredFlavors = flavorsCompatibleWithReal(realResources, false); - double worstMemoryOverhead = 0; - double worstDiskOverhead = 0; - for (VespaFlavor flavor : consideredFlavors) { + public NodeResources realToRequest(NodeResources realResources, boolean exclusive, boolean bestCase) { + double chosenMemoryOverhead = bestCase ? Integer.MAX_VALUE : 0; + double chosenDiskOverhead = bestCase ? Integer.MAX_VALUE : 0; + for (VespaFlavor flavor : consideredFlavorsGivenReal(realResources)) { double memoryOverhead = resourcesCalculator.memoryOverhead(flavor, realResources, true); double diskOverhead = resourcesCalculator.diskOverhead(flavor, realResources, true, exclusive); NodeResources advertised = realResources.withMemoryGb(realResources.memoryGb() + memoryOverhead) .withDiskGb(realResources.diskGb() + diskOverhead); if ( ! flavor.advertisedResources().satisfies(advertised)) continue; - if (memoryOverhead > worstMemoryOverhead) - worstMemoryOverhead = memoryOverhead; - if (diskOverhead > worstDiskOverhead) - worstDiskOverhead = diskOverhead; + if (bestCase ? memoryOverhead < chosenMemoryOverhead : memoryOverhead > chosenDiskOverhead) + chosenMemoryOverhead = memoryOverhead; + if (bestCase ? diskOverhead < chosenDiskOverhead : diskOverhead > chosenDiskOverhead) + chosenDiskOverhead = diskOverhead; } - return realResources.withMemoryGb(realResources.memoryGb() + worstMemoryOverhead) - .withDiskGb(realResources.diskGb() + worstDiskOverhead); + return realResources.withMemoryGb(realResources.memoryGb() + chosenMemoryOverhead) + .withDiskGb(realResources.diskGb() + chosenDiskOverhead); } @Override @@ -90,15 +79,49 @@ public class AwsHostResourcesCalculatorImpl implements HostResourcesCalculator { return 1; } + private List<VespaFlavor> consideredFlavorsGivenReal(NodeResources realResources) { + // Only consider exactly matched flavors if any to avoid concluding we have slightly too little resources + // on an exactly matched flavor if we move from exclusive to shared hosts + List<VespaFlavor> consideredFlavors = flavorsCompatibleWithReal(realResources, true); + if ( ! consideredFlavors.isEmpty()) return consideredFlavors; + + // If both are applicable we prefer local storage + if (realResources.storageType() == NodeResources.StorageType.any) + consideredFlavors = flavorsCompatibleWithReal(realResources.with(NodeResources.StorageType.local), false); + if ( ! consideredFlavors.isEmpty()) return consideredFlavors; + + return flavorsCompatibleWithReal(realResources, false); + } + + private List<VespaFlavor> consideredFlavorsGivenAdvertised(NodeResources advertisedResources) { + // Only consider exactly matched flavors if any to avoid concluding we have slightly too little resources + // on an exactly matched flavor if we move from exclusive to shared hosts + List<VespaFlavor> consideredFlavors = flavorsCompatibleWithAdvertised(advertisedResources, true); + if ( ! consideredFlavors.isEmpty()) return consideredFlavors; + + // If both are applicable we prefer local storage + if (advertisedResources.storageType() == NodeResources.StorageType.any) + consideredFlavors = flavorsCompatibleWithAdvertised(advertisedResources.with(NodeResources.StorageType.local), false); + if ( ! consideredFlavors.isEmpty()) return consideredFlavors; + + return flavorsCompatibleWithAdvertised(advertisedResources, false); + } + /** Returns the flavors of hosts which are eligible and matches the given advertised resources */ private List<VespaFlavor> flavorsCompatibleWithAdvertised(NodeResources advertisedResources, boolean exactOnly) { return flavors.values().stream() .filter(flavor -> exactOnly - ? flavor.advertisedResources().equalsWhereSpecified(advertisedResources) + ? equals(flavor.advertisedResources(), advertisedResources) : flavor.advertisedResources().satisfies(advertisedResources)) .toList(); } + private boolean equals(NodeResources hostResources, NodeResources advertisedResources) { + if (hostResources.storageType() == NodeResources.StorageType.remote) + hostResources = hostResources.withDiskGb(advertisedResources.diskGb()); + return hostResources.equalsWhereSpecified(advertisedResources); + } + /** Returns the flavors of hosts which are eligible and matches the given real resources */ private List<VespaFlavor> flavorsCompatibleWithReal(NodeResources realResources, boolean exactOnly) { return flavors.values().stream() diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsResourcesCalculator.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsResourcesCalculator.java index 96fa143dc57..69469bb03c7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsResourcesCalculator.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/awsnodes/AwsResourcesCalculator.java @@ -48,6 +48,7 @@ public class AwsResourcesCalculator { ( hostFlavor.advertisedResources().memoryGb() - ( real ? hostMemoryOverhead : 0)); if (memoryShare > 1) // The real resources of the host cannot fit the requested real resources after overhead memoryShare = 1; + return hostMemoryOverhead * memoryShare; } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java index 1e1ccf37c8c..40ca30d758e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; @@ -62,10 +63,10 @@ public class AutoscalingMaintainerTest { tester.deploy(app1, cluster1, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), - IntRange.empty(), false, true, Optional.empty())); + IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty())); tester.deploy(app2, cluster2, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), new ClusterResources(10, 1, new NodeResources(6.5, 9, 20, 0.1)), - IntRange.empty(), false, true, Optional.empty())); + IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty())); tester.clock().advance(Duration.ofMinutes(10)); tester.maintainer().maintain(); // noop diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java index f8ec271ce5f..606bc55fdd2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/CapacityCheckerTester.java @@ -156,8 +156,8 @@ public class CapacityCheckerTester { .collect(Collectors.toSet()); NodeResources nr = containingNodeResources(childResources, excessCapacity); - Node node = Node.create(hostname, IP.Config.of(Set.of("::"), availableIps), hostname, - new Flavor(nr), NodeType.host).build(); + Node node = Node.create(hostname, new IP.Config(Set.of("::"), availableIps), hostname, + new Flavor(nr), NodeType.host).build(); hosts.computeIfAbsent(tenantHostApp, (ignored) -> new ArrayList<>()) .add(node); } @@ -175,8 +175,8 @@ public class CapacityCheckerTester { Set<String> availableIps = IntStream.range(2000, 2000 + ips) .mapToObj(n -> String.format("%04X::%04X", hostId, n)) .collect(Collectors.toSet()); - Node node = Node.create(hostname, IP.Config.of(Set.of("::" + (1000 + hostId)), availableIps), hostname, - new Flavor(capacity), NodeType.host).build(); + Node node = Node.create(hostname, new IP.Config(Set.of("::" + (1000 + hostId)), availableIps), hostname, + new Flavor(capacity), NodeType.host).build(); hosts.add(node); } return hosts; @@ -290,7 +290,7 @@ public class CapacityCheckerTester { Flavor f = new Flavor(nr); Node.Builder builder = Node.create(nodeModel.id, nodeModel.hostname, f, nodeModel.state, nodeModel.type) - .ipConfig(IP.Config.of(nodeModel.ipAddresses, nodeModel.additionalIpAddresses)); + .ipConfig(new IP.Config(nodeModel.ipAddresses, nodeModel.additionalIpAddresses)); nodeModel.parentHostname.ifPresent(builder::parentHostname); Node node = builder.build(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java index e6d056d126d..880a69b61e5 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainerTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; @@ -26,6 +27,7 @@ import com.yahoo.vespa.flags.custom.ClusterCapacity; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.Generation; @@ -122,7 +124,7 @@ public class HostCapacityMaintainerTest { public void preprovision_with_shared_host() { var tester = new DynamicProvisioningTester().addInitialNodes(); // Makes provisioned hosts 48-128-1000-10 - tester.hostProvisioner.overrideHostFlavor("host4"); + tester.hostProvisioner.setHostFlavor("host4"); var clusterCapacity = new ClusterCapacity(2, 1.0, 30.0, 20.0, 3.0, "fast", "local", "x86_64"); tester.flagSource.withListFlag(PermanentFlags.PREPROVISION_CAPACITY.id(), List.of(clusterCapacity), @@ -235,7 +237,7 @@ public class HostCapacityMaintainerTest { // Pretend shared-host flag has been set to host4's flavor var sharedHostNodeResources = new NodeResources(48, 128, 1000, 10, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote); - tester.hostProvisioner.overrideHostFlavor("host4"); + tester.hostProvisioner.setHostFlavor("host4"); // Next maintenance run does nothing tester.assertNodesUnchanged(); @@ -365,7 +367,7 @@ public class HostCapacityMaintainerTest { Cloud cloud = Cloud.builder().dynamicProvisioning(true).build(); DynamicProvisioningTester dynamicProvisioningTester = new DynamicProvisioningTester(cloud, new MockNameResolver().mockAnyLookup()); ProvisioningTester tester = dynamicProvisioningTester.provisioningTester; - dynamicProvisioningTester.hostProvisioner.overrideHostFlavor("default"); + dynamicProvisioningTester.hostProvisioner.setHostFlavor("default"); // Initial config server hosts are provisioned manually List<Node> provisionedHosts = tester.makeReadyNodes(3, "default", hostType, 1).stream() @@ -466,7 +468,7 @@ public class HostCapacityMaintainerTest { ClusterSpec spec = ProvisioningTester.contentClusterSpec(); ClusterResources resources = new ClusterResources(2, 1, new NodeResources(16, 24, 100, 1)); CloudAccount cloudAccount0 = CloudAccount.from("000000000000"); - Capacity capacity0 = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount0)); + Capacity capacity0 = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount0), ClusterInfo.empty()); List<HostSpec> prepared = provisioningTester.prepare(applicationId, spec, capacity0); // Hosts are provisioned in requested account @@ -476,7 +478,7 @@ public class HostCapacityMaintainerTest { // Redeployment in different account provisions a new set of hosts CloudAccount cloudAccount1 = CloudAccount.from("100000000000"); - Capacity capacity1 = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount1)); + Capacity capacity1 = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount1), ClusterInfo.empty()); prepared = provisioningTester.prepare(applicationId, spec, capacity1); provisionHostsIn(cloudAccount1, 2, tester); assertEquals(2, provisioningTester.activate(applicationId, prepared).size()); @@ -649,9 +651,9 @@ public class HostCapacityMaintainerTest { flavor.resources(), Generation.initial(), false)); - List<com.yahoo.config.provision.HostName> hostnames = Stream.of(additionalHostnames).map(com.yahoo.config.provision.HostName::of).toList(); + List<Address> addresses = Stream.of(additionalHostnames).map(Address::new).toList(); Node.Builder builder = Node.create("fake-id-" + hostname, hostname, flavor, state, nodeType) - .ipConfig(IP.Config.of(state == Node.State.active ? Set.of("::1") : Set.of(), Set.of(), hostnames)); + .ipConfig(new IP.Config(state == Node.State.active ? Set.of("::1") : Set.of(), Set.of(), addresses)); parentHostname.ifPresent(builder::parentHostname); allocation.ifPresent(builder::allocation); if (hostname.equals("host2-1")) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisionerTest.java index 8280c0e33fc..5e507d447ab 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostResumeProvisionerTest.java @@ -78,12 +78,12 @@ public class HostResumeProvisionerTest { hostResumeProvisioner.maintain(); assertTrue("No IP addresses written as DNS updates are failing", - provisioning.get().stream().allMatch(host -> host.ipConfig().pool().asSet().isEmpty())); + provisioning.get().stream().allMatch(host -> host.ipConfig().pool().ipSet().isEmpty())); hostProvisioner.without(MockHostProvisioner.Behaviour.failDnsUpdate); hostResumeProvisioner.maintain(); assertTrue("IP addresses written as DNS updates are succeeding", - provisioning.get().stream().noneMatch(host -> host.ipConfig().pool().asSet().isEmpty())); + provisioning.get().stream().noneMatch(host -> host.ipConfig().pool().ipSet().isEmpty())); } @Test diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index 2187611b702..d7ffda542ff 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -87,7 +87,7 @@ public class MetricsReporterTest { Map<String, Number> expectedMetrics = new TreeMap<>(); expectedMetrics.put("zone.working", 1); - expectedMetrics.put("hostedVespa.provisionedHosts", 1); + expectedMetrics.put("hostedVespa.provisionedHosts", 0); expectedMetrics.put("hostedVespa.parkedHosts", 0); expectedMetrics.put("hostedVespa.readyHosts", 0); expectedMetrics.put("hostedVespa.reservedHosts", 0); @@ -97,6 +97,16 @@ public class MetricsReporterTest { expectedMetrics.put("hostedVespa.failedHosts", 0); expectedMetrics.put("hostedVespa.deprovisionedHosts", 0); expectedMetrics.put("hostedVespa.breakfixedHosts", 0); + expectedMetrics.put("hostedVespa.provisionedNodes", 1); + expectedMetrics.put("hostedVespa.parkedNodes", 0); + expectedMetrics.put("hostedVespa.readyNodes", 0); + expectedMetrics.put("hostedVespa.reservedNodes", 0); + expectedMetrics.put("hostedVespa.activeNodes", 0); + expectedMetrics.put("hostedVespa.inactiveNodes", 0); + expectedMetrics.put("hostedVespa.dirtyNodes", 0); + expectedMetrics.put("hostedVespa.failedNodes", 0); + expectedMetrics.put("hostedVespa.deprovisionedNodes", 0); + expectedMetrics.put("hostedVespa.breakfixedNodes", 0); expectedMetrics.put("hostedVespa.pendingRedeployments", 42); expectedMetrics.put("hostedVespa.docker.totalCapacityDisk", 0.0); expectedMetrics.put("hostedVespa.docker.totalCapacityMem", 0.0); @@ -176,7 +186,7 @@ public class MetricsReporterTest { // Allow 4 containers Set<String> ipAddressPool = Set.of("::2", "::3", "::4", "::5"); - Node dockerHost = Node.create("node-id-1", IP.Config.of(Set.of("::1"), ipAddressPool), "dockerHost", + Node dockerHost = Node.create("node-id-1", new IP.Config(Set.of("::1"), ipAddressPool), "dockerHost", nodeFlavors.getFlavorOrThrow("host"), NodeType.host).build(); nodeRepository.nodes().addNodes(List.of(dockerHost), Agent.system); nodeRepository.nodes().deallocateRecursively("dockerHost", Agent.system, getClass().getSimpleName()); @@ -207,8 +217,8 @@ public class MetricsReporterTest { MetricsReporter metricsReporter = metricsReporter(metric, tester); metricsReporter.maintain(); - assertEquals(0, metric.values.get("hostedVespa.readyHosts")); // Only tenants counts - assertEquals(2, metric.values.get("hostedVespa.reservedHosts")); + assertEquals(0, metric.values.get("hostedVespa.readyNodes")); // Only tenants counts + assertEquals(2, metric.values.get("hostedVespa.reservedNodes")); assertEquals(120.0, metric.values.get("hostedVespa.docker.totalCapacityDisk")); assertEquals(100.0, metric.values.get("hostedVespa.docker.totalCapacityMem")); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java index a07a4f2c72a..1b0826a8323 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailTester.java @@ -244,8 +244,8 @@ public class NodeFailTester { for (int i = startIndex; i < startIndex + count; i++) { String hostname = "host" + i; Set<String> ipPool = nodeType.isHost() ? Set.of("127.0." + i + "." + (++lastOctetOfPoolAddress)) : Set.of(); - IP.Config ipConfig = IP.Config.of(nodeRepository.nameResolver().resolveAll(hostname), - ipPool); + IP.Config ipConfig = new IP.Config(nodeRepository.nameResolver().resolveAll(hostname), + ipPool); Node.Builder builder = Node.create("node" + i, ipConfig, hostname, flavor, nodeType); parentHostname.ifPresent(builder::parentHostname); nodes.add(builder.build()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java index 47803594148..491485b78fc 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/NodeFailerTest.java @@ -177,17 +177,17 @@ public class NodeFailerTest { @Test public void zone_is_not_working_if_too_many_nodes_down() { - NodeFailTester tester = NodeFailTester.withTwoApplications(); - - tester.serviceMonitor.setHostDown(tester.nodeRepository.nodes().list(Node.State.active).owner(NodeFailTester.app1).asList().get(0).hostname()); - tester.runMaintainers(); - assertTrue(tester.nodeRepository.nodes().isWorking()); + NodeFailTester tester = NodeFailTester.withTwoApplications(10, 5, 5); - tester.serviceMonitor.setHostDown(tester.nodeRepository.nodes().list(Node.State.active).owner(NodeFailTester.app1).asList().get(1).hostname()); - tester.runMaintainers(); - assertTrue(tester.nodeRepository.nodes().isWorking()); + int i = 0; + while (i < 4) { + tester.serviceMonitor.setHostDown(tester.nodeRepository.nodes().list(Node.State.active).owner(NodeFailTester.app1).asList().get(i).hostname()); + tester.runMaintainers(); + assertTrue(tester.nodeRepository.nodes().isWorking()); + i++; + } - tester.serviceMonitor.setHostDown(tester.nodeRepository.nodes().list(Node.State.active).owner(NodeFailTester.app1).asList().get(2).hostname()); + tester.serviceMonitor.setHostDown(tester.nodeRepository.nodes().list(Node.State.active).owner(NodeFailTester.app1).asList().get(i).hostname()); tester.runMaintainers(); assertFalse(tester.nodeRepository.nodes().isWorking()); @@ -199,6 +199,11 @@ public class NodeFailerTest { @Test public void node_failing() { NodeFailTester tester = NodeFailTester.withTwoApplications(6); + String downHost1 = tester.nodeRepository.nodes().list(Node.State.active).owner(NodeFailTester.app1).asList().get(1).hostname(); + String downHost2 = tester.nodeRepository.nodes().list(Node.State.active).owner(NodeFailTester.app2).asList().get(3).hostname(); + // No liveness evidence yet: + assertFalse(tester.nodeRepository.nodes().node(downHost1).get().isDown()); + assertFalse(tester.nodeRepository.nodes().node(downHost1).get().isUp()); // For a day all nodes work so nothing happens for (int minutes = 0; minutes < 24 * 60; minutes +=5 ) { @@ -208,10 +213,10 @@ public class NodeFailerTest { assertEquals(0, tester.deployer.redeployments); assertEquals(8, tester.nodeRepository.nodes().list(Node.State.active).nodeType(NodeType.tenant).size()); assertEquals(0, tester.nodeRepository.nodes().list(Node.State.failed).nodeType(NodeType.tenant).size()); + assertFalse(tester.nodeRepository.nodes().node(downHost1).get().isDown()); + assertTrue(tester.nodeRepository.nodes().node(downHost1).get().isUp()); } - String downHost1 = tester.nodeRepository.nodes().list(Node.State.active).owner(NodeFailTester.app1).asList().get(1).hostname(); - String downHost2 = tester.nodeRepository.nodes().list(Node.State.active).owner(NodeFailTester.app2).asList().get(3).hostname(); tester.serviceMonitor.setHostDown(downHost1); tester.serviceMonitor.setHostDown(downHost2); // nothing happens the first 45 minutes @@ -221,12 +226,16 @@ public class NodeFailerTest { assertEquals(0, tester.deployer.redeployments); assertEquals(8, tester.nodeRepository.nodes().list(Node.State.active).nodeType(NodeType.tenant).size()); assertEquals(0, tester.nodeRepository.nodes().list(Node.State.failed).nodeType(NodeType.tenant).size()); + assertTrue(tester.nodeRepository.nodes().node(downHost1).get().isDown()); + assertFalse(tester.nodeRepository.nodes().node(downHost1).get().isUp()); } tester.serviceMonitor.setHostUp(downHost1); // downHost2 should now be failed and replaced, but not downHost1 tester.clock.advance(Duration.ofDays(1)); tester.runMaintainers(); + assertFalse(tester.nodeRepository.nodes().node(downHost1).get().isDown()); + assertTrue(tester.nodeRepository.nodes().node(downHost1).get().isUp()); assertEquals(1, tester.deployer.redeployments); assertEquals(8, tester.nodeRepository.nodes().list(Node.State.active).nodeType(NodeType.tenant).size()); assertEquals(1, tester.nodeRepository.nodes().list(Node.State.failed).nodeType(NodeType.tenant).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java index 58f4be18992..f5e524a90cc 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RetiredExpirerTest.java @@ -239,8 +239,8 @@ public class RetiredExpirerTest { MockNameResolver nameResolver = (MockNameResolver) tester.nodeRepository().nameResolver(); String ipv4 = "127.0.1.4"; nameResolver.addRecord(retiredNode.hostname(), ipv4); - Node node = Node.create(retiredNode.hostname(), IP.Config.of(Set.of(ipv4), Set.of()), retiredNode.hostname(), - tester.asFlavor("default", NodeType.config), NodeType.config).build(); + Node node = Node.create(retiredNode.hostname(), new IP.Config(Set.of(ipv4), Set.of()), retiredNode.hostname(), + tester.asFlavor("default", NodeType.config), NodeType.config).build(); var nodes = List.of(node); nodes = nodeRepository.nodes().addNodes(nodes, Agent.system); nodes = nodeRepository.nodes().deallocate(nodes, Agent.system, getClass().getSimpleName()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java index 9c69543a9d6..2786da4b69e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.maintenance; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.collections.Pair; import com.yahoo.config.provision.ApplicationId; @@ -56,10 +57,10 @@ public class ScalingSuggestionsMaintainerTest { tester.deploy(app1, cluster1, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), - IntRange.empty(), false, true, Optional.empty())); + IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty())); tester.deploy(app2, cluster2, Capacity.from(new ClusterResources(5, 1, new NodeResources(4, 4, 10, 0.1)), new ClusterResources(10, 1, new NodeResources(6.5, 5, 15, 0.1)), - IntRange.empty(), false, true, Optional.empty())); + IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty())); tester.clock().advance(Duration.ofHours(13)); Duration timeAdded = addMeasurements(0.90f, 0.90f, 0.90f, 0, 500, app1, tester.nodeRepository()); @@ -97,7 +98,7 @@ public class ScalingSuggestionsMaintainerTest { maintainer.maintain(); var suggested = tester.nodeRepository().applications().get(app1).get().cluster(cluster1.id()).get().suggested().resources().get(); tester.deploy(app1, cluster1, Capacity.from(suggested, suggested, - IntRange.empty(), false, true, Optional.empty())); + IntRange.empty(), false, true, Optional.empty(), ClusterInfo.empty())); tester.clock().advance(Duration.ofDays(2)); addMeasurements(0.2f, 0.65f, 0.6f, 0, 500, app1, tester.nodeRepository()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java index b54975cbf41..c9421f098e7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java @@ -327,8 +327,8 @@ public class SpareCapacityMaintainerTest { } private IP.Config ipConfig(int id, boolean host) { - return IP.Config.of(Set.of(String.format("%04X::%04X", id, 0)), - host ? IntStream.range(0, 10) + return new IP.Config(Set.of(String.format("%04X::%04X", id, 0)), + host ? IntStream.range(0, 10) .mapToObj(n -> String.format("%04X::%04X", id, n)) .collect(Collectors.toSet()) : Set.of()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java index 88fe88dbaad..c26ffdaa023 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/node/IPTest.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.provision.node; import com.google.common.collect.ImmutableSet; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.LockedNodeList; @@ -15,6 +14,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -156,7 +156,7 @@ public class IPTest { IP.Config config = IP.Config.of(Set.of("2600:1f10:::1"), Set.of("2600:1f10:::2", "2600:1f10:::3"), - List.of(HostName.of("node1"), HostName.of("node2"))); + List.of(new Address("node1"), new Address("node2"))); IP.Pool pool = config.pool(); Optional<IP.Allocation> allocation = pool.findAllocation(emptyList, resolver, false); } @@ -193,12 +193,12 @@ public class IPTest { } IP.Pool pool = node.ipConfig().pool(); - assertNotEquals(dualStack, pool.ipAddresses().protocol() == IP.IpAddresses.Protocol.ipv4); + assertNotEquals(dualStack, pool.getProtocol() == IP.IpAddresses.Protocol.ipv4); return pool; } private static Node createNode(Set<String> ipAddresses) { - return Node.create("id1", IP.Config.of(Set.of("127.0.0.1"), ipAddresses), + return Node.create("id1", new IP.Config(Set.of("127.0.0.1"), ipAddresses), "host1", nodeFlavors.getFlavorOrThrow("default"), NodeType.host).build(); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java index d04fe1bdda2..c429f88cfa1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.persistence; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.IntRange; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterResources; @@ -15,6 +16,7 @@ import com.yahoo.vespa.hosted.provision.autoscale.Autoscaling; import com.yahoo.vespa.hosted.provision.autoscale.Load; import org.junit.Test; +import java.time.Duration; import java.time.Instant; import java.util.ArrayList; import java.util.List; @@ -40,6 +42,7 @@ public class ApplicationSerializerTest { true, Autoscaling.empty(), Autoscaling.empty(), + ClusterInfo.empty(), BcpGroupInfo.empty(), List.of())); var minResources = new NodeResources(1, 2, 3, 4); @@ -65,6 +68,7 @@ public class ApplicationSerializerTest { Load.zero(), Load.one(), Autoscaling.Metrics.zero()), + new ClusterInfo.Builder().bcpDeadline(Duration.ofMinutes(33)).build(), new BcpGroupInfo(0.1, 0.2, 0.3), List.of(new ScalingEvent(new ClusterResources(10, 5, minResources), new ClusterResources(12, 6, minResources), @@ -95,6 +99,7 @@ public class ApplicationSerializerTest { assertEquals(originalCluster.required(), serializedCluster.required()); assertEquals(originalCluster.suggested(), serializedCluster.suggested()); assertEquals(originalCluster.target(), serializedCluster.target()); + assertEquals(originalCluster.clusterInfo(), serializedCluster.clusterInfo()); assertEquals(originalCluster.bcpGroupInfo(), serializedCluster.bcpGroupInfo()); assertEquals(originalCluster.scalingEvents(), serializedCluster.scalingEvents()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ArchiveUriSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ArchiveUriSerializerTest.java new file mode 100644 index 00000000000..6204ace8a51 --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ArchiveUriSerializerTest.java @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.persistence; + +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.provision.archive.ArchiveUris; +import org.junit.Test; + +import java.util.Map; + +import static org.junit.Assert.assertEquals; + +/** + * @author freva + */ +public class ArchiveUriSerializerTest { + + @Test + public void test_serialization() { + ArchiveUris archiveUris = new ArchiveUris(Map.of( + TenantName.from("tenant1"), "ftp://host123.test/dir/", + TenantName.from("tenant2"), "ftp://archive.test/vespa/"), + Map.of(CloudAccount.from("321456987012"), "ftp://host123.test/dir/")); + + ArchiveUris serialized = ArchiveUriSerializer.fromJson(ArchiveUriSerializer.toJson(archiveUris)); + assertEquals(archiveUris, serialized); + } + +}
\ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java index 1086f2026a8..d61a3d95a65 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NetworkPorts; import com.yahoo.config.provision.NodeFlavors; @@ -24,6 +23,7 @@ import com.yahoo.test.ManualClock; import com.yahoo.text.Utf8; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.Node.State; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Generation; import com.yahoo.vespa.hosted.provision.node.History; @@ -249,7 +249,7 @@ public class NodeSerializerTest { @Test public void serialize_parent_hostname() { final String parentHostname = "parent.yahoo.com"; - Node node = Node.create("myId", IP.Config.of(Set.of("127.0.0.1"), Set.of()), "myHostname", nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant) + Node node = Node.create("myId", new IP.Config(Set.of("127.0.0.1"), Set.of()), "myHostname", nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant) .parentHostname(parentHostname) .build(); @@ -272,14 +272,16 @@ public class NodeSerializerTest { // Test round-trip with address pool node = node.with(node.ipConfig().withPool(IP.Pool.of( Set.of("::1", "::2", "::3"), - List.of(HostName.of("a"), HostName.of("b"), HostName.of("c"))))); + List.of(new Address("a"), new Address("b"), new Address("c"))))); Node copy = nodeSerializer.fromJson(nodeSerializer.toJson(node)); - assertEquals(node.ipConfig(), copy.ipConfig()); + assertEquals(node.ipConfig().pool().ipSet(), copy.ipConfig().pool().ipSet()); + assertEquals(Set.copyOf(node.ipConfig().pool().getAddressList()), Set.copyOf(copy.ipConfig().pool().getAddressList())); // Test round-trip without address pool (handle empty pool) node = createNode(); copy = nodeSerializer.fromJson(nodeSerializer.toJson(node)); - assertEquals(node.ipConfig(), copy.ipConfig()); + assertEquals(node.ipConfig().pool().ipSet(), copy.ipConfig().pool().ipSet()); + assertEquals(Set.copyOf(node.ipConfig().pool().getAddressList()), Set.copyOf(copy.ipConfig().pool().getAddressList())); } @Test @@ -527,7 +529,7 @@ public class NodeSerializerTest { private Node createNode() { return Node.create("myId", - IP.Config.of(Set.of("127.0.0.1"), Set.of()), + new IP.Config(Set.of("127.0.0.1"), Set.of()), "myHostname", nodeFlavors.getFlavorOrThrow("default"), NodeType.tenant).build(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/TenantArchiveUriSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/TenantArchiveUriSerializerTest.java deleted file mode 100644 index 2ae4f6363e0..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/TenantArchiveUriSerializerTest.java +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.persistence; - -import com.yahoo.config.provision.TenantName; -import org.junit.Test; - -import java.util.Map; -import java.util.TreeMap; - -import static org.junit.Assert.assertEquals; - -/** - * @author freva - */ -public class TenantArchiveUriSerializerTest { - - @Test - public void test_serialization() { - Map<TenantName, String> archiveUris = new TreeMap<>(); - archiveUris.put(TenantName.from("tenant1"), "ftp://host123.test/dir/"); - archiveUris.put(TenantName.from("tenant2"), "ftp://archive.test/vespa/"); - - Map<TenantName, String> serialized = TenantArchiveUriSerializer.fromJson(TenantArchiveUriSerializer.toJson(archiveUris)); - assertEquals(archiveUris, serialized); - } - -}
\ No newline at end of file diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ArchiveUrisTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ArchiveUrisTest.java deleted file mode 100644 index 7751e906c48..00000000000 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ArchiveUrisTest.java +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.provisioning; - -import com.yahoo.component.Version; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterMembership; -import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.NodeResources; -import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.hosted.provision.Node; -import com.yahoo.vespa.hosted.provision.node.Allocation; -import com.yahoo.vespa.hosted.provision.node.Generation; -import org.junit.Test; - -import java.util.Optional; - -import static com.yahoo.vespa.hosted.provision.provisioning.ArchiveUris.normalizeUri; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.fail; - -/** - * @author freva - */ -public class ArchiveUrisTest { - - @Test - public void archive_uri() { - ApplicationId app = ApplicationId.from("vespa", "music", "main"); - Node allocated = createNode(app); - Node unallocated = createNode(null); - ArchiveUris archiveUris = new ProvisioningTester.Builder().build().nodeRepository().archiveUris(); - - assertFalse(archiveUris.archiveUriFor(unallocated).isPresent()); - assertFalse(archiveUris.archiveUriFor(allocated).isPresent()); - - archiveUris.setArchiveUri(app.tenant(), Optional.of("scheme://hostname/dir")); - assertEquals("scheme://hostname/dir/music/main/default/h432a/", archiveUris.archiveUriFor(allocated).get()); - } - - private Node createNode(ApplicationId appId) { - Node.Builder nodeBuilder = Node.create("id", "h432a.prod.us-south-1.vespa.domain.tld", new Flavor(NodeResources.unspecified()), Node.State.parked, NodeType.tenant); - Optional.ofNullable(appId) - .map(app -> new Allocation(app, - ClusterMembership.from("container/default/0/0", Version.fromString("1.2.3"), Optional.empty()), - NodeResources.unspecified(), - Generation.initial(), - false)) - .ifPresent(nodeBuilder::allocation); - return nodeBuilder.build(); - } - - @Test - public void normalize_test() { - assertEquals("ftp://domain/legal-dir123/", normalizeUri("ftp://domain/legal-dir123")); - assertEquals("ftp://domain/legal-dir123/", normalizeUri("ftp://domain/legal-dir123/")); - assertEquals("s3://my-bucket-prod.region/my-tenant-123/", normalizeUri("s3://my-bucket-prod.region/my-tenant-123/")); - assertEquals("s3://my-bucket-prod.region/my-tenant_123/", normalizeUri("s3://my-bucket-prod.region/my-tenant_123/")); - assertThrows(IllegalArgumentException.class, () -> normalizeUri("domain/dir/")); - assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp:/domain/dir/")); - assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp:/domain//dir/")); - assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/illegal:dir/")); - assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/-illegal-dir/")); - assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/_illegal-dir/")); - assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/illegal-dir-/")); - assertThrows(IllegalArgumentException.class, () -> normalizeUri("ftp://domain/illegal-dir_/")); - } - - private static void assertThrows(Class<? extends Throwable> clazz, Runnable runnable) { - try { - runnable.run(); - fail("Expected " + clazz); - } catch (Throwable e) { - if (!clazz.isInstance(e)) throw e; - } - } -} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java index 47d34a76dd6..23c2d0fc47a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java @@ -534,7 +534,7 @@ public class DynamicAllocationTest { } private void addAndAssignNode(ApplicationId id, String hostname, String parentHostname, ClusterSpec clusterSpec, NodeResources flavor, int index, ProvisioningTester tester) { - Node node1a = Node.create("open1", IP.Config.of(Set.of("127.0.233." + index), Set.of()), hostname, + Node node1a = Node.create("open1", new IP.Config(Set.of("127.0.233." + index), Set.of()), hostname, new Flavor(flavor), NodeType.tenant).parentHostname(parentHostname).build(); ClusterMembership clusterMembership1 = ClusterMembership.from( clusterSpec.with(Optional.of(ClusterSpec.Group.from(0))), index); // Need to add group here so that group is serialized in node allocation diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java index f406f44f02f..6362a07ae00 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java @@ -256,13 +256,13 @@ public class DynamicProvisioningTest { ClusterSpec cluster = ClusterSpec.request(ClusterSpec.Type.content, new ClusterSpec.Id("cluster1")).vespaVersion("8").build(); Capacity capacity = Capacity.from(new ClusterResources(4, 2, new NodeResources(2, 4, 50, 0.1, DiskSpeed.any, StorageType.any, Architecture.any))); - hostProvisioner.overrideHostFlavor("x86"); + hostProvisioner.setHostFlavor("x86", ClusterSpec.Type.content); tester.activate(app, cluster, capacity); NodeList nodes = tester.nodeRepository().nodes().list(); assertEquals(4, nodes.owner(app).state(Node.State.active).size()); assertEquals(Set.of("x86"), nodes.parentsOf(nodes.owner(app).state(Node.State.active)).stream().map(n -> n.flavor().name()).collect(Collectors.toSet())); - hostProvisioner.overrideHostFlavor("arm"); + hostProvisioner.setHostFlavor("arm", ClusterSpec.Type.content); flagSource.withStringFlag(PermanentFlags.HOST_FLAVOR.id(), "arm"); tester.activate(app, cluster, capacity); nodes = tester.nodeRepository().nodes().list(); @@ -453,7 +453,7 @@ public class DynamicProvisioningTest { if (!provisionedHosts.isEmpty()) { List<Node> hosts = provisionedHosts.asList() .stream() - .map(h -> h.with(((MockHostProvisioner) tester.hostProvisioner()).createIpConfig(h))) + .map(h -> ((MockHostProvisioner)tester.hostProvisioner()).withIpAssigned(h)) .toList(); tester.move(Node.State.ready, hosts); tester.activateTenantHosts(); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java index 7c75b07eb47..a24eb61bb79 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.provision.autoscale; +package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; @@ -19,6 +19,10 @@ import com.yahoo.vespa.hosted.provision.Nodelike; import com.yahoo.vespa.hosted.provision.applications.Application; import com.yahoo.vespa.hosted.provision.applications.Cluster; import com.yahoo.vespa.hosted.provision.applications.ScalingEvent; +import com.yahoo.vespa.hosted.provision.autoscale.Autoscaler; +import com.yahoo.vespa.hosted.provision.autoscale.Autoscaling; +import com.yahoo.vespa.hosted.provision.autoscale.Fixture; +import com.yahoo.vespa.hosted.provision.autoscale.MetricsDb; import com.yahoo.vespa.hosted.provision.node.IP; import com.yahoo.vespa.hosted.provision.provisioning.CapacityPolicies; import com.yahoo.vespa.hosted.provision.provisioning.HostResourcesCalculator; @@ -34,28 +38,43 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; /** + * A provisioniong tester which + * - Supports dynamic provisioning (only). + * - Optionally replicates the actual AWS setup and logic used on Vespa Cloud. + * - Supports autoscaling testing. + * + * TODO: All provisioning testing should migrate to use this, and then the provisionging tester should be collapsed + * into this. + * * @author bratseth */ -class AutoscalingTester { +public class DynamicProvisioningTester { private final ProvisioningTester provisioningTester; private final Autoscaler autoscaler; private final HostResourcesCalculator hostResourcesCalculator; private final CapacityPolicies capacityPolicies; - public AutoscalingTester(Zone zone, HostResourcesCalculator resourcesCalculator, List<Flavor> hostFlavors, InMemoryFlagSource flagSource, int hostCount) { + public DynamicProvisioningTester(Zone zone, HostResourcesCalculator resourcesCalculator, List<Flavor> hostFlavors, InMemoryFlagSource flagSource, int hostCount) { this(zone, hostFlavors, resourcesCalculator, flagSource); for (Flavor flavor : hostFlavors) provisioningTester.makeReadyNodes(hostCount, flavor.name(), NodeType.host, 8); provisioningTester.activateTenantHosts(); } - private AutoscalingTester(Zone zone, List<Flavor> flavors, HostResourcesCalculator resourcesCalculator, InMemoryFlagSource flagSource) { + private DynamicProvisioningTester(Zone zone, List<Flavor> flavors, HostResourcesCalculator resourcesCalculator, InMemoryFlagSource flagSource) { + MockHostProvisioner hostProvisioner = null; + if (zone.cloud().dynamicProvisioning()) { + hostProvisioner = new MockHostProvisioner(flavors); + hostProvisioner.setHostFlavorIfAvailable(new NodeResources(2, 8, 75, 10, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote), resourcesCalculator, ClusterSpec.Type.admin + ); + } + provisioningTester = new ProvisioningTester.Builder().zone(zone) .flavors(flavors) .resourcesCalculator(resourcesCalculator) .flagSource(flagSource) - .hostProvisioner(zone.cloud().dynamicProvisioning() ? new MockHostProvisioner(flavors) : null) + .hostProvisioner(hostProvisioner) .build(); hostResourcesCalculator = resourcesCalculator; @@ -105,9 +124,9 @@ class AutoscalingTester { public void makeReady(String hostname) { Node node = nodeRepository().nodes().node(hostname).get(); - provisioningTester.patchNode(node, (n) -> n.with(IP.Config.of(Set.of("::" + 0 + ":0"), Set.of()))); + provisioningTester.patchNode(node, (n) -> n.with(new IP.Config(Set.of("::" + 0 + ":0"), Set.of()))); Node host = nodeRepository().nodes().node(node.parentHostname().get()).get(); - host = host.with(IP.Config.of(Set.of("::" + 0 + ":0"), Set.of("::" + 0 + ":2"))); + host = host.with(new IP.Config(Set.of("::" + 0 + ":0"), Set.of("::" + 0 + ":2"))); if (host.state() == Node.State.provisioned) provisioningTester.move(Node.State.ready, host); } @@ -134,6 +153,7 @@ class AutoscalingTester { cluster.required(), cluster.suggested(), cluster.target(), + cluster.clusterInfo(), cluster.bcpGroupInfo(), List.of()); // Remove scaling events cluster = cluster.with(ScalingEvent.create(cluster.minResources(), cluster.minResources(), @@ -234,12 +254,12 @@ class AutoscalingTester { } @Override - public NodeResources requestToReal(NodeResources resources, boolean exclusive) { + public NodeResources requestToReal(NodeResources resources, boolean exclusive, boolean bestCase) { return resources.withMemoryGb(resources.memoryGb()); } @Override - public NodeResources realToRequest(NodeResources resources, boolean exclusive) { + public NodeResources realToRequest(NodeResources resources, boolean exclusive, boolean bestCase) { return resources.withMemoryGb(resources.memoryGb()); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java index 12a8e4d9386..ea2af5f3fca 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java @@ -2,12 +2,12 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.Flavor; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.IP; import org.junit.Before; import org.junit.Test; @@ -16,6 +16,7 @@ import java.util.ArrayList; import java.util.LinkedHashSet; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import java.util.stream.Stream; import static org.junit.Assert.assertEquals; @@ -166,7 +167,7 @@ public class HostCapacityTest { } private Node setupHostWithAdditionalHostnames(String hostHostname, String... additionalHostnames) { - List<HostName> hostnames = Stream.of(additionalHostnames).map(HostName::of).toList(); + List<Address> addresses = Stream.of(additionalHostnames).map(Address::new).toList(); doAnswer(invocation -> ((Flavor)invocation.getArguments()[0]).resources()) .when(hostResourcesCalculator).advertisedResourcesOf(any()); @@ -175,7 +176,7 @@ public class HostCapacityTest { "host", // 7-100-120-5 "docker"); // 2- 40- 40-0.5 = resources1 - return Node.create(hostHostname, IP.Config.of(Set.of("::1"), Set.of(), hostnames), hostHostname, + return Node.create(hostHostname, IP.Config.of(Set.of("::1"), Set.of(), addresses), hostHostname, nodeFlavors.getFlavorOrThrow("host"), NodeType.host).build(); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index c791c7848d7..d7b5f30a9bc 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -6,6 +6,7 @@ import com.google.common.collect.Iterators; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.ClusterInfo; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; @@ -326,7 +327,7 @@ public class LoadBalancerProvisionerTest { @Test public void load_balancer_with_custom_settings() { ClusterResources resources = new ClusterResources(3, 1, nodeResources); - Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(CloudAccount.empty)); + Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(CloudAccount.empty), ClusterInfo.empty()); tester.activate(app1, prepare(app1, capacity, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("c1")))); LoadBalancerList loadBalancers = tester.nodeRepository().loadBalancers().list(); assertEquals(1, loadBalancers.size()); @@ -343,7 +344,7 @@ public class LoadBalancerProvisionerTest { @Test public void load_balancer_with_changing_visibility() { ClusterResources resources = new ClusterResources(3, 1, nodeResources); - Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(CloudAccount.empty)); + Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(CloudAccount.empty), ClusterInfo.empty()); tester.activate(app1, prepare(app1, capacity, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("c1")))); LoadBalancerList loadBalancers = tester.nodeRepository().loadBalancers().list(); assertEquals(1, loadBalancers.size()); @@ -379,7 +380,7 @@ public class LoadBalancerProvisionerTest { ClusterResources resources = new ClusterResources(3, 1, nodeResources); CloudAccount cloudAccount0 = CloudAccount.empty; { - Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount0)); + Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount0), ClusterInfo.empty()); tester.activate(app1, prepare(app1, capacity, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("c1")))); } LoadBalancerList loadBalancers = tester.nodeRepository().loadBalancers().list(); @@ -388,7 +389,7 @@ public class LoadBalancerProvisionerTest { // Changing account fails if there is an existing LB in the previous account. CloudAccount cloudAccount1 = CloudAccount.from("111111111111"); - Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount1)); + Capacity capacity = Capacity.from(resources, resources, IntRange.empty(), false, true, Optional.of(cloudAccount1), ClusterInfo.empty()); try { prepare(app1, capacity, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("c1"))); fail("Expected exception"); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java index 32db213c445..20819daf356 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidateTest.java @@ -146,7 +146,7 @@ public class NodeCandidateTest { .parentHostname(hostname + "parent") .ipConfigWithEmptyPool(Set.of("::1")).build(); Node parent = Node.create(hostname + "parent", hostname, new Flavor(totalHostResources), Node.State.ready, NodeType.host) - .ipConfig(IP.Config.of(Set.of("::1"), Set.of("::2"))) + .ipConfig(new IP.Config(Set.of("::1"), Set.of("::2"))) .build(); return new NodeCandidate.ConcreteNodeCandidate(node, totalHostResources.subtract(allocatedHostResources), Optional.of(parent), false, exclusiveSwitch, false, true, false); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 68857719bf0..4c404e3030c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -919,10 +919,10 @@ public class ProvisioningTest { // Add 2 config server hosts and 2 config servers Flavor flavor = tester.nodeRepository().flavors().getFlavorOrThrow("default"); List<Node> nodes = List.of( - Node.create("cfghost1", IP.Config.of(Set.of("::1:0"), Set.of("::1:1")), "cfghost1", flavor, NodeType.confighost).build(), - Node.create("cfghost2", IP.Config.of(Set.of("::2:0"), Set.of("::2:1")), "cfghost2", flavor, NodeType.confighost).ipConfig(IP.Config.of(Set.of("::2:0"), Set.of("::2:1"), List.of())).build(), - Node.create("cfg1", IP.Config.of(Set.of("::1:1"), Set.of()), "cfg1", flavor, NodeType.config).parentHostname("cfghost1").build(), - Node.create("cfg2", IP.Config.of(Set.of("::2:1"), Set.of()), "cfg2", flavor, NodeType.config).parentHostname("cfghost2").build()); + Node.create("cfghost1", new IP.Config(Set.of("::1:0"), Set.of("::1:1")), "cfghost1", flavor, NodeType.confighost).build(), + Node.create("cfghost2", new IP.Config(Set.of("::2:0"), Set.of("::2:1")), "cfghost2", flavor, NodeType.confighost).ipConfig(IP.Config.of(Set.of("::2:0"), Set.of("::2:1"), List.of())).build(), + Node.create("cfg1", new IP.Config(Set.of("::1:1"), Set.of()), "cfg1", flavor, NodeType.config).parentHostname("cfghost1").build(), + Node.create("cfg2", new IP.Config(Set.of("::2:1"), Set.of()), "cfg2", flavor, NodeType.config).parentHostname("cfghost2").build()); tester.move(Node.State.ready, tester.nodeRepository().nodes().addNodes(nodes, Agent.system)); InfraApplication cfgHostApp = new ConfigServerHostApplication(); @@ -1041,6 +1041,19 @@ public class ProvisioningTest { assertEquals(new NodeResources(3, 3, 3, 3), CapacityPolicies.versioned(spec.vespaVersion("9.0").build(), resources)); } + @Test + public void testAdminProvisioning() { + var nodeResources = new NodeResources(0.25, 1.32, 10, 0.3); + var resources = new ClusterResources(1, 1, nodeResources); + var fixture = DynamicProvisioningTester.fixture() + .awsProdSetup(true) + .clusterType(ClusterSpec.Type.admin) + .initialResources(Optional.empty()) + .capacity(Capacity.from(resources)) + .build(); + fixture.deploy(); + } + private SystemState prepare(ApplicationId application, int container0Size, int container1Size, int content0Size, int content1Size, NodeResources flavor, ProvisioningTester tester) { return prepare(application, tester, container0Size, container1Size, content0Size, content1Size, flavor, "6.42"); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index e1747a910c9..06f26bc1d8b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -201,7 +201,7 @@ public class ProvisioningTester { try (var lock = nodeRepository.nodes().lockAndGetRequired(prepared.hostname())) { Node node = lock.node(); if (node.ipConfig().primary().isEmpty()) { - node = node.with(IP.Config.of(Set.of("::" + 0 + ":0"), Set.of())); + node = node.with(new IP.Config(Set.of("::" + 0 + ":0"), Set.of())); nodeRepository.nodes().write(node, lock); } if (node.parentHostname().isEmpty()) continue; @@ -209,7 +209,7 @@ public class ProvisioningTester { if (parent.state() == Node.State.active) continue; NestedTransaction t = new NestedTransaction(); if (parent.ipConfig().primary().isEmpty()) - parent = parent.with(IP.Config.of(Set.of("::" + 0 + ":0"), Set.of("::" + 0 + ":2"))); + parent = parent.with(new IP.Config(Set.of("::" + 0 + ":0"), Set.of("::" + 0 + ":2"))); nodeRepository.nodes().activate(List.of(parent), t); t.commit(); } @@ -447,7 +447,7 @@ public class ProvisioningTester { nameResolver.addRecord(nodeHostname, ipv4Addr); } } - Node.Builder builder = Node.create(hostname, IP.Config.of(hostIps, ipAddressPool), hostname, flavor, type); + Node.Builder builder = Node.create(hostname, new IP.Config(hostIps, ipAddressPool), hostname, flavor, type); reservedTo.ifPresent(builder::reservedTo); nodes.add(builder.build()); } @@ -464,8 +464,8 @@ public class ProvisioningTester { String ipv4 = "127.0.1." + i; nameResolver.addRecord(hostname, ipv4); - Node node = Node.create(hostname, IP.Config.of(Set.of(ipv4), Set.of()), hostname, - nodeFlavors.getFlavorOrThrow(flavor), NodeType.config).build(); + Node node = Node.create(hostname, new IP.Config(Set.of(ipv4), Set.of()), hostname, + nodeFlavors.getFlavorOrThrow(flavor), NodeType.config).build(); nodes.add(node); } @@ -532,7 +532,7 @@ public class ProvisioningTester { List<Node> nodes = new ArrayList<>(count); for (int i = startIndex; i < count + startIndex; i++) { String hostname = nodeNamer.apply(i); - IP.Config ipConfig = IP.Config.of(nodeRepository.nameResolver().resolveAll(hostname), Set.of()); + IP.Config ipConfig = new IP.Config(nodeRepository.nameResolver().resolveAll(hostname), Set.of()); Node node = Node.create("node-id", ipConfig, hostname, new Flavor(resources), nodeType) .parentHostname(parentHostname) .build(); @@ -773,13 +773,13 @@ public class ProvisioningTester { } @Override - public NodeResources requestToReal(NodeResources resources, boolean exclusive) { + public NodeResources requestToReal(NodeResources resources, boolean exclusive, boolean bestCase) { return resources.withMemoryGb(resources.memoryGb() - memoryTaxGb) .withDiskGb(resources.diskGb() - ( resources.storageType() == local ? localDiskTax : 0) ); } @Override - public NodeResources realToRequest(NodeResources resources, boolean exclusive) { + public NodeResources realToRequest(NodeResources resources, boolean exclusive, boolean bestCase) { return resources.withMemoryGb(resources.memoryGb() + memoryTaxGb) .withDiskGb(resources.diskGb() + ( resources.storageType() == local ? localDiskTax : 0) ); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningCompleteHostCalculatorTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningCompleteHostCalculatorTest.java index d703ecf44e8..4d9f7d51538 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningCompleteHostCalculatorTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningCompleteHostCalculatorTest.java @@ -62,8 +62,8 @@ public class VirtualNodeProvisioningCompleteHostCalculatorTest { Flavor hostFlavor = new Flavor(new NodeResources(20, 40, 1000, 4)); var calculator = new CompleteResourcesCalculator(hostFlavor); var originalReal = new NodeResources(0.7, 6.0, 12.9, 1.0); - var realToRequest = calculator.realToRequest(originalReal, false); - var requestToReal = calculator.requestToReal(realToRequest, false); + var realToRequest = calculator.realToRequest(originalReal, false, false); + var requestToReal = calculator.requestToReal(realToRequest, false, false); var realResourcesOf = calculator.realResourcesOf(realToRequest); assertEquals(originalReal, requestToReal); assertEquals(originalReal, realResourcesOf); @@ -93,7 +93,7 @@ public class VirtualNodeProvisioningCompleteHostCalculatorTest { } @Override - public NodeResources requestToReal(NodeResources advertisedResources, boolean exclusive) { + public NodeResources requestToReal(NodeResources advertisedResources, boolean exclusive, boolean bestCase) { double memoryOverhead = memoryOverhead(advertisedResourcesOf(hostFlavor).memoryGb(), advertisedResources, false); double diskOverhead = diskOverhead(advertisedResourcesOf(hostFlavor).diskGb(), advertisedResources, false); return advertisedResources.withMemoryGb(advertisedResources.memoryGb() - memoryOverhead) @@ -108,7 +108,7 @@ public class VirtualNodeProvisioningCompleteHostCalculatorTest { } @Override - public NodeResources realToRequest(NodeResources realResources, boolean exclusive) { + public NodeResources realToRequest(NodeResources realResources, boolean exclusive, boolean bestCase) { double memoryOverhead = memoryOverhead(advertisedResourcesOf(hostFlavor).memoryGb(), realResources, true); double diskOverhead = diskOverhead(advertisedResourcesOf(hostFlavor).diskGb(), realResources, true); return realResources.withMemoryGb(realResources.memoryGb() + memoryOverhead) diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java new file mode 100644 index 00000000000..03581146b9f --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/ArchiveApiTest.java @@ -0,0 +1,67 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.restapi; + +import com.yahoo.application.container.handler.Request; +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.SystemName; +import com.yahoo.text.Utf8; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; + +/** + * Test of the REST APIs provided by the node repository. + * + * Note: This class is referenced from our operations documentation and must not be renamed/moved without updating that. + * + * @author bratseth + */ +public class ArchiveApiTest { + + private RestApiTester tester; + + @Before + public void createTester() { + tester = new RestApiTester(SystemName.Public, CloudAccount.from("111222333444")); + } + + @After + public void closeTester() { + tester.close(); + } + + @Test + public void archive_uris() throws IOException { + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "archiveUri", false); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/archive"), "{\"archives\":[]}"); + + assertResponse(new Request("http://localhost:8080/nodes/v2/archive/tenant/tenant3", Utf8.toBytes("{\"uri\": \"ftp://host/dir\"}"), Request.Method.PATCH), + "{\"message\":\"Updated archive URI for tenant3\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/archive/tenant2", Utf8.toBytes("{\"uri\": \"s3://my-bucket/dir\"}"), Request.Method.PATCH), + "{\"message\":\"Updated archive URI for tenant2\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/archive/account/777888999000", Utf8.toBytes("{\"uri\": \"s3://acc-bucket\"}"), Request.Method.PATCH), + "{\"message\":\"Updated archive URI for 777888999000\"}"); + + + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "\"archiveUri\":\"ftp://host/dir/tenant3/application3/instance3/id3/host4/\"", true); + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com"), "\"archiveUri\":\"s3://acc-bucket/zoneapp/zoneapp/zoneapp/node-admin/dockerhost2/\"", true); + assertFile(new Request("http://localhost:8080/nodes/v2/archive"), "archives.json"); + + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/archive/tenant/tenant3", new byte[0], Request.Method.DELETE), "{\"message\":\"Removed archive URI for tenant3\"}"); + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "archiveUri", false); + tester.assertResponse(new Request("http://localhost:8080/nodes/v2/archive/account/777888999000", new byte[0], Request.Method.DELETE), "{\"message\":\"Removed archive URI for 777888999000\"}"); + tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com"), "archiveUri", false); + } + + + private void assertFile(Request request, String file) throws IOException { + tester.assertFile(request, file); + } + + private void assertResponse(Request request, String response) throws IOException { + tester.assertResponse(request, response); + } + +} diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiTest.java index 240d0daf96f..729b6b813cd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersV1ApiTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.application.container.handler.Request; import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.SystemName; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -16,7 +17,7 @@ public class LoadBalancersV1ApiTest { @Before public void createTester() { - tester = new RestApiTester(CloudAccount.empty); + tester = new RestApiTester(SystemName.main, CloudAccount.empty); } @After diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index 1d7b7ec7454..c9e57c22d11 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -6,6 +6,7 @@ import com.yahoo.application.container.handler.Response; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.text.Utf8; import com.yahoo.vespa.applicationmodel.HostName; @@ -41,7 +42,7 @@ public class NodesV2ApiTest { @Before public void createTester() { - tester = new RestApiTester(CloudAccount.from("111222333444")); + tester = new RestApiTester(SystemName.main, CloudAccount.from("111222333444")); } @After @@ -112,8 +113,8 @@ public class NodesV2ApiTest { // POST duplicate node tester.assertResponse(new Request("http://localhost:8080/nodes/v2/node", ("[" + asNodeJson("host8.yahoo.com", "default", "127.0.254.8") + "]").getBytes(StandardCharsets.UTF_8), - Request.Method.POST), 400, - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot add provisioned host host8.yahoo.com: A node with this name already exists\"}"); + Request.Method.POST), 500, + "{\"error-code\":\"INTERNAL_SERVER_ERROR\",\"message\":\"Cannot add provisioned host host8.yahoo.com: A node with this name already exists\"}"); // DELETE a provisioned node assertResponse(new Request("http://localhost:8080/nodes/v2/node/host9.yahoo.com", @@ -990,23 +991,6 @@ public class NodesV2ApiTest { } @Test - public void archive_uris() throws IOException { - tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "archiveUri", false); - tester.assertResponse(new Request("http://localhost:8080/nodes/v2/archive"), "{\"archives\":[]}"); - - assertResponse(new Request("http://localhost:8080/nodes/v2/archive/tenant3", Utf8.toBytes("{\"uri\": \"ftp://host/dir\"}"), Request.Method.PATCH), - "{\"message\":\"Updated archive URI for tenant3\"}"); - assertResponse(new Request("http://localhost:8080/nodes/v2/archive/tenant2", Utf8.toBytes("{\"uri\": \"s3://my-bucket/dir\"}"), Request.Method.PATCH), - "{\"message\":\"Updated archive URI for tenant2\"}"); - - tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "\"archiveUri\":\"ftp://host/dir/application3/instance3/id3/host4/\"", true); - assertFile(new Request("http://localhost:8080/nodes/v2/archive"), "archives.json"); - - tester.assertResponse(new Request("http://localhost:8080/nodes/v2/archive/tenant3", new byte[0], Request.Method.DELETE), "{\"message\":\"Removed archive URI for tenant3\"}"); - tester.assertPartialResponse(new Request("http://localhost:8080/nodes/v2/node/host4.yahoo.com"), "archiveUri", false); - } - - @Test public void trusted_certificates_patch() throws IOException { String url = "http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"; tester.assertPartialResponse(new Request(url), "\"trustStore\":[]", false); // initially empty list diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/RestApiTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/RestApiTester.java index e424b04aeaf..47745d25467 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/RestApiTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/RestApiTester.java @@ -6,6 +6,7 @@ import com.yahoo.application.container.JDisc; import com.yahoo.application.container.handler.Request; import com.yahoo.application.container.handler.Response; import com.yahoo.config.provision.CloudAccount; +import com.yahoo.config.provision.SystemName; import com.yahoo.io.IOUtils; import com.yahoo.vespa.hosted.provision.testutils.ContainerConfig; import org.junit.ComparisonFailure; @@ -25,8 +26,8 @@ public class RestApiTester { private final JDisc container; - public RestApiTester(CloudAccount defaultCloudAccount) { - container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(0, defaultCloudAccount), Networking.disable); + public RestApiTester(SystemName systemName, CloudAccount defaultCloudAccount) { + container = JDisc.fromServicesXml(ContainerConfig.servicesXmlV2(0, systemName, defaultCloudAccount), Networking.disable); } public void close() { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/archives.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/archives.json index 1ce54b54f6a..738d8ee1bb3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/archives.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/archives.json @@ -7,6 +7,10 @@ { "tenant": "tenant3", "uri": "ftp://host/dir/" + }, + { + "account": "777888999000", + "uri": "s3://acc-bucket/" } ] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg2.json index c970725d015..3bd45acb856 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/cfg2.json @@ -116,6 +116,5 @@ "127.0.202.1", "::202:1" ], - "additionalIpAddresses": [], - "wireguardPubkey":"olololololololololololololololololololololo=" + "additionalIpAddresses": [] } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node2.json index b39aba199b7..f7e02261065 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-node2.json @@ -98,5 +98,6 @@ "::101:3", "::101:4" ], - "cloudAccount": "777888999000" + "cloudAccount": "777888999000", + "wireguardPubkey":"000011112222333344445555666677778888999900c=" } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/wireguard.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/wireguard.json index c2853536c5d..660b92d92ba 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/wireguard.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/wireguard.json @@ -5,11 +5,6 @@ "hostname":"cfg1.yahoo.com", "wireguardPubkey":"lololololololololololololololololololololoo=", "ipAddresses":["127.0.201.1","::201:1"] - }, - { - "hostname":"cfg2.yahoo.com", - "wireguardPubkey":"olololololololololololololololololololololo=", - "ipAddresses":["127.0.202.1","::202:1"] } ] } diff --git a/parent/pom.xml b/parent/pom.xml index 2fcf5527302..f5bb4ab8855 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -468,6 +468,11 @@ <version>3.10.0</version> </dependency> <dependency> + <groupId>com.fasterxml.jackson.dataformat</groupId> + <artifactId>jackson-dataformat-cbor</artifactId> + <version>${jackson2.version}</version> + </dependency> + <dependency> <groupId>com.github.cverges.expect4j</groupId> <artifactId>expect4j</artifactId> <version>1.6</version> @@ -1170,7 +1175,7 @@ <prometheus.client.version>0.6.0</prometheus.client.version> <protobuf.version>3.21.7</protobuf.version> <spifly.version>1.3.5</spifly.version> - <surefire.version>2.22.2</surefire.version> + <surefire.version>3.0.0-M9</surefire.version> <wiremock.version>2.35.0</wiremock.version> <zero-allocation-hashing.version>0.16</zero-allocation-hashing.version> <zookeeper.client.version>3.8.0</zookeeper.client.version> diff --git a/persistence/CMakeLists.txt b/persistence/CMakeLists.txt index d87f172bfb5..072e273338b 100644 --- a/persistence/CMakeLists.txt +++ b/persistence/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib document diff --git a/screwdriver.yaml b/screwdriver.yaml index d918078b80e..9d423cacfe7 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -222,6 +222,7 @@ jobs: echo "Must have valid Vespa version and reference to continue (got VESPA_VERSION=$VESPA_VERSION, VESPA_REF=$VESPA_REF)." exit 1 fi + meta set vespa.version $VESPA_VERSION - install-dependencies: | dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo dnf install -y docker-ce docker-ce-cli containerd.io diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index d6c353e46c9..131460b0384 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos fnet vespalog vespalib diff --git a/searchcore/src/apps/proton/proton.cpp b/searchcore/src/apps/proton/proton.cpp index 06df12f73b7..281d7585274 100644 --- a/searchcore/src/apps/proton/proton.cpp +++ b/searchcore/src/apps/proton/proton.cpp @@ -10,7 +10,6 @@ #include <vespa/config/common/exceptions.h> #include <vespa/config/common/configcontext.h> #include <vespa/fnet/transport.h> -#include <vespa/fastos/thread.h> #include <vespa/fastos/file.h> #include <filesystem> #include <iostream> @@ -45,7 +44,7 @@ private: static void setupSignals(); static void setup_fadvise(); Params parseParams(int argc, char **argv); - void startAndRun(FastOS_ThreadPool & threadPool, FNET_Transport & transport, int argc, char **argv); + void startAndRun(FNET_Transport & transport, int argc, char **argv); public: int main(int argc, char **argv); }; @@ -206,10 +205,10 @@ buildTransportConfig() { class Transport { public: - Transport(const fnet::TransportConfig & config, FastOS_ThreadPool & threadPool) + Transport(const fnet::TransportConfig & config) : _transport(config) { - _transport.Start(&threadPool); + _transport.Start(); } ~Transport() { _transport.ShutDown(true); @@ -222,7 +221,7 @@ private: } void -App::startAndRun(FastOS_ThreadPool & threadPool, FNET_Transport & transport, int argc, char **argv) { +App::startAndRun(FNET_Transport & transport, int argc, char **argv) { Params params = parseParams(argc, argv); LOG(debug, "identity: '%s'", params.identity.c_str()); LOG(debug, "serviceidentity: '%s'", params.serviceidentity.c_str()); @@ -231,7 +230,7 @@ App::startAndRun(FastOS_ThreadPool & threadPool, FNET_Transport & transport, int config::ConfigServerSpec configServerSpec(transport); config::ConfigUri identityUri(params.identity, std::make_shared<config::ConfigContext>(configServerSpec)); - auto protonUP = std::make_unique<proton::Proton>(threadPool, transport, identityUri, + auto protonUP = std::make_unique<proton::Proton>(transport, identityUri, (argc > 0) ? argv[0] : "proton", subscribeTimeout); proton::Proton & proton = *protonUP; proton::BootstrapConfig::SP configSnapshot = proton.init(); @@ -254,7 +253,7 @@ App::startAndRun(FastOS_ThreadPool & threadPool, FNET_Transport & transport, int spiProton->createNode(); EV_STARTED("servicelayer"); } else { - proton.getMetricManager().init(identityUri, threadPool); + proton.getMetricManager().init(identityUri); } EV_STARTED("proton"); while (!(SIG::INT.check() || SIG::TERM.check() || (spiProton && spiProton->getNode().attemptedStopped()))) { @@ -283,9 +282,8 @@ App::main(int argc, char **argv) try { setupSignals(); setup_fadvise(); - FastOS_ThreadPool threadPool; - Transport transport(buildTransportConfig(), threadPool); - startAndRun(threadPool, transport.transport(), argc, argv); + Transport transport(buildTransportConfig()); + startAndRun(transport.transport(), argc, argv); } catch (const vespalib::InvalidCommandLineArgumentsException &e) { LOG(warning, "Invalid commandline arguments: '%s'", e.what()); return 1; diff --git a/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp b/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp index 00f62aefc28..50b5dadddbc 100644 --- a/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp +++ b/searchcore/src/apps/vespa-transactionlog-inspect/vespa-transactionlog-inspect.cpp @@ -378,7 +378,6 @@ class BaseUtility : public Utility protected: const BaseOptions &_bopts; DummyFileHeaderContext _fileHeader; - FastOS_ThreadPool _threadPool; FNET_Transport _transport; TransLogServer _server; client::TransLogClient _client; @@ -387,12 +386,11 @@ public: BaseUtility(const BaseOptions &bopts) : _bopts(bopts), _fileHeader(), - _threadPool(), _transport(), _server(_transport, _bopts.tlsName, _bopts.listenPort, _bopts.tlsDir, _fileHeader), _client(_transport, vespalib::make_string("tcp/localhost:%d", _bopts.listenPort)) { - _transport.Start(&_threadPool); + _transport.Start(); } ~BaseUtility() override { _transport.ShutDown(true); diff --git a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp index b94df75c7be..a10fec131e0 100644 --- a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp +++ b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp @@ -23,6 +23,7 @@ #include <vespa/vespalib/util/threadstackexecutor.h> #include <filesystem> #include <thread> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP("attributeflush_test"); diff --git a/searchcore/src/tests/proton/common/timer/timer_test.cpp b/searchcore/src/tests/proton/common/timer/timer_test.cpp index ac82767cd7c..4ff970df84e 100644 --- a/searchcore/src/tests/proton/common/timer/timer_test.cpp +++ b/searchcore/src/tests/proton/common/timer/timer_test.cpp @@ -1,6 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/thread.h> #include <vespa/fnet/transport.h> #include <vespa/searchcore/proton/common/scheduled_forward_executor.h> #include <vespa/searchcore/proton/common/scheduledexecutor.h> @@ -46,17 +45,15 @@ make_scheduled_executor<ScheduledForwardExecutor>(FNET_Transport& transport, ves template <typename ScheduledT> class ScheduledExecutorTest : public testing::Test { public: - FastOS_ThreadPool threadPool; FNET_Transport transport; vespalib::ThreadStackExecutor executor; std::unique_ptr<ScheduledT> timer; ScheduledExecutorTest() - : threadPool(), - transport(), + : transport(), executor(1) { - transport.Start(&threadPool); + transport.Start(); timer = make_scheduled_executor<ScheduledT>(transport, executor); } ~ScheduledExecutorTest() { diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp index 0234d5fbedf..5afe0a0c5a2 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp @@ -4,7 +4,6 @@ #include <vespa/document/repo/documenttyperepo.h> #include <vespa/document/datatype/documenttype.h> #include <vespa/document/test/make_bucket_space.h> -#include <vespa/fastos/thread.h> #include <vespa/persistence/dummyimpl/dummy_bucket_executor.h> #include <vespa/searchcore/proton/attribute/attribute_config_inspector.h> #include <vespa/searchcore/proton/attribute/attribute_usage_filter.h> @@ -82,9 +81,9 @@ namespace { VESPA_THREAD_STACK_TAG(my_executor_init); void -sampleThreadId(FastOS_ThreadId *threadId) +sampleThreadId(std::thread::id *threadId) { - *threadId = FastOS_Thread::GetCurrentThreadId(); + *threadId = std::this_thread::get_id(); } } // namespace @@ -207,12 +206,12 @@ class MyFeedHandler : public IDocumentMoveHandler, public IHeartBeatHandler, public IOperationStorer { - FastOS_ThreadId _executorThreadId; + std::thread::id _executorThreadId; std::vector<MyDocumentSubDB *> _subDBs; SerialNum _serialNum; std::atomic<uint32_t> _heartBeats; public: - explicit MyFeedHandler(FastOS_ThreadId &executorThreadId); + explicit MyFeedHandler(std::thread::id executorThreadId); ~MyFeedHandler() override; @@ -241,7 +240,7 @@ public: class MyExecutor : public vespalib::ThreadStackExecutorBase { public: - FastOS_ThreadId _threadId; + std::thread::id _threadId; MyExecutor(); bool acceptNewTask(unique_lock &, std::condition_variable &) override { @@ -604,7 +603,7 @@ MyDocumentSubDB::getNumUsedLids() const } -MyFeedHandler::MyFeedHandler(FastOS_ThreadId &executorThreadId) +MyFeedHandler::MyFeedHandler(std::thread::id executorThreadId) : IDocumentMoveHandler(), IPruneRemovedDocumentsHandler(), IHeartBeatHandler(), @@ -622,8 +621,7 @@ MyFeedHandler::~MyFeedHandler() = default; bool MyFeedHandler::isExecutorThread() const { - FastOS_ThreadId threadId(FastOS_Thread::GetCurrentThreadId()); - return FastOS_Thread::CompareThreadIds(_executorThreadId, threadId); + return (_executorThreadId == std::this_thread::get_id()); } diff --git a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp index afd224c55de..13371521718 100644 --- a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp +++ b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp @@ -305,7 +305,7 @@ TEST_FFF("require that proton config fetcher follows changes to bootstrap", ConfigTestFixture("search"), ProtonConfigOwner(), ProtonConfigFetcher(f1.transport.transport(), ConfigUri(f1.configId, f1.context), f2, 60s)) { - f3.start(f1.transport.threadPool()); + f3.start(); ASSERT_TRUE(f2._configured); ASSERT_TRUE(f1.configEqual(f2.getBootstrapConfig())); f2._configured = false; @@ -320,7 +320,7 @@ TEST_FFF("require that proton config fetcher follows changes to doctypes", ConfigTestFixture("search"), ProtonConfigOwner(), ProtonConfigFetcher(f1.transport.transport(), ConfigUri(f1.configId, f1.context), f2, 60s)) { - f3.start(f1.transport.threadPool()); + f3.start(); f2._configured = false; f1.addDocType("typea"); @@ -340,7 +340,7 @@ TEST_FFF("require that proton config fetcher reconfigures dbowners", ConfigTestFixture("search"), ProtonConfigOwner(), ProtonConfigFetcher(f1.transport.transport(), ConfigUri(f1.configId, f1.context), f2, 60s)) { - f3.start(f1.transport.threadPool()); + f3.start(); ASSERT_FALSE(f2.getDocumentDBConfig("typea")); // Add db and verify that config for db is provided diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp index 16be2c1bd25..d39d2873edb 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_initializer.cpp @@ -14,6 +14,7 @@ #include <vespa/searchlib/attribute/attribute_header.h> #include <vespa/searchlib/attribute/attributevector.h> #include <vespa/fastos/file.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".proton.attribute.attribute_initializer"); diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp index 4b83b7d5af9..3e577bf6cbe 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.cpp @@ -10,7 +10,6 @@ #include <vespa/searchcore/proton/common/eventlogger.h> #include <vespa/searchlib/common/flush_token.h> #include <vespa/vespalib/util/cpu_usage.h> -#include <thread> #include <vespa/log/log.h> LOG_SETUP(".proton.flushengine.flushengine"); @@ -86,7 +85,8 @@ FlushEngine::FlushEngine(std::shared_ptr<flushengine::ITlsStatsFactory> tlsStats _maxConcurrent(numThreads), _idleInterval(idleInterval), _taskId(0), - _threadPool(), + _thread(), + _has_thread(false), _strategy(std::move(strategy)), _priorityStrategy(), _executor(numThreads, CpuUsage::wrap(flush_engine_executor, CpuUsage::Category::COMPACT)), @@ -111,9 +111,7 @@ FlushEngine::~FlushEngine() FlushEngine & FlushEngine::start() { - if (_threadPool.NewThread(this) == nullptr) { - throw vespalib::IllegalStateException("Failed to start engine thread."); - } + _thread = std::thread([this](){run();}); return *this; } @@ -127,7 +125,9 @@ FlushEngine::close() _closed = true; _cond.notify_all(); } - _threadPool.Close(); + if (_thread.joinable()) { + _thread.join(); + } _executor.shutdown(); _executor.sync(); return *this; @@ -168,8 +168,9 @@ FlushEngine::wait(vespalib::duration minimumWaitTimeIfReady, bool ignorePendingP } void -FlushEngine::Run(FastOS_ThreadInterface *, void *) +FlushEngine::run() { + _has_thread = true; bool shouldIdle = false; vespalib::string prevFlushName; while (wait(shouldIdle ? _idleInterval : vespalib::duration::zero(), false)) { @@ -190,6 +191,7 @@ FlushEngine::Run(FastOS_ThreadInterface *, void *) } _executor.sync(); prune(); + _has_thread = false; } namespace { diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h index 4b15c3503f3..1d6ed763ff6 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flushengine.h @@ -7,7 +7,7 @@ #include <vespa/searchcore/proton/common/doctypename.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/time.h> -#include <vespa/fastos/thread.h> +#include <thread> #include <set> #include <mutex> #include <condition_variable> @@ -18,7 +18,7 @@ namespace proton { namespace flushengine { class ITlsStatsFactory; } -class FlushEngine final : public FastOS_Runnable +class FlushEngine { public: class FlushMeta { @@ -54,7 +54,8 @@ private: const uint32_t _maxConcurrent; const vespalib::duration _idleInterval; uint32_t _taskId; - FastOS_ThreadPool _threadPool; + std::thread _thread; + std::atomic<bool> _has_thread; IFlushStrategy::SP _strategy; mutable IFlushStrategy::SP _priorityStrategy; vespalib::ThreadStackExecutor _executor; @@ -109,7 +110,7 @@ public: /** * Destructor. Waits for all pending tasks to complete. */ - ~FlushEngine() override; + ~FlushEngine(); /** * Observe and reset internal executor stats @@ -129,7 +130,8 @@ public: * @return This, to allow chaining. */ FlushEngine &start(); - + bool has_thread() const { return _has_thread; } + /** * Stops the scheduling thread and. This will prevent any more flush * requests being performed on the attached handlers, allowing you to flush @@ -168,7 +170,7 @@ public: */ IFlushHandler::SP removeFlushHandler(const DocTypeName &docTypeName); - void Run(FastOS_ThreadInterface *thread, void *arg) override; + void run(); FlushMetaSet getCurrentlyFlushingSet() const; diff --git a/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp b/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp index 3512d2eebad..2c26b043fad 100644 --- a/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp +++ b/searchcore/src/vespa/searchcore/proton/index/index_writer.cpp @@ -2,6 +2,7 @@ #include "index_writer.h" #include <vespa/document/fieldvalue/document.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".proton.server.indexadapter"); diff --git a/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h b/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h index 177d19b06f9..cd4d1686eeb 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h +++ b/searchcore/src/vespa/searchcore/proton/matching/docid_range_scheduler.h @@ -3,13 +3,14 @@ #pragma once #include <vespa/searchlib/queryeval/begin_and_end_id.h> -#include <vespa/fastos/types.h> #include <mutex> #include <condition_variable> #include <atomic> #include <algorithm> #include <vector> +#define VESPA_DLL_LOCAL __attribute__ ((visibility("hidden"))) + namespace proton::matching { /** diff --git a/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.cpp index c2d09fa341b..9e9d51abd57 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.cpp @@ -29,6 +29,7 @@ ContentProtonMetrics::ProtonExecutorMetrics::~ProtonExecutorMetrics() = default; ContentProtonMetrics::ContentProtonMetrics() : metrics::MetricSet("content.proton", {}, "Search engine metrics", nullptr), + configGeneration("config.generation", {}, "The oldest config generation used by this process", this), transactionLog(this), resourceUsage(this), executor(this), diff --git a/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.h index 127e32ada07..c82a6804380 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/content_proton_metrics.h @@ -41,6 +41,7 @@ struct ContentProtonMetrics : metrics::MetricSet ~SessionCacheMetrics() override; }; + metrics::LongValueMetric configGeneration; TransLogServerMetrics transactionLog; ResourceUsageMetrics resourceUsage; ProtonExecutorMetrics executor; diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_reconfig.h b/searchcore/src/vespa/searchcore/proton/server/document_db_reconfig.h index 9462c018cc2..8d2701f430f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_db_reconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_reconfig.h @@ -12,15 +12,17 @@ class DocumentSubDBReconfig; * Class representing the result of the prepare step of a DocumentDB reconfig. * * The reconfig is performed in three steps: - * Prepare: + * 1) Prepare: * Based on the config that is changed, new components are instantiated in each subdb. - * This can be costly and is handled by helper threads from the shared executor pool. + * This can be costly and is done by the proton reconfigure thread. * - * Complete prepare: - * Docid limit and serial number are used to complete prepared reconfig. + * 2) Complete prepare: + * Docid limit and serial number are used to complete the prepared reconfig. + * This is done by the DocumentDB master write thread. * - * Apply: - * The new components are swapped with the old ones in the DocumentDB master write thread. + * 3) Apply: + * The new components are swapped with the old ones. + * This is done by the DocumentDB master write thread. */ class DocumentDBReconfig { private: diff --git a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp index 74f6a622661..27650b27c77 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.cpp @@ -3,7 +3,6 @@ #include "executor_thread_service.h" #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/gate.h> -#include <vespa/fastos/thread.h> using vespalib::makeLambdaTask; using vespalib::Executor; @@ -14,27 +13,20 @@ using vespalib::SyncableThreadExecutor; namespace proton { -namespace internal { - -struct ThreadId { - FastOS_ThreadId _id; -}; -} - namespace { void -sampleThreadId(FastOS_ThreadId *threadId) +sampleThreadId(std::thread::id *threadId) { - *threadId = FastOS_Thread::GetCurrentThreadId(); + *threadId = std::this_thread::get_id(); } -std::unique_ptr<internal::ThreadId> +std::thread::id getThreadId(ThreadExecutor &executor) { - std::unique_ptr<internal::ThreadId> id = std::make_unique<internal::ThreadId>(); + std::thread::id id; vespalib::Gate gate; - executor.execute(makeLambdaTask([threadId=&id->_id, &gate] { + executor.execute(makeLambdaTask([threadId = &id, &gate] { sampleThreadId(threadId); gate.countDown(); })); @@ -74,8 +66,7 @@ ExecutorThreadService::run(Runnable &runnable) bool ExecutorThreadService::isCurrentThread() const { - FastOS_ThreadId currentThreadId = FastOS_Thread::GetCurrentThreadId(); - return FastOS_Thread::CompareThreadIds(_threadId->_id, currentThreadId); + return (_threadId == std::this_thread::get_id()); } vespalib::ExecutorStats ExecutorThreadService::getStats() { @@ -118,8 +109,7 @@ SyncableExecutorThreadService::run(Runnable &runnable) bool SyncableExecutorThreadService::isCurrentThread() const { - FastOS_ThreadId currentThreadId = FastOS_Thread::GetCurrentThreadId(); - return FastOS_Thread::CompareThreadIds(_threadId->_id, currentThreadId); + return (_threadId == std::this_thread::get_id()); } vespalib::ExecutorStats diff --git a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h index 7298b81611a..48b8d8be9b9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h +++ b/searchcore/src/vespa/searchcore/proton/server/executor_thread_service.h @@ -3,10 +3,10 @@ #include <vespa/searchcorespi/index/i_thread_service.h> #include <vespa/vespalib/util/threadexecutor.h> +#include <thread> namespace proton { -namespace internal { struct ThreadId; } /** * Implementation of IThreadService using an underlying thread stack executor * with a single thread. @@ -15,7 +15,7 @@ class ExecutorThreadService : public searchcorespi::index::IThreadService { private: vespalib::ThreadExecutor &_executor; - std::unique_ptr<internal::ThreadId> _threadId; + std::thread::id _threadId; public: ExecutorThreadService(vespalib::ThreadExecutor &executor); @@ -39,8 +39,8 @@ public: class SyncableExecutorThreadService : public searchcorespi::index::ISyncableThreadService { private: - vespalib::SyncableThreadExecutor &_executor; - std::unique_ptr<internal::ThreadId> _threadId; + vespalib::SyncableThreadExecutor &_executor; + std::thread::id _threadId; public: SyncableExecutorThreadService(vespalib::SyncableThreadExecutor &executor); @@ -66,5 +66,3 @@ public: }; } // namespace proton - - diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp index f22f57979b4..e4a4e6761aa 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp @@ -21,6 +21,7 @@ #include <filesystem> #include <sstream> #include <cassert> +#include <cinttypes> #include <fcntl.h> #include <vespa/log/log.h> @@ -94,9 +95,8 @@ class ConfigFile { using SP = std::shared_ptr<ConfigFile>; - vespalib::string _name; - time_t _modTime; - std::vector<char> _content; + vespalib::string _name; + std::vector<char> _content; public: ConfigFile(); @@ -111,7 +111,6 @@ public: ConfigFile::ConfigFile() : _name(), - _modTime(0), _content() { } @@ -120,7 +119,6 @@ ConfigFile::~ConfigFile() = default; ConfigFile::ConfigFile(const vespalib::string &name, const vespalib::string &fullName) : _name(name), - _modTime(0), _content() { FastOS_File file; @@ -130,7 +128,6 @@ ConfigFile::ConfigFile(const vespalib::string &name, const vespalib::string &ful int64_t fileSize = file.getSize(); _content.resize(fileSize); file.ReadBuf(_content.data(), fileSize); - _modTime = file.GetModificationTime(); } nbostream & @@ -138,7 +135,7 @@ ConfigFile::serialize(nbostream &stream) const { assert(strchr(_name.c_str(), '/') == nullptr); stream << _name; - stream << static_cast<int64_t>(_modTime);; + stream << int64_t(0ul); // Used to be modtime => unused uint32_t sz = _content.size(); stream << sz; stream.write(_content.data(), sz); @@ -150,9 +147,8 @@ ConfigFile::deserialize(nbostream &stream) { stream >> _name; assert(strchr(_name.c_str(), '/') == nullptr); - int64_t modTime; - stream >> modTime; - _modTime = modTime; + int64_t unused_modTime; + stream >> unused_modTime; uint32_t sz; stream >> sz; _content.resize(sz); diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp index c86019005dc..fa8b86416d0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp @@ -6,8 +6,7 @@ #include <vespa/searchcorespi/index/i_thread_service.h> #include <vespa/searchcore/proton/common/scheduledexecutor.h> #include <vespa/vespalib/util/lambdatask.h> -#include <vespa/fastos/thread.h> -#include <thread> +#include <vespa/vespalib/util/thread.h> #include <vespa/log/log.h> LOG_SETUP(".proton.server.maintenancecontroller"); @@ -16,6 +15,7 @@ using document::BucketId; using vespalib::Executor; using vespalib::MonitoredRefCount; using vespalib::makeLambdaTask; +using vespalib::thread::as_zu; namespace proton { @@ -69,7 +69,7 @@ MaintenanceController::killJobs() } // Called by master write thread assert(_masterThread.isCurrentThread()); - LOG(debug, "killJobs(): threadId=%zu", (size_t)FastOS_Thread::GetCurrentThreadId()); + LOG(debug, "killJobs(): threadId=%zu", as_zu(std::this_thread::get_id())); _periodicTaskHandles.clear(); // No need to take _jobsLock as modification of _jobs also happens in master write thread. for (auto &job : _jobs) { @@ -99,7 +99,7 @@ void MaintenanceController::performHoldJobs(JobList jobs) { // Called by master write thread - LOG(debug, "performHoldJobs(): threadId=%zu", (size_t)FastOS_Thread::GetCurrentThreadId()); + LOG(debug, "performHoldJobs(): threadId=%zu", as_zu(std::this_thread::get_id())); (void) jobs; } diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.cpp index 5e6cee94292..e580066ec17 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenancejobrunner.cpp @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "maintenancejobrunner.h" -#include <vespa/fastos/thread.h> #include <vespa/vespalib/util/cpu_usage.h> #include <vespa/vespalib/util/lambdatask.h> @@ -54,10 +53,9 @@ MaintenanceJobRunner::runJobInExecutor() } bool finished = _job->run(); if (LOG_WOULD_LOG(debug)) { - FastOS_ThreadId threadId = FastOS_Thread::GetCurrentThreadId(); LOG(debug, - "runJobInExecutor(): job='%s', task='%p', threadId=%" PRIu64 "", - _job->getName().c_str(), this, (uint64_t)threadId); + "runJobInExecutor(): job='%s', task='%p'", + _job->getName().c_str(), this); } if (!finished) { addExecutorTask(); diff --git a/searchcore/src/vespa/searchcore/proton/server/memory_flush_config_updater.cpp b/searchcore/src/vespa/searchcore/proton/server/memory_flush_config_updater.cpp index 50a499c8a73..3222cbc3a06 100644 --- a/searchcore/src/vespa/searchcore/proton/server/memory_flush_config_updater.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/memory_flush_config_updater.cpp @@ -2,6 +2,7 @@ #include "memory_flush_config_updater.h" #include <vespa/vespalib/util/size_literals.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".proton.server.memory_flush_config_updater"); diff --git a/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp b/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp index f8d7519fd0c..fbc6e2beaf5 100644 --- a/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/memoryflush.cpp @@ -7,6 +7,7 @@ #include <vespa/vespalib/stllike/hash_set.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/time.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".proton.server.memoryflush"); diff --git a/searchcore/src/vespa/searchcore/proton/server/prepare_restart_handler.cpp b/searchcore/src/vespa/searchcore/proton/server/prepare_restart_handler.cpp index 97b66600817..0ec0e6a1147 100644 --- a/searchcore/src/vespa/searchcore/proton/server/prepare_restart_handler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/prepare_restart_handler.cpp @@ -21,7 +21,7 @@ bool PrepareRestartHandler::prepareRestart(const ProtonConfig &protonCfg) { std::unique_lock lock(_mutex); - if (!_flushEngine.HasThread()) { + if (!_flushEngine.has_thread()) { return false; } if (!_running) { diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 54009ef60f4..c508706ad28 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -215,7 +215,7 @@ Proton::ProtonFileHeaderContext::setClusterName(const vespalib::string & cluster } -Proton::Proton(FastOS_ThreadPool & threadPool, FNET_Transport & transport, const config::ConfigUri & configUri, +Proton::Proton(FNET_Transport & transport, const config::ConfigUri & configUri, const vespalib::string &progName, vespalib::duration subscribeTimeout) : IProtonConfigurerOwner(), search::engine::MonitorServer(), @@ -225,7 +225,6 @@ Proton::Proton(FastOS_ThreadPool & threadPool, FNET_Transport & transport, const ComponentConfigProducer(), _cpu_util(), _hw_info(), - _threadPool(threadPool), _transport(transport), _configUri(configUri), _mutex(), @@ -277,7 +276,7 @@ Proton::init() { assert( ! _initStarted && ! _initComplete ); _initStarted = true; - _protonConfigFetcher.start(_threadPool); + _protonConfigFetcher.start(); auto configSnapshot = _protonConfigurer.getPendingConfigSnapshot(); assert(configSnapshot); auto bootstrapConfig = configSnapshot->getBootstrapConfig(); @@ -757,7 +756,7 @@ Proton::ping(std::unique_ptr<MonitorRequest>, MonitorClient &) bool Proton::triggerFlush() { - if (!_flushEngine || ! _flushEngine->HasThread()) { + if (!_flushEngine || ! _flushEngine->has_thread()) { return false; } _flushEngine->triggerFlush(); @@ -796,6 +795,7 @@ Proton::updateMetrics(const metrics::MetricLockGuard &) { { ContentProtonMetrics &metrics = _metricsEngine->root(); + metrics.configGeneration.set(getConfigGeneration()); auto tls = _tls->getTransLogServer(); if (tls) { metrics.transactionLog.update(tls->getDomainStats()); diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index bf84650867a..a5da72671d2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -88,7 +88,6 @@ private: vespalib::CpuUtil _cpu_util; HwInfo _hw_info; - FastOS_ThreadPool & _threadPool; FNET_Transport & _transport; const config::ConfigUri _configUri; mutable std::shared_mutex _mutex; @@ -156,7 +155,7 @@ public: using UP = std::unique_ptr<Proton>; using SP = std::shared_ptr<Proton>; - Proton(FastOS_ThreadPool & threadPool, FNET_Transport & transport, const config::ConfigUri & configUri, + Proton(FNET_Transport & transport, const config::ConfigUri & configUri, const vespalib::string &progName, vespalib::duration subscribeTimeout); ~Proton() override; @@ -193,7 +192,6 @@ public: const std::shared_ptr<DocumentDBConfig> &documentDBConfig, InitializeThreads initializeThreads); metrics::MetricManager & getMetricManager(); - FastOS_ThreadPool & getThreadPool() { return _threadPool; } bool triggerFlush(); bool prepareRestart(); diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp index 64a09a0bb25..0311cf4a48b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.cpp @@ -29,6 +29,7 @@ ProtonConfigFetcher::ProtonConfigFetcher(FNET_Transport & transport, const confi _cond(), _dbManagerMap(), _running(false), + _thread(), _oldDocumentTypeRepos(), _currentDocumentTypeRepo() { @@ -40,10 +41,8 @@ ProtonConfigFetcher::~ProtonConfigFetcher() } void -ProtonConfigFetcher::Run(FastOS_ThreadInterface * thread, void *arg) +ProtonConfigFetcher::run() { - (void) arg; - (void) thread; while (!_retriever.isClosed()) { try { fetchConfigs(); @@ -166,17 +165,13 @@ ProtonConfigFetcher::getGeneration() const } void -ProtonConfigFetcher::start(FastOS_ThreadPool & threadPool) +ProtonConfigFetcher::start() { fetchConfigs(); lock_guard guard(_mutex); if (_running) return; _running = true; - if (threadPool.NewThread(this, nullptr) == nullptr) { - _running = false; - throw vespalib::IllegalStateException( - "Failed starting thread for proton config fetcher"); - } + _thread = std::thread([this](){run();}); } void @@ -189,6 +184,9 @@ ProtonConfigFetcher::close() while (_running) { _cond.wait(guard); } + if (_thread.joinable()) { + _thread.join(); + } } void diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h index b0046e17a63..9f9104df0af 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton_config_fetcher.h @@ -4,7 +4,6 @@ #include "bootstrapconfigmanager.h" #include "i_document_db_config_owner.h" -#include <vespa/fastos/thread.h> #include <vespa/searchcore/proton/common/doctypename.h> #include <vespa/config/retriever/configretriever.h> #include <vespa/config/subscription/configuri.h> @@ -24,13 +23,13 @@ class IProtonConfigurer; * A ProtonConfigFetcher monitors all config in proton and document dbs for change * and starts proton reconfiguration if config has been reloaded. */ -class ProtonConfigFetcher : public FastOS_Runnable +class ProtonConfigFetcher { public: using BootstrapConfigSP = std::shared_ptr<BootstrapConfig>; ProtonConfigFetcher(FNET_Transport & transport, const config::ConfigUri & configUri, IProtonConfigurer &owner, vespalib::duration subscribeTimeout); - ~ProtonConfigFetcher() override; + ~ProtonConfigFetcher(); /** * Get the current config generation. */ @@ -39,14 +38,14 @@ public: /** * Start config fetcher, callbacks may come from now on. */ - void start(FastOS_ThreadPool & threadPool); + void start(); /** * Shutdown config fetcher, ensuring that no more callbacks arrive */ void close(); - void Run(FastOS_ThreadInterface * thread, void *arg) override; + void run(); private: using DBManagerMap = std::map<DocTypeName, std::shared_ptr<DocumentDBConfigManager>>; @@ -63,7 +62,8 @@ private: std::condition_variable _cond; DBManagerMap _dbManagerMap; bool _running; - + std::thread _thread; + std::deque<OldDocumentTypeRepo> _oldDocumentTypeRepos; std::shared_ptr<const document::DocumentTypeRepo> _currentDocumentTypeRepo; diff --git a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp index 587da244937..8ab71637684 100644 --- a/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/rpc_hooks.cpp @@ -136,7 +136,7 @@ RPCHooksBase::open(Params & params) initRPC(); _regAPI.registerName((params.identity + "/realtimecontroller").c_str()); _orb->Listen(params.rtcPort); - _transport->Start(&_proton.getThreadPool()); + _transport->Start(); LOG(debug, "started monitoring interface"); } diff --git a/searchcore/src/vespa/searchcore/proton/server/simpleflush.cpp b/searchcore/src/vespa/searchcore/proton/server/simpleflush.cpp index a1234ccc8fc..4de426b56d1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/simpleflush.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/simpleflush.cpp @@ -2,15 +2,14 @@ #include "simpleflush.h" #include <algorithm> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".proton.server.simpleflush"); namespace proton { -SimpleFlush::SimpleFlush() -{ -} +SimpleFlush::SimpleFlush() = default; FlushContext::List SimpleFlush::getFlushTargets(const FlushContext::List& targetList, diff --git a/searchcore/src/vespa/searchcore/proton/server/summaryadapter.cpp b/searchcore/src/vespa/searchcore/proton/server/summaryadapter.cpp index 038af801b80..b4565003e9e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/summaryadapter.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/summaryadapter.cpp @@ -4,6 +4,7 @@ #include <vespa/searchcore/proton/docsummary/summarymanager.h> #include <vespa/vespalib/objects/nbostream.h> #include <cassert> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".proton.server.summaryadapter"); diff --git a/searchcore/src/vespa/searchcore/proton/test/transport_helper.cpp b/searchcore/src/vespa/searchcore/proton/test/transport_helper.cpp index d0bc4dfd4d8..92e9dadf898 100644 --- a/searchcore/src/vespa/searchcore/proton/test/transport_helper.cpp +++ b/searchcore/src/vespa/searchcore/proton/test/transport_helper.cpp @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "transport_helper.h" -#include <vespa/fastos/thread.h> #include <vespa/fnet/transport.h> #include <vespa/searchcore/proton/server/executorthreadingservice.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> @@ -12,11 +11,10 @@ namespace proton { Transport::Transport() - : _threadPool(std::make_unique<FastOS_ThreadPool>()), - _transport(std::make_unique<FNET_Transport>()), + : _transport(std::make_unique<FNET_Transport>()), _clock(std::make_unique<vespalib::TestClock>()) { - _transport->Start(_threadPool.get()); + _transport->Start(); } Transport::~Transport() { diff --git a/searchcore/src/vespa/searchcore/proton/test/transport_helper.h b/searchcore/src/vespa/searchcore/proton/test/transport_helper.h index 46ca8131041..8a724009fc1 100644 --- a/searchcore/src/vespa/searchcore/proton/test/transport_helper.h +++ b/searchcore/src/vespa/searchcore/proton/test/transport_helper.h @@ -3,8 +3,6 @@ #include <vespa/searchcorespi/index/ithreadingservice.h> -class FastOS_ThreadPool; - namespace vespalib { class TestClock; } namespace proton { @@ -19,11 +17,9 @@ public: Transport(); virtual ~Transport(); FNET_Transport & transport() { return *_transport; } - FastOS_ThreadPool & threadPool() { return *_threadPool; } const vespalib::Clock & clock() const; virtual void shutdown(); private: - std::unique_ptr<FastOS_ThreadPool> _threadPool; std::unique_ptr<FNET_Transport> _transport; std::unique_ptr<vespalib::TestClock> _clock; }; diff --git a/searchcore/src/vespa/searchcorespi/index/indexflushtarget.cpp b/searchcore/src/vespa/searchcorespi/index/indexflushtarget.cpp index e72525d0aaa..53fb21bf1ed 100644 --- a/searchcore/src/vespa/searchcorespi/index/indexflushtarget.cpp +++ b/searchcore/src/vespa/searchcorespi/index/indexflushtarget.cpp @@ -2,6 +2,7 @@ #include "indexflushtarget.h" #include <vespa/vespalib/util/size_literals.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".searchcorespi.index.indexflushtarget"); diff --git a/searchcore/src/vespa/searchcorespi/index/indexfusiontarget.cpp b/searchcore/src/vespa/searchcorespi/index/indexfusiontarget.cpp index 1df6d321f99..6755976939b 100644 --- a/searchcore/src/vespa/searchcorespi/index/indexfusiontarget.cpp +++ b/searchcore/src/vespa/searchcorespi/index/indexfusiontarget.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "indexfusiontarget.h" +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".searchcorespi.index.indexfusiontarget"); diff --git a/searchcore/src/vespa/searchcorespi/index/indexwriteutilities.cpp b/searchcore/src/vespa/searchcorespi/index/indexwriteutilities.cpp index 54d2cf6e1f5..97afce79861 100644 --- a/searchcore/src/vespa/searchcorespi/index/indexwriteutilities.cpp +++ b/searchcore/src/vespa/searchcorespi/index/indexwriteutilities.cpp @@ -164,7 +164,6 @@ IndexWriteUtilities::updateDiskIndexSchema(const vespalib::string &indexDir, LOG(error, "Could not save schema to '%s'", schemaTmpName.c_str()); } - // XXX: FastOS layer violation FastOS_StatInfo statInfo; bool statres; statres = FastOS_File::Stat(schemaOrigName.c_str(), &statInfo); @@ -183,7 +182,6 @@ IndexWriteUtilities::updateDiskIndexSchema(const vespalib::string &indexDir, } vespalib::File::sync(indexDir); } - // XXX: FastOS layer violation int renameres = ::rename(schemaTmpName.c_str(), schemaName.c_str()); if (renameres != 0) { int error = errno; diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index 03429b956a4..44051a96578 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib vespaeval @@ -129,6 +128,7 @@ vespa_define_module( src/tests/features src/tests/features/beta src/tests/features/bm25 + src/tests/features/closest src/tests/features/constant src/tests/features/element_completeness src/tests/features/element_similarity_feature diff --git a/searchlib/src/apps/uniform/uniform.cpp b/searchlib/src/apps/uniform/uniform.cpp index 784c69f4647..807b8d61a9e 100644 --- a/searchlib/src/apps/uniform/uniform.cpp +++ b/searchlib/src/apps/uniform/uniform.cpp @@ -2,8 +2,7 @@ #include <vespa/vespalib/util/signalhandler.h> #include <vespa/searchlib/bitcompression/compression.h> -#include <vespa/log/log.h> - +#include <cinttypes> static uint64_t maxExpGolombVal(uint64_t kValue, uint64_t maxBits) diff --git a/searchlib/src/apps/vespa-index-inspect/vespa-index-inspect.cpp b/searchlib/src/apps/vespa-index-inspect/vespa-index-inspect.cpp index a5d950c0f31..6278b216b52 100644 --- a/searchlib/src/apps/vespa-index-inspect/vespa-index-inspect.cpp +++ b/searchlib/src/apps/vespa-index-inspect/vespa-index-inspect.cpp @@ -16,6 +16,7 @@ #include <iostream> #include <getopt.h> #include <cstdlib> +#include <cinttypes> #include <unistd.h> #include <vespa/log/log.h> @@ -256,12 +257,12 @@ ShowPostingListSubApp::getOptions(int argc, char **argv) int c; int longopt_index = 0; static struct option longopts[] = { - { "indexdir", 1, NULL, 0 }, - { "field", 1, NULL, 0 }, - { "transpose", 0, NULL, 0 }, - { "docidlimit", 1, NULL, 0 }, - { "mindocid", 1, NULL, 0 }, - { NULL, 0, NULL, 0 } + { "indexdir", 1, nullptr, 0 }, + { "field", 1, nullptr, 0 }, + { "transpose", 0, nullptr, 0 }, + { "docidlimit", 1, nullptr, 0 }, + { "mindocid", 1, nullptr, 0 }, + { nullptr, 0, nullptr, 0 } }; enum longopts_enum { LONGOPT_INDEXDIR, @@ -293,7 +294,7 @@ ShowPostingListSubApp::getOptions(int argc, char **argv) _minDocId = atoi(optarg); break; default: - if (optarg != NULL) { + if (optarg != nullptr) { LOG(error, "longopt %s with arg %s", longopts[longopt_index].name, optarg); @@ -683,9 +684,7 @@ DumpWordsSubApp::DumpWordsSubApp() } -DumpWordsSubApp::~DumpWordsSubApp() -{ -} +DumpWordsSubApp::~DumpWordsSubApp() = default; void @@ -708,12 +707,12 @@ DumpWordsSubApp::getOptions(int argc, char **argv) int c; int longopt_index = 0; static struct option longopts[] = { - { "indexdir", 1, NULL, 0 }, - { "field", 1, NULL, 0 }, - { "minnumdocs", 1, NULL, 0 }, - { "verbose", 0, NULL, 0 }, - { "wordnum", 0, NULL, 0 }, - { NULL, 0, NULL, 0 } + { "indexdir", 1, nullptr, 0 }, + { "field", 1, nullptr, 0 }, + { "minnumdocs", 1, nullptr, 0 }, + { "verbose", 0, nullptr, 0 }, + { "wordnum", 0, nullptr, 0 }, + { nullptr, 0, nullptr, 0 } }; enum longopts_enum { LONGOPT_INDEXDIR, @@ -745,7 +744,7 @@ DumpWordsSubApp::getOptions(int argc, char **argv) _showWordNum = true; break; default: - if (optarg != NULL) { + if (optarg != nullptr) { LOG(error, "longopt %s with arg %s", longopts[longopt_index].name, optarg); diff --git a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Group.java b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Group.java index 3054f53e7b3..0b629d43446 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Group.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/aggregation/Group.java @@ -15,12 +15,13 @@ import java.util.ArrayList; import java.util.Comparator; import java.util.Iterator; import java.util.List; -import java.util.stream.Collectors; public class Group extends Identifiable { public static final int classId = registerClass(0x4000 + 90, Group.class); private static final ObjectPredicate REF_LOCATOR = new RefLocator(); + private static final int MAX_AGGREGATIONS = 0x10000; // Backend limitation + private static final int MAX_ORDERBY_EXPRESSIONS = 8; // Backend limitation private List<Integer> orderByIdx = List.of(); private List<ExpressionNode> orderByExp = List.of(); private List<AggregationResult> aggregationResults = List.of(); @@ -274,6 +275,9 @@ public class Group extends Identifiable { * @return this, to allow chaining */ public Group addAggregationResult(AggregationResult result) { + if (aggregationResults.size() >= MAX_AGGREGATIONS) { + throw new IllegalArgumentException("You have reached the limit for number of aggregations of " + MAX_AGGREGATIONS); + } aggregationResults = add(aggregationResults, result); return this; } @@ -288,6 +292,9 @@ public class Group extends Identifiable { * @return this, to allow chaining */ public Group addOrderBy(ExpressionNode exp, boolean asc) { + if (orderByExp.size() >= MAX_ORDERBY_EXPRESSIONS) { + throw new IllegalArgumentException("You have reached the limit for number of orderby expressions of " + MAX_ORDERBY_EXPRESSIONS); + } if (exp instanceof AggregationResult) { exp = new AggregationRefNode((AggregationResult)exp); } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ExpressionOptimizer.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ExpressionOptimizer.java index c568db34b4f..ffc05f966fa 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ExpressionOptimizer.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/ExpressionOptimizer.java @@ -32,9 +32,9 @@ import com.yahoo.searchlib.rankingexpression.evaluation.tensoroptimization.Tenso */ public class ExpressionOptimizer { - private GBDTOptimizer gbdtOptimizer = new GBDTOptimizer(); - private GBDTForestOptimizer gbdtForestOptimizer = new GBDTForestOptimizer(); - private TensorOptimizer tensorOptimizer = new TensorOptimizer(); + private final GBDTOptimizer gbdtOptimizer = new GBDTOptimizer(); + private final GBDTForestOptimizer gbdtForestOptimizer = new GBDTForestOptimizer(); + private final TensorOptimizer tensorOptimizer = new TensorOptimizer(); /** Gets an optimizer instance used by this by class name, or null if the optimizer is not known */ public Optimizer getOptimizer(Class<?> clazz) { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TensorValue.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TensorValue.java index 25e03c75376..90556bf4b00 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TensorValue.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/TensorValue.java @@ -57,132 +57,87 @@ public class TensorValue extends Value { @Override public Value or(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.join(((TensorValue)argument).value, (a, b) -> ((a!=0.0) || (b!=0.0)) ? 1.0 : 0.0 )); - else - return new TensorValue(value.map((value) -> ((value!=0.0) || argument.asBoolean()) ? 1 : 0)); + return new TensorValue(value.join(argument.asTensor(), (a, b) -> ((a!=0.0) || (b!=0.0)) ? 1.0 : 0.0 )); } @Override public Value and(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.join(((TensorValue)argument).value, (a, b) -> ((a!=0.0) && (b!=0.0)) ? 1.0 : 0.0 )); - else - return new TensorValue(value.map((value) -> ((value!=0.0) && argument.asBoolean()) ? 1 : 0)); + return new TensorValue(value.join(argument.asTensor(), (a, b) -> ((a!=0.0) && (b!=0.0)) ? 1.0 : 0.0 )); } @Override public Value largerOrEqual(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.largerOrEqual(((TensorValue)argument).value)); - else - return new TensorValue(value.map((value) -> value >= argument.asDouble() ? 1.0 : 0.0)); + return new TensorValue(value.largerOrEqual(argument.asTensor())); } @Override public Value larger(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.larger(((TensorValue)argument).value)); - else - return new TensorValue(value.map((value) -> value > argument.asDouble() ? 1.0 : 0.0)); + return new TensorValue(value.larger(argument.asTensor())); } @Override public Value smallerOrEqual(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.smallerOrEqual(((TensorValue)argument).value)); - else - return new TensorValue(value.map((value) -> value <= argument.asDouble() ? 1.0 : 0.0)); + return new TensorValue(value.smallerOrEqual(argument.asTensor())); } @Override public Value smaller(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.smaller(((TensorValue)argument).value)); - else - return new TensorValue(value.map((value) -> value < argument.asDouble() ? 1.0 : 0.0)); + return new TensorValue(value.smaller(argument.asTensor())); } @Override public Value approxEqual(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.approxEqual(((TensorValue)argument).value)); - else - return new TensorValue(value.map((value) -> DoubleCompatibleValue.approxEqual(value, argument.asDouble()) ? 1.0 : 0.0)); + return new TensorValue(value.approxEqual(argument.asTensor())); } @Override public Value notEqual(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.notEqual(((TensorValue)argument).value)); - else - return new TensorValue(value.map((value) -> value != argument.asDouble() ? 1.0 : 0.0)); + return new TensorValue(value.notEqual(argument.asTensor())); } @Override public Value equal(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.equal(((TensorValue)argument).value)); - else - return new TensorValue(value.map((value) -> value == argument.asDouble() ? 1.0 : 0.0)); + return new TensorValue(value.equal(argument.asTensor())); } @Override public Value add(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.add(((TensorValue)argument).value)); - else - return new TensorValue(value.map((value) -> value + argument.asDouble())); + return new TensorValue(value.add(argument.asTensor())); } @Override public Value subtract(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.subtract(((TensorValue) argument).value)); - else - return new TensorValue(value.map((value) -> value - argument.asDouble())); + return new TensorValue(value.subtract(argument.asTensor())); } @Override public Value multiply(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.multiply(((TensorValue) argument).value)); - else - return new TensorValue(value.map((value) -> value * argument.asDouble())); + return new TensorValue(value.multiply(argument.asTensor())); } @Override public Value divide(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.divide(((TensorValue) argument).value)); - else - return new TensorValue(value.map((value) -> value / argument.asDouble())); + return new TensorValue(value.divide(argument.asTensor())); } @Override public Value modulo(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.fmod(((TensorValue) argument).value)); - else - return new TensorValue(value.map((value) -> value % argument.asDouble())); + return new TensorValue(value.fmod(argument.asTensor())); } @Override public Value power(Value argument) { - if (argument instanceof TensorValue) - return new TensorValue(value.pow(((TensorValue)argument).value)); - else - return new TensorValue(value.map((value) -> Math.pow(value, argument.asDouble()))); + return new TensorValue(value.pow(argument.asTensor())); } public Tensor asTensor() { return value; } @Override public Value function(Function function, Value arg) { - if (arg instanceof TensorValue) + if (function.arity() != 1) return new TensorValue(functionOnTensor(function, arg.asTensor())); else - return new TensorValue(value.map((value) -> function.evaluate(value, arg.asDouble()))); + return new TensorValue(value.map((value) -> function.evaluate(value, 0.0))); } private Tensor functionOnTensor(Function function, Tensor argument) { @@ -195,7 +150,7 @@ public class TensorValue extends Value { case ldexp -> value.ldexp(argument); case bit -> value.bit(argument); case hamming -> value.hamming(argument); - default -> throw new UnsupportedOperationException("Cannot combine two tensors using " + function); + default -> value.join(argument, (a, b) -> function.evaluate(a, b)); }; } diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java index 5de2138147e..ed53b82f1d5 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/Value.java @@ -114,8 +114,8 @@ public abstract class Value { return new TensorValue(Tensor.from(value)); else if ((value.indexOf('.') == -1) && (value.indexOf('e') == -1) && (value.indexOf('E') == -1)) return new LongValue(Long.parseLong(value)); - else - return new DoubleValue(Double.parseDouble(value)); + else + return new DoubleValue(Double.parseDouble(value)); } public static Value of(Tensor tensor) { diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/tensoroptimization/TensorOptimizer.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/tensoroptimization/TensorOptimizer.java index a091cc0287c..33b1824fdeb 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/tensoroptimization/TensorOptimizer.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/evaluation/tensoroptimization/TensorOptimizer.java @@ -59,7 +59,6 @@ public class TensorOptimizer extends Optimizer { * The ReduceJoin class determines whether or not the arguments are * compatible with the optimization. */ - @SuppressWarnings("unchecked") private ExpressionNode optimizeReduceJoin(ExpressionNode node) { if ( ! (node instanceof TensorFunctionNode)) { return node; diff --git a/searchlib/src/tests/attribute/benchmark/attributebenchmark.cpp b/searchlib/src/tests/attribute/benchmark/attributebenchmark.cpp index 0b90b4eae95..d5ea6fdaffc 100644 --- a/searchlib/src/tests/attribute/benchmark/attributebenchmark.cpp +++ b/searchlib/src/tests/attribute/benchmark/attributebenchmark.cpp @@ -5,7 +5,7 @@ #include <vespa/searchlib/attribute/attributeguard.h> #include <vespa/searchlib/attribute/attributefactory.h> #include <vespa/searchcommon/attribute/config.h> -#include <vespa/fastos/thread.h> +#include <thread> #include <vespa/vespalib/util/signalhandler.h> #include <iostream> #include "attributesearcher.h" @@ -96,7 +96,6 @@ private: static struct rusage computeDifference(struct rusage & first, struct rusage & second); }; - FastOS_ThreadPool * _threadPool; Config _config; RandomGenerator _rndGen; @@ -129,12 +128,8 @@ private: public: - AttributeBenchmark() : _threadPool(NULL), _config(), _rndGen() {} - ~AttributeBenchmark() { - if (_threadPool != NULL) { - delete _threadPool; - } - } + AttributeBenchmark() : _config(), _rndGen() {} + ~AttributeBenchmark() = default; int main(int argc, char **argv); }; @@ -268,7 +263,7 @@ AttributeBenchmark::benchmarkSearch(const AttributePtr & ptr, const std::vector< } else { searchers.push_back(new AttributeFindSearcher<T>(ptr, values, _config._numQueries)); } - _threadPool->NewThread(searchers.back()); + searchers.back()->start(); } for (uint32_t i = 0; i < searchers.size(); ++i) { @@ -299,7 +294,7 @@ AttributeBenchmark::benchmarkSearchWithUpdater(const AttributePtr & ptr, AttributeUpdaterThread<Vector, T, BT> updater(ptr, values, _rndGen, _config._validate, _config._commitFreq, _config._minValueCount, _config._maxValueCount); - _threadPool->NewThread(&updater); + updater.start(); benchmarkSearch(ptr, values); updater.stop(); updater.join(); @@ -337,8 +332,6 @@ AttributeBenchmark::benchmarkAttribute(const AttributePtr & ptr, const std::vect } else { benchmarkSearchWithUpdater<Vector, T, BT>(ptr, values); } - - _threadPool->Close(); } @@ -569,8 +562,6 @@ AttributeBenchmark::main(int argc, char **argv) dc._attribute = vespalib::string(argv[optind]); - _threadPool = new FastOS_ThreadPool(); - std::cout << "<attribute-benchmark>" << std::endl; init(dc); _config.printXML(); diff --git a/searchlib/src/tests/attribute/benchmark/attributesearcher.h b/searchlib/src/tests/attribute/benchmark/attributesearcher.h index d6e14d6793d..ea2d7190f25 100644 --- a/searchlib/src/tests/attribute/benchmark/attributesearcher.h +++ b/searchlib/src/tests/attribute/benchmark/attributesearcher.h @@ -2,7 +2,6 @@ #pragma once -#include <vespa/searchlib/util/runnable.h> #include <vespa/searchlib/attribute/attribute.h> #include <vespa/searchlib/attribute/attributeguard.h> #include <vespa/searchlib/attribute/search_context.h> @@ -59,7 +58,7 @@ public: }; -class AttributeSearcher : public Runnable +class AttributeSearcher { protected: using AttributePtr = AttributeVector::SP; @@ -67,17 +66,23 @@ protected: const AttributePtr & _attrPtr; vespalib::Timer _timer; AttributeSearcherStatus _status; - + std::thread _thread; + public: - AttributeSearcher(const AttributePtr & attrPtr) : - Runnable(), _attrPtr(attrPtr), _timer(), _status() + AttributeSearcher(const AttributePtr & attrPtr) + : _attrPtr(attrPtr), _timer(), _status(), _thread() { _status._numClients = 1; } - virtual void doRun() override = 0; + virtual ~AttributeSearcher(); + virtual void doRun() = 0; + void start() { _thread = std::thread([this](){doRun();}); } + void join() { _thread.join(); } AttributeSearcherStatus & getStatus() { return _status; } void buildTermQuery(std::vector<char> & buffer, const vespalib::string & index, const char * term, bool prefix = false); }; +AttributeSearcher::~AttributeSearcher() = default; + void AttributeSearcher::buildTermQuery(std::vector<char> & buffer, const vespalib::string & index, const char * term, bool prefix) diff --git a/searchlib/src/tests/attribute/benchmark/attributeupdater.h b/searchlib/src/tests/attribute/benchmark/attributeupdater.h index 88220a8cfb8..ada9c423cd1 100644 --- a/searchlib/src/tests/attribute/benchmark/attributeupdater.h +++ b/searchlib/src/tests/attribute/benchmark/attributeupdater.h @@ -4,7 +4,6 @@ #include <vespa/vespalib/util/hdr_abort.h> #include <vespa/searchlib/util/randomgenerator.h> -#include <vespa/searchlib/util/runnable.h> #include <vespa/searchlib/attribute/attribute.h> #define VALIDATOR_STR(str) #str @@ -153,26 +152,30 @@ template <typename Vector, typename T, typename BT> AttributeUpdater<Vector, T, BT>::~AttributeUpdater() = default; template <typename Vector, typename T, typename BT> -class AttributeUpdaterThread : public AttributeUpdater<Vector, T, BT>, public Runnable +class AttributeUpdaterThread : public AttributeUpdater<Vector, T, BT> { private: using AttributePtr = AttributeVector::SP; - + std::atomic<bool> _done; + std::thread _thread; public: AttributeUpdaterThread(const AttributePtr & attrPtr, const std::vector<T> & values, RandomGenerator & rndGen, bool validate, uint32_t commitFreq, uint32_t minValueCount, uint32_t maxValueCount); ~AttributeUpdaterThread(); - - virtual void doRun() override; + void doRun(); + void start() { _thread = std::thread([this](){doRun();}); } + void stop() { _done = true; } + void join() { _thread.join(); } }; template <typename Vector, typename T, typename BT> AttributeUpdaterThread<Vector, T, BT>::AttributeUpdaterThread(const AttributePtr & attrPtr, const std::vector<T> & values, RandomGenerator & rndGen, bool validate, uint32_t commitFreq, uint32_t minValueCount, uint32_t maxValueCount) - : AttributeUpdater<Vector, T, BT>(attrPtr, values, rndGen, validate, commitFreq, minValueCount, maxValueCount), - Runnable() + : AttributeUpdater<Vector, T, BT>(attrPtr, values, rndGen, validate, commitFreq, minValueCount, maxValueCount), + _done(false), + _thread() {} template <typename Vector, typename T, typename BT> AttributeUpdaterThread<Vector, T, BT>::~AttributeUpdaterThread() = default; diff --git a/searchlib/src/tests/attribute/postinglist/postinglist.cpp b/searchlib/src/tests/attribute/postinglist/postinglist.cpp index 3f07faa159f..7d2a89b6e5b 100644 --- a/searchlib/src/tests/attribute/postinglist/postinglist.cpp +++ b/searchlib/src/tests/attribute/postinglist/postinglist.cpp @@ -11,6 +11,7 @@ #include <vespa/vespalib/testkit/testapp.h> #include <set> #include <map> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP("postinglist_test"); diff --git a/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp b/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp index b5101f1ea58..e356187a19f 100644 --- a/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp +++ b/searchlib/src/tests/attribute/reference_attribute/reference_attribute_test.cpp @@ -15,6 +15,7 @@ #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/test/insertion_operators.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP("reference_attribute_test"); diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index 2f51459ebfa..28c50891225 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -26,9 +26,11 @@ #include <vespa/searchlib/util/bufferwriter.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/document/base/exceptions.h> +#include <vespa/eval/eval/fast_value.h> #include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/tensor_spec.h> #include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/value_codec.h> #include <vespa/eval/eval/test/value_compare.h> #include <vespa/fastos/file.h> #include <filesystem> @@ -60,7 +62,9 @@ using search::tensor::PrepareResult; using search::tensor::SerializedFastValueAttribute; using search::tensor::TensorAttribute; using search::tensor::VectorBundle; +using vespalib::SharedStringRepo; using vespalib::datastore::CompactionStrategy; +using vespalib::eval::FastValueBuilderFactory; using vespalib::eval::CellType; using vespalib::eval::SimpleValue; using vespalib::eval::TensorSpec; @@ -76,7 +80,17 @@ vespalib::string vec_2d_spec("tensor(x[2])"); vespalib::string vec_mixed_2d_spec("tensor(a{},x[2])"); Value::UP createTensor(const TensorSpec &spec) { - return SimpleValue::from_spec(spec); + return value_from_spec(spec, FastValueBuilderFactory::get()); +} + +std::vector<vespalib::string> +to_string_labels(vespalib::ConstArrayRef<vespalib::string_id> labels) +{ + std::vector<vespalib::string> result; + for (auto& label : labels) { + result.emplace_back(SharedStringRepo::Handle::string_from_id(label)); + } + return result; } TensorSpec @@ -569,6 +583,7 @@ struct Fixture { void testCompaction(); void testTensorTypeFileHeaderTag(); void testEmptyTensor(); + void testSerializedTensorRef(); void testOnHoldAccounting(); void test_populate_address_space_usage(); void test_mmap_file_allocator(); @@ -776,6 +791,44 @@ Fixture::testEmptyTensor() } void +Fixture::testSerializedTensorRef() +{ + const TensorAttribute &tensorAttr = *_tensorAttr; + if (_traits.use_dense_tensor_attribute || _traits.use_direct_tensor_attribute) { + EXPECT_FALSE(tensorAttr.supports_get_serialized_tensor_ref()); + return; + } + EXPECT_TRUE(tensorAttr.supports_get_serialized_tensor_ref()); + if (_denseTensors) { + set_tensor(3, expDenseTensor3()); + } else { + set_tensor(3, TensorSpec(sparseSpec) + .add({{"x", "one"}, {"y", "two"}}, 11) + .add({{"x", "three"}, {"y", "four"}}, 17)); + } + auto ref = tensorAttr.get_serialized_tensor_ref(3); + auto vectors = ref.get_vectors(); + if (_denseTensors) { + EXPECT_EQUAL(1u, vectors.subspaces()); + auto cells = vectors.cells(0).typify<double>(); + auto labels = ref.get_labels(0); + EXPECT_EQUAL(0u, labels.size()); + EXPECT_EQUAL((std::vector<double>{0.0, 11.0, 0.0, 0.0, 0.0, 0.0}), (std::vector<double>{ cells.begin(), cells.end() })); + } else { + EXPECT_EQUAL(2u, vectors.subspaces()); + auto cells = vectors.cells(0).typify<double>(); + auto labels = ref.get_labels(0); + EXPECT_EQUAL((std::vector<vespalib::string>{"one", "two"}), to_string_labels(labels)); + EXPECT_EQUAL((std::vector<double>{11.0}), (std::vector<double>{ cells.begin(), cells.end() })); + cells = vectors.cells(1).typify<double>(); + labels = ref.get_labels(1); + EXPECT_EQUAL((std::vector<vespalib::string>{"three", "four"}), to_string_labels(labels)); + EXPECT_EQUAL((std::vector<double>{17.0}), (std::vector<double>{ cells.begin(), cells.end() })); + } + TEST_DO(clearTensor(3)); +} + +void Fixture::testOnHoldAccounting() { { @@ -829,6 +882,7 @@ void testAll(MakeFixture &&f) TEST_DO(f()->testCompaction()); TEST_DO(f()->testTensorTypeFileHeaderTag()); TEST_DO(f()->testEmptyTensor()); + TEST_DO(f()->testSerializedTensorRef()); TEST_DO(f()->testOnHoldAccounting()); TEST_DO(f()->test_populate_address_space_usage()); TEST_DO(f()->test_mmap_file_allocator()); diff --git a/searchlib/src/tests/bitcompression/expgolomb/expgolomb_test.cpp b/searchlib/src/tests/bitcompression/expgolomb/expgolomb_test.cpp index 5bb36a8462e..9a726f9d8a6 100644 --- a/searchlib/src/tests/bitcompression/expgolomb/expgolomb_test.cpp +++ b/searchlib/src/tests/bitcompression/expgolomb/expgolomb_test.cpp @@ -5,6 +5,7 @@ #include <vespa/vespalib/util/size_literals.h> #include <vector> #include <algorithm> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP("expglomb_test"); diff --git a/searchlib/src/tests/common/location_iterator/location_iterator_test.cpp b/searchlib/src/tests/common/location_iterator/location_iterator_test.cpp index 44a2f1697cd..bf372c0e62f 100644 --- a/searchlib/src/tests/common/location_iterator/location_iterator_test.cpp +++ b/searchlib/src/tests/common/location_iterator/location_iterator_test.cpp @@ -9,6 +9,7 @@ #include <vespa/searchlib/common/locationiterators.h> #include <vespa/searchcommon/attribute/config.h> #include <vespa/vespalib/gtest/gtest.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP("location_iterator_test"); diff --git a/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp b/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp index f1729f21f39..408cf370c59 100644 --- a/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp +++ b/searchlib/src/tests/diskindex/pagedict4/pagedict4test.cpp @@ -16,6 +16,7 @@ #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/vespalib/util/signalhandler.h> #include <sstream> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP("pagedict4test"); diff --git a/searchlib/src/tests/features/closest/CMakeLists.txt b/searchlib/src/tests/features/closest/CMakeLists.txt new file mode 100644 index 00000000000..71572c5e5a2 --- /dev/null +++ b/searchlib/src/tests/features/closest/CMakeLists.txt @@ -0,0 +1,10 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +vespa_add_executable(searchlib_closest_test_app TEST + SOURCES + closest_test.cpp + DEPENDS + searchlib + searchlib_test +) +vespa_add_test(NAME searchlib_closest_test_app COMMAND searchlib_closest_test_app) diff --git a/searchlib/src/tests/features/closest/closest_test.cpp b/searchlib/src/tests/features/closest/closest_test.cpp new file mode 100644 index 00000000000..c53e627b528 --- /dev/null +++ b/searchlib/src/tests/features/closest/closest_test.cpp @@ -0,0 +1,208 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/eval/value.h> +#include <vespa/eval/eval/value_codec.h> +#include <vespa/searchlib/features/closest_feature.h> +#include <vespa/searchlib/features/setup.h> +#include <vespa/searchlib/fef/test/dummy_dependency_handler.h> +#include <vespa/searchlib/fef/test/labels.h> +#include <vespa/searchlib/test/features/distance_closeness_fixture.h> +#include <vespa/vespalib/stllike/asciistream.h> +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/stllike/asciistream.h> + +using search::feature_t; +using search::features::test::BlueprintFactoryFixture; +using search::features::test::DistanceClosenessFixture; +using search::features::test::FeatureDumpFixture; +using search::features::test::IndexEnvironmentFixture; +using search::features::ClosestBlueprint; +using vespalib::eval::FastValueBuilderFactory; +using vespalib::eval::TensorSpec; +using vespalib::eval::Value; +using vespalib::eval::spec_from_value; +using vespalib::eval::value_from_spec; + +const vespalib::string field_and_label_feature_name("closest(bar,nns)"); +const vespalib::string field_feature_name("closest(bar)"); + +const vespalib::string dense_tensor_type("tensor(x[2])"); +const vespalib::string mixed_tensor_type("tensor(a{},x[2])"); +const vespalib::string sparse_tensor_type("tensor(a{})"); + +TensorSpec no_subspace(sparse_tensor_type); +TensorSpec subspace_a = TensorSpec::from_expr("tensor(a{}):{{a:\"a\"}:1}"); +TensorSpec subspace_b = TensorSpec::from_expr("tensor(a{}):{{a:\"b\"}:1}"); + +TensorSpec doc_tensor = TensorSpec::from_expr("tensor(a{},x[2]):{{a:\"a\",x:0}:3,{a:\"a\",x:1}:10,{a:\"b\",x:0}:5,{a:\"b\",x:1}:10}"); + +using RankFixture = DistanceClosenessFixture; + +TensorSpec get_spec(RankFixture& f, uint32_t docid) { + return spec_from_value(f.getObject(docid).get()); +} + +struct TestParam +{ + vespalib::string _name; + bool _direct_tensor; + TestParam(vespalib::string name, bool direct_tensor) + : _name(std::move(name)), + _direct_tensor(direct_tensor) + { + } + ~TestParam(); +}; + +TestParam::~TestParam() = default; + +std::ostream& operator<<(std::ostream& os, const TestParam param) +{ + os << param._name; + return os; +} + +void +assert_setup(vespalib::string field_name, + bool exp_setup_result, + std::optional<vespalib::string> attr_type_spec, + std::optional<vespalib::string> label) +{ + vespalib::asciistream feature_name; + std::vector<vespalib::string> setup_args; + ClosestBlueprint f1; + IndexEnvironmentFixture f2; + DummyDependencyHandler deps(f1); + setup_args.emplace_back(field_name); + feature_name << f1.getBaseName() << "(" << field_name; + if (label.has_value()) { + feature_name << "," << label.value(); + setup_args.emplace_back(label.value()); + } + feature_name << ")"; + f1.setName(feature_name.str()); + if (attr_type_spec.has_value()) { + search::fef::indexproperties::type::Attribute::set(f2.indexEnv.getProperties(), field_name, attr_type_spec.value()); + } + EXPECT_EQ(exp_setup_result, static_cast<Blueprint&>(f1).setup(f2.indexEnv, setup_args)); +} + +class ClosestTest : public ::testing::TestWithParam<TestParam> +{ +protected: + ClosestTest(); + ~ClosestTest(); + bool direct_tensor() const noexcept { return GetParam()._direct_tensor; } + void assert_closest(const Labels& labels, const vespalib::string& feature_name, const vespalib::string& query_tensor, const TensorSpec& exp_spec); + void assert_closest(const Labels& labels, const vespalib::string& feature_name, const std::vector<TensorSpec>& exp_specs); +}; + +ClosestTest::ClosestTest() + : testing::TestWithParam<TestParam>() +{ +} + +ClosestTest::~ClosestTest() = default; + +void +ClosestTest::assert_closest(const Labels& labels, const vespalib::string& feature_name, const vespalib::string& query_tensor, const TensorSpec& exp_spec) +{ + RankFixture f(mixed_tensor_type, direct_tensor(), 0, 1, labels, feature_name, + dense_tensor_type + ":" + query_tensor); + ASSERT_FALSE(f.failed()); + SCOPED_TRACE(query_tensor); + f.set_attribute_tensor(9, doc_tensor); + EXPECT_EQ(exp_spec, get_spec(f, 9)); +} + +void +ClosestTest::assert_closest(const Labels& labels, const vespalib::string& feature_name, const std::vector<TensorSpec>& exp_specs) +{ + assert_closest(labels, feature_name, "[9,10]", exp_specs[0]); + assert_closest(labels, feature_name, "[1,10]", exp_specs[1]); +} + +INSTANTIATE_TEST_SUITE_P(ClosestMultiTest, + ClosestTest, + testing::Values(TestParam("Serialized", false), + TestParam("Direct", true)), + testing::PrintToStringParamName()); + +TEST(ClosestTest, require_that_blueprint_can_be_created_from_factory) +{ + BlueprintFactoryFixture f; + auto bp = f.factory.createBlueprint("closest"); + EXPECT_TRUE(bp); + EXPECT_TRUE(dynamic_cast<ClosestBlueprint*>(bp.get()) != 0); +} + +TEST(ClosestTest, require_that_no_features_are_dumped) +{ + ClosestBlueprint f1; + IndexEnvironmentFixture f2; + FeatureDumpFixture f3; + f1.visitDumpFeatures(f2.indexEnv, f3); +} + +TEST(ClosestTest, require_that_setup_fails_for_unknown_field) +{ + assert_setup("random_field", false, mixed_tensor_type, std::nullopt); +} + +TEST(ClosestTest, require_that_setup_fails_if_field_type_is_not_attribute) +{ + assert_setup("ibar", false, sparse_tensor_type, std::nullopt); +} + +TEST(ClosestTest, require_that_setup_fails_if_field_data_type_is_not_tensor) +{ + assert_setup("foo", false, sparse_tensor_type, std::nullopt); +} + +TEST(ClosestTest, require_that_setup_can_be_done_on_random_label) +{ + assert_setup("bar", true, mixed_tensor_type, "random_label"); +} + +TEST(ClosestTest, require_that_setup_fails_if_tensor_type_is_missing) +{ + assert_setup("bar", false, std::nullopt, std::nullopt); +} + +TEST(ClosestTest, require_that_setup_fails_if_tensor_type_is_dense) +{ + assert_setup("bar", false, dense_tensor_type, std::nullopt); +} + +TEST(ClosestTest, require_that_setup_fails_if_tensor_type_is_sparse) +{ + assert_setup("bar", false, sparse_tensor_type, std::nullopt); +} + +TEST_P(ClosestTest, require_that_no_label_gives_empty_result) +{ + NoLabel f; + assert_closest(f, field_and_label_feature_name, {no_subspace, no_subspace}); +} + +TEST_P(ClosestTest, require_that_unrelated_label_gives_empty_result) +{ + SingleLabel f("unrelated", 1); + assert_closest(f, field_and_label_feature_name, {no_subspace, no_subspace}); +} + +TEST_P(ClosestTest, closest_using_field_setup) +{ + NoLabel f; + assert_closest(f, field_feature_name, {subspace_b, subspace_a}); +} + +TEST_P(ClosestTest, closest_using_field_and_label_setup) +{ + SingleLabel f("nns", 1); + assert_closest(f, field_and_label_feature_name, {subspace_b, subspace_a}); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/features/nns_closeness/nns_closeness_test.cpp b/searchlib/src/tests/features/nns_closeness/nns_closeness_test.cpp index e67370d48f6..8cb060c08e4 100644 --- a/searchlib/src/tests/features/nns_closeness/nns_closeness_test.cpp +++ b/searchlib/src/tests/features/nns_closeness/nns_closeness_test.cpp @@ -7,7 +7,7 @@ #include <vespa/searchlib/fef/test/labels.h> #include <vespa/searchlib/test/features/distance_closeness_fixture.h> #include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/util/stringfmt.h> using search::feature_t; @@ -23,60 +23,97 @@ const vespalib::string fieldFeatureName("closeness(bar)"); using RankFixture = DistanceClosenessFixture; -TEST_F("require that blueprint can be created from factory", BlueprintFactoryFixture) { +TEST(NnsClosenessTest, require_that_blueprint_can_be_created_from_factory) +{ + BlueprintFactoryFixture f; Blueprint::SP bp = f.factory.createBlueprint("closeness"); EXPECT_TRUE(bp.get() != 0); EXPECT_TRUE(dynamic_cast<ClosenessBlueprint*>(bp.get()) != 0); } -TEST_FFF("require that no features are dumped", ClosenessBlueprint, IndexEnvironmentFixture, FeatureDumpFixture) { +TEST(NnsClosenessTest, require_that_no_features_are_dumped) +{ + ClosenessBlueprint f1; + IndexEnvironmentFixture f2; + FeatureDumpFixture f3; f1.visitDumpFeatures(f2.indexEnv, f3); } -TEST_FF("require that setup can be done on random label", ClosenessBlueprint, IndexEnvironmentFixture) { +TEST(NnsClosenessTest, require_that_setup_can_be_done_on_random_label) +{ + ClosenessBlueprint f1; + IndexEnvironmentFixture f2; DummyDependencyHandler deps(f1); f1.setName(vespalib::make_string("%s(label,random_label)", f1.getBaseName().c_str())); EXPECT_TRUE(static_cast<Blueprint&>(f1).setup(f2.indexEnv, std::vector<vespalib::string>{"label", "random_label"})); } -TEST_FF("require that no label gives 0 closeness", NoLabel(), RankFixture(2, 2, f1, labelFeatureName)) { - EXPECT_EQUAL(0.0, f2.getScore(10)); +TEST(NnsClosenessTest, require_that_no_label_gives_0_closeness) +{ + NoLabel f1; + RankFixture f2(2, 2, f1, labelFeatureName); + ASSERT_FALSE(f2.failed()); + EXPECT_EQ(0.0, f2.getScore(10)); } -TEST_FF("require that unrelated label gives 0 closeness", SingleLabel("unrelated", 1), RankFixture(2, 2, f1, labelFeatureName)) { - EXPECT_EQUAL(0.0, f2.getScore(10)); +TEST(NnsClosenessTest, require_that_unrelated_label_gives_0_closeness) +{ + SingleLabel f1("unrelated", 1); + RankFixture f2(2, 2, f1, labelFeatureName); + ASSERT_FALSE(f2.failed()); + EXPECT_EQ(0.0, f2.getScore(10)); } -TEST_FF("require that labeled item raw score can be obtained", SingleLabel("nns", 1), RankFixture(2, 2, f1, labelFeatureName)) { +TEST(NnsClosenessTest, require_that_labeled_item_raw_score_can_be_obtained) +{ + SingleLabel f1("nns", 1); + RankFixture f2(2, 2, f1, labelFeatureName); + ASSERT_FALSE(f2.failed()); f2.setFooScore(0, 10, 5.0); - EXPECT_EQUAL(1/(1+5.0), f2.getScore(10)); + EXPECT_EQ(1/(1+5.0), f2.getScore(10)); } -TEST_FF("require that field raw score can be obtained", NoLabel(), RankFixture(2, 2, f1, fieldFeatureName)) { +TEST(NnsClosenessTest, require_that_field_raw_score_can_be_obtained) +{ + NoLabel f1; + RankFixture f2(2, 2, f1, fieldFeatureName); + ASSERT_FALSE(f2.failed()); f2.setBarScore(0, 10, 5.0); - EXPECT_EQUAL(1/(1+5.0), f2.getScore(10)); + EXPECT_EQ(1/(1+5.0), f2.getScore(10)); } -TEST_FF("require that other raw scores are ignored", SingleLabel("nns", 2), RankFixture(2, 2, f1, labelFeatureName)) { +TEST(NnsClosenessTest, require_that_other_raw_scores_are_ignored) +{ + SingleLabel f1("nns", 2); + RankFixture f2(2, 2, f1, labelFeatureName); + ASSERT_FALSE(f2.failed()); f2.setFooScore(0, 10, 1.0); f2.setFooScore(1, 10, 2.0); f2.setBarScore(0, 10, 5.0); f2.setBarScore(1, 10, 6.0); - EXPECT_EQUAL(1/(1+2.0), f2.getScore(10)); + EXPECT_EQ(1/(1+2.0), f2.getScore(10)); } -TEST_FF("require that the correct raw score is used", NoLabel(), RankFixture(2, 2, f1, fieldFeatureName)) { +TEST(NnsClosenessTest, require_that_the_correct_raw_score_is_used) +{ + NoLabel f1; + RankFixture f2(2, 2, f1, fieldFeatureName); + ASSERT_FALSE(f2.failed()); f2.setFooScore(0, 10, 3.0); f2.setFooScore(1, 10, 4.0); f2.setBarScore(0, 10, 8.0); f2.setBarScore(1, 10, 7.0); - EXPECT_EQUAL(1/(1+7.0), f2.getScore(10)); + EXPECT_EQ(1/(1+7.0), f2.getScore(10)); } -TEST_FF("require that stale data is ignored", SingleLabel("nns", 2), RankFixture(2, 2, f1, labelFeatureName)) { +TEST(NnsClosenessTest, require_that_stale_data_is_ignored) +{ + SingleLabel f1("nns", 2); + RankFixture f2(2, 2, f1, labelFeatureName); + ASSERT_FALSE(f2.failed()); f2.setFooScore(0, 10, 1.0); f2.setFooScore(1, 5, 2.0); - EXPECT_EQUAL(0, f2.getScore(10)); + EXPECT_EQ(0, f2.getScore(10)); } void @@ -88,21 +125,25 @@ expect_raw_score_calculated_on_the_fly(RankFixture& f) // For docids 9 and 10 the raw score is calculated on the fly // using a distance calculator over the attribute and query tensors. - EXPECT_EQUAL(1/(1+13.0), f.getScore(8)); - EXPECT_EQUAL(1/(1+(5.0-3.0)), f.getScore(9)); - EXPECT_EQUAL(1/(1+(7.0-3.0)), f.getScore(10)); + EXPECT_EQ(1/(1+13.0), f.getScore(8)); + EXPECT_EQ(1/(1+(5.0-3.0)), f.getScore(9)); + EXPECT_EQ(1/(1+(7.0-3.0)), f.getScore(10)); } -TEST_FF("raw score is calculated on the fly (using field setup)", - NoLabel(), RankFixture(0, 1, f1, fieldFeatureName, "tensor(x[2]):[3,11]")) +TEST(NnsClosenessTest, raw_score_is_calculated_on_the_fly_using_field_setup) { + NoLabel f1; + RankFixture f2(0, 1, f1, fieldFeatureName, "tensor(x[2]):[3,11]"); + ASSERT_FALSE(f2.failed()); expect_raw_score_calculated_on_the_fly(f2); } -TEST_FF("raw score is calculated on the fly (using label setup)", - SingleLabel("nns", 1), RankFixture(0, 1, f1, labelFeatureName, "tensor(x[2]):[3,11]")) +TEST(NnsClosenessTest, raw_score_is_calculated_on_the_fly_using_label_setup) { + SingleLabel f1("nns", 1); + RankFixture f2(0, 1, f1, labelFeatureName, "tensor(x[2]):[3,11]"); + ASSERT_FALSE(f2.failed()); expect_raw_score_calculated_on_the_fly(f2); } -TEST_MAIN() { TEST_RUN_ALL(); } +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/features/nns_distance/nns_distance_test.cpp b/searchlib/src/tests/features/nns_distance/nns_distance_test.cpp index fff4c9f1c0e..acc67803886 100644 --- a/searchlib/src/tests/features/nns_distance/nns_distance_test.cpp +++ b/searchlib/src/tests/features/nns_distance/nns_distance_test.cpp @@ -6,7 +6,7 @@ #include <vespa/searchlib/fef/test/labels.h> #include <vespa/searchlib/test/features/distance_closeness_fixture.h> #include <vespa/vespalib/stllike/asciistream.h> -#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/util/stringfmt.h> using search::feature_t; @@ -22,66 +22,106 @@ const vespalib::string fieldFeatureName("distance(bar)"); using RankFixture = DistanceClosenessFixture; -TEST_F("require that blueprint can be created from factory", BlueprintFactoryFixture) { +TEST(NnsDistanceTest, require_that_blueprint_can_be_created_from_factory) +{ + BlueprintFactoryFixture f; Blueprint::SP bp = f.factory.createBlueprint("distance"); EXPECT_TRUE(bp.get() != 0); EXPECT_TRUE(dynamic_cast<DistanceBlueprint*>(bp.get()) != 0); } -TEST_FFF("require that no features are dumped", DistanceBlueprint, IndexEnvironmentFixture, FeatureDumpFixture) { +TEST(NnsDistanceTest, require_that_no_features_are_dumped) +{ + DistanceBlueprint f1; + IndexEnvironmentFixture f2; + FeatureDumpFixture f3; f1.visitDumpFeatures(f2.indexEnv, f3); } -TEST_FF("require that setup can be done on random label", DistanceBlueprint, IndexEnvironmentFixture) { +TEST(NnsDistanceTest, require_that_setup_can_be_done_on_random_label) +{ + DistanceBlueprint f1; + IndexEnvironmentFixture f2; DummyDependencyHandler deps(f1); f1.setName(vespalib::make_string("%s(label,random_label)", f1.getBaseName().c_str())); EXPECT_TRUE(static_cast<Blueprint&>(f1).setup(f2.indexEnv, std::vector<vespalib::string>{"label", "random_label"})); } -TEST_FF("require that setup with unknown field fails", DistanceBlueprint, IndexEnvironmentFixture) { +TEST(NnsDistanceTest, require_that_setup_with_unknown_field_fails) +{ + DistanceBlueprint f1; + IndexEnvironmentFixture f2; DummyDependencyHandler deps(f1); f1.setName(vespalib::make_string("%s(field,random_fieldname)", f1.getBaseName().c_str())); EXPECT_FALSE(static_cast<Blueprint&>(f1).setup(f2.indexEnv, std::vector<vespalib::string>{"field", "random_fieldname"})); } -TEST_FF("require that no label gives max-double distance", NoLabel(), RankFixture(2, 2, f1, labelFeatureName)) { - EXPECT_EQUAL(std::numeric_limits<feature_t>::max(), f2.getScore(10)); +TEST(NnsDistanceTest, require_that_no_label_gives_max_double_distance) +{ + NoLabel f1; + RankFixture f2(2, 2, f1, labelFeatureName); + ASSERT_FALSE(f2.failed()); + EXPECT_EQ(std::numeric_limits<feature_t>::max(), f2.getScore(10)); } -TEST_FF("require that unrelated label gives max-double distance", SingleLabel("unrelated", 1), RankFixture(2, 2, f1, labelFeatureName)) { - EXPECT_EQUAL(std::numeric_limits<feature_t>::max(), f2.getScore(10)); +TEST(NnsDistanceTest, require_that_unrelated_label_gives_max_double_distance) +{ + SingleLabel f1("unrelated", 1); + RankFixture f2(2, 2, f1, labelFeatureName); + ASSERT_FALSE(f2.failed()); + EXPECT_EQ(std::numeric_limits<feature_t>::max(), f2.getScore(10)); } -TEST_FF("require that labeled item raw score can be obtained", SingleLabel("nns", 1), RankFixture(2, 2, f1, labelFeatureName)) { +TEST(NnsDistanceTest, require_that_labeled_item_raw_score_can_be_obtained) +{ + SingleLabel f1("nns", 1); + RankFixture f2(2, 2, f1, labelFeatureName); + ASSERT_FALSE(f2.failed()); f2.setFooScore(0, 10, 5.0); - EXPECT_EQUAL(5.0, f2.getScore(10)); + EXPECT_EQ(5.0, f2.getScore(10)); } -TEST_FF("require that field raw score can be obtained", NoLabel(), RankFixture(2, 2, f1, fieldFeatureName)) { +TEST(NnsDistanceTest, require_that_field_raw_score_can_be_obtained) +{ + NoLabel f1; + RankFixture f2(2, 2, f1, fieldFeatureName); + ASSERT_FALSE(f2.failed()); f2.setBarScore(0, 10, 5.0); - EXPECT_EQUAL(5.0, f2.getScore(10)); + EXPECT_EQ(5.0, f2.getScore(10)); } -TEST_FF("require that other raw scores are ignored", SingleLabel("nns", 2), RankFixture(2, 2, f1, labelFeatureName)) { +TEST(NnsDistanceTest, require_that_other_raw_scores_are_ignored) +{ + SingleLabel f1("nns", 2); + RankFixture f2(2, 2, f1, labelFeatureName); + ASSERT_FALSE(f2.failed()); f2.setFooScore(0, 10, 1.0); f2.setFooScore(1, 10, 2.0); f2.setBarScore(0, 10, 5.0); f2.setBarScore(1, 10, 6.0); - EXPECT_EQUAL(2.0, f2.getScore(10)); + EXPECT_EQ(2.0, f2.getScore(10)); } -TEST_FF("require that the correct raw score is used", NoLabel(), RankFixture(2, 2, f1, fieldFeatureName)) { +TEST(NnsDistanceTest, require_that_the_correct_raw_score_is_used) +{ + NoLabel f1; + RankFixture f2(2, 2, f1, fieldFeatureName); + ASSERT_FALSE(f2.failed()); f2.setFooScore(0, 10, 3.0); f2.setFooScore(1, 10, 4.0); f2.setBarScore(0, 10, 8.0); f2.setBarScore(1, 10, 7.0); - EXPECT_EQUAL(7.0, f2.getScore(10)); + EXPECT_EQ(7.0, f2.getScore(10)); } -TEST_FF("require that stale data is ignored", SingleLabel("nns", 2), RankFixture(2, 2, f1, labelFeatureName)) { +TEST(NnsDistanceTest, require_that_stale_data_is_ignored) +{ + SingleLabel f1("nns", 2); + RankFixture f2(2, 2, f1, labelFeatureName); + ASSERT_FALSE(f2.failed()); f2.setFooScore(0, 10, 1.0); f2.setFooScore(1, 5, 2.0); - EXPECT_EQUAL(std::numeric_limits<feature_t>::max(), f2.getScore(10)); + EXPECT_EQ(std::numeric_limits<feature_t>::max(), f2.getScore(10)); } void @@ -93,21 +133,25 @@ expect_raw_score_calculated_on_the_fly(RankFixture& f) // For docids 9 and 10 the raw score is calculated on the fly // using a distance calculator over the attribute and query tensors. - EXPECT_EQUAL(13.0, f.getScore(8)); - EXPECT_EQUAL((5-3), f.getScore(9)); - EXPECT_EQUAL((7-3), f.getScore(10)); + EXPECT_EQ(13.0, f.getScore(8)); + EXPECT_EQ((5-3), f.getScore(9)); + EXPECT_EQ((7-3), f.getScore(10)); } -TEST_FF("raw score is calculated on the fly (using field setup)", - NoLabel(), RankFixture(0, 1, f1, fieldFeatureName, "tensor(x[2]):[3,11]")) +TEST(NnsDistanceTest, raw_score_is_calculated_on_the_fly_using_field_setup) { + NoLabel f1; + RankFixture f2(0, 1, f1, fieldFeatureName, "tensor(x[2]):[3,11]"); + ASSERT_FALSE(f2.failed()); expect_raw_score_calculated_on_the_fly(f2); } -TEST_FF("raw score is calculated on the fly (using label setup)", - SingleLabel("nns", 1), RankFixture(0, 1, f1, labelFeatureName, "tensor(x[2]):[3,11]")) +TEST(NnsDistanceTest, raw_score_is_calculated_on_the_fly_using_label_setup) { + SingleLabel f1("nns", 1); + RankFixture f2(0, 1, f1, labelFeatureName, "tensor(x[2]):[3,11]"); + ASSERT_FALSE(f2.failed()); expect_raw_score_calculated_on_the_fly(f2); } -TEST_MAIN() { TEST_RUN_ALL(); } +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/tests/grouping/grouping_test.cpp b/searchlib/src/tests/grouping/grouping_test.cpp index adf4320ede3..5f227ffc68a 100644 --- a/searchlib/src/tests/grouping/grouping_test.cpp +++ b/searchlib/src/tests/grouping/grouping_test.cpp @@ -29,8 +29,6 @@ namespace { const int64_t undefinedInteger = getUndefined<int64_t>(); -} - //----------------------------------------------------------------------------- template<typename A, typename T> @@ -145,75 +143,30 @@ public: }; AggregationContext::AggregationContext() : _attrMan(), _result(), _attrCtx(_attrMan.createContext()) {} -AggregationContext::~AggregationContext() {} +AggregationContext::~AggregationContext() = default; -//----------------------------------------------------------------------------- +#define MU std::make_unique -class Test : public TestApp +class CheckAttributeReferences : public vespalib::ObjectOperation, public vespalib::ObjectPredicate { public: - bool testAggregation(AggregationContext &ctx, - const Grouping &request, - const Group &expect); - bool testMerge(const Grouping &a, const Grouping &b, - const Group &expect); - bool testMerge(const Grouping &a, const Grouping &b, const Grouping &c, - const Group &expect); - bool testPrune(const Grouping &a, const Grouping &b, - const Group &expect); - bool testPartialMerge(const Grouping &a, const Grouping &b, - const Group &expect); - void testAggregationSimple(); - void testAggregationLevels(); - void testAggregationMaxGroups(); - void testAggregationGroupOrder(); - void testAggregationGroupRank(); - void testAggregationGroupCapping(); - void testMergeSimpleSum(); - void testMergeLevels(); - void testMergeGroups(); - void testMergeTrees(); - void testPruneSimple(); - void testPruneComplex(); - void testPartialMerging(); - void testCount(); - void testTopN(); - void testFS4HitCollection(); - bool checkBucket(const NumericResultNode &width, const NumericResultNode &value, const BucketResultNode &bucket); - bool checkHits(const Grouping &g, uint32_t first, uint32_t last, uint32_t cnt); - void testFixedWidthBuckets(); - void testThatNanIsConverted(); - void testNanSorting(); - void testAttributeMapLookup(); - int Main() override; + CheckAttributeReferences() : _numrefs(0) { } + int _numrefs; private: - void testAggregationSimple(AggregationContext & ctx, const AggregationResult & aggr, const ResultNode & ir, const vespalib::string &name); - void testAggregationSimpleSum(AggregationContext & ctx, const AggregationResult & aggr, const ResultNode & ir, const ResultNode & fr, const ResultNode & sr); - class CheckAttributeReferences : public vespalib::ObjectOperation, public vespalib::ObjectPredicate - { - public: - CheckAttributeReferences() : _numrefs(0) { } - int _numrefs; - private: - virtual void execute(vespalib::Identifiable &obj) override { - if (static_cast<AttributeNode &>(obj).getAttribute() != NULL) { - _numrefs++; - } + void execute(vespalib::Identifiable &obj) override { + if (static_cast<AttributeNode &>(obj).getAttribute() != nullptr) { + _numrefs++; } - virtual bool check(const vespalib::Identifiable &obj) const override { return obj.inherits(AttributeNode::classId); } - }; + } + bool check(const vespalib::Identifiable &obj) const override { return obj.inherits(AttributeNode::classId); } }; -//----------------------------------------------------------------------------- - /** * Run the given grouping request and verify that the resulting group * tree matches the expected value. **/ bool -Test::testAggregation(AggregationContext &ctx, - const Grouping &request, - const Group &expect) +testAggregation(AggregationContext &ctx, const Grouping &request, const Group &expect) { Grouping tmp = request; // create local copy ctx.setup(tmp); @@ -229,13 +182,45 @@ Test::testAggregation(AggregationContext &ctx, return ok; } +std::unique_ptr<AggregationResult> +prepareAggr(const AggregationResult & aggr, ExpressionNode::UP expr) { + std::unique_ptr<AggregationResult> clone(aggr.clone()); + clone->setExpression(std::move(expr)); + return clone; +} + +std::unique_ptr<ExpressionNode> +prepareAggr(const AggregationResult & aggr, ExpressionNode::UP expr, const ResultNode & r) { + auto prepared = prepareAggr(aggr, std::move(expr)); + prepared->setResult(r); + return prepared; +} + +void +testAggregationSimpleSum(AggregationContext & ctx, const AggregationResult & aggr, const ResultNode & ir, const ResultNode & fr, const ResultNode & sr) +{ + ExpressionNode::CP clone(aggr); + Grouping request; + request.setRoot(Group().addResult(prepareAggr(aggr, MU<AttributeNode>("int"))) + .addResult(prepareAggr(aggr, MU<AttributeNode>("float"))) + .addResult(prepareAggr(aggr, MU<AttributeNode>("string")))); + + Group expect; + expect.addResult(prepareAggr(aggr, MU<AttributeNode>("int"), ir)) + .addResult(prepareAggr(aggr, MU<AttributeNode>("float"), fr)) + .addResult(prepareAggr(aggr, MU<AttributeNode>("string"), sr)); + + EXPECT_TRUE(testAggregation(ctx, request, expect)); +} + +//----------------------------------------------------------------------------- + /** * Merge the given grouping requests and verify that the resulting * group tree matches the expected value. **/ bool -Test::testMerge(const Grouping &a, const Grouping &b, - const Group &expect) +testMerge(const Grouping &a, const Grouping &b, const Group &expect) { Grouping tmp = a; // create local copy Grouping tmpB = b; @@ -250,8 +235,7 @@ Test::testMerge(const Grouping &a, const Grouping &b, * group tree matches the expected value. **/ bool -Test::testPrune(const Grouping &a, const Grouping &b, - const Group &expect) +testPrune(const Grouping &a, const Grouping &b, const Group &expect) { Grouping tmp = a; // create local copy tmp.prune(b); @@ -267,8 +251,7 @@ Test::testPrune(const Grouping &a, const Grouping &b, * partial request is correct. **/ bool -Test::testPartialMerge(const Grouping &a, const Grouping &b, - const Group &expect) +testPartialMerge(const Grouping &a, const Grouping &b, const Group &expect) { Grouping tmp = a; // create local copy tmp.mergePartial(b); @@ -284,9 +267,7 @@ Test::testPartialMerge(const Grouping &a, const Grouping &b, * group tree matches the expected value. **/ bool -Test::testMerge(const Grouping &a, const Grouping &b, const Grouping &c, - const Group &expect) -{ +testMerge(const Grouping &a, const Grouping &b, const Grouping &c, const Group &expect) { Grouping tmp = a; // create local copy Grouping tmpB = b; // create local copy Grouping tmpC = c; // create local copy @@ -297,45 +278,8 @@ Test::testMerge(const Grouping &a, const Grouping &b, const Grouping &c, return EXPECT_EQUAL(tmp.getRoot().asString(), expect.asString()); } -//----------------------------------------------------------------------------- - -/** - * Test collecting the sum of the values from a single attribute - * vector directly into the root node. Consider this a smoke test. - **/ void -Test::testAggregationSimple() -{ - EXPECT_EQUAL(64u, sizeof(Group)); - AggregationContext ctx; - ctx.result().add(0).add(1).add(2); - ctx.add(IntAttrBuilder("int").add(3).add(7).add(15).sp()); - ctx.add(FloatAttrBuilder("float").add(3).add(7).add(15).sp()); - ctx.add(StringAttrBuilder("string").add("3").add("7").add("15").sp()); - - TEST_DO(testAggregationSimpleSum(ctx, SumAggregationResult(), Int64ResultNode(25), FloatResultNode(25), StringResultNode("25"))); - TEST_DO(testAggregationSimpleSum(ctx, MinAggregationResult(), Int64ResultNode(3), FloatResultNode(3), StringResultNode("15"))); - TEST_DO(testAggregationSimpleSum(ctx, MaxAggregationResult(), Int64ResultNode(15), FloatResultNode(15), StringResultNode("7"))); -} - -#define MU std::make_unique -using EUP = ExpressionNode::UP; - -std::unique_ptr<AggregationResult> -prepareAggr(const AggregationResult & aggr, ExpressionNode::UP expr) { - std::unique_ptr<AggregationResult> clone(aggr.clone()); - clone->setExpression(std::move(expr)); - return clone; -} - -ExpressionNode::UP -prepareAggr(const AggregationResult & aggr, ExpressionNode::UP expr, const ResultNode & r) { - auto prepared = prepareAggr(aggr, std::move(expr)); - prepared->setResult(r); - return prepared; -} - -void Test::testAggregationSimple(AggregationContext & ctx, const AggregationResult & aggr, const ResultNode & ir, const vespalib::string &name) +testAggregationSimple(AggregationContext & ctx, const AggregationResult & aggr, const ResultNode & ir, const vespalib::string &name) { ExpressionNode::CP clone(aggr); Grouping request; @@ -346,22 +290,6 @@ void Test::testAggregationSimple(AggregationContext & ctx, const AggregationResu EXPECT_TRUE(testAggregation(ctx, request, expect)); } -void Test::testAggregationSimpleSum(AggregationContext & ctx, const AggregationResult & aggr, const ResultNode & ir, const ResultNode & fr, const ResultNode & sr) -{ - ExpressionNode::CP clone(aggr); - Grouping request; - request.setRoot(Group().addResult(prepareAggr(aggr, MU<AttributeNode>("int"))) - .addResult(prepareAggr(aggr, MU<AttributeNode>("float"))) - .addResult(prepareAggr(aggr, MU<AttributeNode>("string")))); - - Group expect; - expect.addResult(prepareAggr(aggr, MU<AttributeNode>("int"), ir)) - .addResult(prepareAggr(aggr, MU<AttributeNode>("float"), fr)) - .addResult(prepareAggr(aggr, MU<AttributeNode>("string"), sr)); - - EXPECT_TRUE(testAggregation(ctx, request, expect)); -} - GroupingLevel createGL(ExpressionNode::UP expr, ExpressionNode::UP resultExpr) { GroupingLevel l; @@ -393,13 +321,87 @@ createGL(size_t maxGroups, ExpressionNode::UP expr, ExpressionNode::UP result) { l.addResult(SumAggregationResult().setExpression(std::move(result))); return l; } + +template<typename T> +ExpressionNode::UP +createAggr(ExpressionNode::UP e) { + std::unique_ptr<T> aggr = MU<T>(); + aggr->setExpression(std::move(e)); + return aggr; +} + +template<typename T> +ExpressionNode::UP +createAggr(SingleResultNode::UP r, ExpressionNode::UP e) { + std::unique_ptr<T> aggr = MU<T>(std::move(r)); + aggr->setExpression(std::move(e)); + return aggr; +} + +template<typename T> +ExpressionNode::UP +createNumAggr(NumericResultNode::UP r, ExpressionNode::UP e) { + std::unique_ptr<T> aggr = MU<T>(std::move(r)); + aggr->setExpression(std::move(e)); + return aggr; +} + +bool +checkHits(const Grouping &g, uint32_t first, uint32_t last, uint32_t cnt) +{ + CountFS4Hits pop; + Grouping tmp = g; + tmp.setFirstLevel(first).setLastLevel(last).select(pop, pop); + return EXPECT_EQUAL(pop.getHitCount(), cnt); +} +bool +checkBucket(const NumericResultNode &width, const NumericResultNode &value, const BucketResultNode &bucket) +{ + AggregationContext ctx; + ctx.result().add(0); + if (value.getClass().inherits(IntegerResultNode::classId)) { + ctx.add(IntAttrBuilder("attr").add(value.getInteger()).sp()); + } else if (value.getClass().inherits(FloatResultNode::classId)) { + ctx.add(FloatAttrBuilder("attr").add(value.getFloat()).sp()); + } else { + return EXPECT_TRUE(false); + } + std::unique_ptr<FixedWidthBucketFunctionNode> fixed = MU<FixedWidthBucketFunctionNode>(MU<AttributeNode>("attr")); + fixed->setWidth(width); + Grouping request = Grouping().addLevel(createGL(std::move(fixed))); + Group expect = Group().addChild(Group().setId(bucket)); + return testAggregation(ctx, request, expect); +} +} + +TEST("Control size of objects") { + EXPECT_EQUAL(64u, sizeof(Group)); + EXPECT_EQUAL(40u, sizeof(Group::Value)); +} + +/** + * Test collecting the sum of the values from a single attribute + * vector directly into the root node. Consider this a smoke test. + **/ +TEST("testAggregationSimple") +{ + AggregationContext ctx; + ctx.result().add(0).add(1).add(2); + ctx.add(IntAttrBuilder("int").add(3).add(7).add(15).sp()); + ctx.add(FloatAttrBuilder("float").add(3).add(7).add(15).sp()); + ctx.add(StringAttrBuilder("string").add("3").add("7").add("15").sp()); + + TEST_DO(testAggregationSimpleSum(ctx, SumAggregationResult(), Int64ResultNode(25), FloatResultNode(25), StringResultNode("25"))); + TEST_DO(testAggregationSimpleSum(ctx, MinAggregationResult(), Int64ResultNode(3), FloatResultNode(3), StringResultNode("15"))); + TEST_DO(testAggregationSimpleSum(ctx, MaxAggregationResult(), Int64ResultNode(15), FloatResultNode(15), StringResultNode("7"))); +} + /** * Verify that the backend aggregation will classify and collect on * the appropriate levels, as indicated by the firstLevel and * lastLevel parameters. **/ -void -Test::testAggregationLevels() +TEST("testAggregationLevels") { AggregationContext ctx; ctx.add(IntAttrBuilder("attr0").add(10).add(10).sp()); @@ -516,8 +518,7 @@ Test::testAggregationLevels() * Verify that the aggregation step does not create more groups than * indicated by the maxgroups parameter. **/ -void -Test::testAggregationMaxGroups() +TEST("testAggregationMaxGroups") { AggregationContext ctx; ctx.add(IntAttrBuilder("attr").add(5).add(10).add(15).sp()); @@ -562,11 +563,7 @@ Test::testAggregationMaxGroups() } } -/** - * Verify that groups are sorted by group id - **/ -void -Test::testAggregationGroupOrder() +TEST("Verify that groups are sorted by group id") { AggregationContext ctx; ctx.add(IntAttrBuilder("attr").add(10).add(25).add(35).add(5).add(20).add(15).add(30).sp()); @@ -587,11 +584,7 @@ Test::testAggregationGroupOrder() EXPECT_TRUE(testAggregation(ctx, request, expect)); } -/** - * Verify that groups are tagged with the appropriate rank value. - **/ -void -Test::testAggregationGroupRank() +TEST("Verify that groups are tagged with the appropriate rank value") { AggregationContext ctx; ctx.add(IntAttrBuilder("attr") @@ -613,32 +606,7 @@ Test::testAggregationGroupRank() EXPECT_TRUE(testAggregation(ctx, request, expect)); } -template<typename T> -ExpressionNode::UP -createAggr(ExpressionNode::UP e) { - std::unique_ptr<T> aggr = MU<T>(); - aggr->setExpression(std::move(e)); - return aggr; -} - -template<typename T> -ExpressionNode::UP -createAggr(SingleResultNode::UP r, ExpressionNode::UP e) { - std::unique_ptr<T> aggr = MU<T>(std::move(r)); - aggr->setExpression(std::move(e)); - return aggr; -} - -template<typename T> -ExpressionNode::UP -createNumAggr(NumericResultNode::UP r, ExpressionNode::UP e) { - std::unique_ptr<T> aggr = MU<T>(std::move(r)); - aggr->setExpression(std::move(e)); - return aggr; -} - -void -Test::testAggregationGroupCapping() +TEST("testAggregationGroupCapping") { AggregationContext ctx; ctx.add(IntAttrBuilder("attr") @@ -754,8 +722,7 @@ Test::testAggregationGroupCapping() * that was collected directly into the root node. Consider this a * smoke test. **/ -void -Test::testMergeSimpleSum() +TEST("testMergeSimpleSum") { Grouping a = Grouping() .setRoot(Group() @@ -780,11 +747,7 @@ Test::testMergeSimpleSum() EXPECT_TRUE(testMerge(a, b, expect)); } -/** - * Verify that frozen levels are not touched during merge. - **/ -void -Test::testMergeLevels() +TEST("Verify that frozen levels are not touched during merge.") { Grouping request; request.addLevel(createGL(MU<AttributeNode>("c1"), MU<AttributeNode>("s1"))) @@ -963,8 +926,7 @@ Test::testMergeLevels() * maxGroups, that the remaining groups are the highest ranked ones, * and that they are sorted by group id. **/ -void -Test::testMergeGroups() +TEST("testMergeGroups") { Grouping request; request.addLevel(createGL(MU<AttributeNode>("attr"))); @@ -1025,8 +987,7 @@ Test::testMergeGroups() * Merge two relatively complex tree structures and verify that the * end result is as expected. **/ -void -Test::testMergeTrees() +TEST("testMergeTrees") { Grouping request; request.addLevel(createGL(3, MU<AttributeNode>("c1"), MU<AttributeNode>("s1"))) @@ -1290,8 +1251,7 @@ Test::testMergeTrees() EXPECT_TRUE(testMerge(request.unchain().setRoot(b), request.unchain().setRoot(a), expect)); } -void -Test::testPruneComplex() +TEST("testPruneComplex") { { // First level Group baseTree = Group() @@ -1430,8 +1390,7 @@ Test::testPruneComplex() * merged. The last level should not contain any children groups, and only empty * results. **/ -void -Test::testPartialMerging() +TEST("testPartialMerging") { Grouping baseRequest; baseRequest.addLevel(createGL(MU<AttributeNode>("c1"), MU<AttributeNode>("s1"))) @@ -1592,11 +1551,7 @@ Test::testPartialMerging() } } -/** - * Test that pruning a simple grouping tree works. - **/ -void -Test::testPruneSimple() +TEST("Test that pruning a simple grouping tree works.") { { Grouping request; @@ -1615,12 +1570,7 @@ Test::testPruneSimple() } } -/** - * Test that simple counting works as long as we use an expression - * that we init, calculate and ignore. - **/ -void -Test::testTopN() +TEST("Test that simple counting works as long as we use an expression that we init, calculate and ignore.") { AggregationContext ctx; ctx.result().add(0).add(1).add(2); @@ -1660,8 +1610,7 @@ Test::testTopN() * Test that simple counting works as long as we use an expression * that we init, calculate and ignore. **/ -void -Test::testCount() +TEST("testCount") { AggregationContext ctx; ctx.result().add(0).add(1).add(2); @@ -1676,19 +1625,7 @@ Test::testCount() EXPECT_TRUE(testAggregation(ctx, request, expect)); } -//----------------------------------------------------------------------------- - -bool -Test::checkHits(const Grouping &g, uint32_t first, uint32_t last, uint32_t cnt) -{ - CountFS4Hits pop; - Grouping tmp = g; - tmp.setFirstLevel(first).setLastLevel(last).select(pop, pop); - return EXPECT_EQUAL(pop.getHitCount(), cnt); -} - -void -Test::testFS4HitCollection() +TEST("testFS4HitCollection") { { // aggregation AggregationContext ctx; @@ -1803,27 +1740,7 @@ Test::testFS4HitCollection() } } -bool -Test::checkBucket(const NumericResultNode &width, const NumericResultNode &value, const BucketResultNode &bucket) -{ - AggregationContext ctx; - ctx.result().add(0); - if (value.getClass().inherits(IntegerResultNode::classId)) { - ctx.add(IntAttrBuilder("attr").add(value.getInteger()).sp()); - } else if (value.getClass().inherits(FloatResultNode::classId)) { - ctx.add(FloatAttrBuilder("attr").add(value.getFloat()).sp()); - } else { - return EXPECT_TRUE(false); - } - std::unique_ptr<FixedWidthBucketFunctionNode> fixed = MU<FixedWidthBucketFunctionNode>(MU<AttributeNode>("attr")); - fixed->setWidth(width); - Grouping request = Grouping().addLevel(createGL(std::move(fixed))); - Group expect = Group().addChild(Group().setId(bucket)); - return testAggregation(ctx, request, expect); -} - -void -Test::testFixedWidthBuckets() +TEST("testFixedWidthBuckets") { using Int = Int64ResultNode; using Float = FloatResultNode; @@ -1879,43 +1796,7 @@ Test::testFixedWidthBuckets() } } - -void -Test::testNanSorting() -{ - // Attempt at reproducing issue with segfault when setting NaN value. Not - // successful yet, so no point in running test. -#if 0 - double myNan = std::sqrt(-1); - EXPECT_TRUE(isnan(myNan)); - EXPECT_TRUE(myNan != myNan); - EXPECT_FALSE(myNan < myNan); - EXPECT_FALSE(myNan > myNan); - EXPECT_FALSE(myNan < 0.2); - EXPECT_FALSE(myNan > 0.2); - EXPECT_FALSE(0.2 < myNan); - EXPECT_FALSE(0.2 > myNan); - - vespalib::Timer timer; - std::vector<double> groups; - while (timer.elapsed() < 60s) { - std::vector<double> vec; - srand((unsigned int)count_us(timer.elapsed())); - size_t limit = 2345678; - size_t mod = rand() % limit; - for (size_t i = 0; i < limit; i++) { - if ((i % mod) == 0) - vec.push_back(myNan); - else - vec.push_back(1.0 * rand()); - } - } - std::sort(groups.begin(), groups.end()); -#endif -} - -void -Test::testThatNanIsConverted() +TEST("testThatNanIsConverted") { Group g; double myNan = std::sqrt(-1); @@ -1924,8 +1805,7 @@ Test::testThatNanIsConverted() ASSERT_EQUAL(g.getRank(), g.getRank()); } -void -Test::testAttributeMapLookup() +TEST("testAttributeMapLookup") { AggregationContext ctx; ctx.result().add(0).add(1); @@ -1946,40 +1826,4 @@ Test::testAttributeMapLookup() testAggregationSimple(ctx, MaxAggregationResult(), Int64ResultNode(100), "smap{attribute(key2)}.weight"); } -//----------------------------------------------------------------------------- - -struct RunDiff { ~RunDiff() { system("diff -u lhs.out rhs.out > diff.txt"); }}; - -//----------------------------------------------------------------------------- - -int -Test::Main() -{ - //RunDiff runDiff; - //(void) runDiff; - //TEST_DEBUG("lhs.out", "rhs.out"); - TEST_INIT("grouping_test"); - TEST_DO(testAggregationSimple()); - testAggregationLevels(); - testAggregationMaxGroups(); - testAggregationGroupOrder(); - testAggregationGroupRank(); - testAggregationGroupCapping(); - testMergeSimpleSum(); - testMergeLevels(); - testMergeGroups(); - testMergeTrees(); - testPruneSimple(); - testPruneComplex(); - testPartialMerging(); - testFS4HitCollection(); - testFixedWidthBuckets(); - testCount(); - testTopN(); - testThatNanIsConverted(); - testNanSorting(); - testAttributeMapLookup(); - TEST_DONE(); -} - -TEST_APPHOOK(Test); +TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp b/searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp index 564824031a6..f4dda88b6f0 100644 --- a/searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp +++ b/searchlib/src/tests/memoryindex/datastore/feature_store_test.cpp @@ -3,6 +3,7 @@ #include <vespa/searchlib/memoryindex/feature_store.h> #include <vespa/searchcommon/common/schema.h> #include <vespa/vespalib/gtest/gtest.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP("feature_store_test"); diff --git a/searchlib/src/tests/postinglistbm/stress_runner.cpp b/searchlib/src/tests/postinglistbm/stress_runner.cpp index 179e4f49ef4..3e3db3701c7 100644 --- a/searchlib/src/tests/postinglistbm/stress_runner.cpp +++ b/searchlib/src/tests/postinglistbm/stress_runner.cpp @@ -2,7 +2,6 @@ #include "stress_runner.h" -#include <vespa/fastos/thread.h> #include <vespa/searchlib/test/fakedata/fake_match_loop.h> #include <vespa/searchlib/test/fakedata/fakeposting.h> #include <vespa/searchlib/test/fakedata/fakeword.h> @@ -13,7 +12,7 @@ #include <condition_variable> #include <mutex> #include <vector> -#include <thread> +#include <vespa/vespalib/util/thread.h> #include <vespa/log/log.h> LOG_SETUP(".stress_runner"); @@ -43,7 +42,7 @@ private: uint32_t _stride; bool _unpack; - FastOS_ThreadPool *_threadPool; + vespalib::ThreadPool _threadPool; std::vector<StressWorkerUP> _workers; uint32_t _workersDone; @@ -88,7 +87,7 @@ public: double runWorkers(const std::string &postingFormat); }; -class StressWorker : public FastOS_Runnable { +class StressWorker : vespalib::Runnable { protected: StressMaster& _master; uint32_t _id; @@ -102,7 +101,7 @@ public: StressWorker(StressMaster& master, uint32_t id); virtual ~StressWorker(); - virtual void Run(FastOS_ThreadInterface* thisThread, void* arg) override; + virtual void run() override; }; class DirectStressWorker : public StressWorker { @@ -147,7 +146,7 @@ StressMaster::StressMaster(vespalib::Rand48 &rnd, _skipCommonPairsRate(skipCommonPairsRate), _stride(stride), _unpack(unpack), - _threadPool(nullptr), + _threadPool(), _workers(), _workersDone(0), _wordSet(wordSet), @@ -159,17 +158,12 @@ StressMaster::StressMaster(vespalib::Rand48 &rnd, _tasks() { LOG(info, "StressMaster::StressMaster()"); - - _threadPool = new FastOS_ThreadPool(400); } StressMaster::~StressMaster() { LOG(info, "StressMaster::~StressMaster()"); - - _threadPool->Close(); - delete _threadPool; - _threadPool = nullptr; + _threadPool.join(); _workers.clear(); dropPostings(); } @@ -329,7 +323,7 @@ StressMaster::runWorkers(const std::string &postingFormat) } for (auto& worker : _workers) { - _threadPool->NewThread(worker.get()); + _threadPool.start([obj = worker.get()](){obj->run();}); } { @@ -357,10 +351,8 @@ StressWorker::StressWorker(StressMaster& master, uint32_t id) StressWorker::~StressWorker() = default; void -StressWorker::Run(FastOS_ThreadInterface* thisThread, void* arg) +StressWorker::run() { - (void) thisThread; - (void) arg; LOG(debug, "StressWorker::Run(), id=%u", _id); bool unpack = _master.getUnpack(); diff --git a/searchlib/src/tests/ranksetup/verify_feature/verify_feature_test.cpp b/searchlib/src/tests/ranksetup/verify_feature/verify_feature_test.cpp index 50c089ce8dc..86097e8872a 100644 --- a/searchlib/src/tests/ranksetup/verify_feature/verify_feature_test.cpp +++ b/searchlib/src/tests/ranksetup/verify_feature/verify_feature_test.cpp @@ -110,7 +110,7 @@ TEST_F("verify too deep dependency graph", RankFixture) { " ... needed by rank feature chain(basic,253,4)\n" " ... needed by rank feature chain(basic,254,4)\n" " ... needed by rank feature chain(basic,255,4)\n" - " ... needed by rank feature chain(basic,256,4)\n"}}, + " ... needed by rank feature chain(basic,256,4)"}}, {regex, {Level::WARNING, "high stack usage: [0-9]+ bytes"}}, {equal, {Level::ERROR, "verification failed: rank feature chain(basic, 256, 4) (feature verification test)"}}})); } @@ -123,7 +123,7 @@ TEST_F("verify dependency cycle", RankFixture) { " ... needed by rank feature chain(cycle,1,2)\n" " ... needed by rank feature chain(cycle,2,2)\n" " ... needed by rank feature chain(cycle,3,2)\n" - " ... needed by rank feature chain(cycle,4,2)\n"}}, + " ... needed by rank feature chain(cycle,4,2)"}}, {equal, {Level::ERROR, "verification failed: rank feature chain(cycle, 4, 2) (feature verification test)"}}})); } diff --git a/searchlib/src/tests/sortspec/multilevelsort.cpp b/searchlib/src/tests/sortspec/multilevelsort.cpp index ec14f0c97e1..001903ff302 100644 --- a/searchlib/src/tests/sortspec/multilevelsort.cpp +++ b/searchlib/src/tests/sortspec/multilevelsort.cpp @@ -11,6 +11,7 @@ #include <vespa/vespalib/util/testclock.h> #include <vespa/vespalib/testkit/testapp.h> #include <type_traits> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP("multilevelsort_test"); diff --git a/searchlib/src/tests/tensor/distance_calculator/distance_calculator_test.cpp b/searchlib/src/tests/tensor/distance_calculator/distance_calculator_test.cpp index 5e556979254..ef4292ddbb4 100644 --- a/searchlib/src/tests/tensor/distance_calculator/distance_calculator_test.cpp +++ b/searchlib/src/tests/tensor/distance_calculator/distance_calculator_test.cpp @@ -18,6 +18,8 @@ using namespace vespalib::eval; using search::AttributeVector; +using OptSubspace = std::optional<uint32_t>; + std::unique_ptr<Value> make_tensor(const vespalib::string& expr) { return SimpleValue::from_spec(TensorSpec::from_expr(expr)); } @@ -49,6 +51,11 @@ public: auto calc = DistanceCalculator::make_with_validation(*attr, *qt); return calc->calc_raw_score(docid); } + OptSubspace calc_closest_subspace(uint32_t docid, const vespalib::string& query_tensor) { + auto qt = make_tensor(query_tensor); + auto calc = DistanceCalculator::make_with_validation(*attr, *qt); + return calc->calc_closest_subspace(attr->asTensorAttribute()->get_vectors(docid)); + } void make_calc_throws(const vespalib::string& query_tensor) { auto qt = make_tensor(query_tensor); DistanceCalculator::make_with_validation(*attr, *qt); @@ -63,9 +70,11 @@ TEST_F(DistanceCalculatorTest, calculation_over_dense_tensor_attribute) vespalib::string qt = "tensor(y[2]):[7,10]"; EXPECT_DOUBLE_EQ(16, calc_distance(1, qt)); EXPECT_DOUBLE_EQ(max_distance, calc_distance(2, qt)); + EXPECT_EQ(OptSubspace(0), calc_closest_subspace(1, qt)); EXPECT_DOUBLE_EQ(1.0/(1.0 + 4.0), calc_rawscore(1, qt)); EXPECT_DOUBLE_EQ(0.0, calc_rawscore(2, qt)); + EXPECT_EQ(OptSubspace(), calc_closest_subspace(2, qt)); } TEST_F(DistanceCalculatorTest, calculation_over_mixed_tensor_attribute) @@ -77,8 +86,12 @@ TEST_F(DistanceCalculatorTest, calculation_over_mixed_tensor_attribute) vespalib::string qt_2 = "tensor(y[2]):[1,10]"; EXPECT_DOUBLE_EQ(16, calc_distance(1, qt_1)); EXPECT_DOUBLE_EQ(4, calc_distance(1, qt_2)); + EXPECT_EQ(OptSubspace(1), calc_closest_subspace(1, qt_1)); + EXPECT_EQ(OptSubspace(0), calc_closest_subspace(1, qt_2)); EXPECT_DOUBLE_EQ(max_distance, calc_distance(2, qt_1)); EXPECT_DOUBLE_EQ(max_distance, calc_distance(3, qt_1)); + EXPECT_EQ(OptSubspace(), calc_closest_subspace(2, qt_1)); + EXPECT_EQ(OptSubspace(), calc_closest_subspace(3, qt_1)); EXPECT_DOUBLE_EQ(1.0/(1.0 + 4.0), calc_rawscore(1, qt_1)); EXPECT_DOUBLE_EQ(1.0/(1.0 + 2.0), calc_rawscore(1, qt_2)); diff --git a/searchlib/src/tests/transactionlog/translogclient_test.cpp b/searchlib/src/tests/transactionlog/translogclient_test.cpp index af214c34be8..af277ecbc68 100644 --- a/searchlib/src/tests/transactionlog/translogclient_test.cpp +++ b/searchlib/src/tests/transactionlog/translogclient_test.cpp @@ -479,16 +479,14 @@ getMaxSessionRunTime(TransLogServer &tls, const vespalib::string &domain) } struct TLS { - FastOS_ThreadPool threadPool; FNET_Transport transport; TransLogServer tls; TLS(const vespalib::string &name, int listenPort, const vespalib::string &baseDir, const common::FileHeaderContext &fileHeaderContext, const DomainConfig & cfg, size_t maxThreads = 4) - : threadPool(), - transport(), + : transport(), tls(transport, name, listenPort, baseDir, fileHeaderContext, cfg, maxThreads) { - transport.Start(&threadPool); + transport.Start(); } ~TLS() { transport.ShutDown(true); diff --git a/searchlib/src/tests/transactionlogstress/translogstress.cpp b/searchlib/src/tests/transactionlogstress/translogstress.cpp index eb457f312e6..124eb39e84b 100644 --- a/searchlib/src/tests/transactionlogstress/translogstress.cpp +++ b/searchlib/src/tests/transactionlogstress/translogstress.cpp @@ -5,7 +5,6 @@ #include <vespa/searchlib/transactionlog/translogclient.h> #include <vespa/vespalib/util/rand48.h> #include <vespa/vespalib/util/size_literals.h> -#include <vespa/searchlib/util/runnable.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/fnet/transport.h> #include <vespa/vespalib/util/signalhandler.h> @@ -20,7 +19,6 @@ LOG_SETUP("translogstress"); using vespalib::nbostream; -using search::Runnable; using std::shared_ptr; using vespalib::make_string; using vespalib::ConstBufferRef; @@ -190,7 +188,7 @@ public: //----------------------------------------------------------------------------- // FeederThread //----------------------------------------------------------------------------- -class FeederThread : public Runnable +class FeederThread { private: std::string _tlsSpec; @@ -203,6 +201,8 @@ private: SerialNum _current; SerialNum _lastCommited; vespalib::Timer _timer; + std::atomic<bool> _done; + std::thread _thread; void commitPacket(); bool addEntry(const Packet::Entry & e); @@ -210,8 +210,11 @@ private: public: FeederThread(FNET_Transport & transport, const std::string & tlsSpec, const std::string & domain, const EntryGenerator & generator, uint32_t feedRate, size_t packetSize); - ~FeederThread() override; - void doRun() override; + ~FeederThread(); + void doRun(); + void start() { _thread = std::thread([this](){doRun();}); } + void stop() { _done = true; } + void join() { _thread.join(); } SerialNumRange getRange() const { return SerialNumRange(1, _lastCommited); } }; @@ -450,7 +453,7 @@ VisitorAgent::receive(const Packet & packet) //----------------------------------------------------------------------------- // ControllerThread //----------------------------------------------------------------------------- -class ControllerThread : public Runnable +class ControllerThread { private: std::string _tlsSpec; @@ -466,7 +469,9 @@ private: SerialNum _begin; SerialNum _end; size_t _count; - + std::atomic<bool> _done; + std::thread _thread; + void getStatus(); void makeRandomVisitorVector(); @@ -475,8 +480,10 @@ public: uint32_t numVisitors, vespalib::duration visitorInterval, vespalib::duration pruneInterval); ~ControllerThread(); std::vector<std::shared_ptr<VisitorAgent> > & getVisitors() { return _visitors; } - virtual void doRun() override; - + void doRun(); + void start() { _thread = std::thread([this](){doRun();}); } + void stop() { _done = true; } + void join() { _thread.join(); } }; ControllerThread::ControllerThread(FNET_Transport & transport, const std::string & tlsSpec, const std::string & domain, @@ -698,7 +705,6 @@ TransLogStress::main(int argc, char **argv) } // start transaction log server - FastOS_ThreadPool threadPool; FNET_Transport transport; DummyFileHeaderContext fileHeaderContext; TransLogServer tls(transport, "server", 17897, ".", fileHeaderContext, DomainConfig().setPartSizeLimit(_cfg.domainPartSize)); @@ -719,12 +725,12 @@ TransLogStress::main(int argc, char **argv) // start feeder and controller FeederThread feeder(transport, tlsSpec, domain, generator, _cfg.feedRate, _cfg.packetSize); - threadPool.NewThread(&feeder); + feeder.start(); std::this_thread::sleep_for(sleepTime); ControllerThread controller(transport, tlsSpec, domain, generator, _cfg.numVisitors, _cfg.visitorInterval, _cfg.pruneInterval); - threadPool.NewThread(&controller); + controller.start(); // stop feeder and controller std::this_thread::sleep_for(_cfg.stressTime); @@ -751,8 +757,6 @@ TransLogStress::main(int argc, char **argv) std::cout << "</visitor>" << std::endl; } - threadPool.Close(); - return 0; } diff --git a/searchlib/src/vespa/searchlib/aggregation/group.cpp b/searchlib/src/vespa/searchlib/aggregation/group.cpp index e025d3b0f94..6ac994874ba 100644 --- a/searchlib/src/vespa/searchlib/aggregation/group.cpp +++ b/searchlib/src/vespa/searchlib/aggregation/group.cpp @@ -243,7 +243,6 @@ Group::Value::addExpressionResult(ExpressionNode::UP expressionNode) void Group::Value::addAggregationResult(ExpressionNode::UP aggr) { - assert(getAggrSize() < 15); size_t newSize = getAggrSize() + 1 + getExprSize(); auto n = new ExpressionNode::CP[newSize]; for (size_t i(0), m(getAggrSize()); i < m; i++) { @@ -259,6 +258,24 @@ Group::Value::addAggregationResult(ExpressionNode::UP aggr) setAggrSize(getAggrSize() + 1); } +void +Group::Value::setAggrSize(uint32_t v) { + assert(v < 0x10000); + _packedLength = (_packedLength & ~0xffff) | v; +} + +void +Group::Value::setExprSize(uint32_t v) { + assert(v < sizeof(_orderBy)*2); + _packedLength = (_packedLength & ~0xf0000) | (v << 16); +} + +void +Group::Value::setOrderBySize(uint32_t v) { + assert(v < sizeof(_orderBy)*2); + _packedLength = (_packedLength & ~0xf00000) | (v << 20); +} + template <typename Doc> void Group::Value::collect(const Doc & doc, HitRank rank) { @@ -272,15 +289,13 @@ Group::Value::addResult(ExpressionNode::UP aggr) { assert(getExprSize() < 15); addAggregationResult(std::move(aggr)); - addExpressionResult(ExpressionNode::UP(new AggregationRefNode(getAggrSize() - 1))); + addExpressionResult(std::make_unique<AggregationRefNode>(getAggrSize() - 1)); setupAggregationReferences(); } void Group::Value::addOrderBy(ExpressionNode::UP orderBy, bool ascending) { - assert(getOrderBySize() < sizeof(_orderBy)*2-1); - assert(getExprSize() < 15); addExpressionResult(std::move(orderBy)); setOrderBy(getOrderBySize(), (ascending ? getExprSize() : -getExprSize())); setOrderBySize(getOrderBySize() + 1); @@ -555,7 +570,6 @@ Group::Value::deserialize(Deserializer & is) { } uint32_t aggrSize(0); is >> aggrSize; - assert(aggrSize < 16); // To avoid protocol changes, we must first deserialize the aggregation // results into a temporary buffer, and then reallocate the actual // vector when we know the total size. Then we copy the temp buffer and @@ -575,7 +589,6 @@ Group::Value::deserialize(Deserializer & is) { } delete [] tmpAggregationResults; - assert(exprSize < 16); setExprSize(exprSize); for (uint32_t i(aggrSize); i < aggrSize + exprSize; i++) { is >> _aggregationResults[i]; diff --git a/searchlib/src/vespa/searchlib/aggregation/group.h b/searchlib/src/vespa/searchlib/aggregation/group.h index b4975dba3c5..3a3b6fedd9a 100644 --- a/searchlib/src/vespa/searchlib/aggregation/group.h +++ b/searchlib/src/vespa/searchlib/aggregation/group.h @@ -30,7 +30,7 @@ class Grouping; * * Total: 50 bytes */ -class Group : public vespalib::Identifiable +class Group final : public vespalib::Identifiable { public: using ResultNode = expression::ResultNode; @@ -57,8 +57,6 @@ public: using GroupingLevelList = std::vector<GroupingLevel>; -private: - class Value { public: Value(); @@ -98,8 +96,9 @@ private: GroupList groups() const { return _children; } void addChild(Group * child); - uint32_t getAggrSize() const { return _packedLength & 0x0f; } - uint32_t getOrderBySize() const { return (_packedLength >> 6) & 0x03; } + uint32_t getAggrSize() const { return _packedLength & 0xffff; } + uint32_t getExprSize() const { return (_packedLength >> 16) & 0x0f; } + uint32_t getOrderBySize() const { return (_packedLength >> 20) & 0x0f; } uint32_t getChildrenSize() const { return _childrenLength; } uint32_t getExpr(uint32_t i) const { return getAggrSize() + i; } int32_t getOrderBy(uint32_t i) const { @@ -109,7 +108,6 @@ private: const AggregationResult & getAggregationResult(size_t i) const { return static_cast<const AggregationResult &>(*_aggregationResults[i]); } AggregationResult & getAggregationResult(size_t i) { return static_cast<AggregationResult &>(*_aggregationResults[i]); } - uint32_t getExprSize() const { return (_packedLength >> 4) & 0x03; } const Group & getChild(size_t i) const { return *_children[i]; } template <typename Doc> @@ -118,9 +116,9 @@ private: using ExpressionVector = ExpressionNode::CP *; using GroupHash = vespalib::hash_set<uint32_t, GroupHasher, GroupEqual >; - void setAggrSize(uint32_t v) { _packedLength = (_packedLength & ~0x0f) | v; } - void setExprSize(uint32_t v) { _packedLength = (_packedLength & ~0x30) | (v << 4); } - void setOrderBySize(uint32_t v) { _packedLength = (_packedLength & ~0xc0) | (v << 6); } + void setAggrSize(uint32_t v); + void setExprSize(uint32_t v); + void setOrderBySize(uint32_t v); void setChildrenSize(uint32_t v) { _childrenLength = v; } AggregationResult * getAggr(size_t i) { return static_cast<AggregationResult *>(_aggregationResults[i].get()); } const AggregationResult & getAggr(size_t i) const { return static_cast<const AggregationResult &>(*_aggregationResults[i]); } @@ -149,11 +147,11 @@ private: size_t _allChildren; // Keep real number of children. } _childInfo; uint32_t _childrenLength; - uint32_t _tag; // Opaque tag used to identify the group by the client. - uint8_t _packedLength; // Length of aggr and expr vectors. - uint8_t _orderBy[2]; // How this group is ranked, negative means reverse rank. + uint32_t _tag; // Opaque tag used to identify the group by the client. + uint32_t _packedLength; // Length of aggr and expr vectors. + uint8_t _orderBy[4]; // How this group is ranked, negative means reverse rank. }; - +private: ResultNode::CP _id; // the label of this group, separating it from other groups RawRank _rank; // The default rank taken from the highest hit relevance. Value _aggr; diff --git a/searchlib/src/vespa/searchlib/common/bitvector.h b/searchlib/src/vespa/searchlib/common/bitvector.h index 67f3e2ad502..149e57390af 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.h +++ b/searchlib/src/vespa/searchlib/common/bitvector.h @@ -3,10 +3,8 @@ #pragma once #include "bitword.h" -#include <memory> #include <vespa/vespalib/util/alloc.h> #include <vespa/vespalib/util/atomic.h> -#include <vespa/fastos/types.h> #include <algorithm> #include <cassert> diff --git a/searchlib/src/vespa/searchlib/common/bitvectorcache.cpp b/searchlib/src/vespa/searchlib/common/bitvectorcache.cpp index cb4d112b77e..8bb24bcbbec 100644 --- a/searchlib/src/vespa/searchlib/common/bitvectorcache.cpp +++ b/searchlib/src/vespa/searchlib/common/bitvectorcache.cpp @@ -3,6 +3,7 @@ #include <vespa/vespalib/stllike/hash_map.hpp> #include <algorithm> #include <cassert> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".searchlib.common.bitvectorcache"); diff --git a/searchlib/src/vespa/searchlib/diskindex/fileheader.cpp b/searchlib/src/vespa/searchlib/diskindex/fileheader.cpp index e1c31c1ed8d..5399d70fbe7 100644 --- a/searchlib/src/vespa/searchlib/diskindex/fileheader.cpp +++ b/searchlib/src/vespa/searchlib/diskindex/fileheader.cpp @@ -5,6 +5,7 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/data/fileheader.h> #include <vespa/fastos/file.h> +#include <cinttypes> #include <arpa/inet.h> #include <vespa/log/log.h> diff --git a/searchlib/src/vespa/searchlib/docstore/compacter.cpp b/searchlib/src/vespa/searchlib/docstore/compacter.cpp index 04ff150c741..c886e52659f 100644 --- a/searchlib/src/vespa/searchlib/docstore/compacter.cpp +++ b/searchlib/src/vespa/searchlib/docstore/compacter.cpp @@ -4,6 +4,7 @@ #include "logdatastore.h" #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/array.hpp> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".searchlib.docstore.compacter"); diff --git a/searchlib/src/vespa/searchlib/engine/proto_converter.cpp b/searchlib/src/vespa/searchlib/engine/proto_converter.cpp index ae00889850b..b03dbf5ff37 100644 --- a/searchlib/src/vespa/searchlib/engine/proto_converter.cpp +++ b/searchlib/src/vespa/searchlib/engine/proto_converter.cpp @@ -6,8 +6,9 @@ #include <vespa/vespalib/data/slime/binary_format.h> #include <vespa/vespalib/data/smart_buffer.h> #include <vespa/vespalib/util/size_literals.h> -#include <vespa/log/log.h> +#include <cinttypes> +#include <vespa/log/log.h> LOG_SETUP(".searchlib.engine.proto_converter"); namespace search::engine { diff --git a/searchlib/src/vespa/searchlib/features/CMakeLists.txt b/searchlib/src/vespa/searchlib/features/CMakeLists.txt index 8acf28f4a2f..4af5c0e561e 100644 --- a/searchlib/src/vespa/searchlib/features/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/features/CMakeLists.txt @@ -7,6 +7,7 @@ vespa_add_library(searchlib_features OBJECT attributematchfeature.cpp bm25_feature.cpp closenessfeature.cpp + closest_feature.cpp constant_feature.cpp debug_attribute_wait.cpp debug_wait.cpp diff --git a/searchlib/src/vespa/searchlib/features/closenessfeature.cpp b/searchlib/src/vespa/searchlib/features/closenessfeature.cpp index e44c94dbb2d..048a507b3fd 100644 --- a/searchlib/src/vespa/searchlib/features/closenessfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/closenessfeature.cpp @@ -36,7 +36,7 @@ ConvertRawScoreToCloseness::ConvertRawScoreToCloseness(const fef::IQueryEnvironm } ConvertRawScoreToCloseness::ConvertRawScoreToCloseness(const fef::IQueryEnvironment &env, const vespalib::string &label) - : _bundle(env, label, "closeness"), + : _bundle(env, std::nullopt, label, "closeness"), _md(nullptr) { } diff --git a/searchlib/src/vespa/searchlib/features/closest_feature.cpp b/searchlib/src/vespa/searchlib/features/closest_feature.cpp new file mode 100644 index 00000000000..c0284c9fa89 --- /dev/null +++ b/searchlib/src/vespa/searchlib/features/closest_feature.cpp @@ -0,0 +1,282 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "closest_feature.h" +#include "constant_tensor_executor.h" +#include "distance_calculator_bundle.h" +#include "valuefeature.h" +#include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/value_codec.h> +#include <vespa/searchcommon/common/schema.h> +#include <vespa/searchlib/fef/indexproperties.h> +#include <vespa/searchlib/fef/parameterdescriptions.h> +#include <vespa/searchlib/fef/test/dummy_dependency_handler.h> +#include <vespa/searchlib/tensor/distance_calculator.h> +#include <vespa/searchlib/tensor/fast_value_view.h> +#include <vespa/searchlib/tensor/i_tensor_attribute.h> +#include <vespa/searchlib/tensor/serialized_tensor_ref.h> +#include <vespa/searchlib/tensor/subspace_type.h> +#include <vespa/vespalib/util/stash.h> + +#include <vespa/log/log.h> +LOG_SETUP(".features.closest_feature"); + +using search::fef::FeatureType; +using search::fef::FieldInfo; +using search::fef::ParameterDataTypeSet; +using search::tensor::FastValueView; +using search::tensor::ITensorAttribute; +using search::tensor::SubspaceType; +using search::tensor::VectorBundle; +using vespalib::eval::CellType; +using vespalib::eval::FastValueBuilderFactory; +using vespalib::eval::TypedCells; +using vespalib::eval::TypifyCellType; +using vespalib::eval::Value; +using vespalib::eval::ValueType; +using vespalib::string_id; +using vespalib::typify_invoke; + +using namespace search::fef::indexproperties; + +namespace { + +struct SetIdentity { + template <typename T> + static void invoke(void *space, size_t size) { + assert(size == sizeof(T)); + *(T *) space = 1.0; + } +}; + +void setup_identity_cells(const ValueType& type, std::vector<char>& space, TypedCells& cells) +{ + if (type.is_double()) { + return; + } + space.resize(vespalib::eval::CellTypeUtils::mem_size(type.cell_type(), 1)); + cells = TypedCells(space.data(), type.cell_type(), 1); + typify_invoke<1,TypifyCellType,SetIdentity>(type.cell_type(), space.data(), space.size()); +} + +} + +namespace search::features { + +class ClosestExecutor : public fef::FeatureExecutor { +protected: + DistanceCalculatorBundle _bundle; + Value& _empty_output; + TypedCells _identity; + const ITensorAttribute& _attr; + std::unique_ptr<Value> _output; +public: + ClosestExecutor(DistanceCalculatorBundle&& bundle, Value& empty_output, TypedCells identity, const ITensorAttribute& attr); + ~ClosestExecutor() override; + static fef::FeatureExecutor& make(DistanceCalculatorBundle&& bundle, Value& empty_output, TypedCells identity, const ITensorAttribute& attr, vespalib::Stash& stash); +}; + +/** + * Implements the executor for the closest feature for SerializedFastValueAttribute. + */ +class ClosestSerializedExecutor : public ClosestExecutor { +public: + ClosestSerializedExecutor(DistanceCalculatorBundle&& bundle, Value& empty_output, TypedCells identity, const ITensorAttribute& attr); + ~ClosestSerializedExecutor() override; + void execute(uint32_t docId) override; +}; + +/** + * Implements the executor for the closest feature for DirectTensorAttribute. + */ +class ClosestDirectExecutor : public ClosestExecutor { + SubspaceType _subspace_type; + std::vector<string_id> _labels; + std::vector<string_id*> _label_ptrs; +public: + ClosestDirectExecutor(DistanceCalculatorBundle&& bundle, Value& empty_output, TypedCells identity, const ITensorAttribute& attr); + ~ClosestDirectExecutor() override; + void execute(uint32_t docId) override; +}; + +ClosestExecutor::ClosestExecutor(DistanceCalculatorBundle&& bundle, Value& empty_output, TypedCells identity, const ITensorAttribute& attr) + : _bundle(std::move(bundle)), + _empty_output(empty_output), + _identity(identity), + _attr(attr), + _output() +{ +} + +ClosestExecutor::~ClosestExecutor() = default; + +fef::FeatureExecutor& +ClosestExecutor::make(DistanceCalculatorBundle&& bundle, Value& empty_output, TypedCells identity, const ITensorAttribute& attr, vespalib::Stash& stash) +{ + if (attr.supports_get_serialized_tensor_ref()) { + return stash.create<ClosestSerializedExecutor>(std::move(bundle), empty_output, identity, attr); + } else if (attr.supports_get_tensor_ref()) { + return stash.create<ClosestDirectExecutor>(std::move(bundle), empty_output, identity, attr); + } else { + return ConstantTensorExecutor::createEmpty(empty_output.type(), stash); + } +} + +ClosestSerializedExecutor::ClosestSerializedExecutor(DistanceCalculatorBundle&& bundle, Value& empty_output, TypedCells identity, const ITensorAttribute& attr) + : ClosestExecutor(std::move(bundle), empty_output, identity, attr) +{ +} + +ClosestSerializedExecutor::~ClosestSerializedExecutor() = default; + +void +ClosestSerializedExecutor::execute(uint32_t docId) +{ + double best_distance = 0.0; + std::optional<uint32_t> closest_subspace; + auto ref = _attr.get_serialized_tensor_ref(docId); + for (const auto& elem : _bundle.elements()) { + elem.calc->calc_closest_subspace(ref.get_vectors(), closest_subspace, best_distance); + } + if (closest_subspace.has_value()) { + auto labels = ref.get_labels(closest_subspace.value()); + _output = std::make_unique<FastValueView>(_empty_output.type(), labels, _identity, labels.size(), 1); + outputs().set_object(0, *_output); + } else { + outputs().set_object(0, _empty_output); + } +} + +ClosestDirectExecutor::ClosestDirectExecutor(DistanceCalculatorBundle&& bundle, Value& empty_output, TypedCells identity, const ITensorAttribute& attr) + : ClosestExecutor(std::move(bundle), empty_output, identity, attr), + _subspace_type(attr.getTensorType()), + _labels(1), + _label_ptrs(_labels.size()) +{ + for (size_t i = 0; i < _labels.size(); ++i) { + _label_ptrs[i] = &_labels[i]; + } +} + +ClosestDirectExecutor::~ClosestDirectExecutor() = default; + +void +ClosestDirectExecutor::execute(uint32_t docId) +{ + double best_distance = 0.0; + std::optional<uint32_t> closest_subspace; + auto& tensor = _attr.get_tensor_ref(docId); + VectorBundle vectors(tensor.cells().data, tensor.index().size(), _subspace_type); + for (const auto& elem : _bundle.elements()) { + elem.calc->calc_closest_subspace(vectors, closest_subspace, best_distance); + } + if (closest_subspace.has_value()) { + size_t subspace_id = 0; + auto view = tensor.index().create_view({}); + view->lookup({}); + while (view->next_result(_label_ptrs, subspace_id)) { + if (subspace_id == closest_subspace.value()) { + _output = std::make_unique<FastValueView>(_empty_output.type(), _labels, _identity, _labels.size(), 1); + outputs().set_object(0, *_output); + return; + } + } + } + outputs().set_object(0, _empty_output); +} + +ClosestBlueprint::ClosestBlueprint() + : Blueprint("closest"), + _field_name(), + _field_tensor_type(ValueType::error_type()), + _output_tensor_type(ValueType::error_type()), + _field_id(search::index::Schema::UNKNOWN_FIELD_ID), + _item_label(), + _empty_output(), + _identity_space(), + _identity_cells() +{ +} + +ClosestBlueprint::~ClosestBlueprint() = default; + +void +ClosestBlueprint::visitDumpFeatures(const fef::IIndexEnvironment&, fef::IDumpFeatureVisitor&) const +{ +} + +std::unique_ptr<fef::Blueprint> +ClosestBlueprint::createInstance() const +{ + return std::make_unique<ClosestBlueprint>(); +} + +fef::ParameterDescriptions +ClosestBlueprint::getDescriptions() const +{ + auto data_type_set = ParameterDataTypeSet::tensor_type_set(); + return fef::ParameterDescriptions(). + desc().attribute(data_type_set, fef::ParameterCollection::SINGLE). + desc().attribute(data_type_set, fef::ParameterCollection::SINGLE).string(); +} + +bool +ClosestBlueprint::setup(const fef::IIndexEnvironment & env, const fef::ParameterList & params) +{ + if (params.size() < 1 || params.size() > 2) { + LOG(error, "%s: Wrong number of parameters, was %d, must be 1 or 2", getName().c_str(), (int) params.size()); + return false; + } + _field_name = params[0].getValue(); + if (params.size() == 2) { + _item_label = params[1].getValue(); + } + auto fi = env.getFieldByName(_field_name); + assert(fi != nullptr); + vespalib::string attr_type_spec = type::Attribute::lookup(env.getProperties(), _field_name); + if (attr_type_spec.empty()) { + LOG(error, "%s: Field %s lacks a type in index properties", getName().c_str(), _field_name.c_str()); + return false; + } + _field_tensor_type = ValueType::from_spec(attr_type_spec); + if (_field_tensor_type.is_error() || _field_tensor_type.is_double() || _field_tensor_type.count_mapped_dimensions() != 1 || _field_tensor_type.count_indexed_dimensions() != 1) { + LOG(error, "%s: Field %s has invalid type: '%s'", getName().c_str(), _field_name.c_str(), attr_type_spec.c_str()); + return false; + } + _output_tensor_type = ValueType::make_type(_field_tensor_type.cell_type(), _field_tensor_type.mapped_dimensions()); + assert(!_output_tensor_type.is_double()); + FeatureType output_type = FeatureType::object(_output_tensor_type); + describeOutput("out", "The closest tensor subspace.", output_type); + _field_id = fi->id(); + _empty_output = vespalib::eval::value_from_spec(_output_tensor_type.to_spec(), FastValueBuilderFactory::get()); + setup_identity_cells(_output_tensor_type, _identity_space, _identity_cells); + return true; +} + +void +ClosestBlueprint::prepareSharedState(const fef::IQueryEnvironment& env, fef::IObjectStore& store) const +{ + if (_item_label.has_value()) { + DistanceCalculatorBundle::prepare_shared_state(env, store, _item_label.value(), "closest"); + } else { + DistanceCalculatorBundle::prepare_shared_state(env, store, _field_id, "closest"); + } +} + +fef::FeatureExecutor& +ClosestBlueprint::createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const +{ + auto bundle = _item_label.has_value() ? DistanceCalculatorBundle(env, _field_id, _item_label.value(), "closest") : DistanceCalculatorBundle(env, _field_id, "closest"); + if (bundle.elements().empty()) { + return ConstantTensorExecutor::createEmpty(_output_tensor_type, stash); + } else { + for (const auto& elem : bundle.elements()) { + if (!elem.calc) { + return ConstantTensorExecutor::createEmpty(_output_tensor_type, stash); + } + } + auto& attr = bundle.elements().front().calc->attribute_tensor(); + return ClosestExecutor::make(std::move(bundle), *_empty_output, _identity_cells, attr, stash); + } +} + +} diff --git a/searchlib/src/vespa/searchlib/features/closest_feature.h b/searchlib/src/vespa/searchlib/features/closest_feature.h new file mode 100644 index 00000000000..840f896abe2 --- /dev/null +++ b/searchlib/src/vespa/searchlib/features/closest_feature.h @@ -0,0 +1,33 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchlib/fef/blueprint.h> +#include <optional> + +namespace search::features { + +/** + * Implements the blueprint for the closest executor. + */ +class ClosestBlueprint : public fef::Blueprint { + vespalib::string _field_name; + vespalib::eval::ValueType _field_tensor_type; + vespalib::eval::ValueType _output_tensor_type; + uint32_t _field_id; + std::optional<vespalib::string> _item_label; + std::unique_ptr<vespalib::eval::Value> _empty_output; + std::vector<char> _identity_space; + vespalib::eval::TypedCells _identity_cells; +public: + ClosestBlueprint(); + ~ClosestBlueprint() override; + void visitDumpFeatures(const fef::IIndexEnvironment & env, fef::IDumpFeatureVisitor & visitor) const override; + std::unique_ptr<fef::Blueprint> createInstance() const override; + fef::ParameterDescriptions getDescriptions() const override; + bool setup(const fef::IIndexEnvironment & env, const fef::ParameterList & params) override; + void prepareSharedState(const fef::IQueryEnvironment& env, fef::IObjectStore& store) const override; + fef::FeatureExecutor &createExecutor(const fef::IQueryEnvironment &env, vespalib::Stash &stash) const override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.cpp b/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.cpp index 4b2d67c933d..fad4c649165 100644 --- a/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.cpp +++ b/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.cpp @@ -115,6 +115,7 @@ DistanceCalculatorBundle::DistanceCalculatorBundle(const fef::IQueryEnvironment& } DistanceCalculatorBundle::DistanceCalculatorBundle(const fef::IQueryEnvironment& env, + std::optional<uint32_t> field_id, const vespalib::string& label, const vespalib::string& feature_name) : _elems() @@ -124,6 +125,9 @@ DistanceCalculatorBundle::DistanceCalculatorBundle(const fef::IQueryEnvironment& // expect numFields() == 1 for (uint32_t i = 0; i < term->numFields(); ++i) { const auto& term_field = term->field(i); + if (field_id.has_value() && field_id.value() != term_field.getFieldId()) { + continue; + } TermFieldHandle handle = term_field.getHandle(); if (handle != IllegalHandle) { std::unique_ptr<DistanceCalculator> calc; diff --git a/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.h b/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.h index 35295c771a6..e3be52aecc5 100644 --- a/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.h +++ b/searchlib/src/vespa/searchlib/features/distance_calculator_bundle.h @@ -5,6 +5,7 @@ #include <vespa/searchlib/fef/handle.h> #include <vespa/vespalib/stllike/string.h> #include <memory> +#include <optional> #include <vector> namespace search::tensor { class DistanceCalculator; } @@ -40,6 +41,7 @@ public: const vespalib::string& feature_name); DistanceCalculatorBundle(const fef::IQueryEnvironment& env, + std::optional<uint32_t> field_id, const vespalib::string& label, const vespalib::string& feature_name); diff --git a/searchlib/src/vespa/searchlib/features/distancefeature.cpp b/searchlib/src/vespa/searchlib/features/distancefeature.cpp index 40f994c18e9..f601c91a0b2 100644 --- a/searchlib/src/vespa/searchlib/features/distancefeature.cpp +++ b/searchlib/src/vespa/searchlib/features/distancefeature.cpp @@ -44,7 +44,7 @@ ConvertRawscoreToDistance::ConvertRawscoreToDistance(const fef::IQueryEnvironmen } ConvertRawscoreToDistance::ConvertRawscoreToDistance(const fef::IQueryEnvironment &env, const vespalib::string &label) - : _bundle(env, label, "distance"), + : _bundle(env, std::nullopt, label, "distance"), _md(nullptr) { } diff --git a/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp b/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp index 91ccd390692..ad92f8ef6e0 100644 --- a/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/random_normal_feature.cpp @@ -5,6 +5,7 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/util/stash.h> #include <chrono> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".features.randomnormalfeature"); diff --git a/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp b/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp index f7fcffca8cb..f1a42da1266 100644 --- a/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/random_normal_stable_feature.cpp @@ -4,6 +4,7 @@ #include "utils.h" #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/util/stash.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".features.randomnormalstablefeature"); diff --git a/searchlib/src/vespa/searchlib/features/randomfeature.cpp b/searchlib/src/vespa/searchlib/features/randomfeature.cpp index a8f2dd275a7..30a313d54d2 100644 --- a/searchlib/src/vespa/searchlib/features/randomfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/randomfeature.cpp @@ -5,6 +5,7 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/vespalib/util/stash.h> #include <chrono> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".features.randomfeature"); diff --git a/searchlib/src/vespa/searchlib/features/setup.cpp b/searchlib/src/vespa/searchlib/features/setup.cpp index 2bc8a349d1b..5e152d4b455 100644 --- a/searchlib/src/vespa/searchlib/features/setup.cpp +++ b/searchlib/src/vespa/searchlib/features/setup.cpp @@ -6,6 +6,7 @@ #include "attributematchfeature.h" #include "bm25_feature.h" #include "closenessfeature.h" +#include "closest_feature.h" #include "constant_feature.h" #include "debug_attribute_wait.h" #include "debug_wait.h" @@ -75,6 +76,7 @@ void setup_search_features(fef::IBlueprintRegistry & registry) registry.addPrototype(std::make_shared<AttributeMatchBlueprint>()); registry.addPrototype(std::make_shared<Bm25Blueprint>()); registry.addPrototype(std::make_shared<ClosenessBlueprint>()); + registry.addPrototype(std::make_shared<ClosestBlueprint>()); registry.addPrototype(std::make_shared<DebugAttributeWaitBlueprint>()); registry.addPrototype(std::make_shared<DebugWaitBlueprint>()); registry.addPrototype(std::make_shared<DistanceBlueprint>()); diff --git a/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp b/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp index cc2cae1c8cb..864dd64b98f 100644 --- a/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp +++ b/searchlib/src/vespa/searchlib/fef/blueprintresolver.cpp @@ -134,11 +134,8 @@ struct Compiler : public Blueprint::DependencyHandler { failed_set.insert(feature_name); auto trace = make_trace(skip_self); vespalib::string msg; - if (trace.empty()) { - msg = fmt("invalid %s: %s", describe(feature_name).c_str(), reason.c_str()); - } else { - msg = fmt("invalid %s: %s\n%s", describe(feature_name).c_str(), reason.c_str(), trace.c_str()); - } + msg = fmt("invalid %s: %s\n%s", describe(feature_name).c_str(), reason.c_str(), trace.c_str()); + msg.chomp(); errors.emplace_back(msg); } probe_stack(); diff --git a/searchlib/src/vespa/searchlib/fef/parameterdescriptions.h b/searchlib/src/vespa/searchlib/fef/parameterdescriptions.h index e47ce0df7a5..46a932696ca 100644 --- a/searchlib/src/vespa/searchlib/fef/parameterdescriptions.h +++ b/searchlib/src/vespa/searchlib/fef/parameterdescriptions.h @@ -71,6 +71,7 @@ private: asMask(DataType::REFERENCE) | asMask(DataType::COMBINED)); } + static uint32_t tensor_type_mask() { return asMask(DataType::TENSOR); } ParameterDataTypeSet(uint32_t typeMask) : _typeMask(typeMask) { @@ -87,8 +88,9 @@ public: return ParameterDataTypeSet(asMask(DataType::INT32) | asMask(DataType::INT64)); } static ParameterDataTypeSet normalOrTensorTypeSet() { - return ParameterDataTypeSet(normalTypesMask() | asMask(DataType::TENSOR)); + return ParameterDataTypeSet(normalTypesMask() | tensor_type_mask()); } + static ParameterDataTypeSet tensor_type_set() { return ParameterDataTypeSet(tensor_type_mask()); } bool allowedType(DataType dataType) const { return ((asMask(dataType) & _typeMask) != 0); } diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_posting_list.h b/searchlib/src/vespa/searchlib/predicate/predicate_posting_list.h index 0bf33f1d0e5..0de9be332fa 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_posting_list.h +++ b/searchlib/src/vespa/searchlib/predicate/predicate_posting_list.h @@ -3,7 +3,8 @@ #include <memory> #include <cstdint> -#include <vespa/fastos/types.h> + +#define VESPA_DLL_LOCAL __attribute__ ((visibility("hidden"))) /** * Interface for posting lists used by PredicateSearch. @@ -25,7 +26,7 @@ protected: public: using UP = std::unique_ptr<PredicatePostingList>; - virtual ~PredicatePostingList() {} + virtual ~PredicatePostingList() = default; /* * Moves to next document after the one supplied. diff --git a/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h b/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h index 42de8646091..6a6b91e20f2 100644 --- a/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h +++ b/searchlib/src/vespa/searchlib/predicate/predicate_range_term_expander.h @@ -5,6 +5,7 @@ #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/issue.h> #include <climits> +#include <cinttypes> namespace search::predicate { diff --git a/searchlib/src/vespa/searchlib/queryeval/hitcollector.h b/searchlib/src/vespa/searchlib/queryeval/hitcollector.h index fe686f3e0bf..e244c856c4a 100644 --- a/searchlib/src/vespa/searchlib/queryeval/hitcollector.h +++ b/searchlib/src/vespa/searchlib/queryeval/hitcollector.h @@ -9,7 +9,6 @@ #include <vespa/vespalib/util/sort.h> #include <algorithm> #include <vector> -#include <vespa/fastos/types.h> namespace search::queryeval { diff --git a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt index a00a50f32c8..a64bd6af4a9 100644 --- a/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/tensor/CMakeLists.txt @@ -13,6 +13,7 @@ vespa_add_library(searchlib_tensor OBJECT distance_function_factory.cpp empty_subspace.cpp euclidean_distance.cpp + fast_value_view.cpp geo_degrees_distance.cpp hamming_distance.cpp hash_set_visited_tracker.cpp @@ -30,6 +31,7 @@ vespa_add_library(searchlib_tensor OBJECT nearest_neighbor_index.cpp nearest_neighbor_index_saver.cpp serialized_fast_value_attribute.cpp + serialized_tensor_ref.cpp small_subspaces_buffer_type.cpp subspace_type.cpp tensor_attribute.cpp diff --git a/searchlib/src/vespa/searchlib/tensor/distance_calculator.h b/searchlib/src/vespa/searchlib/tensor/distance_calculator.h index 320f071cbbb..6b4cf142264 100644 --- a/searchlib/src/vespa/searchlib/tensor/distance_calculator.h +++ b/searchlib/src/vespa/searchlib/tensor/distance_calculator.h @@ -4,6 +4,7 @@ #include "distance_function.h" #include "i_tensor_attribute.h" #include "vector_bundle.h" +#include <optional> namespace vespalib::eval { struct Value; } @@ -64,6 +65,23 @@ public: return result; } + void calc_closest_subspace(VectorBundle vectors, std::optional<uint32_t>& closest_subspace, double& best_distance) { + for (uint32_t i = 0; i < vectors.subspaces(); ++i) { + double distance = _dist_fun->calc(_query_tensor_cells, vectors.cells(i)); + if (!closest_subspace.has_value() || distance < best_distance) { + best_distance = distance; + closest_subspace = i; + } + } + } + + std::optional<uint32_t> calc_closest_subspace(VectorBundle vectors) { + double best_distance = 0.0; + std::optional<uint32_t> closest_subspace; + calc_closest_subspace(vectors, closest_subspace, best_distance); + return closest_subspace; + } + /** * Create a calculator for the given attribute tensor and query tensor, if possible. * diff --git a/searchlib/src/vespa/searchlib/tensor/fast_value_view.cpp b/searchlib/src/vespa/searchlib/tensor/fast_value_view.cpp new file mode 100644 index 00000000000..29cc47cb543 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/fast_value_view.cpp @@ -0,0 +1,39 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "fast_value_view.h" +#include <vespa/vespalib/stllike/hash_map.hpp> + +using vespalib::ConstArrayRef; +using vespalib::MemoryUsage; +using vespalib::string_id; +using vespalib::eval::FastAddrMap; +using vespalib::eval::TypedCells; +using vespalib::eval::Value; +using vespalib::eval::ValueType; +using vespalib::eval::self_memory_usage; + +namespace search::tensor { + +FastValueView::FastValueView(const ValueType& type, ConstArrayRef<string_id> labels, TypedCells cells, size_t num_mapped_dimensions, size_t num_subspaces) + : Value(), + _type(type), + _labels(labels.begin(), labels.end()), + _index(num_mapped_dimensions, _labels, num_subspaces), + _cells(cells) +{ + for (size_t i = 0; i < num_subspaces; ++i) { + ConstArrayRef<string_id> addr(_labels.data() + (i * num_mapped_dimensions), num_mapped_dimensions); + _index.map.add_mapping(FastAddrMap::hash_labels(addr)); + } + assert(_index.map.size() == num_subspaces); +} + +MemoryUsage +FastValueView::get_memory_usage() const +{ + MemoryUsage usage = self_memory_usage<FastValueView>(); + usage.merge(_index.map.estimate_extra_memory_usage()); + return usage; +} + +} diff --git a/searchlib/src/vespa/searchlib/tensor/fast_value_view.h b/searchlib/src/vespa/searchlib/tensor/fast_value_view.h new file mode 100644 index 00000000000..c3f13ac5856 --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/fast_value_view.h @@ -0,0 +1,24 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/fast_value_index.h> + +namespace search::tensor { + +/* + * Tensor view that is not self-contained. It references external cell values. + */ +struct FastValueView final : vespalib::eval::Value { + const vespalib::eval::ValueType& _type; + vespalib::StringIdVector _labels; + vespalib::eval::FastValueIndex _index; + vespalib::eval::TypedCells _cells; + FastValueView(const vespalib::eval::ValueType& type, vespalib::ConstArrayRef<vespalib::string_id> labels, vespalib::eval::TypedCells cells, size_t num_mapped_dimensions, size_t num_subspaces); + const vespalib::eval::ValueType& type() const override { return _type; } + const vespalib::eval::Value::Index& index() const override { return _index; } + vespalib::eval::TypedCells cells() const override { return _cells; } + vespalib::MemoryUsage get_memory_usage() const override; +}; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h index 9b5f80b2ece..ec6774c9517 100644 --- a/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/i_tensor_attribute.h @@ -13,6 +13,7 @@ namespace vespalib::slime { struct Inserter; } namespace search::tensor { class NearestNeighborIndex; +class SerializedTensorRef; /** * Interface for tensor attribute used by feature executors to get information. @@ -24,8 +25,10 @@ public: virtual std::unique_ptr<vespalib::eval::Value> getEmptyTensor() const = 0; virtual vespalib::eval::TypedCells extract_cells_ref(uint32_t docid) const = 0; virtual const vespalib::eval::Value& get_tensor_ref(uint32_t docid) const = 0; + virtual SerializedTensorRef get_serialized_tensor_ref(uint32_t docid) const = 0; virtual bool supports_extract_cells_ref() const = 0; virtual bool supports_get_tensor_ref() const = 0; + virtual bool supports_get_serialized_tensor_ref() const = 0; virtual const vespalib::eval::ValueType & getTensorType() const = 0; diff --git a/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.cpp b/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.cpp index f9459823ce4..9a7b81ae1fa 100644 --- a/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.cpp +++ b/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "imported_tensor_attribute_vector_read_guard.h" +#include "serialized_tensor_ref.h" #include "vector_bundle.h" #include <vespa/searchlib/attribute/attributevector.h> #include <vespa/eval/eval/value.h> @@ -79,6 +80,18 @@ ImportedTensorAttributeVectorReadGuard::getTensorType() const return _target_tensor_attribute.getTensorType(); } +SerializedTensorRef +ImportedTensorAttributeVectorReadGuard::get_serialized_tensor_ref(uint32_t docid) const +{ + return _target_tensor_attribute.get_serialized_tensor_ref(getTargetLid(docid)); +} + +bool +ImportedTensorAttributeVectorReadGuard::supports_get_serialized_tensor_ref() const +{ + return _target_tensor_attribute.supports_get_serialized_tensor_ref(); +} + void ImportedTensorAttributeVectorReadGuard::get_state(const vespalib::slime::Inserter& inserter) const { diff --git a/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h b/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h index f277d39e97d..4e1cc9efd96 100644 --- a/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h +++ b/searchlib/src/vespa/searchlib/tensor/imported_tensor_attribute_vector_read_guard.h @@ -35,9 +35,11 @@ public: std::unique_ptr<vespalib::eval::Value> getEmptyTensor() const override; vespalib::eval::TypedCells extract_cells_ref(uint32_t docid) const override; const vespalib::eval::Value& get_tensor_ref(uint32_t docid) const override; + SerializedTensorRef get_serialized_tensor_ref(uint32_t docid) const override; bool supports_extract_cells_ref() const override { return _target_tensor_attribute.supports_extract_cells_ref(); } bool supports_get_tensor_ref() const override { return _target_tensor_attribute.supports_get_tensor_ref(); } DistanceMetric distance_metric() const override { return _target_tensor_attribute.distance_metric(); } + bool supports_get_serialized_tensor_ref() const override; uint32_t get_num_docs() const override { return getNumDocs(); } vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const override; diff --git a/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp index 6612db1d27e..51ebc22c269 100644 --- a/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "serialized_fast_value_attribute.h" +#include "serialized_tensor_ref.h" #include <vespa/eval/eval/value.h> #include <vespa/searchcommon/attribute/config.h> @@ -26,6 +27,19 @@ SerializedFastValueAttribute::~SerializedFastValueAttribute() _tensorStore.reclaim_all_memory(); } +SerializedTensorRef +SerializedFastValueAttribute::get_serialized_tensor_ref(uint32_t docid) const +{ + EntryRef ref = acquire_entry_ref(docid); + return _tensorBufferStore.get_serialized_tensor_ref(ref); +} + +bool +SerializedFastValueAttribute::supports_get_serialized_tensor_ref() const +{ + return true; +} + vespalib::eval::TypedCells SerializedFastValueAttribute::get_vector(uint32_t docid, uint32_t subspace) const { diff --git a/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.h b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.h index 4cfcc3d19a2..9066766fbc4 100644 --- a/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/serialized_fast_value_attribute.h @@ -23,6 +23,9 @@ public: SerializedFastValueAttribute(vespalib::stringref baseFileName, const Config &cfg, const NearestNeighborIndexFactory& index_factory = DefaultNearestNeighborIndexFactory()); ~SerializedFastValueAttribute() override; + SerializedTensorRef get_serialized_tensor_ref(uint32_t docid) const override; + bool supports_get_serialized_tensor_ref() const override; + // Implements DocVectorAccess vespalib::eval::TypedCells get_vector(uint32_t docid, uint32_t subspace) const override; VectorBundle get_vectors(uint32_t docid) const override; diff --git a/searchlib/src/vespa/searchlib/tensor/serialized_tensor_ref.cpp b/searchlib/src/vespa/searchlib/tensor/serialized_tensor_ref.cpp new file mode 100644 index 00000000000..1f8ca9ed2fd --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/serialized_tensor_ref.cpp @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "serialized_tensor_ref.h" + +namespace search::tensor { + +SerializedTensorRef::SerializedTensorRef() + : _vectors(), + _num_mapped_dimensions(0), + _labels() +{ +} + +SerializedTensorRef::SerializedTensorRef(VectorBundle vectors, uint32_t num_mapped_dimensions, vespalib::ConstArrayRef<vespalib::string_id> labels) + : _vectors(vectors), + _num_mapped_dimensions(num_mapped_dimensions), + _labels(labels) +{ +} + +SerializedTensorRef::~SerializedTensorRef() = default; + +vespalib::ConstArrayRef<vespalib::string_id> +SerializedTensorRef::get_labels(uint32_t subspace) const +{ + assert(subspace < _vectors.subspaces()); + return {_labels.data() + subspace * _num_mapped_dimensions, _num_mapped_dimensions}; +} + +} diff --git a/searchlib/src/vespa/searchlib/tensor/serialized_tensor_ref.h b/searchlib/src/vespa/searchlib/tensor/serialized_tensor_ref.h new file mode 100644 index 00000000000..01ddaadb2ff --- /dev/null +++ b/searchlib/src/vespa/searchlib/tensor/serialized_tensor_ref.h @@ -0,0 +1,26 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "vector_bundle.h" +#include <vespa/vespalib/util/string_id.h> + +namespace search::tensor { + +/* + * This class contains a reference to a tensor stored in a TensorBufferStore. + */ +class SerializedTensorRef +{ + VectorBundle _vectors; + uint32_t _num_mapped_dimensions; + vespalib::ConstArrayRef<vespalib::string_id> _labels; // all subspaces +public: + SerializedTensorRef(); + SerializedTensorRef(VectorBundle vectors, uint32_t num_mapped_dimensions, vespalib::ConstArrayRef<vespalib::string_id> labels); + ~SerializedTensorRef(); + const VectorBundle& get_vectors() const noexcept { return _vectors; } + vespalib::ConstArrayRef<vespalib::string_id> get_labels(uint32_t subspace) const; +}; + +} diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp index 9ee8d9fdf46..13dad7fc1f2 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.cpp @@ -4,6 +4,7 @@ #include "nearest_neighbor_index.h" #include "nearest_neighbor_index_factory.h" #include "nearest_neighbor_index_saver.h" +#include "serialized_tensor_ref.h" #include "tensor_attribute_constants.h" #include "tensor_attribute_loader.h" #include "tensor_attribute_saver.h" @@ -261,6 +262,18 @@ TensorAttribute::get_tensor_ref(uint32_t /*docid*/) const notImplemented(); } +SerializedTensorRef +TensorAttribute::get_serialized_tensor_ref(uint32_t) const +{ + notImplemented(); +} + +bool +TensorAttribute::supports_get_serialized_tensor_ref() const +{ + return false; +} + const vespalib::eval::ValueType & TensorAttribute::getTensorType() const { diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h index a4c30a574e5..20c8ae60107 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_attribute.h @@ -63,8 +63,10 @@ public: std::unique_ptr<vespalib::eval::Value> getEmptyTensor() const override; vespalib::eval::TypedCells extract_cells_ref(uint32_t docid) const override; const vespalib::eval::Value& get_tensor_ref(uint32_t docid) const override; + SerializedTensorRef get_serialized_tensor_ref(uint32_t docid) const override; bool supports_extract_cells_ref() const override { return false; } bool supports_get_tensor_ref() const override { return false; } + bool supports_get_serialized_tensor_ref() const override; const vespalib::eval::ValueType & getTensorType() const override; const NearestNeighborIndex* nearest_neighbor_index() const override; void get_state(const vespalib::slime::Inserter& inserter) const override; diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.cpp b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.cpp index fcdb9311ec6..135c62b3cfa 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.cpp +++ b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.cpp @@ -1,8 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "tensor_buffer_operations.h" -#include <vespa/eval/eval/fast_value.hpp> -#include <vespa/eval/eval/value.h> +#include "fast_value_view.h" #include <vespa/eval/eval/value_codec.h> #include <vespa/eval/eval/value_type.h> #include <vespa/eval/streamed/streamed_value_view.h> @@ -35,36 +34,6 @@ adjust_min_alignment(size_t min_alignment) return std::max(std::max(sizeof(uint32_t), sizeof(string_id)), min_alignment); } -struct FastValueView final : Value { - const ValueType& _type; - StringIdVector _labels; - FastValueIndex _index; - TypedCells _cells; - FastValueView(const ValueType& type, ConstArrayRef<string_id> labels, TypedCells cells, size_t num_mapped_dimensions, size_t num_subspaces); - const ValueType& type() const override { return _type; } - const Value::Index& index() const override { return _index; } - TypedCells cells() const override { return _cells; } - MemoryUsage get_memory_usage() const override { - MemoryUsage usage = self_memory_usage<FastValueView>(); - usage.merge(_index.map.estimate_extra_memory_usage()); - return usage; - } -}; - -FastValueView::FastValueView(const ValueType& type, ConstArrayRef<string_id> labels, TypedCells cells, size_t num_mapped_dimensions, size_t num_subspaces) - : Value(), - _type(type), - _labels(labels.begin(), labels.end()), - _index(num_mapped_dimensions, _labels, num_subspaces), - _cells(cells) -{ - for (size_t i = 0; i < num_subspaces; ++i) { - ConstArrayRef<string_id> addr(_labels.data() + (i * num_mapped_dimensions), num_mapped_dimensions); - _index.map.add_mapping(FastAddrMap::hash_labels(addr)); - } - assert(_index.map.size() == num_subspaces); -} - } TensorBufferOperations::TensorBufferOperations(const vespalib::eval::ValueType& tensor_type) diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.h b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.h index 3928b41c2d1..72940cbd6a0 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_operations.h @@ -3,6 +3,7 @@ #pragma once #include "empty_subspace.h" +#include "serialized_tensor_ref.h" #include "subspace_type.h" #include "vector_bundle.h" #include <vespa/vespalib/datastore/aligner.h> @@ -110,6 +111,13 @@ public: auto aligner = select_aligner(cells_mem_size); return VectorBundle(buf.data() + get_cells_offset(num_subspaces, aligner), num_subspaces, _subspace_type); } + SerializedTensorRef get_serialized_tensor_ref(vespalib::ConstArrayRef<char> buf) const { + auto num_subspaces = get_num_subspaces(buf); + auto cells_mem_size = get_cells_mem_size(num_subspaces); + auto aligner = select_aligner(cells_mem_size); + vespalib::ConstArrayRef<vespalib::string_id> labels(reinterpret_cast<const vespalib::string_id*>(buf.data() + get_labels_offset()), num_subspaces * _num_mapped_dimensions); + return SerializedTensorRef(VectorBundle(buf.data() + get_cells_offset(num_subspaces, aligner), num_subspaces, _subspace_type), _num_mapped_dimensions, labels); + } }; } diff --git a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.h b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.h index f602836bd32..2e86ff5fb67 100644 --- a/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.h +++ b/searchlib/src/vespa/searchlib/tensor/tensor_buffer_store.h @@ -44,6 +44,13 @@ public: auto buf = _array_store.get(ref); return _ops.get_vectors(buf); } + SerializedTensorRef get_serialized_tensor_ref(EntryRef ref) const { + if (!ref.valid()) { + return SerializedTensorRef(); + } + auto buf = _array_store.get(ref); + return _ops.get_serialized_tensor_ref(buf); + } // Used by unit test static constexpr uint32_t get_offset_bits() noexcept { return RefType::offset_bits; } diff --git a/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp b/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp index e3cf067bd49..6bba9d96d02 100644 --- a/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp +++ b/searchlib/src/vespa/searchlib/test/fakedata/fakeegcompr64filterocc.cpp @@ -5,6 +5,7 @@ #include "bitdecode64.h" #include "fpfactory.h" #include <vespa/searchlib/queryeval/iterators.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".searchlib.test.fake_eg_compr64_filter_occ"); diff --git a/searchlib/src/vespa/searchlib/test/features/CMakeLists.txt b/searchlib/src/vespa/searchlib/test/features/CMakeLists.txt index ba70fe57e88..f51ccea5678 100644 --- a/searchlib/src/vespa/searchlib/test/features/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/test/features/CMakeLists.txt @@ -4,4 +4,5 @@ vespa_add_library(searchlib_searchlib_test_features distance_closeness_fixture.cpp DEPENDS searchlib + GTest::GTest ) diff --git a/searchlib/src/vespa/searchlib/test/features/distance_closeness_fixture.cpp b/searchlib/src/vespa/searchlib/test/features/distance_closeness_fixture.cpp index e161a4e9839..e0444e8dca7 100644 --- a/searchlib/src/vespa/searchlib/test/features/distance_closeness_fixture.cpp +++ b/searchlib/src/vespa/searchlib/test/features/distance_closeness_fixture.cpp @@ -7,6 +7,8 @@ #include <vespa/eval/eval/value_type.h> #include <vespa/searchcommon/attribute/config.h> #include <vespa/searchlib/tensor/dense_tensor_attribute.h> +#include <vespa/searchlib/tensor/direct_tensor_attribute.h> +#include <vespa/searchlib/tensor/serialized_fast_value_attribute.h> using search::attribute::BasicType; using search::attribute::CollectionType; @@ -15,6 +17,9 @@ using search::attribute::DistanceMetric; using search::fef::test::IndexEnvironment; using search::fef::test::QueryEnvironment; using search::tensor::DenseTensorAttribute; +using search::tensor::DirectTensorAttribute; +using search::tensor::SerializedFastValueAttribute; +using search::tensor::TensorAttribute; using vespalib::eval::SimpleValue; using vespalib::eval::TensorSpec; using vespalib::eval::Value; @@ -24,15 +29,23 @@ namespace search::features::test { namespace { -std::shared_ptr<DenseTensorAttribute> +std::shared_ptr<TensorAttribute> create_tensor_attribute(const vespalib::string& attr_name, const vespalib::string& tensor_type, + bool direct_tensor, uint32_t docid_limit) { Config cfg(BasicType::TENSOR, CollectionType::SINGLE); cfg.setTensorType(ValueType::from_spec(tensor_type)); cfg.set_distance_metric(DistanceMetric::Euclidean); - auto result = std::make_shared<DenseTensorAttribute>(attr_name, cfg); + std::shared_ptr<TensorAttribute> result; + if (cfg.tensorType().is_dense()) { + result = std::make_shared<DenseTensorAttribute>(attr_name, cfg); + } else if (direct_tensor) { + result = std::make_shared<DirectTensorAttribute>(attr_name, cfg); + } else { + result = std::make_shared<SerializedFastValueAttribute>(attr_name, cfg); + } result->addReservedDoc(); result->addDocs(docid_limit-1); result->commit(); @@ -47,10 +60,21 @@ DistanceClosenessFixture::DistanceClosenessFixture(size_t fooCnt, size_t barCnt, const Labels& labels, const vespalib::string& featureName, const vespalib::string& query_tensor) + : DistanceClosenessFixture("tensor(x[2])", false, fooCnt, barCnt, labels, featureName, query_tensor) +{ +} + +DistanceClosenessFixture::DistanceClosenessFixture(const vespalib::string& tensor_type, + bool direct_tensor, + size_t fooCnt, size_t barCnt, + const Labels& labels, + const vespalib::string& featureName, + const vespalib::string& query_tensor) : queryEnv(&indexEnv), rankSetup(factory, indexEnv), mdl(), match_data(), rankProgram(), fooHandles(), barHandles(), tensor_attr(), - docid_limit(11) + docid_limit(11), + _failed(false) { for (size_t i = 0; i < fooCnt; ++i) { uint32_t fieldId = indexEnv.getFieldByName("foo")->id(); @@ -72,14 +96,18 @@ DistanceClosenessFixture::DistanceClosenessFixture(size_t fooCnt, size_t barCnt, queryEnv.getTerms().push_back(term); } if (!query_tensor.empty()) { - tensor_attr = create_tensor_attribute("bar", "tensor(x[2])", docid_limit); + tensor_attr = create_tensor_attribute("bar", tensor_type, direct_tensor, docid_limit); indexEnv.getAttributeMap().add(tensor_attr); + search::fef::indexproperties::type::Attribute::set(indexEnv.getProperties(), "bar", tensor_type); set_query_tensor("qbar", "tensor(x[2])", TensorSpec::from_expr(query_tensor)); } labels.inject(queryEnv.getProperties()); rankSetup.setFirstPhaseRank(featureName); rankSetup.setIgnoreDefaultRankFeatures(true); - ASSERT_TRUE(rankSetup.compile()); + EXPECT_TRUE(rankSetup.compile()) << (_failed = true, ""); + if (_failed) { + return; + } rankSetup.prepareSharedState(queryEnv, queryEnv.getObjectStore()); match_data = mdl.createMatchData(); rankProgram = rankSetup.create_first_phase_program(); diff --git a/searchlib/src/vespa/searchlib/test/features/distance_closeness_fixture.h b/searchlib/src/vespa/searchlib/test/features/distance_closeness_fixture.h index cc1c0a6fb15..8aae1ecb942 100644 --- a/searchlib/src/vespa/searchlib/test/features/distance_closeness_fixture.h +++ b/searchlib/src/vespa/searchlib/test/features/distance_closeness_fixture.h @@ -8,12 +8,12 @@ #include <vespa/searchlib/fef/test/indexenvironmentbuilder.h> #include <vespa/searchlib/fef/test/labels.h> #include <vespa/searchlib/fef/test/queryenvironment.h> -#include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/gtest/gtest.h> using namespace search::fef; using namespace search::fef::test; -namespace search::tensor { class DenseTensorAttribute; } +namespace search::tensor { class TensorAttribute; } namespace vespalib::eval { class TensorSpec; } namespace search::features::test { @@ -33,12 +33,13 @@ struct IndexEnvironmentFixture { IndexEnvironmentBuilder builder(indexEnv); builder.addField(FieldType::ATTRIBUTE, FieldInfo::CollectionType::SINGLE, FieldInfo::DataType::INT64, "foo"); builder.addField(FieldType::ATTRIBUTE, FieldInfo::CollectionType::SINGLE, FieldInfo::DataType::TENSOR, "bar"); + builder.addField(FieldType::INDEX, FieldInfo::CollectionType::SINGLE, FieldInfo::DataType::TENSOR, "ibar"); } }; struct FeatureDumpFixture : public IDumpFeatureVisitor { virtual void visitDumpFeature(const vespalib::string &) override { - TEST_ERROR("no features should be dumped"); + FAIL() << "no features should be dumped"; } FeatureDumpFixture() : IDumpFeatureVisitor() {} ~FeatureDumpFixture() override; @@ -55,11 +56,17 @@ struct DistanceClosenessFixture : BlueprintFactoryFixture, IndexEnvironmentFixtu RankProgram::UP rankProgram; std::vector<TermFieldHandle> fooHandles; std::vector<TermFieldHandle> barHandles; - std::shared_ptr<search::tensor::DenseTensorAttribute> tensor_attr; + std::shared_ptr<search::tensor::TensorAttribute> tensor_attr; uint32_t docid_limit; + bool _failed; DistanceClosenessFixture(size_t fooCnt, size_t barCnt, const Labels &labels, const vespalib::string &featureName, const vespalib::string& query_tensor = ""); + DistanceClosenessFixture(const vespalib::string& tensor_type, + bool direct_tensor, + size_t fooCnt, size_t barCnt, + const Labels &labels, const vespalib::string &featureName, + const vespalib::string& query_tensor = ""); ~DistanceClosenessFixture(); void set_attribute_tensor(uint32_t docid, const vespalib::eval::TensorSpec& spec); void set_query_tensor(const vespalib::string& query_tensor_name, @@ -68,17 +75,21 @@ struct DistanceClosenessFixture : BlueprintFactoryFixture, IndexEnvironmentFixtu feature_t getScore(uint32_t docId) { return Utils::getScoreFeature(*rankProgram, docId); } + vespalib::eval::Value::CREF getObject(uint32_t docId) { + return Utils::getObjectFeature(*rankProgram, docId); + } void setScore(TermFieldHandle handle, uint32_t docId, feature_t score) { match_data->resolveTermField(handle)->setRawScore(docId, score); } void setFooScore(uint32_t i, uint32_t docId, feature_t distance) { - ASSERT_LESS(i, fooHandles.size()); + ASSERT_LT(i, fooHandles.size()); setScore(fooHandles[i], docId, 1.0/(1.0+distance)); } void setBarScore(uint32_t i, uint32_t docId, feature_t distance) { - ASSERT_LESS(i, barHandles.size()); + ASSERT_LT(i, barHandles.size()); setScore(barHandles[i], docId, 1.0/(1.0+distance)); } + bool failed() const noexcept { return _failed; } }; } diff --git a/searchlib/src/vespa/searchlib/transactionlog/session.cpp b/searchlib/src/vespa/searchlib/transactionlog/session.cpp index b11f4027ae7..ec77a5f150e 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/session.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/session.cpp @@ -4,6 +4,7 @@ #include "domainpart.h" #include <vespa/fastlib/io/bufferedfile.h> #include <cassert> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".transactionlog.session"); diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogclient.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogclient.cpp index 133fabd3e5f..17f06b189c6 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogclient.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogclient.cpp @@ -8,7 +8,6 @@ #include <vespa/fnet/transport.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/size_literals.h> -#include <vespa/fastos/thread.h> #include <vespa/log/log.h> diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogclient.h b/searchlib/src/vespa/searchlib/transactionlog/translogclient.h index c3dcecf93b3..72a55f6ae77 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogclient.h +++ b/searchlib/src/vespa/searchlib/transactionlog/translogclient.h @@ -12,7 +12,6 @@ class FNET_Transport; class FRT_Supervisor; class FRT_Target; -class FastOS_ThreadPool; namespace vespalib { class ThreadStackExecutorBase; } namespace search::transactionlog::client { diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index 98a9568e4e8..c96b0cdcd61 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -99,7 +99,7 @@ TransLogServer::TransLogServer(FNET_Transport & transport, const vespalib::strin _baseDir(baseDir), _domainConfig(cfg), _executor(maxThreads, CpuUsage::wrap(tls_executor, CpuUsage::Category::WRITE)), - _threadPool(std::make_unique<FastOS_ThreadPool>()), + _thread(), _supervisor(std::make_unique<FRT_Supervisor>(&transport)), _domains(), _reqQ(), @@ -143,25 +143,24 @@ TransLogServer::TransLogServer(FNET_Transport & transport, const vespalib::strin } else { throw std::runtime_error(make_string("Failed creating tls base dir %s r(%d), e(%d). Requires manual intervention.", _baseDir.c_str(), retval, errno)); } - start(*_threadPool); + _thread = std::thread([this](){run();}); } TransLogServer::~TransLogServer() { - _closed = true; - stop(); - join(); + request_stop(); + _thread.join(); _executor.sync(); _executor.shutdown(); _executor.sync(); } -bool -TransLogServer::onStop() +void +TransLogServer::request_stop() { + _closed = true; LOG(info, "Stopping TLS"); _reqQ.push(nullptr); - return true; } void diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.h b/searchlib/src/vespa/searchlib/transactionlog/translogserver.h index f7ea80c9248..83993da5964 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.h +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.h @@ -2,12 +2,12 @@ #pragma once #include "domainconfig.h" -#include <vespa/vespalib/util/document_runnable.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/document/util/queue.h> #include <vespa/fnet/frt/invokable.h> #include <shared_mutex> #include <atomic> +#include <thread> class FRT_Supervisor; class FNET_Transport; @@ -18,7 +18,7 @@ namespace search::transactionlog { class TransLogServerExplorer; class Domain; -class TransLogServer : private FRT_Invokable, public document::Runnable, public WriterFactory +class TransLogServer : private FRT_Invokable, public WriterFactory { public: friend class TransLogServerExplorer; @@ -36,8 +36,8 @@ public: TransLogServer & setDomainConfig(const DomainConfig & cfg); private: - bool onStop() override; - void run() override; + void request_stop(); + void run(); void exportRPC(FRT_Supervisor & supervisor); void relayToThreadRPC(FRT_RPCRequest *req); @@ -63,11 +63,13 @@ private: using ReadGuard = std::shared_lock<std::shared_mutex>; using WriteGuard = std::unique_lock<std::shared_mutex>; + bool running() const { return !_closed.load(std::memory_order_relaxed); } + vespalib::string _name; vespalib::string _baseDir; DomainConfig _domainConfig; vespalib::ThreadStackExecutor _executor; - std::unique_ptr<FastOS_ThreadPool> _threadPool; + std::thread _thread; std::unique_ptr<FRT_Supervisor> _supervisor; DomainList _domains; mutable std::shared_mutex _domainMutex;; // Protects _domains diff --git a/searchlib/src/vespa/searchlib/util/filesizecalculator.cpp b/searchlib/src/vespa/searchlib/util/filesizecalculator.cpp index 16a96c7b439..c50d402db0e 100644 --- a/searchlib/src/vespa/searchlib/util/filesizecalculator.cpp +++ b/searchlib/src/vespa/searchlib/util/filesizecalculator.cpp @@ -2,6 +2,7 @@ #include "filesizecalculator.h" #include <vespa/vespalib/data/fileheader.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".searchlib.util.filesizecalculator"); diff --git a/searchlib/src/vespa/searchlib/util/runnable.h b/searchlib/src/vespa/searchlib/util/runnable.h deleted file mode 100644 index e268b13e09a..00000000000 --- a/searchlib/src/vespa/searchlib/util/runnable.h +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <mutex> -#include <condition_variable> - -namespace search { - -class Runnable : public FastOS_Runnable -{ -protected: - std::mutex _lock; - std::condition_variable _cond; - bool _done; - bool _stopped; - -public: - Runnable() : - _lock(), _cond(), _done(false), _stopped(false) - { } - void Run(FastOS_ThreadInterface *, void *) override { - doRun(); - - std::lock_guard<std::mutex> guard(_lock); - _stopped = true; - _cond.notify_all(); - } - virtual void doRun() = 0; - void stop() { - std::lock_guard<std::mutex> guard(_lock); - _done = true; - } - void join() { - std::unique_lock<std::mutex> guard(_lock); - while (!_stopped) { - _cond.wait(guard); - } - } -}; - -} // search - diff --git a/searchlib/src/vespa/searchlib/util/url.cpp b/searchlib/src/vespa/searchlib/util/url.cpp index 5eeb16e3f3b..141f54363e9 100644 --- a/searchlib/src/vespa/searchlib/util/url.cpp +++ b/searchlib/src/vespa/searchlib/util/url.cpp @@ -3,6 +3,7 @@ #include "url.h" #include <algorithm> #include <cstdio> +#include <cstring> #include <vespa/log/log.h> LOG_SETUP(".searchlib.util.url"); diff --git a/searchsummary/src/tests/juniper/auxTest.cpp b/searchsummary/src/tests/juniper/auxTest.cpp index 8eda6a2132a..a43294e709f 100644 --- a/searchsummary/src/tests/juniper/auxTest.cpp +++ b/searchsummary/src/tests/juniper/auxTest.cpp @@ -11,9 +11,7 @@ LOG_SETUP(".auxtest"); #define COLOR_HIGH_ON "\e[1;31m" #define COLOR_HIGH_OFF "\e[0m" -#ifndef FASTOS_DEBUG static int debug_level = 0; -#endif bool color_highlight = false; bool verbose = false; diff --git a/searchsummary/src/tests/juniper/testenv.cpp b/searchsummary/src/tests/juniper/testenv.cpp index cc6a6458376..1f8f52d8766 100644 --- a/searchsummary/src/tests/juniper/testenv.cpp +++ b/searchsummary/src/tests/juniper/testenv.cpp @@ -26,11 +26,7 @@ TestEnv::TestEnv(int argc, char **argv, const char* propfile) : switch (c) { case 'd': -#ifdef FASTOS_DEBUG - debug_level = strtol(optarg, NULL, 0); -#else fprintf(stderr, "This version of Juniper compiled without debug\n"); -#endif break; case 'c': color_highlight = true; diff --git a/searchsummary/src/vespa/juniper/Matcher.cpp b/searchsummary/src/vespa/juniper/Matcher.cpp index 22d1bbc7e96..73362aac5a3 100644 --- a/searchsummary/src/vespa/juniper/Matcher.cpp +++ b/searchsummary/src/vespa/juniper/Matcher.cpp @@ -1,7 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <algorithm> -#include <string> #include "query.h" #include "juniperdebug.h" #include "sumdesc.h" @@ -10,6 +8,10 @@ #include "juniperparams.h" #include "config.h" #include <sstream> +#include <algorithm> +#include <string> +#include <cinttypes> + #include <vespa/log/log.h> LOG_SETUP(".juniper.matcher"); diff --git a/searchsummary/src/vespa/juniper/juniperdebug.h b/searchsummary/src/vespa/juniper/juniperdebug.h index d475134dae3..f6d4eda2c46 100644 --- a/searchsummary/src/vespa/juniper/juniperdebug.h +++ b/searchsummary/src/vespa/juniper/juniperdebug.h @@ -27,23 +27,10 @@ /* Logging to log object (juniperlog summary field) */ #define JL(level, stmt) do { if (_log_mask & level) { stmt; } } while (0) -#ifdef FASTOS_DEBUG -extern unsigned debug_level; -#define JD(level, stmt) do { if (debug_level & level) { stmt; } } while (0) -# warning "FASTOS_DEBUG is defined" - -/* Invariant checking */ - -#define JD_INVAR(level, condition, action, log) \ - do { if (!(condition)) { if (debug_level & level) { log; } action; } } while (0) -#else - #define JD_INVAR(level, condition, action, log) \ do { if (!(condition)) { action; } } while (0) #define JD(level, stmt) -#endif - template <class _container> void dump_list(_container& __c) { diff --git a/searchsummary/src/vespa/juniper/rpinterface.cpp b/searchsummary/src/vespa/juniper/rpinterface.cpp index 202b96a442d..afda82c1110 100644 --- a/searchsummary/src/vespa/juniper/rpinterface.cpp +++ b/searchsummary/src/vespa/juniper/rpinterface.cpp @@ -26,13 +26,6 @@ bool AnalyseCompatible(Config* conf1, Config* conf2) void SetDebug(unsigned int mask) { -#ifdef FASTOS_DEBUG - if (mask & ~1 && mask != debug_level) - LOG(info, "Juniper debug mode enabled (0x%x)", mask); - else if (! (debug_level & ~1)) - LOG(info, "Juniper debug mode disabled (0x%x)", mask); - debug_level = mask; -#else // Make sure we do not get 200 of these warnings per query.. static bool warning_printed = false; if (mask && !warning_printed) @@ -41,7 +34,6 @@ void SetDebug(unsigned int mask) "Juniper debug mode requested in binary compiled without debug support!"); warning_printed = true; } -#endif } diff --git a/searchsummary/src/vespa/juniper/sumdesc.cpp b/searchsummary/src/vespa/juniper/sumdesc.cpp index 18e1b7bbd11..aa6aededa0c 100644 --- a/searchsummary/src/vespa/juniper/sumdesc.cpp +++ b/searchsummary/src/vespa/juniper/sumdesc.cpp @@ -6,6 +6,7 @@ #include "Matcher.h" #include "appender.h" #include <vespa/fastlib/text/unicodeutil.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".juniper.sumdesc"); diff --git a/searchsummary/src/vespa/juniper/tokenizer.cpp b/searchsummary/src/vespa/juniper/tokenizer.cpp index 9253f81cf25..965befe01e3 100644 --- a/searchsummary/src/vespa/juniper/tokenizer.cpp +++ b/searchsummary/src/vespa/juniper/tokenizer.cpp @@ -3,6 +3,7 @@ #include "tokenizer.h" #include "juniperdebug.h" #include <vespa/fastlib/text/wordfolder.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".juniper.tokenizer"); @@ -40,7 +41,6 @@ void JuniperTokenizer::scan() { if (_registry == NULL) { // explicit prefetching seems to have negative effect with many threads - // FastOS_Prefetch::NT(const_cast<void *>((const void *)(src + 32))); src = _wordfolder->UCS4Tokenize(src, src_end, dst, dst_end, startpos, result_len); } else { const char * tmpSrc = _registry->tokenize(src, src_end, dst, dst_end, startpos, result_len); diff --git a/security-utils/src/main/java/com/yahoo/security/tls/Capability.java b/security-utils/src/main/java/com/yahoo/security/tls/Capability.java index e60598a9dc2..dce40681b90 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/Capability.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/Capability.java @@ -2,6 +2,7 @@ package com.yahoo.security.tls; import java.util.Arrays; +import java.util.Optional; /** * @author bjorncs @@ -29,6 +30,7 @@ public enum Capability implements ToCapabilitySet { CONTENT__METRICS_API("vespa.content.metrics_api"), CONTENT__PROTON_ADMIN_API("vespa.content.proton_admin_api"), CONTENT__SEARCH_API("vespa.content.search_api"), + CONTENT__STATE_API("vespa.content.state_api"), CONTENT__STATUS_PAGES("vespa.content.status_pages"), CONTENT__STORAGE_API("vespa.content.storage_api"), LOGSERVER_API("vespa.logserver.api"), @@ -48,11 +50,6 @@ public enum Capability implements ToCapabilitySet { @Override public CapabilitySet toCapabilitySet() { return CapabilitySet.of(this); } - public static Capability fromName(String name) { - return Arrays.stream(values()) - .filter(c -> c.name.equals(name)) - .findAny().orElseThrow(() -> - new IllegalArgumentException("Cannot find predefined capability set with name '" + name + "'")); - } + public static Optional<Capability> fromName(String n) { return Arrays.stream(values()).filter(c -> c.name.equals(n)).findAny(); } } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/CapabilitySet.java b/security-utils/src/main/java/com/yahoo/security/tls/CapabilitySet.java index b4674e2ac38..197088ff434 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/CapabilitySet.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/CapabilitySet.java @@ -1,17 +1,17 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.security.tls; -import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; -import java.util.SortedSet; -import java.util.TreeSet; +import java.util.logging.Logger; import java.util.stream.Collectors; /** @@ -19,6 +19,8 @@ import java.util.stream.Collectors; */ public class CapabilitySet implements ToCapabilitySet { + private static final Logger log = Logger.getLogger(CapabilitySet.class.getName()); + private static final Map<String, CapabilitySet> PREDEFINED = new HashMap<>(); @@ -27,13 +29,14 @@ public class CapabilitySet implements ToCapabilitySet { "vespa.all", Capability.values()); public static final CapabilitySet TELEMETRY = predefined( "vespa.telemetry", - Capability.CONTENT__STATUS_PAGES, Capability.CONTENT__METRICS_API, Capability.CONTAINER__STATE_API, - Capability.METRICSPROXY__METRICS_API, Capability.SENTINEL__CONNECTIVITY_CHECK); + Capability.CONTENT__STATUS_PAGES, Capability.CONTENT__STATE_API, Capability.CONTENT__METRICS_API, + Capability.CONTAINER__STATE_API, Capability.METRICSPROXY__METRICS_API, + Capability.SENTINEL__CONNECTIVITY_CHECK); - private static final CapabilitySet SHARED_CAPABILITIES_APP_NODE = CapabilitySet.of( + private static final CapabilitySet SHARED_CAPABILITIES_APP_NODE = CapabilitySet.unionOf(List.of( Capability.LOGSERVER_API, Capability.CONFIGSERVER__CONFIG_API, Capability.CONFIGSERVER__FILEDISTRIBUTION_API, Capability.CONFIGPROXY__CONFIG_API, - Capability.CONFIGPROXY__FILEDISTRIBUTION_API, Capability.SLOBROK__API, TELEMETRY); + Capability.CONFIGPROXY__FILEDISTRIBUTION_API, Capability.SLOBROK__API, TELEMETRY)); public static final CapabilitySet CONTENT_NODE = predefined( "vespa.content_node", @@ -41,7 +44,8 @@ public class CapabilitySet implements ToCapabilitySet { SHARED_CAPABILITIES_APP_NODE); public static final CapabilitySet CONTAINER_NODE = predefined( "vespa.container_node", - Capability.CONTENT__DOCUMENT_API, Capability.CONTENT__SEARCH_API, SHARED_CAPABILITIES_APP_NODE); + Capability.CONTAINER__DOCUMENT_API, Capability.CONTENT__DOCUMENT_API, Capability.CONTENT__SEARCH_API, + SHARED_CAPABILITIES_APP_NODE); public static final CapabilitySet CLUSTER_CONTROLLER_NODE = predefined( "vespa.cluster_controller_node", Capability.CONTENT__CLUSTER_CONTROLLER__INTERNAL_STATE_API, @@ -51,10 +55,11 @@ public class CapabilitySet implements ToCapabilitySet { public static final CapabilitySet CONFIGSERVER_NODE = predefined( "vespa.config_server_node", Capability.CLIENT__FILERECEIVER_API, Capability.CONTAINER__MANAGEMENT_API, Capability.SLOBROK__API, - Capability.CLUSTER_CONTROLLER__REINDEXING, Capability.CLUSTER_CONTROLLER__STATE, TELEMETRY); + Capability.CLUSTER_CONTROLLER__REINDEXING, Capability.CLUSTER_CONTROLLER__STATE, Capability.LOGSERVER_API, + TELEMETRY); private static CapabilitySet predefined(String name, ToCapabilitySet... capabilities) { - var instance = CapabilitySet.of(capabilities); + var instance = CapabilitySet.unionOf(List.of(capabilities)); PREDEFINED.put(name, instance); return instance; } @@ -71,21 +76,23 @@ public class CapabilitySet implements ToCapabilitySet { public static CapabilitySet fromNames(Collection<String> names) { EnumSet<Capability> caps = EnumSet.noneOf(Capability.class); for (String name : names) { - var predefined = PREDEFINED.get(name); - if (predefined != null) caps.addAll(predefined.caps); - else caps.add(Capability.fromName(name)); + var predefinedSet = PREDEFINED.get(name); + var capability = Capability.fromName(name).orElse(null); + if (capability != null) caps.add(capability); + else if (predefinedSet != null) caps.addAll(predefinedSet.caps); + else log.warning("Cannot find capability or capability set with name '%s'".formatted(name)); } return new CapabilitySet(caps); } - public static CapabilitySet unionOf(Collection<CapabilitySet> capSets) { + public static CapabilitySet ofSets(Collection<CapabilitySet> capSets) { EnumSet<Capability> union = EnumSet.noneOf(Capability.class); capSets.forEach(cs -> union.addAll(cs.caps)); return new CapabilitySet(union); } - public static CapabilitySet of(ToCapabilitySet... capabilities) { - return CapabilitySet.unionOf(Arrays.stream(capabilities).map(ToCapabilitySet::toCapabilitySet).toList()); + public static CapabilitySet unionOf(Collection<ToCapabilitySet> caps) { + return CapabilitySet.ofSets(caps.stream().map(ToCapabilitySet::toCapabilitySet).toList()); } public static CapabilitySet of(EnumSet<Capability> caps) { return new CapabilitySet(EnumSet.copyOf(caps)); } @@ -100,8 +107,33 @@ public class CapabilitySet implements ToCapabilitySet { public boolean has(Collection<Capability> caps) { return this.caps.containsAll(caps); } public boolean has(Capability... caps) { return this.caps.containsAll(List.of(caps)); } - public SortedSet<String> toNames() { - return caps.stream().map(Capability::asString).collect(Collectors.toCollection(TreeSet::new)); + public Set<String> toCapabilityNames() { + return caps.stream().map(Capability::asString).collect(Collectors.toSet()); + } + + /** return name of the capability set if predefined, otherwise names of the individual capabilities */ + public Set<String> resolveNames() { + var predefinedName = toPredefinedName().orElse(null); + if (predefinedName != null) return Set.of(predefinedName); + return toCapabilityNames(); + } + + /** @return the name if this is a predefined capability set, or empty if not */ + public Optional<String> toPredefinedName() { + return PREDEFINED.entrySet().stream() + .filter(e -> e.getValue().equals(this)) + .map(Map.Entry::getKey) + .findFirst(); + } + + public static Set<String> resolveNames(Collection<ToCapabilitySet> capabilities) { + var names = new HashSet<String>(); + for (ToCapabilitySet tcs : capabilities) { + if (tcs instanceof Capability c) names.add(c.asString()); + else if (tcs instanceof CapabilitySet cs) names.addAll(cs.resolveNames()); + else throw new IllegalArgumentException(tcs.toString()); + } + return Set.copyOf(names); } public Set<Capability> asSet() { return Collections.unmodifiableSet(caps); } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/ConnectionAuthContext.java b/security-utils/src/main/java/com/yahoo/security/tls/ConnectionAuthContext.java index d7ea93955af..9252b5619f9 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/ConnectionAuthContext.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/ConnectionAuthContext.java @@ -8,6 +8,7 @@ import java.util.List; import java.util.Optional; import java.util.Set; import java.util.logging.Logger; +import java.util.stream.Collectors; import static com.yahoo.security.SubjectAlternativeName.Type.DNS; import static com.yahoo.security.SubjectAlternativeName.Type.URI; @@ -78,10 +79,14 @@ public record ConnectionAuthContext(List<X509Certificate> peerCertificateChain, b.append(". Peer "); if (peer != null) b.append("'").append(peer).append("' "); return b.append("with ").append(peerCertificateString().orElse("<missing-certificate>")).append(". Requires capabilities ") - .append(required.toNames()).append(" but peer has ").append(capabilities.toNames()) + .append(toCapabilityNames(required)).append(" but peer has ").append(toCapabilityNames(capabilities)) .append(".").toString(); } + private static String toCapabilityNames(CapabilitySet capabilities) { + return capabilities.toCapabilityNames().stream().sorted().collect(Collectors.joining(", ", "[", "]")); + } + public Optional<X509Certificate> peerCertificate() { return peerCertificateChain.isEmpty() ? Optional.empty() : Optional.of(peerCertificateChain.get(0)); } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/PeerAuthorizer.java b/security-utils/src/main/java/com/yahoo/security/tls/PeerAuthorizer.java index 746fce0e290..d0e1a33fcac 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/PeerAuthorizer.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/PeerAuthorizer.java @@ -49,7 +49,7 @@ public class PeerAuthorizer { // TODO Pass this through constructor CapabilityMode capabilityMode = TransportSecurityUtils.getCapabilityMode(); return new ConnectionAuthContext( - certChain, CapabilitySet.unionOf(grantedCapabilities), matchedPolicies, capabilityMode); + certChain, CapabilitySet.ofSets(grantedCapabilities), matchedPolicies, capabilityMode); } private static boolean matchesPolicy(PeerPolicy peerPolicy, String cn, List<String> sans) { diff --git a/security-utils/src/main/java/com/yahoo/security/tls/PeerPolicy.java b/security-utils/src/main/java/com/yahoo/security/tls/PeerPolicy.java index ea3d4cfe002..f713bcb0b08 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/PeerPolicy.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/PeerPolicy.java @@ -1,17 +1,25 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.security.tls; +import java.util.Collection; import java.util.List; import java.util.Optional; +import java.util.Set; /** * @author bjorncs */ -public record PeerPolicy(String policyName, Optional<String> description, CapabilitySet capabilities, - List<RequiredPeerCredential> requiredCredentials) { +public record PeerPolicy(String policyName, Optional<String> description, Set<String> capabilityNames, + CapabilitySet capabilities, List<RequiredPeerCredential> requiredCredentials) { public PeerPolicy { requiredCredentials = List.copyOf(requiredCredentials); + capabilityNames = Set.copyOf(capabilityNames); + } + + public PeerPolicy(String policyName, Optional<String> description, + CapabilitySet capabilities, List<RequiredPeerCredential> requiredCredentials) { + this(policyName, description, capabilities.resolveNames(), capabilities, requiredCredentials); } public PeerPolicy(String policyName, List<RequiredPeerCredential> requiredCredentials) { @@ -21,4 +29,16 @@ public record PeerPolicy(String policyName, Optional<String> description, Capabi public PeerPolicy(String policyName, String description, List<RequiredPeerCredential> requiredCredentials) { this(policyName, Optional.ofNullable(description), CapabilitySet.all(), requiredCredentials); } + + public PeerPolicy(String policyName, Optional<String> description, Collection<ToCapabilitySet> capabilities, + List<RequiredPeerCredential> requiredCredentials) { + this(policyName, description, CapabilitySet.resolveNames(capabilities), + CapabilitySet.unionOf(capabilities), requiredCredentials); + } + + public PeerPolicy(String policyName, Optional<String> description, Set<String> capabilities, + List<RequiredPeerCredential> requiredCredentials) { + this(policyName, description, capabilities, CapabilitySet.fromNames(capabilities), + requiredCredentials); + } } diff --git a/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityOptionsJsonSerializer.java b/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityOptionsJsonSerializer.java index 34626e23e7a..66b90b32f79 100644 --- a/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityOptionsJsonSerializer.java +++ b/security-utils/src/main/java/com/yahoo/security/tls/TransportSecurityOptionsJsonSerializer.java @@ -96,15 +96,15 @@ class TransportSecurityOptionsJsonSerializer { throw missingFieldException("required-credentials"); } return new PeerPolicy(authorizedPeer.name, Optional.ofNullable(authorizedPeer.description), - toCapabilities(authorizedPeer.capabilities), toRequestPeerCredentials(authorizedPeer.requiredCredentials)); + toCapabilities(authorizedPeer.capabilities), toRequestPeerCredentials(authorizedPeer.requiredCredentials)); } - private static CapabilitySet toCapabilities(List<String> capabilities) { - if (capabilities == null) return CapabilitySet.all(); + private static Set<String> toCapabilities(List<String> capabilities) { + if (capabilities == null) return Set.of(CapabilitySet.ALL.toPredefinedName().get()); if (capabilities.isEmpty()) throw new IllegalArgumentException("\"capabilities\" array must either be not present " + "(implies all capabilities) or contain at least one capability name"); - return CapabilitySet.fromNames(capabilities); + return Set.copyOf(capabilities); } private static List<RequiredPeerCredential> toRequestPeerCredentials(List<RequiredCredential> requiredCredentials) { @@ -148,7 +148,7 @@ class TransportSecurityOptionsJsonSerializer { authorizedPeer.description = peerPolicy.description().orElse(null); CapabilitySet caps = peerPolicy.capabilities(); if (!caps.hasAll()) { - authorizedPeer.capabilities = List.copyOf(caps.toNames()); + authorizedPeer.capabilities = peerPolicy.capabilityNames().stream().sorted().toList(); } for (RequiredPeerCredential requiredPeerCredential : peerPolicy.requiredCredentials()) { RequiredCredential requiredCredential = new RequiredCredential(); diff --git a/security-utils/src/test/java/com/yahoo/security/tls/CapabilitySetTest.java b/security-utils/src/test/java/com/yahoo/security/tls/CapabilitySetTest.java index 87b16dbff1f..3fa75df27e1 100644 --- a/security-utils/src/test/java/com/yahoo/security/tls/CapabilitySetTest.java +++ b/security-utils/src/test/java/com/yahoo/security/tls/CapabilitySetTest.java @@ -4,8 +4,6 @@ package com.yahoo.security.tls; import org.junit.jupiter.api.Test; import java.util.Arrays; -import java.util.SortedSet; -import java.util.TreeSet; import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -17,10 +15,10 @@ class CapabilitySetTest { @Test void contains_all_capabilities() { - SortedSet<String> expectedNames = Arrays.stream(Capability.values()) + var expectedNames = Arrays.stream(Capability.values()) .map(Capability::asString) - .collect(Collectors.toCollection(TreeSet::new)); - SortedSet<String> actualNames = CapabilitySet.all().toNames(); + .collect(Collectors.toSet()); + var actualNames = CapabilitySet.all().toCapabilityNames(); assertEquals(expectedNames, actualNames); } diff --git a/slobrok/CMakeLists.txt b/slobrok/CMakeLists.txt index 332e1e90282..64f728a7008 100644 --- a/slobrok/CMakeLists.txt +++ b/slobrok/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalib fnet configdefinitions diff --git a/slobrok/src/tests/mirrorapi/mirrorapi.cpp b/slobrok/src/tests/mirrorapi/mirrorapi.cpp index 2ba54fdb9d0..2e03ba52bd4 100644 --- a/slobrok/src/tests/mirrorapi/mirrorapi.cpp +++ b/slobrok/src/tests/mirrorapi/mirrorapi.cpp @@ -9,7 +9,6 @@ #include <vespa/fnet/frt/target.h> #include <vespa/fnet/transport.h> #include <thread> -#include <vespa/fastos/thread.h> #include <vespa/log/log.h> LOG_SETUP("mirrorapi_test"); @@ -139,12 +138,11 @@ Test::Main() cloud::config::SlobroksConfig::Slobrok slobrok; slobrok.connectionspec = "tcp/localhost:18501"; specBuilder.slobrok.push_back(slobrok); - FastOS_ThreadPool threadPool; FNET_Transport transport; FRT_Supervisor supervisor(&transport); MirrorAPI mirror(supervisor, slobrok::ConfiguratorFactory(config::ConfigUri::createFromInstance(specBuilder))); EXPECT_TRUE(!mirror.ready()); - transport.Start(&threadPool); + transport.Start(); std::this_thread::sleep_for(1s); a.reg(); diff --git a/slobrok/src/vespa/slobrok/server/sbenv.h b/slobrok/src/vespa/slobrok/server/sbenv.h index 212c163d0cc..cdfe5a21667 100644 --- a/slobrok/src/vespa/slobrok/server/sbenv.h +++ b/slobrok/src/vespa/slobrok/server/sbenv.h @@ -15,7 +15,6 @@ #include <vespa/vespalib/net/http/simple_health_producer.h> #include <vespa/vespalib/net/http/simple_component_config_producer.h> -class FastOS_ThreadPool; class FNET_Transport; class FNET_Scheduler; class FRT_Supervisor; diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt index 4b7c12b0f31..1062a89f055 100644 --- a/storage/CMakeLists.txt +++ b/storage/CMakeLists.txt @@ -2,7 +2,6 @@ vespa_define_module( DEPENDS vespadefaults - fastos metrics config_cloudconfig configdefinitions diff --git a/storage/src/tests/bucketdb/lockablemaptest.cpp b/storage/src/tests/bucketdb/lockablemaptest.cpp index 582e6957c22..3a16ee170fe 100644 --- a/storage/src/tests/bucketdb/lockablemaptest.cpp +++ b/storage/src/tests/bucketdb/lockablemaptest.cpp @@ -1,6 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/util/document_runnable.h> #include <vespa/storage/bucketdb/btree_lockable_map.hpp> #include <vespa/storage/bucketdb/striped_btree_lockable_map.hpp> #include <vespa/vespalib/datastore/buffer_type.hpp> diff --git a/storage/src/tests/common/dummystoragelink.h b/storage/src/tests/common/dummystoragelink.h index e8ccc38df76..8da92917c08 100644 --- a/storage/src/tests/common/dummystoragelink.h +++ b/storage/src/tests/common/dummystoragelink.h @@ -11,8 +11,6 @@ #include <vespa/storage/common/bucketmessages.h> #include <vespa/storageapi/message/internal.h> -class FastOS_ThreadPool; - namespace storage { class DummyStorageLink : public StorageLink { diff --git a/storage/src/tests/common/metricstest.cpp b/storage/src/tests/common/metricstest.cpp index 97d1c22364f..7231a071319 100644 --- a/storage/src/tests/common/metricstest.cpp +++ b/storage/src/tests/common/metricstest.cpp @@ -52,8 +52,7 @@ namespace { { framework::Clock& _clock; explicit MetricClock(framework::Clock& c) : _clock(c) {} - [[nodiscard]] time_t getTime() const override { return vespalib::count_s(_clock.getMonotonicTime().time_since_epoch()); } - [[nodiscard]] time_t getTimeInMilliSecs() const override { return vespalib::count_ms(_clock.getMonotonicTime().time_since_epoch()); } + [[nodiscard]] metrics::time_point getTime() const override { return _clock.getSystemTime(); } }; } @@ -85,10 +84,7 @@ void MetricsTest::SetUp() { _metricManager->registerMetric(guard, *_topSet); } - _metricsConsumer = std::make_unique<StatusMetricConsumer>( - _node->getComponentRegister(), - *_metricManager, - "status"); + _metricsConsumer = std::make_unique<StatusMetricConsumer>(_node->getComponentRegister(), *_metricManager, "status"); _filestorMetrics = std::make_shared<FileStorMetrics>(); _filestorMetrics->initDiskMetrics(1, 1); @@ -100,7 +96,7 @@ void MetricsTest::SetUp() { _visitorMetrics = std::make_shared<VisitorMetrics>(); _visitorMetrics->initThreads(4); _topSet->registerMetric(*_visitorMetrics); - _metricManager->init(config::ConfigUri(_config->getConfigId()), _node->getThreadPool()); + _metricManager->init(config::ConfigUri(_config->getConfigId())); } void MetricsTest::TearDown() { diff --git a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp index 50ad7b54382..7f3fe06fc29 100644 --- a/storage/src/tests/persistence/filestorage/filestormanagertest.cpp +++ b/storage/src/tests/persistence/filestorage/filestormanagertest.cpp @@ -533,17 +533,18 @@ TEST_F(FileStorManagerTest, handler_priority) { ASSERT_EQ(75, filestorHandler.getNextMessage(stripeId).msg->getPriority()); } -class MessagePusherThread : public document::Runnable { +class MessagePusherThread { public: FileStorHandler& _handler; Document::SP _doc; std::atomic<bool> _done; std::atomic<bool> _threadDone; - + std::thread _thread; + MessagePusherThread(FileStorHandler& handler, Document::SP doc); - ~MessagePusherThread() override; + ~MessagePusherThread(); - void run() override { + void run() { while (!_done) { document::BucketIdFactory factory; document::BucketId bucket(16, factory.getBucketId(_doc->getId()).getRawId()); @@ -558,11 +559,16 @@ public: }; MessagePusherThread::MessagePusherThread(FileStorHandler& handler, Document::SP doc) - : _handler(handler), _doc(std::move(doc)), _done(false), _threadDone(false) -{} -MessagePusherThread::~MessagePusherThread() = default; + : _handler(handler), _doc(std::move(doc)), _done(false), _threadDone(false), _thread() +{ + _thread = std::thread([this](){run();}); +} +MessagePusherThread::~MessagePusherThread() +{ + _thread.join(); +} -class MessageFetchingThread : public document::Runnable { +class MessageFetchingThread { public: const uint32_t _threadId; FileStorHandler& _handler; @@ -571,13 +577,17 @@ public: std::atomic<bool> _done; std::atomic<bool> _failed; std::atomic<bool> _threadDone; - + std::thread _thread; + explicit MessageFetchingThread(FileStorHandler& handler) : _threadId(0), _handler(handler), _config(0), _fetchedCount(0), _done(false), - _failed(false), _threadDone(false) - {} - - void run() override { + _failed(false), _threadDone(false), _thread() + { + _thread = std::thread([this](){run();}); + } + ~MessageFetchingThread(); + + void run() { while (!_done) { FileStorHandler::LockedMessage msg = _handler.getNextMessage(_threadId); if (msg.msg.get()) { @@ -596,6 +606,10 @@ public: _threadDone = true; }; }; +MessageFetchingThread::~MessageFetchingThread() +{ + _thread.join(); +} TEST_F(FileStorManagerTest, handler_paused_multi_thread) { FileStorHandlerComponents c(*this); @@ -606,12 +620,8 @@ TEST_F(FileStorManagerTest, handler_paused_multi_thread) { Document::SP doc(createDocument(content, "id:footype:testdoctype1:n=1234:bar").release()); - FastOS_ThreadPool pool; MessagePusherThread pushthread(filestorHandler, doc); - pushthread.start(pool); - MessageFetchingThread thread(filestorHandler); - thread.start(pool); for (uint32_t i = 0; i < 50; ++i) { std::this_thread::sleep_for(2ms); diff --git a/storage/src/tests/storageserver/statereportertest.cpp b/storage/src/tests/storageserver/statereportertest.cpp index b7903de0fe2..3a772c1ddde 100644 --- a/storage/src/tests/storageserver/statereportertest.cpp +++ b/storage/src/tests/storageserver/statereportertest.cpp @@ -30,7 +30,6 @@ public: }; struct StateReporterTest : Test { - FastOS_ThreadPool _threadPool; framework::defaultimplementation::FakeClock* _clock; std::unique_ptr<TestServiceLayerApp> _node; std::unique_ptr<DummyStorageLink> _top; @@ -54,15 +53,13 @@ struct MetricClock : public metrics::MetricManager::Timer { framework::Clock& _clock; explicit MetricClock(framework::Clock& c) : _clock(c) {} - [[nodiscard]] time_t getTime() const override { return vespalib::count_s(_clock.getMonotonicTime().time_since_epoch()); } - [[nodiscard]] time_t getTimeInMilliSecs() const override { return vespalib::count_ms(_clock.getMonotonicTime().time_since_epoch()); } + [[nodiscard]] metrics::time_point getTime() const override { return _clock.getSystemTime(); } }; } StateReporterTest::StateReporterTest() - : _threadPool(), - _clock(nullptr), + : _clock(nullptr), _top(), _stateReporter() { @@ -87,17 +84,14 @@ void StateReporterTest::SetUp() { _metricManager->registerMetric(guard, *_topSet); } - _stateReporter = std::make_unique<StateReporter>( - _node->getComponentRegister(), - *_metricManager, - _generationFetcher, - "status"); + _stateReporter = std::make_unique<StateReporter>(_node->getComponentRegister(), *_metricManager, + _generationFetcher, "status"); _filestorMetrics = std::make_shared<FileStorMetrics>(); _filestorMetrics->initDiskMetrics(1, 1); _topSet->registerMetric(*_filestorMetrics); - _metricManager->init(config::ConfigUri(_config->getConfigId()), _node->getThreadPool()); + _metricManager->init(config::ConfigUri(_config->getConfigId())); } void StateReporterTest::TearDown() { @@ -127,20 +121,14 @@ vespalib::Slime slime; \ #define ASSERT_GENERATION(jsonData, component, generation) \ { \ PARSE_JSON(jsonData); \ - ASSERT_EQ( \ - generation, \ - slime.get()["config"][component]["generation"].asDouble()); \ + ASSERT_EQ(generation, slime.get()["config"][component]["generation"].asDouble()); \ } #define ASSERT_NODE_STATUS(jsonData, code, message) \ { \ PARSE_JSON(jsonData); \ - ASSERT_EQ( \ - vespalib::string(code), \ - slime.get()["status"]["code"].asString().make_string()); \ - ASSERT_EQ( \ - vespalib::string(message), \ - slime.get()["status"]["message"].asString().make_string()); \ + ASSERT_EQ(vespalib::string(code), slime.get()["status"]["code"].asString().make_string()); \ + ASSERT_EQ(vespalib::string(message), slime.get()["status"]["message"].asString().make_string()); \ } #define ASSERT_METRIC_GET_PUT(jsonData, expGetCount, expPutCount) \ @@ -150,16 +138,11 @@ vespalib::Slime slime; \ double putCount = -1; \ size_t metricCount = slime.get()["metrics"]["values"].children(); \ for (size_t j=0; j<metricCount; j++) { \ - const vespalib::string name = slime.get()["metrics"]["values"][j]["name"] \ - .asString().make_string(); \ - if (name.compare("vds.filestor.allthreads.get.count") == 0) \ - { \ - getCount = slime.get()["metrics"]["values"][j]["values"]["count"] \ - .asDouble(); \ - } else if (name.compare("vds.filestor.allthreads.put.count") == 0) \ - { \ - putCount = slime.get()["metrics"]["values"][j]["values"]["count"] \ - .asDouble(); \ + const vespalib::string name = slime.get()["metrics"]["values"][j]["name"].asString().make_string(); \ + if (name.compare("vds.filestor.allthreads.get.count") == 0) { \ + getCount = slime.get()["metrics"]["values"][j]["values"]["count"].asDouble(); \ + } else if (name.compare("vds.filestor.allthreads.put.count") == 0) { \ + putCount = slime.get()["metrics"]["values"][j]["values"]["count"].asDouble(); \ } \ } \ ASSERT_EQ(expGetCount, getCount); \ @@ -228,8 +211,7 @@ TEST_F(StateReporterTest, report_metrics) { for (uint32_t i = 0; i < 6; ++i) { _clock->addSecondsToTime(60); _metricManager->timeChangedNotification(); - while (int64_t(_metricManager->getLastProcessedTime()) < vespalib::count_s(_clock->getMonotonicTime().time_since_epoch())) - { + while (int64_t(_metricManager->getLastProcessedTime()) < vespalib::count_s(_clock->getMonotonicTime().time_since_epoch())) { std::this_thread::sleep_for(1ms); } } diff --git a/storage/src/vespa/storage/common/statusmetricconsumer.cpp b/storage/src/vespa/storage/common/statusmetricconsumer.cpp index 8eb3e9f3ab6..c6f73540605 100644 --- a/storage/src/vespa/storage/common/statusmetricconsumer.cpp +++ b/storage/src/vespa/storage/common/statusmetricconsumer.cpp @@ -5,7 +5,6 @@ #include <boost/lexical_cast.hpp> #include <vespa/metrics/jsonwriter.h> #include <vespa/metrics/textwriter.h> -#include <vespa/metrics/xmlwriter.h> #include <vespa/metrics/metricmanager.h> #include <vespa/storageapi/messageapi/storagemessage.h> #include <vespa/vespalib/stllike/asciistream.h> @@ -37,10 +36,6 @@ StatusMetricConsumer::getReportContentType(const framework::HttpUrlPath& path) c return "text/plain"; } - if (path.getAttribute("format") == "xml") { - return "application/xml"; - } - if (path.getAttribute("format") == "text") { return "text/plain"; } @@ -67,7 +62,6 @@ StatusMetricConsumer::reportStatus(std::ostream& out, LOG(debug, "Not calling update hooks as dontcallupdatehooks option has been given"); } int64_t currentTimeS(vespalib::count_s(_component.getClock().getMonotonicTime().time_since_epoch())); - bool xml = (path.getAttribute("format") == "xml"); bool json = (path.getAttribute("format") == "json"); int verbosity(path.get("verbosity", 0)); @@ -131,13 +125,7 @@ StatusMetricConsumer::reportStatus(std::ostream& out, } std::string consumer = path.getAttribute("consumer", ""); - if (xml) { - out << "<?xml version=\"1.0\"?>\n"; - vespalib::XmlOutputStream xos(out); - metrics::XmlWriter xmlWriter(xos, snapshot->getPeriod(), verbosity); - _manager.visit(metricLock, *snapshot, xmlWriter, consumer); - out << "\n"; - } else if (json) { + if (json) { vespalib::asciistream jsonStreamData; vespalib::JsonStream stream(jsonStreamData, true); stream << Object() << "metrics"; diff --git a/storage/src/vespa/storage/common/storagelinkqueued.h b/storage/src/vespa/storage/common/storagelinkqueued.h index 74434c0116b..17a344a368a 100644 --- a/storage/src/vespa/storage/common/storagelinkqueued.h +++ b/storage/src/vespa/storage/common/storagelinkqueued.h @@ -16,7 +16,6 @@ #include "storagelink.h" #include <vespa/storageframework/generic/thread/runnable.h> -#include <vespa/vespalib/util/document_runnable.h> #include <deque> #include <limits> #include <mutex> diff --git a/storage/src/vespa/storage/distributor/distributor_stripe_pool.cpp b/storage/src/vespa/storage/distributor/distributor_stripe_pool.cpp index 26ca8963783..ceadd20baca 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe_pool.cpp +++ b/storage/src/vespa/storage/distributor/distributor_stripe_pool.cpp @@ -8,8 +8,7 @@ namespace storage::distributor { DistributorStripePool::DistributorStripePool(bool test_mode, PrivateCtorTag) - : _thread_pool(std::make_unique<FastOS_ThreadPool>()), - _n_stripe_bits(0), + : _n_stripe_bits(0), _stripes(), _threads(), _mutex(), @@ -119,7 +118,7 @@ void DistributorStripePool::start(const std::vector<TickableStripe*>& stripes) { } std::unique_lock lock(_mutex); // Ensure _threads is visible to all started threads for (auto& s : _stripes) { - _threads.emplace_back(_thread_pool->NewThread(s.get())); + _threads.start([ptr = s.get()](){ ptr->run(); }); } } @@ -131,9 +130,7 @@ void DistributorStripePool::stop_and_join() { for (auto& s : _stripes) { s->signal_should_stop(); } - for (auto* t : _threads) { - t->Join(); - } + _threads.join(); } void DistributorStripePool::set_tick_wait_duration(vespalib::duration new_tick_wait_duration) noexcept { diff --git a/storage/src/vespa/storage/distributor/distributor_stripe_pool.h b/storage/src/vespa/storage/distributor/distributor_stripe_pool.h index 00f5f57edf9..6ac95c27b76 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe_pool.h +++ b/storage/src/vespa/storage/distributor/distributor_stripe_pool.h @@ -2,14 +2,12 @@ #pragma once #include <vespa/vespalib/util/time.h> +#include <vespa/vespalib/util/thread.h> #include <atomic> #include <condition_variable> #include <mutex> #include <vector> -class FastOS_ThreadInterface; -class FastOS_ThreadPool; - namespace storage::distributor { class DistributorStripeThread; @@ -37,12 +35,10 @@ class TickableStripe; */ class DistributorStripePool { using StripeVector = std::vector<std::unique_ptr<DistributorStripeThread>>; - using NativeThreadVector = std::vector<FastOS_ThreadInterface*>; - std::unique_ptr<FastOS_ThreadPool> _thread_pool; uint8_t _n_stripe_bits; StripeVector _stripes; - NativeThreadVector _threads; + vespalib::ThreadPool _threads; std::mutex _mutex; std::condition_variable _parker_cond; size_t _parked_threads; // Must be protected by _park_mutex diff --git a/storage/src/vespa/storage/distributor/distributor_stripe_thread.cpp b/storage/src/vespa/storage/distributor/distributor_stripe_thread.cpp index 8f37dbbbf5d..72854d9af75 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe_thread.cpp +++ b/storage/src/vespa/storage/distributor/distributor_stripe_thread.cpp @@ -23,7 +23,7 @@ DistributorStripeThread::DistributorStripeThread(TickableStripe& stripe, DistributorStripeThread::~DistributorStripeThread() = default; -void DistributorStripeThread::Run(FastOS_ThreadInterface*, void*) { +void DistributorStripeThread::run() { uint32_t tick_waits_inhibited = 0; while (!should_stop_thread_relaxed()) { while (should_park_relaxed()) { diff --git a/storage/src/vespa/storage/distributor/distributor_stripe_thread.h b/storage/src/vespa/storage/distributor/distributor_stripe_thread.h index 7015d27a53e..8b9453ab3f3 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe_thread.h +++ b/storage/src/vespa/storage/distributor/distributor_stripe_thread.h @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/fastos/thread.h> #include <vespa/vespalib/util/time.h> #include <atomic> #include <condition_variable> @@ -21,7 +20,7 @@ class TickableStripe; * A DistributorStripeThread instance is bidirectionally bound to a particular pool and * should therefore always be created by the pool itself (never standalone). */ -class DistributorStripeThread : private FastOS_Runnable { +class DistributorStripeThread { using AtomicDuration = std::atomic<vespalib::duration>; TickableStripe& _stripe; @@ -41,7 +40,7 @@ public: DistributorStripePool& stripe_pool); ~DistributorStripeThread(); - void Run(FastOS_ThreadInterface*, void*) override; + void run(); // Wakes up stripe thread if it's currently waiting for an external event to be triggered, // such as the arrival of a new RPC message. If thread is parked this call will have no diff --git a/storage/src/vespa/storage/distributor/messagetracker.cpp b/storage/src/vespa/storage/distributor/messagetracker.cpp index 93db31bdc29..8830e5ecabc 100644 --- a/storage/src/vespa/storage/distributor/messagetracker.cpp +++ b/storage/src/vespa/storage/distributor/messagetracker.cpp @@ -3,6 +3,7 @@ #include "messagetracker.h" #include <vespa/storageapi/messageapi/bucketcommand.h> #include <vespa/storageapi/messageapi/bucketreply.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".messagetracker"); diff --git a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp index 55fe2e039e1..bdf4fa2ba72 100644 --- a/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/twophaseupdateoperation.cpp @@ -12,7 +12,7 @@ #include <vespa/storage/distributor/distributor_bucket_space_repo.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/vdslib/state/cluster_state_bundle.h> -#include <vespa/vespalib/stllike/hash_map.hpp> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".distributor.callback.twophaseupdate"); diff --git a/storage/src/vespa/storage/distributor/sentmessagemap.cpp b/storage/src/vespa/storage/distributor/sentmessagemap.cpp index 44dd4fbde89..4b7292c1e81 100644 --- a/storage/src/vespa/storage/distributor/sentmessagemap.cpp +++ b/storage/src/vespa/storage/distributor/sentmessagemap.cpp @@ -4,6 +4,7 @@ #include <vespa/storage/distributor/operations/operation.h> #include <sstream> #include <set> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".distributor.callback.map"); diff --git a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp index 8690f6e122d..0b4e32d637d 100644 --- a/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp +++ b/storage/src/vespa/storage/frameworkimpl/status/statuswebserver.cpp @@ -10,6 +10,7 @@ #include <vespa/vespalib/component/vtag.h> #include <vespa/vespalib/net/connection_auth_context.h> #include <vespa/vespalib/net/crypto_engine.h> +#include <vespa/vespalib/net/tls/statistics.h> #include <vespa/config/subscription/configuri.h> #include <vespa/config/helper/configfetcher.hpp> #include <functional> @@ -203,6 +204,7 @@ StatusWebServer::handlePage(const framework::HttpUrlPath& urlpath, vespalib::Por if (auth_ctx.capabilities().contains_all(reporter->required_capabilities())) { invoke_reporter(*reporter, urlpath, request); } else { + vespalib::net::tls::CapabilityStatistics::get().inc_status_capability_checks_failed(); // TODO should print peer address as well; not currently exposed LOG(warning, "Peer with %s denied status page access to '%s' due to insufficient " "credentials (had %s, needed %s)", diff --git a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h index 08a48cc2d8a..99f61c62cd1 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestormanager.h +++ b/storage/src/vespa/storage/persistence/filestorage/filestormanager.h @@ -10,7 +10,6 @@ #include "filestorhandler.h" #include "service_layer_host_info_reporter.h" -#include <vespa/vespalib/util/document_runnable.h> #include <vespa/vespalib/util/isequencedtaskexecutor.h> #include <vespa/document/bucket/bucketid.h> #include <vespa/persistence/spi/bucketexecutor.h> diff --git a/storage/src/vespa/storage/storageserver/CMakeLists.txt b/storage/src/vespa/storage/storageserver/CMakeLists.txt index 8b82bb251b2..009f8170669 100644 --- a/storage/src/vespa/storage/storageserver/CMakeLists.txt +++ b/storage/src/vespa/storage/storageserver/CMakeLists.txt @@ -14,7 +14,6 @@ vespa_add_library(storage_storageserver OBJECT documentapiconverter.cpp fnet_metrics_wrapper.cpp mergethrottler.cpp - messagesink.cpp opslogger.cpp priorityconverter.cpp rpcrequestwrapper.cpp diff --git a/storage/src/vespa/storage/storageserver/communicationmanager.h b/storage/src/vespa/storage/storageserver/communicationmanager.h index 593640e7d03..156ec8bc031 100644 --- a/storage/src/vespa/storage/storageserver/communicationmanager.h +++ b/storage/src/vespa/storage/storageserver/communicationmanager.h @@ -23,7 +23,6 @@ #include <vespa/messagebus/imessagehandler.h> #include <vespa/messagebus/ireplyhandler.h> #include <vespa/config/helper/ifetchercallback.h> -#include <vespa/vespalib/util/document_runnable.h> #include <vespa/config/subscription/configuri.h> #include <vespa/config-bucketspaces.h> #include <map> diff --git a/storage/src/vespa/storage/storageserver/mergethrottler.h b/storage/src/vespa/storage/storageserver/mergethrottler.h index 0adfb209e66..dddcd42aad7 100644 --- a/storage/src/vespa/storage/storageserver/mergethrottler.h +++ b/storage/src/vespa/storage/storageserver/mergethrottler.h @@ -15,7 +15,6 @@ #include <vespa/storageframework/generic/thread/runnable.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/document/bucket/bucket.h> -#include <vespa/vespalib/util/document_runnable.h> #include <vespa/metrics/metricset.h> #include <vespa/metrics/summetric.h> #include <vespa/metrics/countmetric.h> diff --git a/storage/src/vespa/storage/storageserver/messagesink.cpp b/storage/src/vespa/storage/storageserver/messagesink.cpp deleted file mode 100644 index 94762e545d9..00000000000 --- a/storage/src/vespa/storage/storageserver/messagesink.cpp +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "messagesink.h" -#include <vespa/storageapi/message/persistence.h> -#include <ostream> - -using std::shared_ptr; - -namespace storage { - -MessageSink::MessageSink() - : StorageLink("Message Sink") -{ -} - -MessageSink::~MessageSink() -{ - closeNextLink(); -} - -void -MessageSink::print(std::ostream& out, bool verbose, - const std::string& indent) const -{ - (void) verbose; (void) indent; - out << "MessageSink"; -} - -namespace { -#if 0 - std::string getTimeString() { - char timeBuf[200]; - time_t tm; - struct tm tms; - time(&tm); - gmtime_r(&tm, &tms); - strftime(timeBuf, sizeof(timeBuf), "%Y-%m-%d:%H:%M:%S %Z", &tms); - return std::string(timeBuf); - } -#endif -} - -IMPL_MSG_COMMAND_H(MessageSink, Get) -{ - //LOG(event, "[%s] Get %s", getTimeString().c_str(), - // cmd->getDocumentId()->toString()); - shared_ptr<api::StorageReply> rmsg(new api::GetReply(*cmd)); - rmsg->setResult(api::ReturnCode::NOT_IMPLEMENTED); - sendUp(rmsg); - return true; -} - -IMPL_MSG_COMMAND_H(MessageSink, Put) -{ - //LOG(event, "[%s] Put %s", getTimeString().c_str(), - // cmd->getDocumentId()->toString()); - shared_ptr<api::StorageReply> rmsg(new api::PutReply(*cmd)); - rmsg->setResult(api::ReturnCode::OK); - sendUp(rmsg); - return true; -} - -IMPL_MSG_COMMAND_H(MessageSink, Remove) -{ - //LOG(event, "[%s] Remove %s", getTimeString().c_str(), - // cmd->getDocumentId()->toString()); - shared_ptr<api::StorageReply> rmsg(new api::RemoveReply(*cmd)); - rmsg->setResult(api::ReturnCode::OK); - sendUp(rmsg); - return true; -} - -IMPL_MSG_COMMAND_H(MessageSink, Revert) -{ - //LOG(event, "[%s] Revert %s", getTimeString().c_str(), - // cmd->getDocumentId()->toString()); - shared_ptr<api::StorageReply> rmsg(new api::RevertReply(*cmd)); - rmsg->setResult(api::ReturnCode::OK); - sendUp(rmsg); - return true; -} - -} // storage diff --git a/storage/src/vespa/storage/storageserver/messagesink.h b/storage/src/vespa/storage/storageserver/messagesink.h deleted file mode 100644 index d98d0439b48..00000000000 --- a/storage/src/vespa/storage/storageserver/messagesink.h +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class storage::MessageSink - * @ingroup storageserver - * - * @brief This class grabs persistence messages, and answers them without doing anything. - * - * @version $Id$ - */ - -#pragma once - -#include <vespa/storage/common/storagelink.h> - -namespace storage { - -class MessageSink : public StorageLink { -public: - explicit MessageSink(); - MessageSink(const MessageSink &) = delete; - MessageSink& operator=(const MessageSink &) = delete; - ~MessageSink(); - - void print(std::ostream& out, bool verbose, const std::string& indent) const override; - -private: - DEF_MSG_COMMAND_H(Get); - DEF_MSG_COMMAND_H(Put); - DEF_MSG_COMMAND_H(Remove); - DEF_MSG_COMMAND_H(Revert); -}; - -} diff --git a/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.cpp b/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.cpp index e940dc71722..3f015d91a4a 100644 --- a/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.cpp +++ b/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.cpp @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "rpc_target.h" #include "shared_rpc_resources.h" -#include <vespa/fastos/thread.h> #include <vespa/fnet/frt/supervisor.h> #include <vespa/fnet/frt/target.h> #include <vespa/fnet/transport.h> @@ -66,8 +65,7 @@ SharedRpcResources::SharedRpcResources(const config::ConfigUri& config_uri, int rpc_server_port, size_t rpc_thread_pool_size, size_t rpc_events_before_wakeup) - : _thread_pool(std::make_unique<FastOS_ThreadPool>()), - _transport(std::make_unique<FNET_Transport>(fnet::TransportConfig(rpc_thread_pool_size). + : _transport(std::make_unique<FNET_Transport>(fnet::TransportConfig(rpc_thread_pool_size). events_before_wakeup(rpc_events_before_wakeup))), _orb(std::make_unique<FRT_Supervisor>(_transport.get())), _slobrok_register(std::make_unique<slobrok::api::RegisterAPI>(*_orb, slobrok::ConfiguratorFactory(config_uri))), @@ -92,7 +90,7 @@ void SharedRpcResources::start_server_and_register_slobrok(vespalib::stringref m if (!_orb->Listen(_rpc_server_port)) { throw IllegalStateException(fmt("Failed to listen to RPC port %d", _rpc_server_port), VESPA_STRLOC); } - _transport->Start(_thread_pool.get()); + _transport->Start(); _slobrok_register->registerName(my_handle); wait_until_slobrok_is_ready(); _handle = my_handle; diff --git a/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.h b/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.h index a30fcdc4ea7..953492089c1 100644 --- a/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.h +++ b/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.h @@ -6,7 +6,6 @@ #include <vespa/vespalib/stllike/string.h> #include <memory> -class FastOS_ThreadPool; class FNET_Transport; class FRT_Supervisor; @@ -19,7 +18,6 @@ namespace storage::rpc { class SharedRpcResources { class RpcTargetFactoryImpl; - std::unique_ptr<FastOS_ThreadPool> _thread_pool; std::unique_ptr<FNET_Transport> _transport; std::unique_ptr<FRT_Supervisor> _orb; std::unique_ptr<slobrok::api::RegisterAPI> _slobrok_register; diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp index 2836ab80acf..9f8456afc37 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.cpp +++ b/storage/src/vespa/storage/storageserver/storagenode.cpp @@ -33,34 +33,36 @@ namespace storage { namespace { - using vespalib::getLastErrorString; +using vespalib::getLastErrorString; - void writePidFile(const vespalib::string& pidfile) - { - ssize_t rv = -1; - vespalib::string mypid = vespalib::make_string("%d\n", getpid()); - size_t lastSlash = pidfile.rfind('/'); - if (lastSlash != vespalib::string::npos) { - std::filesystem::create_directories(std::filesystem::path(pidfile.substr(0, lastSlash))); - } - int fd = open(pidfile.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); - if (fd != -1) { - rv = write(fd, mypid.c_str(), mypid.size()); - close(fd); - } - if (rv < 1) { - LOG(warning, "Failed to write pidfile '%s': %s", - pidfile.c_str(), getLastErrorString().c_str()); - } +void +writePidFile(const vespalib::string& pidfile) +{ + ssize_t rv = -1; + vespalib::string mypid = vespalib::make_string("%d\n", getpid()); + size_t lastSlash = pidfile.rfind('/'); + if (lastSlash != vespalib::string::npos) { + std::filesystem::create_directories(std::filesystem::path(pidfile.substr(0, lastSlash))); + } + int fd = open(pidfile.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (fd != -1) { + rv = write(fd, mypid.c_str(), mypid.size()); + close(fd); } + if (rv < 1) { + LOG(warning, "Failed to write pidfile '%s': %s", + pidfile.c_str(), getLastErrorString().c_str()); + } +} - void removePidFile(const vespalib::string& pidfile) - { - if (unlink(pidfile.c_str()) != 0) { - LOG(warning, "Failed to delete pidfile '%s': %s", - pidfile.c_str(), getLastErrorString().c_str()); - } +void +removePidFile(const vespalib::string& pidfile) +{ + if (unlink(pidfile.c_str()) != 0) { + LOG(warning, "Failed to delete pidfile '%s': %s", + pidfile.c_str(), getLastErrorString().c_str()); } +} } // End of anonymous namespace @@ -201,7 +203,7 @@ StorageNode::initialize() // have been created, such that we don't need to pay the extra cost of // reinitializing metric manager often. if ( ! _context.getComponentRegister().getMetricManager().isInitialized() ) { - _context.getComponentRegister().getMetricManager().init(_configUri, _context.getThreadPool()); + _context.getComponentRegister().getMetricManager().init(_configUri); } if (_chain) { @@ -429,7 +431,8 @@ StorageNode::shutdown() LOG(debug, "Done shutting down node"); } -void StorageNode::configure(std::unique_ptr<StorServerConfig> config) { +void +StorageNode::configure(std::unique_ptr<StorServerConfig> config) { log_config_received(*config); // When we get config, we try to grab the config lock to ensure noone // else is doing configuration work, and then we write the new config @@ -445,7 +448,8 @@ void StorageNode::configure(std::unique_ptr<StorServerConfig> config) { } } -void StorageNode::configure(std::unique_ptr<UpgradingConfig> config) { +void +StorageNode::configure(std::unique_ptr<UpgradingConfig> config) { log_config_received(*config); { std::lock_guard configLockGuard(_configLock); @@ -457,7 +461,8 @@ void StorageNode::configure(std::unique_ptr<UpgradingConfig> config) { } } -void StorageNode::configure(std::unique_ptr<StorDistributionConfig> config) { +void +StorageNode::configure(std::unique_ptr<StorDistributionConfig> config) { log_config_received(*config); { std::lock_guard configLockGuard(_configLock); @@ -486,7 +491,8 @@ StorageNode::configure(std::unique_ptr<document::config::DocumenttypesConfig> co } } -void StorageNode::configure(std::unique_ptr<BucketspacesConfig> config) { +void +StorageNode::configure(std::unique_ptr<BucketspacesConfig> config) { log_config_received(*config); { std::lock_guard configLockGuard(_configLock); diff --git a/storage/src/vespa/storage/storageserver/storagenodecontext.h b/storage/src/vespa/storage/storageserver/storagenodecontext.h index f07bdd37cd4..52709fb1d9b 100644 --- a/storage/src/vespa/storage/storageserver/storagenodecontext.h +++ b/storage/src/vespa/storage/storageserver/storagenodecontext.h @@ -28,12 +28,6 @@ struct StorageNodeContext { */ ComponentRegister& getComponentRegister() { return *_componentRegister; } - /** - * There currently exist threads that doesn't use the component model. - * Let the backend threadpool be accessible for now. - */ - FastOS_ThreadPool& getThreadPool() { return _threadPool.getThreadPool(); } - ~StorageNodeContext(); protected: diff --git a/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.cpp b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.cpp index 5e281152b2b..ad74e020a82 100644 --- a/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.cpp +++ b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.cpp @@ -27,9 +27,14 @@ TlsStatisticsMetricsWrapper::TlsStatisticsMetricsWrapper(metrics::MetricSet* own "connections broken due to failures during frame encoding or decoding", this), failed_tls_config_reloads("failed-tls-config-reloads", {}, "Number of times " "background reloading of TLS config has failed", this), + rpc_capability_checks_failed("rpc-capability-checks-failed", {}, + "Number of RPC operations that failed to due one or more missing capabilities", this), + status_capability_checks_failed("status-capability-checks-failed", {}, + "Number of status page operations that failed to due one or more missing capabilities", this), last_client_stats_snapshot(), last_server_stats_snapshot(), - last_config_stats_snapshot() + last_config_stats_snapshot(), + last_capability_stats_snapshot() {} TlsStatisticsMetricsWrapper::~TlsStatisticsMetricsWrapper() = default; @@ -60,9 +65,16 @@ void TlsStatisticsMetricsWrapper::update_metrics_with_snapshot_delta() { failed_tls_config_reloads.set(config_delta.failed_config_reloads); + auto capability_current = vespalib::net::tls::CapabilityStatistics::get().snapshot(); + auto capability_delta = capability_current.subtract(last_capability_stats_snapshot); + + rpc_capability_checks_failed.set(capability_delta.rpc_capability_checks_failed); + status_capability_checks_failed.set(capability_delta.status_capability_checks_failed); + last_server_stats_snapshot = server_current; last_client_stats_snapshot = client_current; last_config_stats_snapshot = config_current; + last_capability_stats_snapshot = capability_current; } } diff --git a/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.h b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.h index 7bb51acd1fe..daf02b53b7a 100644 --- a/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.h +++ b/storage/src/vespa/storage/storageserver/tls_statistics_metrics_wrapper.h @@ -29,9 +29,13 @@ class TlsStatisticsMetricsWrapper : public metrics::MetricSet { metrics::LongCountMetric failed_tls_config_reloads; + metrics::LongCountMetric rpc_capability_checks_failed; + metrics::LongCountMetric status_capability_checks_failed; + vespalib::net::tls::ConnectionStatistics::Snapshot last_client_stats_snapshot; vespalib::net::tls::ConnectionStatistics::Snapshot last_server_stats_snapshot; vespalib::net::tls::ConfigStatistics::Snapshot last_config_stats_snapshot; + vespalib::net::tls::CapabilityStatistics::Snapshot last_capability_stats_snapshot; public: explicit TlsStatisticsMetricsWrapper(metrics::MetricSet* owner); diff --git a/storage/src/vespa/storage/visiting/visitormanager.h b/storage/src/vespa/storage/visiting/visitormanager.h index 0c5ec08fb4c..02bb37db59f 100644 --- a/storage/src/vespa/storage/visiting/visitormanager.h +++ b/storage/src/vespa/storage/visiting/visitormanager.h @@ -30,7 +30,6 @@ #include <vespa/storageapi/message/internal.h> #include <vespa/storageapi/message/visitor.h> #include <vespa/config/helper/ifetchercallback.h> -#include <vespa/vespalib/util/document_runnable.h> namespace config { class ConfigUri; diff --git a/storage/src/vespa/storage/visiting/visitorthread.h b/storage/src/vespa/storage/visiting/visitorthread.h index 729b675df3a..f6204fed438 100644 --- a/storage/src/vespa/storage/visiting/visitorthread.h +++ b/storage/src/vespa/storage/visiting/visitorthread.h @@ -22,7 +22,6 @@ #include <vespa/storageframework/generic/thread/runnable.h> #include <vespa/storageapi/messageapi/messagehandler.h> #include <vespa/metrics/metrictimer.h> -#include <vespa/vespalib/util/document_runnable.h> #include <atomic> #include <deque> diff --git a/storage/src/vespa/storageapi/mbusprot/serializationhelper.h b/storage/src/vespa/storageapi/mbusprot/serializationhelper.h index 457a6178704..671ffbddd6f 100644 --- a/storage/src/vespa/storageapi/mbusprot/serializationhelper.h +++ b/storage/src/vespa/storageapi/mbusprot/serializationhelper.h @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/fastos/types.h> #include <vespa/document/base/globalid.h> #include <vespa/document/fieldvalue/document.h> #include <vespa/document/util/bytebuffer.h> @@ -13,12 +12,6 @@ namespace storage::mbusprot { class SerializationHelper { public: - static int64_t getLong(document::ByteBuffer& buf) { - int64_t tmp; - buf.getLongNetwork(tmp); - return tmp; - } - static int32_t getInt(document::ByteBuffer& buf) { int32_t tmp; buf.getIntNetwork(tmp); @@ -46,26 +39,6 @@ public: return s; } - static bool getBoolean(document::ByteBuffer& buf) { - uint8_t tmp; - buf.getByte(tmp); - return (tmp == 1); - } - - static api::ReturnCode getReturnCode(document::ByteBuffer& buf) { - api::ReturnCode::Result result = (api::ReturnCode::Result) getInt(buf); - vespalib::stringref message = getString(buf); - return api::ReturnCode(result, message); - } - - static void putReturnCode(const api::ReturnCode& code, vespalib::GrowableByteBuffer& buf) - { - buf.putInt(code.getResult()); - buf.putString(code.getMessage()); - } - - static const uint32_t BUCKET_INFO_SERIALIZED_SIZE = sizeof(uint32_t) * 3; - static document::GlobalId getGlobalId(document::ByteBuffer& buf) { std::vector<char> buffer(getShort(buf)); for (uint32_t i=0; i<buffer.size(); ++i) { @@ -74,13 +47,6 @@ public: return document::GlobalId(&buffer[0]); } - static void putGlobalId(const document::GlobalId& gid, vespalib::GrowableByteBuffer& buf) - { - buf.putShort(document::GlobalId::LENGTH); - for (uint32_t i=0; i<document::GlobalId::LENGTH; ++i) { - buf.putByte(gid.get()[i]); - } - } static document::Document::UP getDocument(document::ByteBuffer& buf, const document::DocumentTypeRepo& repo) { uint32_t size = getInt(buf); diff --git a/storage/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.h b/storage/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.h index d228dace1ed..1aede4d12e8 100644 --- a/storage/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.h +++ b/storage/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.h @@ -31,7 +31,6 @@ public: virtual ComponentRegisterImpl& getComponentRegister() { return *_compReg; } FakeClock& getClock() { return _clock; } ThreadPoolImpl& getThreadPoolImpl() { return _threadPool; } - FastOS_ThreadPool& getThreadPool() { return _threadPool.getThreadPool(); } }; } diff --git a/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp b/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp index 925c9cda248..c1fa2aac708 100644 --- a/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp +++ b/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp @@ -5,6 +5,7 @@ #include <vespa/storageframework/generic/clock/clock.h> #include <vespa/vespalib/util/atomic.h> #include <vespa/vespalib/util/signalhandler.h> +#include <cinttypes> #include <vespa/log/bufferedlogger.h> LOG_SETUP(".framework.thread.impl"); @@ -28,11 +29,11 @@ ThreadImpl::ThreadImpl(ThreadPoolImpl& pool, _tickDataPtr(0), _interrupted(false), _joined(false), - _thread(*this), + _thread(), _cpu_category(cpu_category) { _tickData[load_relaxed(_tickDataPtr)]._lastTick = pool.getClock().getMonotonicTime(); - _thread.start(_pool.getThreadPool()); + _thread = std::thread([this](){run();}); } ThreadImpl::~ThreadImpl() @@ -70,19 +71,21 @@ void ThreadImpl::interrupt() { _interrupted.store(true, std::memory_order_relaxed); - _thread.stop(); } void ThreadImpl::join() { - _thread.join(); + if (_thread.joinable()) { + _thread.join(); + } } vespalib::string ThreadImpl::get_live_thread_stack_trace() const { - return vespalib::SignalHandler::get_cross_thread_stack_trace(_thread.native_thread_id()); + auto native_handle = const_cast<std::thread&>(_thread).native_handle(); + return vespalib::SignalHandler::get_cross_thread_stack_trace(native_handle); } void diff --git a/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.h b/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.h index d95ba2a37ef..68ed63ea17c 100644 --- a/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.h +++ b/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.h @@ -4,7 +4,6 @@ #include <vespa/storageframework/generic/thread/thread.h> #include <vespa/vespalib/util/cpu_usage.h> -#include <vespa/vespalib/util/document_runnable.h> #include <array> #include <atomic> #include <optional> @@ -15,12 +14,6 @@ struct ThreadPoolImpl; class ThreadImpl final : public Thread { - struct BackendThread : public document::Runnable { - ThreadImpl& _impl; - explicit BackendThread(ThreadImpl& impl) : _impl(impl) {} - void run() override { _impl.run(); } - }; - /** * Internal data race free implementation of tick data that maps to and * from ThreadTickData. We hide the atomicity of this since atomic vars @@ -52,7 +45,7 @@ class ThreadImpl final : public Thread std::atomic<uint32_t> _tickDataPtr; std::atomic<bool> _interrupted; bool _joined; - BackendThread _thread; + std::thread _thread; std::optional<vespalib::CpuUsage::Category> _cpu_category; void run(); diff --git a/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.cpp b/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.cpp index 95959d06b54..068de8f5880 100644 --- a/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.cpp +++ b/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.cpp @@ -15,8 +15,7 @@ using vespalib::IllegalStateException; namespace storage::framework::defaultimplementation { ThreadPoolImpl::ThreadPoolImpl(Clock& clock) - : _backendThreadPool(std::make_unique<FastOS_ThreadPool>()), - _clock(clock), + : _clock(clock), _stopping(false) { } @@ -44,7 +43,6 @@ ThreadPoolImpl::~ThreadPoolImpl() } std::this_thread::sleep_for(10ms); } - _backendThreadPool->Close(); } Thread::UP diff --git a/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h b/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h index b788a3eed78..4319b4a0efe 100644 --- a/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h +++ b/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h @@ -4,8 +4,6 @@ #include <vespa/storageframework/generic/thread/threadpool.h> -class FastOS_ThreadPool; - namespace storage::framework { struct Clock; } @@ -15,7 +13,6 @@ class ThreadImpl; struct ThreadPoolImpl final : public ThreadPool { - std::unique_ptr<FastOS_ThreadPool> _backendThreadPool; std::vector<ThreadImpl*> _threads; mutable std::mutex _threadVectorLock; Clock & _clock; @@ -30,7 +27,6 @@ public: std::optional<vespalib::CpuUsage::Category> cpu_category) override; void visitThreads(ThreadVisitor&) const override; void unregisterThread(ThreadImpl&); - FastOS_ThreadPool& getThreadPool() { return *_backendThreadPool; } Clock& getClock() { return _clock; } }; diff --git a/storage/src/vespa/storageframework/generic/component/component.h b/storage/src/vespa/storageframework/generic/component/component.h index 9a5e524e504..47469cce05d 100644 --- a/storage/src/vespa/storageframework/generic/component/component.h +++ b/storage/src/vespa/storageframework/generic/component/component.h @@ -45,12 +45,12 @@ * optimize clock fetching as we see fit later. * * - A thread pool is given. This makes us able to use a thread pool. - * (Allthough currently we don't really need a thread pool, as threads - * typically live for the whole lifetime of the server. But currently we are - * forced to use a thread pool due to fastos.) Another feature of this is - * that the thread interface has built in information needed to detect - * deadlocks and report status about thread behavior, such that deadlock - * detecting and thread status can be shown without the threads themselves + * (Allthough currently we don't really need a thread pool, as + * threads typically live for the whole lifetime of the + * server. Another feature of this is that the thread interface has + * built in information needed to detect deadlocks and report + * status about thread behavior, such that deadlock detecting and + * thread status can be shown without the threads themselves * depending on how this is done. * * - A memory manager may also be provided, allowing components to request diff --git a/storageserver/CMakeLists.txt b/storageserver/CMakeLists.txt index ee3335c4921..a2f9d0b776e 100644 --- a/storageserver/CMakeLists.txt +++ b/storageserver/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos storage streamingvisitors diff --git a/streamingvisitors/CMakeLists.txt b/streamingvisitors/CMakeLists.txt index 2c7f01ddf37..fede7087d8d 100644 --- a/streamingvisitors/CMakeLists.txt +++ b/streamingvisitors/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog storage config_cloudconfig diff --git a/vbench/CMakeLists.txt b/vbench/CMakeLists.txt index 3fb5df8cd20..e78913262be 100644 --- a/vbench/CMakeLists.txt +++ b/vbench/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalib LIBS diff --git a/vdslib/CMakeLists.txt b/vdslib/CMakeLists.txt index 0f8144b99e9..1276323f83b 100644 --- a/vdslib/CMakeLists.txt +++ b/vdslib/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib config_cloudconfig diff --git a/vdstestlib/CMakeLists.txt b/vdstestlib/CMakeLists.txt index d0a921672c2..7f478989332 100644 --- a/vdstestlib/CMakeLists.txt +++ b/vdstestlib/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalib TESTS diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java index cf46cad57b1..21c8f4ddd31 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.athenz.api.NToken; import com.yahoo.vespa.athenz.api.ZToken; import com.yahoo.vespa.athenz.client.ErrorHandler; import com.yahoo.vespa.athenz.client.common.ClientBase; +import com.yahoo.vespa.athenz.client.zms.bindings.AccessResponseEntity; import com.yahoo.vespa.athenz.client.zts.bindings.AccessTokenResponseEntity; import com.yahoo.vespa.athenz.client.zts.bindings.AwsTemporaryCredentialsResponseEntity; import com.yahoo.vespa.athenz.client.zts.bindings.IdentityRefreshRequestEntity; @@ -221,6 +222,19 @@ public class DefaultZtsClient extends ClientBase implements ZtsClient { }); } + @Override + public boolean hasAccess(AthenzResourceName resource, String action, AthenzIdentity identity) { + URI uri = ztsUrl.resolve(String.format("access/%s/%s?principal=%s", + action, resource.toResourceNameString(), identity.getFullName())); + HttpUriRequest request = RequestBuilder.get() + .setUri(uri) + .build(); + return execute(request, response -> { + AccessResponseEntity result = readEntity(response, AccessResponseEntity.class); + return result.granted; + }); + } + private InstanceIdentity getInstanceIdentity(HttpResponse response) throws IOException { InstanceIdentityCredentials entity = readEntity(response, InstanceIdentityCredentials.class); return entity.getServiceToken() != null diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java index c4be6d8ced7..eade6229123 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/ZtsClient.java @@ -5,6 +5,7 @@ import com.yahoo.security.Pkcs10Csr; import com.yahoo.vespa.athenz.api.AthenzAccessToken; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.athenz.api.AthenzResourceName; import com.yahoo.vespa.athenz.api.AthenzRole; import com.yahoo.vespa.athenz.api.AwsRole; import com.yahoo.vespa.athenz.api.AwsTemporaryCredentials; @@ -187,5 +188,16 @@ public interface ZtsClient extends AutoCloseable { */ AwsTemporaryCredentials getAwsTemporaryCredentials(AthenzDomain athenzDomain, AwsRole awsRole, Duration duration, String externalId); + /** + * Check access to resource for a given principal + * + * @param resource The resource to verify access to + * @param action Action to verify + * @param identity Principal that requests access + * @return <code>true</code> if access is allowed, <code>false</code> otherwise + */ + boolean hasAccess(AthenzResourceName resource, String action, AthenzIdentity identity); + void close(); + } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java index b18ff238b07..49a39d25e87 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/identityprovider/api/SignedIdentityDocument.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.athenz.identityprovider.api; import com.yahoo.vespa.athenz.api.AthenzService; import java.time.Instant; +import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -20,7 +21,13 @@ public record SignedIdentityDocument(String signature, int signingKeyVersion, Ve public SignedIdentityDocument { ipAddresses = Set.copyOf(ipAddresses); - unknownAttributes = Map.copyOf(unknownAttributes); + + Map<String, Object> nonNull = new HashMap<>(); + unknownAttributes.forEach((key, value) -> { + if (value != null) nonNull.put(key, value); + }); + // Map.copyOf() does not allow null values + unknownAttributes = Map.copyOf(nonNull); } public SignedIdentityDocument(String signature, int signingKeyVersion, VespaUniqueInstanceId providerUniqueId, diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index 2cd7c6247dd..d5166a7e22f 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -14,15 +14,15 @@ com.amazonaws:aws-java-sdk-ssm:1.12.331 com.amazonaws:aws-java-sdk-sts:1.12.331 com.amazonaws:jmespath-java:1.12.331 com.auth0:java-jwt:3.10.0 -com.fasterxml.jackson.core:jackson-annotations:2.13.4 -com.fasterxml.jackson.core:jackson-core:2.13.4 -com.fasterxml.jackson.core:jackson-databind:2.13.4.2 -com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.12.6 -com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.13.4 -com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.13.4 -com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.13.4 -com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.13.4 -com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.13.4 +com.fasterxml.jackson.core:jackson-annotations:2.14.2 +com.fasterxml.jackson.core:jackson-core:2.14.2 +com.fasterxml.jackson.core:jackson-databind:2.14.2 +com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.14.2 +com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.14.2 +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.14.2 +com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.14.2 +com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.14.2 +com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.14.2 com.github.spotbugs:spotbugs-annotations:3.1.9 com.google.code.findbugs:jsr305:3.0.2 com.google.errorprone:error_prone_annotations:2.18.0 @@ -49,7 +49,7 @@ com.yahoo.athenz:athenz-zts-core:1.10.54 com.yahoo.rdl:rdl-java:1.5.4 commons-cli:commons-cli:1.5.0 commons-codec:commons-codec:1.15 -commons-fileupload:commons-fileupload:1.4 +commons-fileupload:commons-fileupload:1.5 commons-io:commons-io:2.11.0 commons-logging:commons-logging:1.2 io.airlift:airline:0.9 diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java index 0f3531720ae..d753f3d9e73 100644 --- a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java +++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java @@ -448,7 +448,7 @@ public class DocumentGenMojo extends AbstractMojo { out.write(ind(1)+"@Override protected boolean isGenerated() { return true; }\n\n"); Collection<Field> allUniqueFields = getAllUniqueFields(multiExtends, docType.getAllFields()); - exportExtendedStructTypeGetter(className, docType.getName(), allUniqueFields, docType.getFieldSets(), + exportExtendedStructTypeGetter(className, docType.getName(), docType.getInherited(), allUniqueFields, docType.getFieldSets(), docType.getImportedFieldNames(), out, 1, "getDocumentType", "com.yahoo.document.DocumentType"); exportCopyConstructor(className, out, 1, true); @@ -612,14 +612,17 @@ public class DocumentGenMojo extends AbstractMojo { out.write(ind(ind) + "importedFieldNames.add(\"" + importedField + "\");\n"); } } - private static void exportExtendedStructTypeGetter(String className, String name, Collection<Field> fields, Set<FieldSet> fieldSets, - Set<String> importedFieldNames, Writer out, int ind, String methodName, String retType) throws IOException { + private static void exportExtendedStructTypeGetter(String className, String name, Collection<NewDocumentType> parentTypes, + Collection<Field> fields, Set<FieldSet> fieldSets, + Set<String> importedFieldNames, Writer out, int ind, + String methodName, String retType) throws IOException { out.write(ind(ind)+"private static "+retType+" "+methodName+"() {\n"); + String bodyIndent = ind(ind + 1); if (importedFieldNames != null) { exportImportedFields(importedFieldNames, out, ind + 1); - out.write(ind(ind+1)+retType+" ret = new "+retType+"(\"" + name + "\", importedFieldNames);\n"); + out.write(bodyIndent+retType+" ret = new "+retType+"(\"" + name + "\", importedFieldNames);\n"); } else { - out.write(ind(ind+1)+retType+" ret = new "+retType+"(\""+name+"\");\n"); + out.write(bodyIndent+retType+" ret = new "+retType+"(\""+name+"\");\n"); } for (Field f : fields) { if (f.getDataType().equals(DataType.STRING)) { @@ -631,8 +634,13 @@ public class DocumentGenMojo extends AbstractMojo { if (fieldSets != null) { exportFieldSetDefinition(fieldSets, out, ind+1); } + for (NewDocumentType parentType : parentTypes) { + if (!parentType.getName().equals("document")) { + out.write("%sret.inherit(%s.type);\n".formatted(bodyIndent, className(parentType.getName()))); + } + } - out.write(ind(ind+1)+"return ret;\n"); + out.write(bodyIndent+"return ret;\n"); out.write(ind(ind)+"}\n\n"); } @@ -762,7 +770,9 @@ public class DocumentGenMojo extends AbstractMojo { ind(ind+2)+"super("+structClassName+".type);\n" + ind(ind+1)+"}\n\n"); exportCopyConstructor(structClassName, out, ind+1, false); - exportExtendedStructTypeGetter(structClassName, structType.getName(), structType.getFields(), null, null, out, ind+1, "getStructType", "com.yahoo.document.StructDataType"); + exportExtendedStructTypeGetter(structClassName, structType.getName(), List.of(), + structType.getFields(), null, null, out, ind+1, "getStructType", + "com.yahoo.document.StructDataType"); exportAssign(structType, structClassName, out, ind+1); exportFieldsAndAccessors(structClassName, structType.getFields(), out, ind+1, true); diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java index fc74cb6d899..288df7e470c 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/StdOutVisitorHandler.java @@ -35,42 +35,45 @@ import java.util.logging.Logger; @SuppressWarnings("deprecation") public class StdOutVisitorHandler extends VdsVisitHandler { - private static final Logger log = Logger.getLogger( - StdOutVisitorHandler.class.getName()); - private final boolean printIds; - private final boolean indentXml; - private final int processTimeMilliSecs; - private final PrintStream out; - private final boolean jsonOutput; - private final boolean tensorShortForm; - private final boolean tensorDirectValues; + private static final Logger log = Logger.getLogger(StdOutVisitorHandler.class.getName()); - private final VisitorDataHandler dataHandler; + public enum OutputFormat { + JSONL, + JSON, + XML // Deprecated + } - public StdOutVisitorHandler(boolean printIds, boolean indentXml, - boolean showProgress, boolean showStatistics, boolean doStatistics, - boolean abortOnClusterDown, int processtime, boolean jsonOutput, - boolean tensorShortForm, - boolean tensorDirectValues) - { - this(printIds, indentXml, showProgress, showStatistics, doStatistics, abortOnClusterDown, processtime, - jsonOutput, tensorShortForm, tensorDirectValues, createStdOutPrintStream()); + // Explicitly _not_ a record since we want the fields to be mutable when building. + public static class Params { + boolean printIds = false; + boolean indentXml = false; + boolean showProgress = false; + boolean showStatistics = false; + boolean doStatistics = false; + boolean abortOnClusterDown = false; + int processTimeMilliSecs = 0; + OutputFormat outputFormat = OutputFormat.JSON; + boolean tensorShortForm = false; // TODO Vespa 9: change default to true + boolean tensorDirectValues = false; // TODO Vespa 9: change default to true + + boolean usesJson() { + return outputFormat == OutputFormat.JSON || outputFormat == OutputFormat.JSONL; + } } - StdOutVisitorHandler(boolean printIds, boolean indentXml, - boolean showProgress, boolean showStatistics, boolean doStatistics, - boolean abortOnClusterDown, int processtime, boolean jsonOutput, - boolean tensorShortForm, boolean tensorDirectValues, PrintStream out) - { - super(showProgress, showStatistics, abortOnClusterDown); - this.printIds = printIds; - this.indentXml = indentXml; - this.processTimeMilliSecs = processtime; - this.jsonOutput = jsonOutput; - this.tensorShortForm = tensorShortForm; - this.tensorDirectValues = tensorDirectValues; + private final Params params; + private final PrintStream out; + private final VisitorDataHandler dataHandler; + + public StdOutVisitorHandler(Params params, PrintStream out) { + super(params.showProgress, params.showStatistics, params.abortOnClusterDown); + this.params = params; this.out = out; - this.dataHandler = new DataHandler(doStatistics); + this.dataHandler = new DataHandler(params.doStatistics); + } + + public StdOutVisitorHandler(Params params) { + this(params, createStdOutPrintStream()); } private static PrintStream createStdOutPrintStream() { @@ -128,9 +131,9 @@ public class StdOutVisitorHandler extends VdsVisitHandler { @Override public void onMessage(Message m, AckToken token) { - if (processTimeMilliSecs > 0) { + if (params.processTimeMilliSecs > 0) { try { - Thread.sleep(processTimeMilliSecs); + Thread.sleep(params.processTimeMilliSecs); } catch (InterruptedException e) {} } @@ -158,16 +161,15 @@ public class StdOutVisitorHandler extends VdsVisitHandler { System.err.print('\r'); } - if (printIds) { + if (params.printIds) { out.print(doc.getId()); out.print(" (Last modified at "); out.println(timestamp + ")"); } else { - if (jsonOutput) { + if (params.usesJson()) { writeJsonDocument(doc); } else { - out.print(doc.toXML( - indentXml ? " " : "")); + out.print(doc.toXML(params.indentXml ? " " : "")); } } } catch (Exception e) { @@ -179,7 +181,7 @@ public class StdOutVisitorHandler extends VdsVisitHandler { private void writeJsonDocument(Document doc) throws IOException { writeFeedStartOrRecordSeparator(); - out.write(JsonWriter.toByteArray(doc, tensorShortForm, tensorDirectValues)); + out.write(JsonWriter.toByteArray(doc, params.tensorShortForm, params.tensorDirectValues)); } @Override @@ -189,10 +191,10 @@ public class StdOutVisitorHandler extends VdsVisitHandler { System.err.print('\r'); } - if (printIds) { + if (params.printIds) { out.println(docId + " (Removed)"); } else { - if (jsonOutput) { + if (params.usesJson()) { writeJsonDocumentRemove(docId); } else { XmlStream stream = new XmlStream(); @@ -218,10 +220,12 @@ public class StdOutVisitorHandler extends VdsVisitHandler { private void writeFeedStartOrRecordSeparator() { if (first) { - out.println("["); + if (params.outputFormat == OutputFormat.JSON) { + out.println("["); + } first = false; } else { - out.println(","); + out.println((params.outputFormat == OutputFormat.JSON) ? "," : ""); } } @@ -259,7 +263,7 @@ public class StdOutVisitorHandler extends VdsVisitHandler { @Override public synchronized void onDone() { - if (jsonOutput && !printIds) { + if ((params.outputFormat == OutputFormat.JSON) && !params.printIds) { if (first) { out.print('['); } diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java index a6e34055fbd..822c64ea5fa 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java @@ -23,9 +23,12 @@ import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; -import java.io.*; -import java.nio.charset.Charset; +import java.io.IOException; +import java.io.PrintStream; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.NoSuchFileException; +import java.nio.file.Path; import java.util.Map; import java.util.stream.Collectors; @@ -334,9 +337,15 @@ public class VdsVisit { .hasArg(false) .build()); + options.addOption(Option.builder() + .longOpt("jsonl") + .desc("Output documents as JSONL (JSON Lines format)") + .hasArg(false) + .build()); + options.addOption(Option.builder("x") .longOpt("xmloutput") - .desc("Output documents as XML") + .desc("Output documents as XML (deprecated)") .hasArg(false) .build()); @@ -370,6 +379,7 @@ public class VdsVisit { private int processTime = 0; private int fullTimeout = 7 * 24 * 60 * 60 * 1000; private boolean jsonOutput = true; + private boolean jsonLinesOutput = false; private boolean tensorShortForm = false; // TODO Vespa 9: change default to true private boolean tensorDirectValues = false; // TODO Vespa 9: change default to true @@ -437,10 +447,32 @@ public class VdsVisit { this.processTime = processTime; } + public boolean jsonOutput() { + return jsonOutput; + } + public void setJsonOutput(boolean jsonOutput) { this.jsonOutput = jsonOutput; } + public boolean jsonLinesOutput() { + return jsonLinesOutput; + } + + public void setJsonLinesOutput(boolean jsonLinesOutput) { + this.jsonLinesOutput = jsonLinesOutput; + } + + public StdOutVisitorHandler.OutputFormat stdOutHandlerOutputFormat() { + if (jsonLinesOutput) { + return StdOutVisitorHandler.OutputFormat.JSONL; + } else if (jsonOutput) { + return StdOutVisitorHandler.OutputFormat.JSON; + } else { + return StdOutVisitorHandler.OutputFormat.XML; + } + } + public boolean tensorShortForm() { return tensorShortForm; } @@ -587,11 +619,18 @@ public class VdsVisit { } boolean jsonOutput = line.hasOption("jsonoutput"); - boolean xmlOutput = line.hasOption("xmloutput"); - if (jsonOutput && xmlOutput) { - throw new IllegalArgumentException("Cannot combine both xml and json output"); + boolean jsonl = line.hasOption("jsonl"); + boolean xmlOutput = line.hasOption("xmloutput"); + if ((jsonOutput || jsonl) && xmlOutput) { + throw new IllegalArgumentException("Cannot combine both XML and JSON output"); + } else if (jsonOutput && jsonl) { + throw new IllegalArgumentException("Cannot combine both JSON and JSONL output"); + } + if (jsonl) { + allParams.setJsonLinesOutput(true); + } else { + allParams.setJsonOutput(!xmlOutput); } - allParams.setJsonOutput(!xmlOutput); allParams.setVisitorParameters(params); return allParams; @@ -716,24 +755,14 @@ public class VdsVisit { !"".equals(visitorParameters.getResumeFileName())) { try { - File file = new File(visitorParameters.getResumeFileName()); - FileInputStream fos = new FileInputStream(file); - - StringBuilder builder = new StringBuilder(); - byte[] b = new byte[100000]; - int length; - - while ((length = fos.read(b)) > 0) { - builder.append(new String(b, 0, length)); - } - fos.close(); - visitorParameters.setResumeToken(new ProgressToken(builder.toString())); + var progressFileContents = Files.readString(Path.of(visitorParameters.getResumeFileName())); + visitorParameters.setResumeToken(new ProgressToken(progressFileContents)); if (params.isVerbose()) { System.err.format("Resuming visitor already %.1f %% finished.\n", visitorParameters.getResumeToken().percentFinished()); } - } catch (FileNotFoundException e) { + } catch (NoSuchFileException e) { // Ignore; file has not been created yet but will be shortly. } catch (IOException e) { System.err.println("Could not open progress file: " + visitorParameters.getResumeFileName()); @@ -747,17 +776,18 @@ public class VdsVisit { VdsVisitHandler handler; - handler = new StdOutVisitorHandler( - params.isPrintIdsOnly(), - params.isVerbose(), - params.isVerbose(), - params.isVerbose(), - params.getStatisticsParts() != null, - params.getAbortOnClusterDown(), - params.getProcessTime(), - params.jsonOutput, - params.tensorShortForm, - params.tensorDirectValues); + var handlerParams = new StdOutVisitorHandler.Params(); + handlerParams.printIds = params.isPrintIdsOnly(); + handlerParams.indentXml = params.isVerbose(); + handlerParams.showProgress = params.isVerbose(); + handlerParams.showStatistics = params.isVerbose(); + handlerParams.doStatistics = params.getStatisticsParts() != null; + handlerParams.abortOnClusterDown = params.getAbortOnClusterDown(); + handlerParams.processTimeMilliSecs = params.getProcessTime(); + handlerParams.outputFormat = params.stdOutHandlerOutputFormat(); + handlerParams.tensorShortForm = params.tensorShortForm(); + handlerParams.tensorDirectValues = params.tensorDirectValues(); + handler = new StdOutVisitorHandler(handlerParams); if (visitorParameters.getResumeFileName() != null) { handler.setProgressFileName(visitorParameters.getResumeFileName()); diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitHandler.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitHandler.java index ea861399e76..ccb0888a654 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitHandler.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisitHandler.java @@ -6,16 +6,20 @@ import com.yahoo.documentapi.VisitorControlHandler; import com.yahoo.documentapi.VisitorDataHandler; import com.yahoo.vdslib.VisitorStatistics; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; import java.util.Date; import java.util.TimeZone; import java.text.DateFormat; import java.text.DecimalFormat; import java.text.SimpleDateFormat; +import static java.nio.file.StandardCopyOption.ATOMIC_MOVE; +import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; + /** * An abstract class that can be subclassed by different visitor handlers. * @@ -30,14 +34,26 @@ public abstract class VdsVisitHandler { String lastPercentage; final Object printLock = new Object(); - protected String progressFileName = ""; + private static class ProgressMeta { + String fileName = ""; + String lastProgressContents; + int unwrittenUpdates = 0; + long lastWriteAtNanos = 0; + Duration writeInterval = Duration.ofSeconds(10); + boolean shouldWriteProgress() { + return !fileName.isEmpty(); + } + } + + ProgressMeta progressMeta = new ProgressMeta(); final VisitorControlHandler controlHandler = new ControlHandler(); public VdsVisitHandler(boolean showProgress, boolean showStatistics, boolean abortOnClusterDown) { this.showProgress = showProgress; this.showStatistics = showStatistics; this.abortOnClusterDown = abortOnClusterDown; + this.progressMeta.lastWriteAtNanos = System.nanoTime(); // Avoid always writing a file on the first progress update } public boolean getShowProgress() { @@ -75,11 +91,11 @@ public abstract class VdsVisitHandler { public void onDone() { } public String getProgressFileName() { - return progressFileName; + return progressMeta.fileName; } public void setProgressFileName(String progressFileName) { - this.progressFileName = progressFileName; + this.progressMeta.fileName = progressFileName; } public VisitorControlHandler getControlHandler() { return controlHandler; } @@ -88,20 +104,29 @@ public abstract class VdsVisitHandler { class ControlHandler extends VisitorControlHandler { VisitorStatistics statistics; + private void rewriteProgressFile() { + try { + var tmpPath = Path.of(progressMeta.fileName + ".tmp"); + Files.writeString(tmpPath, progressMeta.lastProgressContents); + Files.move(tmpPath, Path.of(progressMeta.fileName), REPLACE_EXISTING, ATOMIC_MOVE); + } catch (IOException e) { + e.printStackTrace(); + abort(); // Don't continue visiting if we're unable to save progress state + } + } + public void onProgress(ProgressToken token) { - if (progressFileName.length() > 0) { - try { - synchronized (token) { - File file = new File(progressFileName + ".tmp"); - FileOutputStream fos = new FileOutputStream(file); - fos.write(token.toString().getBytes()); - fos.close(); - file.renameTo(new File(progressFileName)); + if (progressMeta.shouldWriteProgress()) { + synchronized (token) { + progressMeta.unwrittenUpdates++; + progressMeta.lastProgressContents = token.toString(); + long nowNanos = System.nanoTime(); + if ((nowNanos - progressMeta.lastWriteAtNanos) > progressMeta.writeInterval.toNanos()) { + rewriteProgressFile(); + progressMeta.unwrittenUpdates = 0; + progressMeta.lastWriteAtNanos = nowNanos; } } - catch (IOException e) { - e.printStackTrace(); - } } if (showProgress) { synchronized (printLock) { @@ -111,7 +136,9 @@ public abstract class VdsVisitHandler { if (lastLineIsProgress) { System.err.print('\r'); } - System.err.print(percentage + " % finished."); + // Pad with a few extra spaces to handle case where current line written is shorter + // than the previous line written. Would otherwise leave stale characters behind. + System.err.print(percentage + " % finished. "); lastLineIsProgress = true; lastPercentage = percentage; } @@ -147,6 +174,11 @@ public abstract class VdsVisitHandler { } } public void onDone(CompletionCode code, String message) { + // Flush any remaining unwritten progress updates. + // It is expected that this happens-after any and all calls to onProgress(). + if (progressMeta.unwrittenUpdates > 0) { + rewriteProgressFile(); + } if (lastLineIsProgress) { System.err.print('\n'); lastLineIsProgress = false; diff --git a/vespaclient-java/src/test/java/com/yahoo/vespavisit/StdOutVisitorHandlerTest.java b/vespaclient-java/src/test/java/com/yahoo/vespavisit/StdOutVisitorHandlerTest.java index c1bbe8711a5..aa708b1fde9 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespavisit/StdOutVisitorHandlerTest.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespavisit/StdOutVisitorHandlerTest.java @@ -1,15 +1,19 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespavisit; +import com.yahoo.document.DataType; import com.yahoo.document.Document; +import com.yahoo.document.DocumentId; import com.yahoo.document.DocumentPut; import com.yahoo.document.DocumentType; import com.yahoo.document.TensorDataType; +import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; import com.yahoo.documentapi.AckToken; import com.yahoo.documentapi.VisitorControlSession; import com.yahoo.documentapi.VisitorDataHandler; import com.yahoo.documentapi.messagebus.protocol.PutDocumentMessage; +import com.yahoo.documentapi.messagebus.protocol.RemoveDocumentMessage; import com.yahoo.tensor.Tensor; import com.yahoo.tensor.TensorType; import org.junit.jupiter.api.Test; @@ -26,23 +30,27 @@ import static org.mockito.Mockito.mock; * @author bjorncs */ public class StdOutVisitorHandlerTest { - private boolean jsonOutput; - - public void initStdOutVisitorHandlerTest(boolean jsonOutput) { - this.jsonOutput = jsonOutput; - } public static Object[] data() { return new Object[]{true, false}; } + private static StdOutVisitorHandler.Params createHandlerParams(boolean jsonOutput, boolean tensorShortForm, boolean tensorDirectValues) { + var params = new StdOutVisitorHandler.Params(); + params.outputFormat = jsonOutput ? StdOutVisitorHandler.OutputFormat.JSON + : StdOutVisitorHandler.OutputFormat.XML; + params.tensorShortForm = tensorShortForm; + params.tensorDirectValues = tensorDirectValues; + return params; + } + @MethodSource("data") @ParameterizedTest(name = "jsonOutput={0}") void printing_ids_for_zero_documents_produces_empty_output(boolean jsonOutput) { - initStdOutVisitorHandlerTest(jsonOutput); ByteArrayOutputStream out = new ByteArrayOutputStream(); - StdOutVisitorHandler visitorHandler = - new StdOutVisitorHandler(/*printIds*/true, false, false, false, false, false, 0, jsonOutput, false, false, new PrintStream(out, true)); + var params = createHandlerParams(jsonOutput, false, false); + params.printIds = true; + StdOutVisitorHandler visitorHandler = new StdOutVisitorHandler(params, new PrintStream(out, true)); VisitorDataHandler dataHandler = visitorHandler.getDataHandler(); dataHandler.onDone(); String output = out.toString(); @@ -52,10 +60,9 @@ public class StdOutVisitorHandlerTest { @MethodSource("data") @ParameterizedTest(name = "jsonOutput={0}") void printing_zero_documents_produces_empty_output(boolean jsonOutput) { - initStdOutVisitorHandlerTest(jsonOutput); ByteArrayOutputStream out = new ByteArrayOutputStream(); StdOutVisitorHandler visitorHandler = - new StdOutVisitorHandler(/*printIds*/false, false, false, false, false, false, 0, jsonOutput, false, false, new PrintStream(out, true)); + new StdOutVisitorHandler(createHandlerParams(jsonOutput, false, false), new PrintStream(out, true)); VisitorDataHandler dataHandler = visitorHandler.getDataHandler(); dataHandler.onDone(); String expectedOutput = jsonOutput ? "[]" : ""; @@ -71,8 +78,7 @@ public class StdOutVisitorHandlerTest { var putMsg = new PutDocumentMessage(new DocumentPut(doc)); var out = new ByteArrayOutputStream(); - var visitorHandler = new StdOutVisitorHandler(/*printIds*/false, false, false, false, false, false, - 0, true, tensorShortForm, tensorDirectValues, new PrintStream(out, true)); + var visitorHandler = new StdOutVisitorHandler(createHandlerParams(true, tensorShortForm, tensorDirectValues), new PrintStream(out, true)); var dataHandler = visitorHandler.getDataHandler(); var controlSession = mock(VisitorControlSession.class); var ackToken = mock(AckToken.class); @@ -100,4 +106,54 @@ public class StdOutVisitorHandlerTest { do_test_json_tensor_fields_rendering(true, false, expectedOutput); } + private static PutDocumentMessage createPutWithDocAndValue(DocumentType docType, String docId, String fieldValue) { + var doc = new Document(docType, docId); + doc.setFieldValue("bar", new StringFieldValue(fieldValue)); + return new PutDocumentMessage(new DocumentPut(doc)); + } + + private static RemoveDocumentMessage createRemoveForDoc(String docId) { + return new RemoveDocumentMessage(new DocumentId(docId)); + } + + @MethodSource("data") + @ParameterizedTest(name = "jsonLinesFormat={0}") + void json_can_be_output_in_json_lines_format(boolean jsonLinesFormat) { + var docType = new DocumentType("foo"); + docType.addField("bar", DataType.STRING); + + var params = createHandlerParams(true, true, true); + params.outputFormat = jsonLinesFormat ? StdOutVisitorHandler.OutputFormat.JSONL + : StdOutVisitorHandler.OutputFormat.JSON; + + var out = new ByteArrayOutputStream(); + var visitorHandler = new StdOutVisitorHandler(params, new PrintStream(out, true)); + var dataHandler = visitorHandler.getDataHandler(); + var controlSession = mock(VisitorControlSession.class); + dataHandler.setSession(controlSession); + + dataHandler.onMessage(createPutWithDocAndValue(docType, "id:baz:foo::1", "fluffy\nbunnies"), mock(AckToken.class)); + dataHandler.onMessage(createRemoveForDoc("id:baz:foo::2"), mock(AckToken.class)); + dataHandler.onMessage(createPutWithDocAndValue(docType, "id:baz:foo::3", "\r\ncool fox\r\n"), mock(AckToken.class)); + dataHandler.onDone(); + + String output = out.toString().trim(); + if (jsonLinesFormat) { + // JSONL; no implicit start/end array chars or trailing commas after objects + var expected = """ + {"id":"id:baz:foo::1","fields":{"bar":"fluffy\\nbunnies"}} + {"remove":"id:baz:foo::2"} + {"id":"id:baz:foo::3","fields":{"bar":"\\r\\ncool fox\\r\\n"}}"""; + assertEquals(expected, output); + } else { + // non-JSONL; usual array of comma-separated objects form + var expected = """ + [ + {"id":"id:baz:foo::1","fields":{"bar":"fluffy\\nbunnies"}}, + {"remove":"id:baz:foo::2"}, + {"id":"id:baz:foo::3","fields":{"bar":"\\r\\ncool fox\\r\\n"}}]"""; + assertEquals(expected, output); + } + } + } diff --git a/vespaclient/CMakeLists.txt b/vespaclient/CMakeLists.txt index 912b35fa763..6e82b83517e 100644 --- a/vespaclient/CMakeLists.txt +++ b/vespaclient/CMakeLists.txt @@ -2,7 +2,6 @@ vespa_define_module( DEPENDS vespadefaults - fastos configdefinitions config_cloudconfig vespalog diff --git a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java index 1e2c0900ff7..33e46ebc75f 100644 --- a/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java +++ b/vespajlib/src/main/java/com/yahoo/concurrent/maintenance/Maintainer.java @@ -93,7 +93,7 @@ public abstract class Maintainer implements Runnable { * * @return the degree to which the run was successful - a number between 0 (no success), to 1 (complete success). * Note that this indicates whether something is wrong, so e.g if the call did nothing because it should do - * nothing, 1.0 should be returned. + * nothing, 1.0 should be returned. */ protected abstract double maintain(); diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 2720d8786cb..8175e75875a 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -6,7 +6,6 @@ endif() vespa_define_module( DEPENDS - fastos vespalog EXTERNAL_DEPENDS @@ -26,6 +25,8 @@ vespa_define_module( src/apps/vespa-validate-hostname TESTS + ${VESPALIB_DIRECTIO_TESTDIR} + ${VESPALIB_PROCESS_MEMORY_STATS_TESTDIR} src/tests/alloc src/tests/approx src/tests/array @@ -38,9 +39,9 @@ vespa_define_module( src/tests/bits src/tests/box src/tests/btree - src/tests/btree/btree_store src/tests/btree/btree-scan-speed src/tests/btree/btree-stress + src/tests/btree/btree_store src/tests/clock src/tests/component src/tests/compress @@ -73,7 +74,6 @@ vespa_define_module( src/tests/datastore/unique_store src/tests/datastore/unique_store_dictionary src/tests/datastore/unique_store_string_allocator - ${VESPALIB_DIRECTIO_TESTDIR} src/tests/detect_type_benchmark src/tests/dotproduct src/tests/drop-file-from-cache @@ -86,6 +86,9 @@ vespa_define_module( src/tests/executor_idle_tracking src/tests/explore_modern_cpp src/tests/false + src/tests/fastlib/io + src/tests/fastlib/text + src/tests/fastos src/tests/fiddle src/tests/fileheader src/tests/floatingpointtype @@ -95,6 +98,7 @@ vespa_define_module( src/tests/guard src/tests/host_name src/tests/hwaccelrated + src/tests/invokeservice src/tests/io/fileutil src/tests/io/mapped_file_input src/tests/issue @@ -134,7 +138,6 @@ vespa_define_module( src/tests/printable src/tests/priority_queue src/tests/process - ${VESPALIB_PROCESS_MEMORY_STATS_TESTDIR} src/tests/programoptions src/tests/random src/tests/referencecounter @@ -144,14 +147,14 @@ vespa_define_module( src/tests/runnable_pair src/tests/rusage src/tests/sequencedtaskexecutor - src/tests/shutdownguard - src/tests/singleexecutor src/tests/sha1 src/tests/shared_operation_throttler src/tests/shared_string_repo src/tests/sharedptr + src/tests/shutdownguard src/tests/signalhandler src/tests/simple_thread_bundle + src/tests/singleexecutor src/tests/slime src/tests/slime/external_data_value src/tests/slime/summary-feature-benchmark @@ -204,17 +207,15 @@ vespa_define_module( src/tests/util/string_escape src/tests/valgrind src/tests/visit_ranges - src/tests/invokeservice src/tests/wakeup src/tests/xmlserializable src/tests/zcurve - src/tests/fastlib/io - src/tests/fastlib/text LIBS src/vespa/fastlib/io src/vespa/fastlib/text src/vespa/fastlib/text/apps + src/vespa/fastos src/vespa/vespalib src/vespa/vespalib/btree src/vespa/vespalib/component diff --git a/vespalib/src/tests/btree/iteratespeed.cpp b/vespalib/src/tests/btree/iteratespeed.cpp index fceaf01a785..48c4b4a1c39 100644 --- a/vespalib/src/tests/btree/iteratespeed.cpp +++ b/vespalib/src/tests/btree/iteratespeed.cpp @@ -1,10 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/btree/btreeroot.h> #include <vespa/vespalib/btree/btreebuilder.h> #include <vespa/vespalib/btree/btreenodeallocator.h> #include <vespa/vespalib/btree/btree.h> -#include <vespa/vespalib/btree/btreestore.h> #include <vespa/vespalib/btree/btreenodeallocator.hpp> #include <vespa/vespalib/btree/btreenode.hpp> #include <vespa/vespalib/btree/btreenodestore.hpp> @@ -12,11 +10,10 @@ #include <vespa/vespalib/btree/btreeroot.hpp> #include <vespa/vespalib/btree/btreebuilder.hpp> #include <vespa/vespalib/btree/btree.hpp> -#include <vespa/vespalib/btree/btreestore.hpp> #include <vespa/vespalib/datastore/buffer_type.hpp> #include <vespa/vespalib/util/rand48.h> #include <vespa/vespalib/util/time.h> - +#include <cinttypes> #include <unistd.h> #include <vespa/log/log.h> diff --git a/vespalib/src/tests/clock/clock_benchmark.cpp b/vespalib/src/tests/clock/clock_benchmark.cpp index a21ad2b05ef..620b4e5c83c 100644 --- a/vespalib/src/tests/clock/clock_benchmark.cpp +++ b/vespalib/src/tests/clock/clock_benchmark.cpp @@ -2,7 +2,6 @@ #include <vespa/vespalib/util/clock.h> #include <vespa/vespalib/util/invokeserviceimpl.h> -#include <vespa/fastos/thread.h> #include <cassert> #include <vector> #include <atomic> @@ -10,6 +9,7 @@ #include <cstring> #include <condition_variable> #include <mutex> +#include <thread> using vespalib::Clock; using vespalib::steady_time; @@ -36,7 +36,7 @@ struct NSAtomic : public UpdateClock { std::atomic<int64_t> _value; }; -class TestClock : public FastOS_Runnable +class TestClock { private: int _timePeriodMS; @@ -44,8 +44,9 @@ private: std::condition_variable _cond; UpdateClock &_clock; bool _stop; + std::thread _thread; - void Run(FastOS_ThreadInterface *thisThread, void *arguments) override; + void run(); public: TestClock(UpdateClock & clock, double timePeriod) @@ -53,74 +54,68 @@ public: _lock(), _cond(), _clock(clock), - _stop(false) - { } + _stop(false), + _thread() + { + _thread = std::thread([this](){run();}); + } ~TestClock() { - std::lock_guard<std::mutex> guard(_lock); - _stop = true; - _cond.notify_all(); + { + std::lock_guard<std::mutex> guard(_lock); + _stop = true; + _cond.notify_all(); + } + _thread.join(); } }; -void TestClock::Run(FastOS_ThreadInterface *thread, void *) +void TestClock::run() { std::unique_lock<std::mutex> guard(_lock); - while ( ! thread->GetBreakFlag() && !_stop) { + while (!_stop) { _clock.update(); _cond.wait_for(guard, std::chrono::milliseconds(_timePeriodMS)); } } -struct SamplerBase : public FastOS_Runnable { - SamplerBase(uint32_t threadId) - : _thread(nullptr), - _threadId(threadId), - _samples(0), - _count() +template<typename Func> +struct Sampler { + Sampler(Func func, uint64_t samples) + : _samples(samples), + _count(), + _func(func), + _thread() { memset(_count, 0, sizeof(_count)); + _thread = std::thread([this](){run();}); } - FastOS_ThreadInterface * _thread; - uint32_t _threadId; - uint64_t _samples; - uint64_t _count[3]; -}; - -template<typename Func> -struct Sampler : public SamplerBase { - Sampler(Func func, uint32_t threadId) - : SamplerBase(threadId), - _func(func) - { } - void Run(FastOS_ThreadInterface *, void *) override { - uint64_t samples; + void run() { steady_time prev = _func(); - for (samples = 0; (samples < _samples); samples++) { + for (uint64_t samples = 0; samples < _samples; ++samples) { steady_time now = _func(); duration diff = now - prev; if (diff > duration::zero()) prev = now; _count[1 + ((diff == duration::zero()) ? 0 : (diff > duration::zero()) ? 1 : -1)]++; } - } - Func _func; + uint64_t _samples; + uint64_t _count[3]; + Func _func; + std::thread _thread; }; template<typename Func> -void benchmark(const char * desc, FastOS_ThreadPool & pool, uint64_t samples, uint32_t numThreads, Func func) { - std::vector<std::unique_ptr<SamplerBase>> threads; +void benchmark(const char * desc, uint64_t samples, uint32_t numThreads, Func func) { + std::vector<std::unique_ptr<Sampler<Func>>> threads; threads.reserve(numThreads); steady_time start = steady_clock::now(); for (uint32_t i(0); i < numThreads; i++) { - SamplerBase * sampler = new Sampler<Func>(func, i); - sampler->_samples = samples; - sampler->_thread = pool.NewThread(sampler, nullptr); - threads.emplace_back(sampler); + threads.push_back(std::make_unique<Sampler<Func>>(func, samples)); } uint64_t count[3]; memset(count, 0, sizeof(count)); for (const auto & sampler : threads) { - sampler->_thread->Join(); + sampler->_thread.join(); for (uint32_t i(0); i < 3; i++) { count[i] += sampler->_count[i]; } @@ -129,12 +124,15 @@ void benchmark(const char * desc, FastOS_ThreadPool & pool, uint64_t samples, ui } int -main(int , char *argv[]) +main(int argc, char *argv[]) { + if (argc != 4) { + fprintf(stderr, "usage: %s <frequency> <numThreads> <samples>\n", argv[0]); + return 1; + } uint64_t frequency = atoll(argv[1]); uint32_t numThreads = atoi(argv[2]); uint64_t samples = atoll(argv[3]); - FastOS_ThreadPool pool; NSValue nsValue; NSVolatile nsVolatile; NSAtomic nsAtomic; @@ -143,36 +141,30 @@ main(int , char *argv[]) TestClock nsClock(nsValue, 1.0/frequency); TestClock nsVolatileClock(nsVolatile, 1.0/frequency); TestClock nsAtomicClock(nsAtomic, 1.0/frequency); - assert(pool.NewThread(&nsClock, nullptr) != nullptr); - assert(pool.NewThread(&nsVolatileClock, nullptr) != nullptr); - assert(pool.NewThread(&nsAtomicClock, nullptr) != nullptr); - benchmark("vespalib::Clock", pool, samples, numThreads, [&clock]() { + benchmark("vespalib::Clock", samples, numThreads, [&clock]() { return clock.getTimeNS(); }); - benchmark("uint64_t", pool, samples, numThreads, [&nsValue]() { + benchmark("uint64_t", samples, numThreads, [&nsValue]() { return steady_time (duration(nsValue._value)); }); - benchmark("volatile uint64_t", pool, samples, numThreads, [&nsVolatile]() { + benchmark("volatile uint64_t", samples, numThreads, [&nsVolatile]() { return steady_time(duration(nsVolatile._value)); }); - benchmark("memory_order_relaxed", pool, samples, numThreads, [&nsAtomic]() { + benchmark("memory_order_relaxed", samples, numThreads, [&nsAtomic]() { return steady_time(duration(nsAtomic._value.load(std::memory_order_relaxed))); }); - benchmark("memory_order_consume", pool, samples, numThreads, [&nsAtomic]() { + benchmark("memory_order_consume", samples, numThreads, [&nsAtomic]() { return steady_time(duration(nsAtomic._value.load(std::memory_order_consume))); }); - benchmark("memory_order_acquire", pool, samples, numThreads, [&nsAtomic]() { + benchmark("memory_order_acquire", samples, numThreads, [&nsAtomic]() { return steady_time(duration(nsAtomic._value.load(std::memory_order_acquire))); }); - benchmark("memory_order_seq_cst", pool, samples, numThreads, [&nsAtomic]() { + benchmark("memory_order_seq_cst", samples, numThreads, [&nsAtomic]() { return steady_time(duration(nsAtomic._value.load(std::memory_order_seq_cst))); }); - - benchmark("vespalib::steady_time::now()", pool, samples, numThreads, []() { + benchmark("vespalib::steady_time::now()", samples, numThreads, []() { return steady_clock::now(); }); - - pool.Close(); return 0; } diff --git a/vespalib/src/tests/datastore/datastore/datastore_test.cpp b/vespalib/src/tests/datastore/datastore/datastore_test.cpp index 645871d3ef6..e17ac94775e 100644 --- a/vespalib/src/tests/datastore/datastore/datastore_test.cpp +++ b/vespalib/src/tests/datastore/datastore/datastore_test.cpp @@ -6,6 +6,7 @@ #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/test/memory_allocator_observer.h> #include <vespa/vespalib/util/size_literals.h> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP("datastore_test"); diff --git a/vespalib/src/tests/fastos/CMakeLists.txt b/vespalib/src/tests/fastos/CMakeLists.txt new file mode 100644 index 00000000000..6ea986ac0b0 --- /dev/null +++ b/vespalib/src/tests/fastos/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(fastos_file_test_app TEST + SOURCES + file_test.cpp + DEPENDS + vespalib + GTest::GTest +) +vespa_add_test(NAME fastos_file_test_app COMMAND fastos_file_test_app) diff --git a/fastos/src/tests/filetest.cpp b/vespalib/src/tests/fastos/file_test.cpp index d0f8bbfd98b..d0f8bbfd98b 100644 --- a/fastos/src/tests/filetest.cpp +++ b/vespalib/src/tests/fastos/file_test.cpp diff --git a/fastos/src/tests/hello.txt b/vespalib/src/tests/fastos/hello.txt index 62a405393a6..62a405393a6 100644 --- a/fastos/src/tests/hello.txt +++ b/vespalib/src/tests/fastos/hello.txt diff --git a/fastos/src/tests/tests.h b/vespalib/src/tests/fastos/tests.h index 3a6f2ef9010..9cd7a10ab48 100644 --- a/fastos/src/tests/tests.h +++ b/vespalib/src/tests/fastos/tests.h @@ -1,8 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/thread.h> #include <cstring> #include <csignal> +#include <cstdio> +#include <cstdint> class BaseTest { @@ -83,13 +84,6 @@ public: return Progress(result, string); } - bool Progress (bool result, const char *str, const FastOS_ThreadInterface *s1) - { - char string[MAX_STR_LEN-100]; - snprintf(string, sizeof(string), str, s1); - return Progress(result, string); - } - bool Progress (bool result, const char *str, const char *s1, const char *s2) { char string[MAX_STR_LEN-100]; diff --git a/vespalib/src/tests/net/tls/capabilities/capabilities_test.cpp b/vespalib/src/tests/net/tls/capabilities/capabilities_test.cpp index 0bf04289a65..5fdddf086ba 100644 --- a/vespalib/src/tests/net/tls/capabilities/capabilities_test.cpp +++ b/vespalib/src/tests/net/tls/capabilities/capabilities_test.cpp @@ -84,6 +84,7 @@ TEST("All known capabilities can be looked up by name, and resolve back to same check_capability_mapping("vespa.content.metrics_api", Capability::content_metrics_api()); check_capability_mapping("vespa.content.proton_admin_api", Capability::content_proton_admin_api()); check_capability_mapping("vespa.content.search_api", Capability::content_search_api()); + check_capability_mapping("vespa.content.state_api", Capability::content_state_api()); check_capability_mapping("vespa.content.status_pages", Capability::content_status_pages()); check_capability_mapping("vespa.content.storage_api", Capability::content_storage_api()); check_capability_mapping("vespa.logserver.api", Capability::logserver_api()); @@ -109,6 +110,7 @@ TEST("CapabilitySet instances can be stringified") { "vespa.container.state_api, " "vespa.content.document_api, " "vespa.content.metrics_api, " + "vespa.content.state_api, " "vespa.content.status_pages, " "vespa.content.storage_api, " "vespa.logserver.api, " @@ -124,7 +126,7 @@ TEST("All known capability sets can be looked up by name") { check_capability_set_mapping("vespa.telemetry", CapabilitySet::telemetry()); check_capability_set_mapping("vespa.cluster_controller_node", CapabilitySet::cluster_controller_node()); check_capability_set_mapping("vespa.logserver_node", CapabilitySet::logserver_node()); - check_capability_set_mapping("vespa.config_server", CapabilitySet::config_server()); + check_capability_set_mapping("vespa.config_server_node", CapabilitySet::config_server_node()); } TEST("Unknown capability set name returns nullopt") { @@ -135,7 +137,7 @@ TEST("Resolving a capability set adds all its underlying capabilities") { CapabilitySet caps; EXPECT_TRUE(caps.resolve_and_add("vespa.content_node")); // Slightly suboptimal; this test will fail if the default set of capabilities for vespa.content_node changes. - EXPECT_EQUAL(caps.count(), 14u); + EXPECT_EQUAL(caps.count(), 15u); EXPECT_FALSE(caps.empty()); EXPECT_TRUE(caps.contains(Capability::content_storage_api())); EXPECT_TRUE(caps.contains(Capability::content_document_api())); @@ -147,6 +149,7 @@ TEST("Resolving a capability set adds all its underlying capabilities") { EXPECT_TRUE(caps.contains(Capability::configproxy_config_api())); EXPECT_TRUE(caps.contains(Capability::configproxy_filedistribution_api())); // vespa.content_node -> shared node caps -> vespa.telemetry + EXPECT_TRUE(caps.contains(Capability::content_state_api())); EXPECT_TRUE(caps.contains(Capability::content_status_pages())); EXPECT_TRUE(caps.contains(Capability::content_metrics_api())); EXPECT_TRUE(caps.contains(Capability::container_state_api())); diff --git a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp index 0178443643e..068345b7254 100644 --- a/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp +++ b/vespalib/src/tests/net/tls/openssl_impl/openssl_impl_test.cpp @@ -735,6 +735,28 @@ TEST_F("Authz policy-derived peer capabilities are propagated to CryptoCodec", C Capability::content_status_pages()})); } +TEST_F("Handshake is allowed if at least one policy matches, even if resulting capability set is empty", CertFixture) { + auto server_ck = f.create_ca_issued_peer_cert({}, {{"DNS:hello.world.example.com"}}); + auto authorized = authorized_peers({policy_with({required_san_dns("stale.memes.example.com")}, + CapabilitySet::make_empty()), + policy_with({required_san_dns("fresh.memes.example.com")}, + CapabilitySet::make_with_all_capabilities())}); + f.reset_server_with_cert_opts(server_ck, std::move(authorized)); + auto client_ck = f.create_ca_issued_peer_cert({}, {{"DNS:stale.memes.example.com"}}); + f.reset_client_with_cert_opts(client_ck, AuthorizedPeers::allow_all_authenticated()); + + ASSERT_TRUE(f.handshake()); + + // Note: "inversion" of client <-> server is because the capabilities are that of the _peer_. + auto client_caps = f.server->granted_capabilities(); + auto server_caps = f.client->granted_capabilities(); + // Server (from client's PoV) implicitly has all capabilities since client doesn't specify any policies + EXPECT_EQUAL(server_caps, CapabilitySet::make_with_all_capabilities()); + // Client (from server's PoV) only has capabilities for the rule matching its DNS SAN entry. + // In this case, it is the empty set. + EXPECT_EQUAL(client_caps, CapabilitySet::make_empty()); +} + void reset_peers_with_server_authz_mode(CertFixture& f, AuthorizationMode authz_mode) { auto ck = f.create_ca_issued_peer_cert({"hello.world.example.com"}, {}); diff --git a/vespalib/src/tests/thread/thread_test.cpp b/vespalib/src/tests/thread/thread_test.cpp index 9533fe1c190..077b85ea1ac 100644 --- a/vespalib/src/tests/thread/thread_test.cpp +++ b/vespalib/src/tests/thread/thread_test.cpp @@ -2,6 +2,7 @@ #include <vespa/vespalib/testkit/test_kit.h> #include <vespa/vespalib/util/thread.h> +#include <iostream> using namespace vespalib; @@ -11,6 +12,7 @@ struct Agent : public Runnable { bool was_run; Agent() : was_run(false) {} void run() override { + fprintf(stderr, "agent run in thread %zu\n", thread::as_zu(std::this_thread::get_id())); was_run = true; } }; @@ -22,11 +24,18 @@ void my_fun(bool *was_run) { Runnable::init_fun_t wrap(Runnable::init_fun_t init, bool *init_called) { return [=](Runnable &target) { + fprintf(stderr, "lambda run in thread %zu\n", thread::as_zu(std::this_thread::get_id())); *init_called = true; return init(target); }; } +TEST("main thread") { + auto my_id = std::this_thread::get_id(); + std::cerr << "main thread(with <<): " << my_id << "\n"; + fprintf(stderr, "main thread(with printf): %zu\n", thread::as_zu(my_id)); +} + TEST("run vespalib::Runnable with init function") { Agent agent; bool init_called = false; @@ -41,9 +50,17 @@ TEST("use thread pool to run multiple things") { bool init_called = false; bool was_run = false; ThreadPool pool; + EXPECT_TRUE(pool.empty()); + EXPECT_EQUAL(pool.size(), 0u); pool.start(my_fun, &was_run); + EXPECT_TRUE(!pool.empty()); + EXPECT_EQUAL(pool.size(), 1u); pool.start(agent, wrap(test_agent_thread, &init_called)); + EXPECT_TRUE(!pool.empty()); + EXPECT_EQUAL(pool.size(), 2u); pool.join(); + EXPECT_TRUE(pool.empty()); + EXPECT_EQUAL(pool.size(), 0u); EXPECT_TRUE(init_called); EXPECT_TRUE(agent.was_run); EXPECT_TRUE(was_run); diff --git a/vespalib/src/tests/util/generationhandler_stress/generation_handler_stress_test.cpp b/vespalib/src/tests/util/generationhandler_stress/generation_handler_stress_test.cpp index 1cc54da7f2e..4abae54e06f 100644 --- a/vespalib/src/tests/util/generationhandler_stress/generation_handler_stress_test.cpp +++ b/vespalib/src/tests/util/generationhandler_stress/generation_handler_stress_test.cpp @@ -1,12 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/log/log.h> -LOG_SETUP("generation_handler_stress_test"); + #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/util/generationhandler.h> #include <vespa/vespalib/util/lambdatask.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/size_literals.h> #include <thread> +#include <cinttypes> + +#include <vespa/log/log.h> +LOG_SETUP("generation_handler_stress_test"); using vespalib::Executor; using vespalib::GenerationHandler; diff --git a/vespalib/src/vespa/fastlib/io/CMakeLists.txt b/vespalib/src/vespa/fastlib/io/CMakeLists.txt index f21cf27b21e..9f6211f3e19 100644 --- a/vespalib/src/vespa/fastlib/io/CMakeLists.txt +++ b/vespalib/src/vespa/fastlib/io/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(fastlib_io OBJECT +vespa_add_library(vespalib_fastlib_io OBJECT SOURCES bufferedfile.cpp DEPENDS diff --git a/vespalib/src/vespa/fastlib/io/bufferedfile.cpp b/vespalib/src/vespa/fastlib/io/bufferedfile.cpp index 31ca735a0bb..aecf08edf6b 100644 --- a/vespalib/src/vespa/fastlib/io/bufferedfile.cpp +++ b/vespalib/src/vespa/fastlib/io/bufferedfile.cpp @@ -107,13 +107,6 @@ Fast_BufferedFile::Sync() return _file->Sync(); } -time_t -Fast_BufferedFile::GetModificationTime() -{ - time_t retval = _file->GetModificationTime(); - return retval; -} - void Fast_BufferedFile::EnableDirectIO() { diff --git a/vespalib/src/vespa/fastlib/io/bufferedfile.h b/vespalib/src/vespa/fastlib/io/bufferedfile.h index 48f90262ad9..2a5e0ec7535 100644 --- a/vespalib/src/vespa/fastlib/io/bufferedfile.h +++ b/vespalib/src/vespa/fastlib/io/bufferedfile.h @@ -172,12 +172,7 @@ public: * Force completion of pending disk writes (flush cache). */ [[nodiscard]] bool Sync() override; - /** - * Get the time the file was last modified. - * - * @return time_t The last modification time. - */ - time_t GetModificationTime() override; + /** * Turn on direct IO. */ diff --git a/vespalib/src/vespa/fastlib/text/CMakeLists.txt b/vespalib/src/vespa/fastlib/text/CMakeLists.txt index d6cb8c29305..06573ee814d 100644 --- a/vespalib/src/vespa/fastlib/text/CMakeLists.txt +++ b/vespalib/src/vespa/fastlib/text/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(fastlib_text OBJECT +vespa_add_library(vespalib_fastlib_text OBJECT SOURCES unicodeutil.cpp normwordfolder.cpp diff --git a/vespalib/src/vespa/fastos/CMakeLists.txt b/vespalib/src/vespa/fastos/CMakeLists.txt new file mode 100644 index 00000000000..2b7a2c3b905 --- /dev/null +++ b/vespalib/src/vespa/fastos/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(vespalib_fastos OBJECT + SOURCES + file.cpp + file_rw_ops.cpp + linux_file.cpp + unix_file.cpp + DEPENDS +) diff --git a/fastos/src/vespa/fastos/file.cpp b/vespalib/src/vespa/fastos/file.cpp index fdbacb570b4..fdbacb570b4 100644 --- a/fastos/src/vespa/fastos/file.cpp +++ b/vespalib/src/vespa/fastos/file.cpp diff --git a/fastos/src/vespa/fastos/file.h b/vespalib/src/vespa/fastos/file.h index 1cf6fee71dd..371a43e27b6 100644 --- a/fastos/src/vespa/fastos/file.h +++ b/vespalib/src/vespa/fastos/file.h @@ -10,10 +10,11 @@ #pragma once -#include "types.h" -#include <cstdint> +#include <vespa/vespalib/util/time.h> #include <string> +#define FASTOS_PREFIX(a) FastOS_##a + constexpr int FASTOS_FILE_OPEN_READ = (1<<0); constexpr int FASTOS_FILE_OPEN_WRITE = (1<<1); constexpr int FASTOS_FILE_OPEN_EXISTING = (1<<2); @@ -100,12 +101,6 @@ public: void setFAdviseOptions(int options) { _fAdviseOptions = options; } /** - * Return path separator string. This will yield "/" on UNIX systems. - * @return pointer to path separator character string - */ - static const char *GetPathSeparator() { return "/"; } - - /** * Constructor. A filename could be supplied at this point, or specified * later using @ref SetFileName() or @ref Open(). * @param filename a filename (optional) @@ -343,12 +338,6 @@ public: int64_t getSize() const { return const_cast<FastOS_FileInterface *>(this)->GetSize(); } /** - * Return the time when file was last modified. - * @return time of last modification - */ - virtual time_t GetModificationTime() = 0; - - /** * Delete the file. This method requires that the file is * currently not opened. * @return Boolean success/failure @@ -412,10 +401,6 @@ public: bool useSyncWrites() const { return _syncWritesEnabled; } - /** - * Set the write chunk size used in WriteBuf. - */ - void setChunkSize(size_t chunkSize) { _chunkSize = chunkSize; } size_t getChunkSize() const { return _chunkSize; } /** @@ -651,14 +636,12 @@ public: */ class FastOS_DirectoryScanInterface { -private: - FastOS_DirectoryScanInterface(const FastOS_DirectoryScanInterface&); - FastOS_DirectoryScanInterface& operator= (const FastOS_DirectoryScanInterface&); - protected: std::string _searchPath; public: + FastOS_DirectoryScanInterface(const FastOS_DirectoryScanInterface&) = delete; + FastOS_DirectoryScanInterface& operator= (const FastOS_DirectoryScanInterface&) = delete; /** * Constructor. @@ -676,13 +659,6 @@ public: virtual ~FastOS_DirectoryScanInterface(); /** - * Get search path. - * This is an internal copy of the path specified in the constructor. - * @return Search path string. - */ - const char *GetSearchPath () { return _searchPath.c_str(); } - - /** * Read the next entry in the directory scan. Failure indicates * that there are no more entries. If the call is successful, * attributes for the entry can be read with @ref IsDirectory(), @@ -725,14 +701,6 @@ public: * @return A pointer to the recently read directory entry. */ virtual const char *GetName() = 0; - - /** - * Check whether the creation of a directory scan succeeded or - * failed (e.g. due to resource constraints). - * - * return True if the directory scan is valid. - */ - virtual bool IsValidScan() const = 0; }; #ifdef __linux__ diff --git a/fastos/src/vespa/fastos/file_rw_ops.cpp b/vespalib/src/vespa/fastos/file_rw_ops.cpp index 79fe95b21f2..79fe95b21f2 100644 --- a/fastos/src/vespa/fastos/file_rw_ops.cpp +++ b/vespalib/src/vespa/fastos/file_rw_ops.cpp diff --git a/fastos/src/vespa/fastos/file_rw_ops.h b/vespalib/src/vespa/fastos/file_rw_ops.h index 4f7aa6f082f..4f7aa6f082f 100644 --- a/fastos/src/vespa/fastos/file_rw_ops.h +++ b/vespalib/src/vespa/fastos/file_rw_ops.h diff --git a/fastos/src/vespa/fastos/linux_file.cpp b/vespalib/src/vespa/fastos/linux_file.cpp index 6fb29782957..6fb29782957 100644 --- a/fastos/src/vespa/fastos/linux_file.cpp +++ b/vespalib/src/vespa/fastos/linux_file.cpp diff --git a/fastos/src/vespa/fastos/linux_file.h b/vespalib/src/vespa/fastos/linux_file.h index 2481b163210..2481b163210 100644 --- a/fastos/src/vespa/fastos/linux_file.h +++ b/vespalib/src/vespa/fastos/linux_file.h diff --git a/fastos/src/vespa/fastos/unix_file.cpp b/vespalib/src/vespa/fastos/unix_file.cpp index 7c4cde19125..417129c99b3 100644 --- a/fastos/src/vespa/fastos/unix_file.cpp +++ b/vespalib/src/vespa/fastos/unix_file.cpp @@ -344,21 +344,6 @@ FastOS_UNIX_File::GetSize() return fileSize; } - -time_t -FastOS_UNIX_File::GetModificationTime() -{ - struct stat stbuf{}; - assert(IsOpened()); - - int res = fstat(_filedes, &stbuf); - assert(res == 0); - (void) res; - - return stbuf.st_mtime; -} - - bool FastOS_UNIX_File::Delete(const char *name) { @@ -581,10 +566,3 @@ FastOS_UNIX_DirectoryScan::GetName() return static_cast<const char *>(_dp->d_name); } - - -bool -FastOS_UNIX_DirectoryScan::IsValidScan() const -{ - return _dir != nullptr; -} diff --git a/fastos/src/vespa/fastos/unix_file.h b/vespalib/src/vespa/fastos/unix_file.h index 31e45f8d2fa..3d1f6b9db3f 100644 --- a/fastos/src/vespa/fastos/unix_file.h +++ b/vespalib/src/vespa/fastos/unix_file.h @@ -82,7 +82,6 @@ public: bool SetPosition(int64_t desiredPosition) override; int64_t GetPosition() override; int64_t GetSize() override; - time_t GetModificationTime() override; bool Delete() override; [[nodiscard]] bool Sync() override; bool SetSize(int64_t newSize) override; @@ -103,9 +102,6 @@ public: class FastOS_UNIX_DirectoryScan : public FastOS_DirectoryScanInterface { private: - FastOS_UNIX_DirectoryScan(const FastOS_UNIX_DirectoryScan&); - FastOS_UNIX_DirectoryScan& operator=(const FastOS_UNIX_DirectoryScan&); - bool _statRun; bool _isDirectory; bool _isRegular; @@ -126,5 +122,4 @@ public: bool IsDirectory() override; bool IsRegular() override; const char *GetName() override; - bool IsValidScan() const override; }; diff --git a/vespalib/src/vespa/vespalib/CMakeLists.txt b/vespalib/src/vespa/vespalib/CMakeLists.txt index 9e31084c067..63876d442ef 100644 --- a/vespalib/src/vespa/vespalib/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/CMakeLists.txt @@ -30,8 +30,9 @@ vespa_add_library(vespalib $<TARGET_OBJECTS:vespalib_vespalib_time> $<TARGET_OBJECTS:vespalib_vespalib_trace> $<TARGET_OBJECTS:vespalib_vespalib_util> - $<TARGET_OBJECTS:fastlib_io> - $<TARGET_OBJECTS:fastlib_text> + $<TARGET_OBJECTS:vespalib_fastlib_io> + $<TARGET_OBJECTS:vespalib_fastlib_text> + $<TARGET_OBJECTS:vespalib_fastos> INSTALL lib64 DEPENDS ${VESPA_GCC_LIB} diff --git a/vespalib/src/vespa/vespalib/data/slime/json_format.cpp b/vespalib/src/vespa/vespalib/data/slime/json_format.cpp index 2681ba6f52b..9cda8a7fae1 100644 --- a/vespalib/src/vespa/vespalib/data/slime/json_format.cpp +++ b/vespalib/src/vespa/vespalib/data/slime/json_format.cpp @@ -7,6 +7,7 @@ #include <vespa/vespalib/data/memory_input.h> #include <vespa/vespalib/locale/c.h> #include <cmath> +#include <cinttypes> #include <sstream> #include <cassert> diff --git a/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp b/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp index f5d87e14802..3bdbb7a81ff 100644 --- a/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp +++ b/vespalib/src/vespa/vespalib/hwaccelrated/avxprivate.hpp @@ -3,7 +3,6 @@ #pragma once #include "private_helpers.hpp" -#include <vespa/fastos/types.h> namespace vespalib::hwaccelrated::avx { diff --git a/vespalib/src/vespa/vespalib/net/native_epoll.cpp b/vespalib/src/vespa/vespalib/net/native_epoll.cpp index 1e68826a145..8b73b092ab2 100644 --- a/vespalib/src/vespa/vespalib/net/native_epoll.cpp +++ b/vespalib/src/vespa/vespalib/net/native_epoll.cpp @@ -3,6 +3,7 @@ #include "native_epoll.h" #include <cassert> #include <cerrno> +#include <cstring> #include <unistd.h> #include <vespa/log/log.h> diff --git a/vespalib/src/vespa/vespalib/net/tls/capability.cpp b/vespalib/src/vespa/vespalib/net/tls/capability.cpp index cfc1cc7a7cc..49f8aa11bad 100644 --- a/vespalib/src/vespa/vespalib/net/tls/capability.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/capability.cpp @@ -35,6 +35,7 @@ constexpr std::array<std::string_view, Capability::max_value_count()> capability "vespa.content.metrics_api"sv, "vespa.content.proton_admin_api"sv, "vespa.content.search_api"sv, + "vespa.content.state_api"sv, "vespa.content.status_pages"sv, "vespa.content.storage_api"sv, "vespa.logserver.api"sv, @@ -83,6 +84,7 @@ std::optional<Capability> Capability::find_capability(const string& cap_name) no {"vespa.content.metrics_api", content_metrics_api()}, {"vespa.content.proton_admin_api", content_proton_admin_api()}, {"vespa.content.search_api", content_search_api()}, + {"vespa.content.state_api", content_state_api()}, {"vespa.content.status_pages", content_status_pages()}, {"vespa.content.storage_api", content_storage_api()}, {"vespa.logserver.api", logserver_api()}, diff --git a/vespalib/src/vespa/vespalib/net/tls/capability.h b/vespalib/src/vespa/vespalib/net/tls/capability.h index a7a1dcd15ac..396fad4cbcd 100644 --- a/vespalib/src/vespa/vespalib/net/tls/capability.h +++ b/vespalib/src/vespa/vespalib/net/tls/capability.h @@ -47,6 +47,7 @@ private: ContentMetricsApi, ContentProtonAdminApi, ContentSearchApi, + ContentStateApi, ContentStatusPages, ContentStorageApi, LogserverApi, @@ -176,6 +177,10 @@ public: return Capability(Id::ContentSearchApi); } + constexpr static Capability content_state_api() noexcept { + return Capability(Id::ContentStateApi); + } + constexpr static Capability content_proton_admin_api() noexcept { return Capability(Id::ContentProtonAdminApi); } diff --git a/vespalib/src/vespa/vespalib/net/tls/capability_set.cpp b/vespalib/src/vespa/vespalib/net/tls/capability_set.cpp index cff56208ae4..06231582461 100644 --- a/vespalib/src/vespa/vespalib/net/tls/capability_set.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/capability_set.cpp @@ -32,7 +32,7 @@ std::optional<CapabilitySet> CapabilitySet::find_capability_set(const string& ca {"vespa.telemetry", telemetry()}, {"vespa.cluster_controller_node", cluster_controller_node()}, {"vespa.logserver_node", logserver_node()}, - {"vespa.config_server", config_server()} + {"vespa.config_server_node", config_server_node()} }); auto iter = name_to_cap_set.find(cap_set_name); return (iter != name_to_cap_set.end()) ? std::optional<CapabilitySet>(iter->second) : std::nullopt; @@ -65,12 +65,14 @@ CapabilitySet CapabilitySet::content_node() noexcept { CapabilitySet CapabilitySet::container_node() noexcept { return CapabilitySet::of({Capability::content_document_api(), + Capability::container_document_api(), Capability::content_search_api()}) .union_of(shared_app_node_capabilities()); } CapabilitySet CapabilitySet::telemetry() noexcept { return CapabilitySet::of({Capability::content_status_pages(), + Capability::content_state_api(), Capability::content_metrics_api(), Capability::container_state_api(), Capability::metricsproxy_metrics_api(), @@ -88,12 +90,13 @@ CapabilitySet CapabilitySet::logserver_node() noexcept { return shared_app_node_capabilities(); } -CapabilitySet CapabilitySet::config_server() noexcept { +CapabilitySet CapabilitySet::config_server_node() noexcept { return CapabilitySet::of({Capability::client_filereceiver_api(), Capability::container_management_api(), Capability::slobrok_api(), Capability::cluster_controller_reindexing(), - Capability::cluster_controller_state()}) + Capability::cluster_controller_state(), + Capability::logserver_api()}) .union_of(telemetry()); } diff --git a/vespalib/src/vespa/vespalib/net/tls/capability_set.h b/vespalib/src/vespa/vespalib/net/tls/capability_set.h index 8aad28a4162..0811739217e 100644 --- a/vespalib/src/vespa/vespalib/net/tls/capability_set.h +++ b/vespalib/src/vespa/vespalib/net/tls/capability_set.h @@ -111,7 +111,7 @@ public: [[nodiscard]] static CapabilitySet telemetry() noexcept; [[nodiscard]] static CapabilitySet cluster_controller_node() noexcept; [[nodiscard]] static CapabilitySet logserver_node() noexcept; - [[nodiscard]] static CapabilitySet config_server() noexcept; + [[nodiscard]] static CapabilitySet config_server_node() noexcept; [[nodiscard]] static CapabilitySet make_with_all_capabilities() noexcept; [[nodiscard]] static constexpr CapabilitySet make_empty() noexcept { return {}; }; diff --git a/vespalib/src/vespa/vespalib/net/tls/policy_checking_certificate_verifier.cpp b/vespalib/src/vespa/vespalib/net/tls/policy_checking_certificate_verifier.cpp index e280434c59f..a3f9b3f52c9 100644 --- a/vespalib/src/vespa/vespalib/net/tls/policy_checking_certificate_verifier.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/policy_checking_certificate_verifier.cpp @@ -74,13 +74,15 @@ VerificationResult PolicyConfiguredCertificateVerifier::verify(const PeerCredent return VerificationResult::make_authorized_with_all_capabilities(); } CapabilitySet caps; + bool matched_any_policy = false; for (const auto& policy : _authorized_peers.peer_policies()) { if (matches_all_policy_requirements(peer_creds, policy)) { caps.add_all(policy.granted_capabilities()); + matched_any_policy = true; } } - if (!caps.empty()) { - return VerificationResult::make_authorized_with_capabilities(std::move(caps)); + if (matched_any_policy) { + return VerificationResult::make_authorized_with_capabilities(caps); } else { return VerificationResult::make_not_authorized(); } diff --git a/vespalib/src/vespa/vespalib/net/tls/statistics.cpp b/vespalib/src/vespa/vespalib/net/tls/statistics.cpp index a308a20bf2f..dc642518abb 100644 --- a/vespalib/src/vespa/vespalib/net/tls/statistics.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/statistics.cpp @@ -9,6 +9,8 @@ ConnectionStatistics ConnectionStatistics::server_stats = {}; ConfigStatistics ConfigStatistics::instance = {}; +CapabilityStatistics CapabilityStatistics::instance = {}; + ConnectionStatistics::Snapshot ConnectionStatistics::snapshot() const noexcept { Snapshot s; s.insecure_connections = insecure_connections.load(std::memory_order_relaxed); @@ -43,4 +45,18 @@ ConfigStatistics::Snapshot ConfigStatistics::Snapshot::subtract(const Snapshot& return s; } +CapabilityStatistics::Snapshot CapabilityStatistics::snapshot() const noexcept { + Snapshot s; + s.rpc_capability_checks_failed = rpc_capability_checks_failed.load(std::memory_order_relaxed); + s.status_capability_checks_failed = status_capability_checks_failed.load(std::memory_order_relaxed); + return s; +} + +CapabilityStatistics::Snapshot CapabilityStatistics::Snapshot::subtract(const Snapshot& rhs) const noexcept { + Snapshot s; + s.rpc_capability_checks_failed = rpc_capability_checks_failed - rhs.rpc_capability_checks_failed; + s.status_capability_checks_failed = status_capability_checks_failed - rhs.status_capability_checks_failed; + return s; +} + } diff --git a/vespalib/src/vespa/vespalib/net/tls/statistics.h b/vespalib/src/vespa/vespalib/net/tls/statistics.h index 3e94ed95590..e693b09a3d2 100644 --- a/vespalib/src/vespa/vespalib/net/tls/statistics.h +++ b/vespalib/src/vespa/vespalib/net/tls/statistics.h @@ -55,12 +55,12 @@ struct ConnectionStatistics { uint64_t invalid_peer_credentials = 0; uint64_t broken_tls_connections = 0; - Snapshot subtract(const Snapshot& rhs) const noexcept; + [[nodiscard]] Snapshot subtract(const Snapshot& rhs) const noexcept; }; - // Acquires a snapshot of statistics that is expected to be reasonably up to date. + // Acquires a snapshot of statistics that is expected to be reasonably up-to-date. // Thread safe. - Snapshot snapshot() const noexcept; + [[nodiscard]] Snapshot snapshot() const noexcept; static ConnectionStatistics client_stats; static ConnectionStatistics server_stats; @@ -85,15 +85,42 @@ struct ConfigStatistics { uint64_t successful_config_reloads = 0; uint64_t failed_config_reloads = 0; - Snapshot subtract(const Snapshot& rhs) const noexcept; + [[nodiscard]] Snapshot subtract(const Snapshot& rhs) const noexcept; }; - // Acquires a snapshot of statistics that is expected to be reasonably up to date. + // Acquires a snapshot of statistics that is expected to be reasonably up-to-date. // Thread safe. - Snapshot snapshot() const noexcept; + [[nodiscard]] Snapshot snapshot() const noexcept; static ConfigStatistics instance; static ConfigStatistics& get() noexcept { return instance; } }; +struct CapabilityStatistics { + std::atomic<uint64_t> rpc_capability_checks_failed = 0; + std::atomic<uint64_t> status_capability_checks_failed = 0; + + void inc_rpc_capability_checks_failed() noexcept { + rpc_capability_checks_failed.fetch_add(1, std::memory_order_relaxed); + } + + void inc_status_capability_checks_failed() noexcept { + status_capability_checks_failed.fetch_add(1, std::memory_order_relaxed); + } + + struct Snapshot { + uint64_t rpc_capability_checks_failed = 0; + uint64_t status_capability_checks_failed = 0; + + [[nodiscard]] Snapshot subtract(const Snapshot& rhs) const noexcept; + }; + + // Acquires a snapshot of statistics that is expected to be reasonably up-to-date. + // Thread safe. + [[nodiscard]] Snapshot snapshot() const noexcept; + + static CapabilityStatistics instance; + static CapabilityStatistics& get() noexcept { return instance; } +}; + } diff --git a/vespalib/src/vespa/vespalib/net/tls/verification_result.cpp b/vespalib/src/vespa/vespalib/net/tls/verification_result.cpp index f1e50d3115e..37b95c3c07a 100644 --- a/vespalib/src/vespa/vespalib/net/tls/verification_result.cpp +++ b/vespalib/src/vespa/vespalib/net/tls/verification_result.cpp @@ -6,14 +6,18 @@ namespace vespalib::net::tls { -VerificationResult::VerificationResult() = default; +VerificationResult::VerificationResult() noexcept + : _granted_capabilities(), + _authorized(false) +{} -VerificationResult::VerificationResult(CapabilitySet granted_capabilities) - : _granted_capabilities(std::move(granted_capabilities)) +VerificationResult::VerificationResult(bool authorized, CapabilitySet granted_capabilities) noexcept + : _granted_capabilities(granted_capabilities), + _authorized(authorized) {} -VerificationResult::VerificationResult(const VerificationResult&) = default; -VerificationResult& VerificationResult::operator=(const VerificationResult&) = default; +VerificationResult::VerificationResult(const VerificationResult&) noexcept = default; +VerificationResult& VerificationResult::operator=(const VerificationResult&) noexcept = default; VerificationResult::VerificationResult(VerificationResult&&) noexcept = default; VerificationResult& VerificationResult::operator=(VerificationResult&&) noexcept = default; VerificationResult::~VerificationResult() = default; @@ -29,18 +33,18 @@ void VerificationResult::print(asciistream& os) const { } VerificationResult -VerificationResult::make_authorized_with_capabilities(CapabilitySet granted_capabilities) { - return VerificationResult(std::move(granted_capabilities)); +VerificationResult::make_authorized_with_capabilities(CapabilitySet granted_capabilities) noexcept { + return {true, granted_capabilities}; } VerificationResult -VerificationResult::make_authorized_with_all_capabilities() { - return VerificationResult(CapabilitySet::make_with_all_capabilities()); +VerificationResult::make_authorized_with_all_capabilities() noexcept { + return {true, CapabilitySet::make_with_all_capabilities()}; } VerificationResult -VerificationResult::make_not_authorized() { - return {}; +VerificationResult::make_not_authorized() noexcept { + return {false, CapabilitySet::make_empty()}; } asciistream& operator<<(asciistream& os, const VerificationResult& res) { diff --git a/vespalib/src/vespa/vespalib/net/tls/verification_result.h b/vespalib/src/vespa/vespalib/net/tls/verification_result.h index 92b32ad92f7..896908f7c13 100644 --- a/vespalib/src/vespa/vespalib/net/tls/verification_result.h +++ b/vespalib/src/vespa/vespalib/net/tls/verification_result.h @@ -16,22 +16,27 @@ namespace vespalib::net::tls { * This result contains the union set of all capabilities granted by the matching * authorization rules. If no rules matched, the set will be empty. The capability * set will also be empty for a default-constructed instance. + * + * It is possible for a VerificationResult to be successful but with an empty + * capability set. If capabilities are enforced, this will effectively only + * allow mTLS handshakes to go through, allowing rudimentary health checking. */ class VerificationResult { CapabilitySet _granted_capabilities; + bool _authorized; - explicit VerificationResult(CapabilitySet granted_capabilities); + VerificationResult(bool authorized, CapabilitySet granted_capabilities) noexcept; public: - VerificationResult(); - VerificationResult(const VerificationResult&); - VerificationResult& operator=(const VerificationResult&); + VerificationResult() noexcept; // Unauthorized by default + VerificationResult(const VerificationResult&) noexcept; + VerificationResult& operator=(const VerificationResult&) noexcept; VerificationResult(VerificationResult&&) noexcept; VerificationResult& operator=(VerificationResult&&) noexcept; ~VerificationResult(); - // Returns true iff at least one capability been granted. + // Returns true iff the peer matched at least one policy or authorization is not enforced. [[nodiscard]] bool success() const noexcept { - return !_granted_capabilities.empty(); + return _authorized; } [[nodiscard]] const CapabilitySet& granted_capabilities() const noexcept { @@ -40,9 +45,9 @@ public: void print(asciistream& os) const; - static VerificationResult make_authorized_with_capabilities(CapabilitySet granted_capabilities); - static VerificationResult make_authorized_with_all_capabilities(); - static VerificationResult make_not_authorized(); + static VerificationResult make_authorized_with_capabilities(CapabilitySet granted_capabilities) noexcept; + static VerificationResult make_authorized_with_all_capabilities() noexcept; + static VerificationResult make_not_authorized() noexcept; }; asciistream& operator<<(asciistream&, const VerificationResult&); diff --git a/vespalib/src/vespa/vespalib/net/wakeup_pipe.cpp b/vespalib/src/vespa/vespalib/net/wakeup_pipe.cpp index 60900289e79..b8025bfcf9f 100644 --- a/vespalib/src/vespa/vespalib/net/wakeup_pipe.cpp +++ b/vespalib/src/vespa/vespalib/net/wakeup_pipe.cpp @@ -2,34 +2,40 @@ #include "wakeup_pipe.h" #include "socket_utils.h" +#include <vespa/vespalib/util/require.h> #include <unistd.h> namespace vespalib { WakeupPipe::WakeupPipe() - : _pipe() + : _reader(), + _writer() { - socketutils::nonblocking_pipe(_pipe); + int pipe[2]; + socketutils::nonblocking_pipe(pipe); + _reader.reset(pipe[0]); + _writer.reset(pipe[1]); } -WakeupPipe::~WakeupPipe() -{ - close(_pipe[0]); - close(_pipe[1]); -} +WakeupPipe::~WakeupPipe() = default; void WakeupPipe::write_token() { char token = 'T'; - [[maybe_unused]] ssize_t res = write(_pipe[1], &token, 1); + ssize_t res = _writer.write(&token, 1); + if (res < 0) { + res = -errno; + } + REQUIRE(res > 0 || res == -EAGAIN || res == -EWOULDBLOCK); } void WakeupPipe::read_tokens() { char token_trash[128]; - [[maybe_unused]] ssize_t res = read(_pipe[0], token_trash, sizeof(token_trash)); + ssize_t res = _reader.read(token_trash, sizeof(token_trash)); + REQUIRE(res > 0); } } diff --git a/vespalib/src/vespa/vespalib/net/wakeup_pipe.h b/vespalib/src/vespa/vespalib/net/wakeup_pipe.h index b52f7f9e32d..36c88b205c0 100644 --- a/vespalib/src/vespa/vespalib/net/wakeup_pipe.h +++ b/vespalib/src/vespa/vespalib/net/wakeup_pipe.h @@ -2,6 +2,8 @@ #pragma once +#include "socket_handle.h" + namespace vespalib { //----------------------------------------------------------------------------- @@ -15,11 +17,12 @@ namespace vespalib { **/ class WakeupPipe { private: - int _pipe[2]; + SocketHandle _reader; + SocketHandle _writer; public: WakeupPipe(); ~WakeupPipe(); - int get_read_fd() const { return _pipe[0]; } + int get_read_fd() const { return _reader.get(); } void write_token(); void read_tokens(); }; diff --git a/vespalib/src/vespa/vespalib/portal/http_connection.cpp b/vespalib/src/vespa/vespalib/portal/http_connection.cpp index 6ea56e2659c..3d8edf2fc2e 100644 --- a/vespalib/src/vespa/vespalib/portal/http_connection.cpp +++ b/vespalib/src/vespa/vespalib/portal/http_connection.cpp @@ -245,14 +245,16 @@ HttpConnection::handle_event(bool, bool) } void -HttpConnection::respond_with_content(const vespalib::string &content_type, - const vespalib::string &content) +HttpConnection::respond_with_content(vespalib::stringref content_type, + vespalib::stringref content) { { OutputWriter dst(_output, CHUNK_SIZE); dst.printf("HTTP/1.1 200 OK\r\n"); dst.printf("Connection: close\r\n"); - dst.printf("Content-Type: %s\r\n", content_type.c_str()); + dst.printf("Content-Type: "); + dst.write(content_type.data(), content_type.size()); + dst.printf("\r\n"); dst.printf("Content-Length: %zu\r\n", content.size()); emit_http_security_headers(dst); dst.printf("\r\n"); @@ -263,11 +265,13 @@ HttpConnection::respond_with_content(const vespalib::string &content_type, } void -HttpConnection::respond_with_error(int code, const vespalib::string &msg) +HttpConnection::respond_with_error(int code, vespalib::stringref msg) { { OutputWriter dst(_output, CHUNK_SIZE); - dst.printf("HTTP/1.1 %d %s\r\n", code, msg.c_str()); + dst.printf("HTTP/1.1 %d ", code); + dst.write(msg.data(), msg.size()); + dst.printf("\r\n"); dst.printf("Connection: close\r\n"); dst.printf("\r\n"); } diff --git a/vespalib/src/vespa/vespalib/portal/http_connection.h b/vespalib/src/vespa/vespalib/portal/http_connection.h index 03d23351e7d..8540cb87e1d 100644 --- a/vespalib/src/vespa/vespalib/portal/http_connection.h +++ b/vespalib/src/vespa/vespalib/portal/http_connection.h @@ -53,9 +53,9 @@ public: // Precondition: handshake must have been completed const net::ConnectionAuthContext &auth_context() const noexcept { return *_auth_ctx; } - void respond_with_content(const vespalib::string &content_type, - const vespalib::string &content); - void respond_with_error(int code, const vespalib::string &msg); + void respond_with_content(vespalib::stringref content_type, + vespalib::stringref content); + void respond_with_error(int code, const vespalib::stringref msg); }; } // namespace vespalib::portal diff --git a/vespalib/src/vespa/vespalib/portal/portal.cpp b/vespalib/src/vespa/vespalib/portal/portal.cpp index a98562f6504..691ddaef495 100644 --- a/vespalib/src/vespa/vespalib/portal/portal.cpp +++ b/vespalib/src/vespa/vespalib/portal/portal.cpp @@ -77,8 +77,8 @@ Portal::GetRequest::export_params() const } void -Portal::GetRequest::respond_with_content(const vespalib::string &content_type, - const vespalib::string &content) +Portal::GetRequest::respond_with_content(vespalib::stringref content_type, + vespalib::stringref content) { assert(active()); _conn->respond_with_content(content_type, content); @@ -86,7 +86,7 @@ Portal::GetRequest::respond_with_content(const vespalib::string &content_type, } void -Portal::GetRequest::respond_with_error(int code, const vespalib::string &msg) +Portal::GetRequest::respond_with_error(int code, vespalib::stringref msg) { assert(active()); _conn->respond_with_error(code, msg); diff --git a/vespalib/src/vespa/vespalib/portal/portal.h b/vespalib/src/vespa/vespalib/portal/portal.h index 314dd6e7de9..6954d800c91 100644 --- a/vespalib/src/vespa/vespalib/portal/portal.h +++ b/vespalib/src/vespa/vespalib/portal/portal.h @@ -65,9 +65,9 @@ public: bool has_param(const vespalib::string &name) const; const vespalib::string &get_param(const vespalib::string &name) const; std::map<vespalib::string, vespalib::string> export_params() const; - void respond_with_content(const vespalib::string &content_type, - const vespalib::string &content); - void respond_with_error(int code, const vespalib::string &msg); + void respond_with_content(vespalib::stringref content_type, + vespalib::stringref content); + void respond_with_error(int code, vespalib::stringref msg); const net::ConnectionAuthContext &auth_context() const noexcept; ~GetRequest(); }; diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index 73e8b93a2ff..c8536fc68c1 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -24,7 +24,6 @@ vespa_add_library(vespalib_vespalib_util OBJECT cpu_usage.cpp crc.cpp destructor_callbacks.cpp - document_runnable.cpp doom.cpp dual_merge_director.cpp error.cpp diff --git a/vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp b/vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp index 14a235f7257..7fa2d357a5e 100644 --- a/vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp +++ b/vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp @@ -54,29 +54,20 @@ AdaptiveSequencedExecutor::Self::~Self() AdaptiveSequencedExecutor::ThreadTools::ThreadTools(AdaptiveSequencedExecutor &parent_in) : parent(parent_in), - pool(std::make_unique<FastOS_ThreadPool>()), + pool(), allow_worker_exit() { } AdaptiveSequencedExecutor::ThreadTools::~ThreadTools() { - assert(pool->isClosed()); -} - -void -AdaptiveSequencedExecutor::ThreadTools::Run(FastOS_ThreadInterface *, void *) -{ - parent.worker_main(); } void AdaptiveSequencedExecutor::ThreadTools::start(size_t num_threads) { for (size_t i = 0; i < num_threads; ++i) { - FastOS_ThreadInterface *thread = pool->NewThread(this); - assert(thread != nullptr); - (void)thread; + pool.start([this](){ parent.worker_main(); }); } } @@ -84,7 +75,7 @@ void AdaptiveSequencedExecutor::ThreadTools::close() { allow_worker_exit.countDown(); - pool->Close(); + pool.join(); } //----------------------------------------------------------------------------- diff --git a/vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h b/vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h index fee9b8a61f8..ee12af77a61 100644 --- a/vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h +++ b/vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h @@ -3,11 +3,11 @@ #pragma once #include "isequencedtaskexecutor.h" +#include "thread.h" #include <vespa/vespalib/util/executor_idle_tracking.h> #include <vespa/vespalib/util/arrayqueue.hpp> #include <vespa/vespalib/util/gate.h> #include <vespa/vespalib/util/eventbarrier.hpp> -#include <vespa/fastos/thread.h> #include <mutex> #include <condition_variable> #include <optional> @@ -113,14 +113,12 @@ private: /** * Stuff related to worker thread startup and shutdown. **/ - struct ThreadTools : FastOS_Runnable { - static constexpr size_t STACK_SIZE = (256 * 1024); + struct ThreadTools { AdaptiveSequencedExecutor &parent; - std::unique_ptr<FastOS_ThreadPool> pool; + ThreadPool pool; Gate allow_worker_exit; ThreadTools(AdaptiveSequencedExecutor &parent_in); ~ThreadTools(); - void Run(FastOS_ThreadInterface *, void *) override; void start(size_t num_threads); void close(); }; diff --git a/vespalib/src/vespa/vespalib/util/document_runnable.cpp b/vespalib/src/vespa/vespalib/util/document_runnable.cpp deleted file mode 100644 index c0af72dbbb1..00000000000 --- a/vespalib/src/vespa/vespalib/util/document_runnable.cpp +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "document_runnable.h" -#include <vespa/vespalib/util/exceptions.h> -#include <cassert> - -namespace document { - -Runnable::Runnable() - : _stateLock(), - _stateCond(), - _state(NOT_RUNNING) -{ -} - -Runnable::~Runnable() { - std::lock_guard monitorGuard(_stateLock); - assert(getState() == NOT_RUNNING); -} - -bool Runnable::start(FastOS_ThreadPool& pool) -{ - std::unique_lock guard(_stateLock); - _stateCond.wait(guard, [&](){ return (getState() != STOPPING);}); - - if (getState() != NOT_RUNNING) return false; - set_state(STARTING); - if (pool.NewThread(this) == nullptr) { - throw vespalib::IllegalStateException("Failed starting a new thread", VESPA_STRLOC); - } - return true; -} - -void Runnable::set_state(State new_state) noexcept -{ - _state.store(new_state, std::memory_order_relaxed); -} - -bool Runnable::stopping() const noexcept -{ - State s(getState()); - return (s == STOPPING) || (s == RUNNING && GetThread()->GetBreakFlag()); -} - -bool Runnable::running() const noexcept -{ - State s(getState()); - // Must check break-flag too, as threadpool will use that to close - // down. - return (s == STARTING || (s == RUNNING && !GetThread()->GetBreakFlag())); -} - -bool Runnable::stop() -{ - std::lock_guard monitor(_stateLock); - if (getState() == STOPPING || getState() == NOT_RUNNING) return false; - GetThread()->SetBreakFlag(); - set_state(STOPPING); - return onStop(); -} - -bool Runnable::onStop() -{ - return true; -} - -bool Runnable::join() const -{ - std::unique_lock guard(_stateLock); - assert ((getState() != STARTING) && (getState() != RUNNING)); - _stateCond.wait(guard, [&](){ return (getState() == NOT_RUNNING);}); - return true; -} - -FastOS_ThreadId Runnable::native_thread_id() const noexcept -{ - return GetThread()->GetThreadId(); -} - -void Runnable::Run(FastOS_ThreadInterface*, void*) -{ - { - std::lock_guard guard(_stateLock); - // Don't set state if its already at stopping. (And let run() be - // called even though about to stop for consistency) - if (getState() == STARTING) { - set_state(RUNNING); - } - } - - // By not catching exceptions, they should abort whole application. - // We should thus not need to have a catch-all to set state to not - // running. - run(); - - { - std::lock_guard guard(_stateLock); - set_state(NOT_RUNNING); - _stateCond.notify_all(); - } -} - -} diff --git a/vespalib/src/vespa/vespalib/util/document_runnable.h b/vespalib/src/vespa/vespalib/util/document_runnable.h deleted file mode 100644 index 89388bac34c..00000000000 --- a/vespalib/src/vespa/vespalib/util/document_runnable.h +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/** - * @class document::Runnable - * @ingroup util - * - * @brief Implementation of FastOS_Runnable that implements threadsafe stop. - * - * FastOS_Runnable can easily be used unsafe. If you use the thread pointer for - * anything after your runnable had returned from Run(), it could affect another - * runnable now using that thread. - * - * Using this class should be foolproof to avoid synchronization issues during - * thread starting and stopping :) - * - * @author H�kon Humberset - * @date 2005-09-19 - */ - -#pragma once - -#include <vespa/fastos/thread.h> -#include <atomic> - -namespace document { - -class Runnable : private FastOS_Runnable { -public: - enum State { NOT_RUNNING, STARTING, RUNNING, STOPPING }; - -private: - mutable std::mutex _stateLock; - mutable std::condition_variable _stateCond; - std::atomic<State> _state; - - void Run(FastOS_ThreadInterface*, void*) override; - void set_state(State new_state) noexcept; // _stateLock must be held -public: - /** - * Create a runnable. - * @param pool If set, runnable will be started in constructor. - */ - Runnable(); - ~Runnable() override; - - /** - * Start this runnable. - * @param pool The threadpool from which a thread is acquired. - * @return True if thread was started, false if thread was already running. - */ - bool start(FastOS_ThreadPool& pool); - - /** - * Stop this runnable. - * @return True if thread was stopped, false if thread was not running. - */ - bool stop(); - - /** - * Called in stop(). Implement, to for instance notify any monitors that - * can be waiting. - */ - virtual bool onStop(); - - /** - * Wait for this thread to finish, if it is in the process of stopping. - * @return True if thread finished (or not running), false if thread is - * running normally and no stop is scheduled. - */ - bool join() const; - - /** - * Implement this to make the runnable actually do something. - */ - virtual void run() = 0; - - /** - * Get the current state of this runnable. - * Thread safe (but relaxed) read; may be stale if done outside _stateLock. - */ - [[nodiscard]] State getState() const noexcept { - return _state.load(std::memory_order_relaxed); - } - - /** Check if system is in the process of stopping. */ - [[nodiscard]] bool stopping() const noexcept; - - /** - * Checks if runnable is running or not. (Started is considered running) - */ - [[nodiscard]] bool running() const noexcept; - - FastOS_ThreadId native_thread_id() const noexcept; -}; - -} - diff --git a/vespalib/src/vespa/vespalib/util/process_memory_stats.cpp b/vespalib/src/vespa/vespalib/util/process_memory_stats.cpp index f7e8e087727..41f1e282c4b 100644 --- a/vespalib/src/vespa/vespalib/util/process_memory_stats.cpp +++ b/vespalib/src/vespa/vespalib/util/process_memory_stats.cpp @@ -4,6 +4,7 @@ #include <vespa/vespalib/stllike/asciistream.h> #include <algorithm> #include <vector> +#include <cinttypes> #include <vespa/log/log.h> diff --git a/vespalib/src/vespa/vespalib/util/shutdownguard.cpp b/vespalib/src/vespa/vespalib/util/shutdownguard.cpp index e3e56dc78cb..3d7404028a1 100644 --- a/vespalib/src/vespa/vespalib/util/shutdownguard.cpp +++ b/vespalib/src/vespa/vespalib/util/shutdownguard.cpp @@ -1,19 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "shutdownguard.h" #include <unistd.h> -#include <thread> #include <vespa/log/log.h> LOG_SETUP(".vespalib.shutdownguard"); namespace vespalib { -namespace { -enum { STACK_SIZE = (1u << 16) }; -} -void ShutdownGuard::Run(FastOS_ThreadInterface *, void *) +void +ShutdownGuard::run() { - while (_dieAtTime > steady_clock::now() && ! GetThread()->GetBreakFlag()) { + while (_dieAtTime > steady_clock::now() && !_cancel.load(std::memory_order_relaxed)) { std::this_thread::sleep_for(5ms); } if (_dieAtTime <= steady_clock::now()) { @@ -22,19 +19,17 @@ void ShutdownGuard::Run(FastOS_ThreadInterface *, void *) } } -ShutdownGuard::ShutdownGuard(duration millis) : - FastOS_Runnable(), - _pool(1), +ShutdownGuard::ShutdownGuard(duration millis) + : _thread(), _dieAtTime(steady_clock::now() + millis) { - _pool.NewThread(this); + _thread = std::thread(&ShutdownGuard::run, this); } ShutdownGuard::~ShutdownGuard() { - GetThread()->SetBreakFlag(); - GetThread()->Join(); - _pool.Close(); + _cancel = true; + _thread.join(); } } diff --git a/vespalib/src/vespa/vespalib/util/shutdownguard.h b/vespalib/src/vespa/vespalib/util/shutdownguard.h index d76d4deb5d2..fb83c2f5977 100644 --- a/vespalib/src/vespa/vespalib/util/shutdownguard.h +++ b/vespalib/src/vespa/vespalib/util/shutdownguard.h @@ -2,7 +2,8 @@ #pragma once #include <vespa/vespalib/util/time.h> -#include <vespa/fastos/thread.h> +#include <thread> +#include <atomic> namespace vespalib { @@ -13,12 +14,13 @@ namespace vespalib { * termination. * A separate shutdown thread will perform the actual _exit() call. **/ -class ShutdownGuard : public FastOS_Runnable +class ShutdownGuard { - FastOS_ThreadPool _pool; - steady_time _dieAtTime; + std::thread _thread; + steady_time _dieAtTime; + std::atomic<bool> _cancel; - void Run(FastOS_ThreadInterface *, void *) override; + void run(); public: /** diff --git a/vespalib/src/vespa/vespalib/util/thread.cpp b/vespalib/src/vespa/vespalib/util/thread.cpp index 1c8fef53ba3..6113924e352 100644 --- a/vespalib/src/vespa/vespalib/util/thread.cpp +++ b/vespalib/src/vespa/vespalib/util/thread.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "thread.h" +#include <cstring> namespace vespalib::thread { @@ -11,4 +12,11 @@ std::thread start(Runnable &runnable, Runnable::init_fun_t init_fun_in) { }); } +size_t as_zu(std::thread::id id) { + size_t res = 0; + static_assert(sizeof(id) <= sizeof(res)); + std::memcpy(&res, &id, sizeof(id)); + return res; +} + } diff --git a/vespalib/src/vespa/vespalib/util/thread.h b/vespalib/src/vespa/vespalib/util/thread.h index d919b8f2ab7..9f3ebd89165 100644 --- a/vespalib/src/vespa/vespalib/util/thread.h +++ b/vespalib/src/vespa/vespalib/util/thread.h @@ -10,6 +10,7 @@ namespace vespalib { namespace thread { [[nodiscard]] std::thread start(Runnable &runnable, Runnable::init_fun_t init_fun); +size_t as_zu(std::thread::id id); } /** @@ -23,15 +24,18 @@ private: public: ThreadPool() noexcept : _threads() {} void start(Runnable &runnable, Runnable::init_fun_t init_fun) { - _threads.reserve(_threads.size() + 1); + reserve(size() + 1); _threads.push_back(thread::start(runnable, std::move(init_fun))); } template<typename F, typename... Args> requires std::invocable<F,Args...> void start(F &&f, Args && ... args) { - _threads.reserve(_threads.size() + 1); + reserve(size() + 1); _threads.emplace_back(std::forward<F>(f), std::forward<Args>(args)...); }; + void reserve(size_t capacity) { _threads.reserve(capacity); } + size_t size() const { return _threads.size(); } + bool empty() const { return _threads.empty(); } void join() { for (auto &thread: _threads) { thread.join(); diff --git a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp index 8b6427d9391..8af14366293 100644 --- a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp +++ b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.cpp @@ -1,29 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "threadstackexecutorbase.h" -#include <vespa/fastos/thread.h> namespace vespalib { -namespace thread { - -struct ThreadInit : public FastOS_Runnable { - Runnable &worker; - ThreadStackExecutorBase::init_fun_t init_fun; - - explicit ThreadInit(Runnable &worker_in, ThreadStackExecutorBase::init_fun_t init_fun_in) - : worker(worker_in), init_fun(std::move(init_fun_in)) {} - - void Run(FastOS_ThreadInterface *, void *) override; -}; - -void -ThreadInit::Run(FastOS_ThreadInterface *, void *) { - init_fun(worker); -} - -} - ThreadStackExecutorBase::Worker::Worker() : lock(), cond(), @@ -155,7 +135,7 @@ ThreadStackExecutorBase::run() ThreadStackExecutorBase::ThreadStackExecutorBase(uint32_t taskLimit, init_fun_t init_fun) : SyncableThreadExecutor(), Runnable(), - _pool(std::make_unique<FastOS_ThreadPool>()), + _pool(), _lock(), _cond(), _stats(), @@ -167,7 +147,7 @@ ThreadStackExecutorBase::ThreadStackExecutorBase(uint32_t taskLimit, init_fun_t _taskCount(0), _taskLimit(taskLimit), _closed(false), - _thread_init(std::make_unique<thread::ThreadInit>(*this, std::move(init_fun))) + _init_fun(init_fun) { assert(taskLimit > 0); } @@ -177,15 +157,13 @@ ThreadStackExecutorBase::start(uint32_t threads) { assert(threads > 0); for (uint32_t i = 0; i < threads; ++i) { - FastOS_ThreadInterface *thread = _pool->NewThread(_thread_init.get()); - assert(thread != nullptr); - (void)thread; + _pool.start(*this, _init_fun); } } size_t ThreadStackExecutorBase::getNumThreads() const { - return _pool->GetNumStartedThreads(); + return _pool.size(); } void @@ -315,12 +293,11 @@ ThreadStackExecutorBase::cleanup() { shutdown().sync(); _executorCompletion.countDown(); - _pool->Close(); + _pool.join(); } ThreadStackExecutorBase::~ThreadStackExecutorBase() { - assert(_pool->isClosed()); assert(_taskCount == 0); assert(_blocked.empty()); } diff --git a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h index 501fde92f4c..765499b73bc 100644 --- a/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h +++ b/vespalib/src/vespa/vespalib/util/threadstackexecutorbase.h @@ -2,21 +2,17 @@ #pragma once +#include "thread.h" #include "threadexecutor.h" #include "eventbarrier.hpp" #include "arrayqueue.hpp" #include "gate.h" -#include "runnable.h" #include "executor_idle_tracking.h" #include <vector> #include <functional> -class FastOS_ThreadPool; - namespace vespalib { -namespace thread { struct ThreadInit; } - /** * An executor service that executes tasks in multiple threads. **/ @@ -73,7 +69,7 @@ private: void unblock(); }; - std::unique_ptr<FastOS_ThreadPool> _pool; + ThreadPool _pool; mutable std::mutex _lock; std::condition_variable _cond; ExecutorStats _stats; @@ -86,7 +82,7 @@ private: uint32_t _taskCount; uint32_t _taskLimit; bool _closed; - std::unique_ptr<thread::ThreadInit> _thread_init; + init_fun_t _init_fun; static thread_local ThreadStackExecutorBase *_master; void block_thread(const unique_lock &, BlockedThread &blocked_thread); @@ -225,4 +221,3 @@ public: }; } // namespace vespalib - diff --git a/vespalog/src/logctl/logctl.cpp b/vespalog/src/logctl/logctl.cpp index 4cf44e9cd22..9a8987fc462 100644 --- a/vespalog/src/logctl/logctl.cpp +++ b/vespalog/src/logctl/logctl.cpp @@ -6,6 +6,7 @@ #include <vespa/log/component.h> #include <optional> +#include <cstring> #include <unistd.h> #include <dirent.h> #include <sys/stat.h> @@ -16,7 +17,7 @@ LOG_SETUP("vespa-logctl"); using namespace ns_log; static void modifyLevels(const char *file, const char *component, const char *levels, - bool shouldCreateFile, bool shouldCreateEntry); + bool shouldCreateFile, bool shouldCreateEntry); static void readLevels(const char *file, const char *component); diff --git a/vespalog/src/logger/llreader.cpp b/vespalog/src/logger/llreader.cpp index cb25b6747bf..200f4bb039d 100644 --- a/vespalog/src/logger/llreader.cpp +++ b/vespalog/src/logger/llreader.cpp @@ -1,17 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <stdlib.h> -#include <string.h> -#include <stdio.h> -#include <fcntl.h> -#include <errno.h> -#include <unistd.h> -#include <sys/time.h> #include "llreader.h" +#include <cstdlib> +#include <cstring> +#include <unistd.h> namespace ns_log { - InputBuf::InputBuf(int fd) : _inputfd(fd), _size(1000), diff --git a/vespalog/src/test/bufferedlogskiptest.cpp b/vespalog/src/test/bufferedlogskiptest.cpp index 8b7f1982678..29e5e119c34 100644 --- a/vespalog/src/test/bufferedlogskiptest.cpp +++ b/vespalog/src/test/bufferedlogskiptest.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/bufferedlogger.h> +#include <vespa/log/internal.h> #include <fstream> #include <iostream> @@ -9,6 +10,17 @@ LOG_SETUP("bufferedlogskiptest"); +using namespace std::literals::chrono_literals; + +/** Test timer returning just a given time. Used in tests to fake time. */ +struct TestTimer : public ns_log::Timer { + uint64_t & _time; + TestTimer(uint64_t & timeVar) : _time(timeVar) { } + ns_log::system_time getTimestamp() const noexcept override { + return ns_log::system_time(std::chrono::microseconds(_time)); + } +}; + std::string readFile(const std::string& file) { std::ostringstream ost; std::ifstream is(file.c_str()); @@ -75,8 +87,8 @@ main(int argc, char **argv) } ns_log::Logger::fakePid = true; uint64_t timer; - ns_log_logger.setTimer(std::unique_ptr<ns_log::Timer>(new ns_log::TestTimer(timer))); - ns_log::BufferedLogger::instance().setTimer(std::unique_ptr<ns_log::Timer>(new ns_log::TestTimer(timer))); + ns_log_logger.setTimer(std::make_unique<TestTimer>(timer)); + ns_log::BufferedLogger::instance().setTimer(std::make_unique<TestTimer>(timer)); reset(timer); testSkipBufferOnDebug(argv[1], timer); diff --git a/vespalog/src/test/bufferedlogtest.cpp b/vespalog/src/test/bufferedlogtest.cpp index 365f8fb85a7..a2dfdd7c6b8 100644 --- a/vespalog/src/test/bufferedlogtest.cpp +++ b/vespalog/src/test/bufferedlogtest.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/log/bufferedlogger.h> +#include <vespa/log/internal.h> #include "bufferedlogtest.logger1.h" #include "bufferedlogtest.logger2.h" @@ -12,6 +13,18 @@ LOG_SETUP("bufferedlogtest"); +using namespace std::literals::chrono_literals; + + +/** Test timer returning just a given time. Used in tests to fake time. */ +struct TestTimer : public ns_log::Timer { + uint64_t & _time; + TestTimer(uint64_t & timeVar) : _time(timeVar) { } + ns_log::system_time getTimestamp() const noexcept override { + return ns_log::system_time(std::chrono::microseconds(_time)); + } +}; + std::string readFile(const std::string& file) { std::ostringstream ost; std::ifstream is(file.c_str()); @@ -386,8 +399,8 @@ main(int argc, char **argv) ns_log::Logger::fakePid = true; ns_log::BufferedLogger::instance().setMaxCacheSize(10); uint64_t timer; - ns_log_logger.setTimer(std::unique_ptr<ns_log::Timer>(new ns_log::TestTimer(timer))); - ns_log::BufferedLogger::instance().setTimer(std::unique_ptr<ns_log::Timer>(new ns_log::TestTimer(timer))); + ns_log_logger.setTimer(std::make_unique<TestTimer>(timer)); + ns_log::BufferedLogger::instance().setTimer(std::make_unique<TestTimer>(timer)); reset(timer); testThatEntriesWithHighCountIsKept(argv[1], timer); diff --git a/vespalog/src/test/java/com/yahoo/log/VespaLevelControllerRepoTest.java b/vespalog/src/test/java/com/yahoo/log/VespaLevelControllerRepoTest.java index 83db5a027a4..1086b99d557 100644 --- a/vespalog/src/test/java/com/yahoo/log/VespaLevelControllerRepoTest.java +++ b/vespalog/src/test/java/com/yahoo/log/VespaLevelControllerRepoTest.java @@ -2,23 +2,24 @@ package com.yahoo.log; import org.junit.Test; -import org.junit.Ignore; import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; +import java.nio.charset.StandardCharsets; import java.util.logging.Level; import java.util.logging.Logger; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; /** * @author Ulf Lilleengen * @since 5.1 */ -@SuppressWarnings({"deprecation", "removal"}) +@SuppressWarnings({"deprecation"}) public class VespaLevelControllerRepoTest { static int findControlString(RandomAccessFile f, String s) { @@ -28,7 +29,7 @@ public class VespaLevelControllerRepoTest { f.seek(0); f.read(contents); f.seek(0); - String c_as_s = new String(contents, "US-ASCII"); + String c_as_s = new String(contents, StandardCharsets.US_ASCII); int off = c_as_s.indexOf(toFind); if (off < 0) { System.err.println("did not find control line for level '"+s+"' in logcontrol file:"); @@ -49,8 +50,7 @@ public class VespaLevelControllerRepoTest { try { lcf.delete(); Logger.getLogger("com.yahoo.log.test").setLevel(null); - assertEquals(null, - Logger.getLogger("com.yahoo.log.test").getLevel()); + assertNull(Logger.getLogger("com.yahoo.log.test").getLevel()); LevelControllerRepo repo = new VespaLevelControllerRepo(lcf.getName(), "all -debug -spam", "TST"); diff --git a/vespalog/src/test/threads/CMakeLists.txt b/vespalog/src/test/threads/CMakeLists.txt index 00de16ff005..19fe2511025 100644 --- a/vespalog/src/test/threads/CMakeLists.txt +++ b/vespalog/src/test/threads/CMakeLists.txt @@ -4,6 +4,5 @@ vespa_add_executable(vespalog_threads_test_app TEST testthreads.cpp DEPENDS vespalog - fastos ) vespa_add_test(NAME vespalog_threads_test_app COMMAND vespalog_threads_test_app vespa.log ENVIRONMENT "VESPA_LOG_TARGET=file:vespa.log") diff --git a/vespalog/src/test/threads/testthreads.cpp b/vespalog/src/test/threads/testthreads.cpp index ed2683b5c35..6a9d5c18a18 100644 --- a/vespalog/src/test/threads/testthreads.cpp +++ b/vespalog/src/test/threads/testthreads.cpp @@ -1,5 +1,4 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/thread.h> #include <vespa/log/bufferedlogger.h> #include <array> #include <iostream> @@ -10,6 +9,8 @@ #include <unistd.h> #include <sys/stat.h> #include <cstdlib> +#include <cstring> +#include <vector> using std::string; using namespace std::chrono_literals; @@ -17,28 +18,28 @@ using namespace std::chrono; LOG_SETUP(".threadtest"); -class FileThread : public FastOS_Runnable +class FileThread { std::atomic<bool> _done; string _file; public: FileThread(string file) : _done(false), _file(file) {} - void Run(FastOS_ThreadInterface *thread, void *arg) override; + void entry(); void stop() { _done.store(true, std::memory_order_relaxed); } }; -class LoggerThread : public FastOS_Runnable +class LoggerThread { std::atomic<bool> _done; public: std::atomic<bool> _useLogBuffer; LoggerThread() : _done(false), _useLogBuffer(false) {} - void Run(FastOS_ThreadInterface *thread, void *arg) override; + void entry(); void stop() { _done.store(true, std::memory_order_relaxed); } }; void -FileThread::Run(FastOS_ThreadInterface *, void *) +FileThread::entry() { unlink(_file.c_str()); while (!_done.load(std::memory_order_relaxed)) { @@ -63,7 +64,7 @@ FileThread::Run(FastOS_ThreadInterface *, void *) void -LoggerThread::Run(FastOS_ThreadInterface *, void *) +LoggerThread::entry() { int counter = 0; while (!_done.load(std::memory_order_relaxed)) { @@ -89,11 +90,12 @@ public: } }; + int ThreadTester::Main() { std::cerr << "Testing that logging is threadsafe. 5 sec test.\n"; - FastOS_ThreadPool pool; + std::vector<std::thread> threads; const int numWriters = 30; const int numLoggers = 10; @@ -105,11 +107,11 @@ ThreadTester::Main() char filename[100]; snprintf(filename, sizeof(filename), "empty.%d", i); writers[i] = std::make_unique<FileThread>(filename); - pool.NewThread(writers[i].get()); + threads.emplace_back([obj = writers[i].get()](){ obj->entry(); }); } for (int i = 0; i < numLoggers; i++) { loggers[i] = std::make_unique<LoggerThread>(); - pool.NewThread(loggers[i].get()); + threads.emplace_back([obj = loggers[i].get()](){ obj->entry(); }); } steady_clock::time_point start = steady_clock::now(); @@ -136,7 +138,9 @@ ThreadTester::Main() writers[i]->stop(); } - pool.Close(); + for (auto &thread: threads) { + thread.join(); + } return 0; } diff --git a/vespalog/src/vespa/log/CMakeLists.txt b/vespalog/src/vespa/log/CMakeLists.txt index dbeba17334b..82536b1f4e9 100644 --- a/vespalog/src/vespa/log/CMakeLists.txt +++ b/vespalog/src/vespa/log/CMakeLists.txt @@ -18,3 +18,4 @@ vespa_add_library(vespalog reject-filter.cpp INSTALL lib64 ) +target_link_libraries(vespalog PUBLIC ${CMAKE_THREAD_LIBS_INIT}) diff --git a/vespalog/src/vespa/log/bufferedlogger.cpp b/vespalog/src/vespa/log/bufferedlogger.cpp index 33ff3da7366..7123d1bb5fa 100644 --- a/vespalog/src/vespa/log/bufferedlogger.cpp +++ b/vespalog/src/vespa/log/bufferedlogger.cpp @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "bufferedlogger.h" +#include "internal.h" #include <boost/multi_index_container.hpp> #include <boost/multi_index/identity.hpp> -#include <boost/multi_index/member.hpp> #include <boost/multi_index/mem_fun.hpp> #include <boost/multi_index/ordered_index.hpp> #include <boost/multi_index/sequenced_index.hpp> @@ -14,6 +14,8 @@ #include <cstdarg> #include <mutex> +using namespace std::literals::chrono_literals; + namespace ns_log { // implementation details for BufferedLogger @@ -25,7 +27,7 @@ public: /** Lock needed to access cache. */ mutable std::mutex _mutex; - static uint64_t _countFactor; + static duration _countFactor; /** Struct keeping information about log message. */ struct Entry { @@ -35,7 +37,7 @@ public: std::string _token; std::string _message; uint32_t _count; - uint64_t _timestamp; + system_time _timestamp; Logger* _logger; Entry(const Entry &); @@ -44,13 +46,13 @@ public: Entry & operator=(Entry &&) noexcept; Entry(Logger::LogLevel level, const char* file, int line, const std::string& token, const std::string& message, - uint64_t timestamp, Logger&); + system_time timestamp, Logger&); ~Entry(); bool operator==(const Entry& entry) const; bool operator<(const Entry& entry) const; - uint64_t getAgeFactor() const; + system_time getAgeFactor() const; std::string toString() const; }; @@ -73,7 +75,7 @@ public: >, boost::multi_index::ordered_non_unique< boost::multi_index::const_mem_fun< - Entry, uint64_t, &Entry::getAgeFactor + Entry, system_time, &Entry::getAgeFactor > > > @@ -85,7 +87,7 @@ public: LogCacheBack _cacheBack; uint32_t _maxCacheSize; - uint64_t _maxEntryAge; + duration _maxEntryAge; /** Empty buffer and write all log entries in it. */ void flush(); @@ -97,7 +99,7 @@ public: * Flush parts of cache, so we're below max size and only have messages of * acceptable age. Calling this, _mutex should already be locked. */ - void trimCache(uint64_t currentTime); + void trimCache(system_time currentTime); /** * Trim the cache up to current time. Used externally to check if we @@ -125,11 +127,11 @@ public: }; // Let each hit count for 5 seconds -uint64_t BackingBuffer::_countFactor = VESPA_LOG_COUNTAGEFACTOR * 1000 * 1000; +duration BackingBuffer::_countFactor = VESPA_LOG_COUNTAGEFACTOR * 1s; BackingBuffer::Entry::Entry(Logger::LogLevel level, const char* file, int line, const std::string& token, const std::string& msg, - uint64_t timestamp, Logger& l) + system_time timestamp, Logger& l) : _level(level), _file(file), _line(line), @@ -171,11 +173,11 @@ BackingBuffer::Entry::toString() const std::ostringstream ost; ost << "Entry(" << _level << ", " << _file << ":" << _line << ": " << _message << " [" << _token << "], count " << _count - << ", timestamp " << _timestamp << ")"; + << ", timestamp " << count_us(_timestamp.time_since_epoch()) << ")"; return ost.str(); } -uint64_t +system_time BackingBuffer::Entry::getAgeFactor() const { return _timestamp + _countFactor * _count; @@ -191,9 +193,7 @@ BackingBuffer::BackingBuffer() { } -BackingBuffer::~BackingBuffer() -{ -} +BackingBuffer::~BackingBuffer() = default; BufferedLogger::BufferedLogger() { @@ -206,16 +206,25 @@ BufferedLogger::~BufferedLogger() } namespace { - typedef boost::multi_index::nth_index< - BackingBuffer::LogCacheFront, 0>::type LogCacheFrontTimestamp; - typedef boost::multi_index::nth_index< - BackingBuffer::LogCacheFront, 1>::type LogCacheFrontToken; - typedef boost::multi_index::nth_index< - BackingBuffer::LogCacheBack, 0>::type LogCacheBackTimestamp; - typedef boost::multi_index::nth_index< - BackingBuffer::LogCacheBack, 1>::type LogCacheBackToken; - typedef boost::multi_index::nth_index< - BackingBuffer::LogCacheBack, 2>::type LogCacheBackAge; + +typedef boost::multi_index::nth_index< + BackingBuffer::LogCacheFront, 0>::type LogCacheFrontTimestamp; +typedef boost::multi_index::nth_index< + BackingBuffer::LogCacheFront, 1>::type LogCacheFrontToken; +typedef boost::multi_index::nth_index< + BackingBuffer::LogCacheBack, 0>::type LogCacheBackTimestamp; +typedef boost::multi_index::nth_index< + BackingBuffer::LogCacheBack, 1>::type LogCacheBackToken; +typedef boost::multi_index::nth_index< + BackingBuffer::LogCacheBack, 2>::type LogCacheBackAge; + +struct TimeStampWrapper : public Timer { + TimeStampWrapper(system_time timeStamp) : _timeStamp(timeStamp) {} + system_time getTimestamp() const noexcept override { return _timeStamp; } + + system_time _timeStamp; +}; + } void @@ -258,7 +267,7 @@ BackingBuffer::logImpl(Logger& l, Logger::LogLevel level, _cacheBack.get<1>().replace(it2, copy); } else { // If entry didn't already exist, add it to the cache and log it - l.doLogCore(entry._timestamp, level, file, line, message.c_str(), message.size()); + l.doLogCore(TimeStampWrapper(entry._timestamp), level, file, line, message.c_str(), message.size()); _cacheFront.push_back(entry); } trimCache(entry._timestamp); @@ -268,16 +277,12 @@ void BackingBuffer::flush() { std::lock_guard<std::mutex> guard(_mutex); - for (LogCacheBack::const_iterator it = _cacheBack.begin(); - it != _cacheBack.end(); ++it) - { - log(*it); + for (const auto & entry : _cacheBack) { + log(entry); } _cacheBack.clear(); - for (LogCacheFront::const_iterator it = _cacheFront.begin(); - it != _cacheFront.end(); ++it) - { - log(*it); + for (const auto & entry : _cacheFront) { + log(entry); } _cacheFront.clear(); } @@ -288,7 +293,7 @@ BufferedLogger::flush() { } void -BackingBuffer::trimCache(uint64_t currentTime) +BackingBuffer::trimCache(system_time currentTime) { // Remove entries that have been in here too long. while (!_cacheBack.empty() && @@ -310,9 +315,7 @@ BackingBuffer::trimCache(uint64_t currentTime) _cacheBack.push_back(e); } // Remove entries from back based on count modified age. - for (uint32_t i = _cacheFront.size() + _cacheBack.size(); - i > _maxCacheSize; --i) - { + for (uint32_t i = _cacheFront.size() + _cacheBack.size(); i > _maxCacheSize; --i) { log(*_cacheBack.get<2>().begin()); _cacheBack.get<2>().erase(_cacheBack.get<2>().begin()); } @@ -330,10 +333,10 @@ BackingBuffer::log(const Entry& e) const if (e._count > 1) { std::ostringstream ost; ost << e._message << " (Repeated " << (e._count - 1) - << " times since " << (e._timestamp / 1000000) << "." - << std::setw(6) << std::setfill('0') << (e._timestamp % 1000000) + << " times since " << count_s(e._timestamp.time_since_epoch()) << "." + << std::setw(6) << std::setfill('0') << (count_us(e._timestamp.time_since_epoch()) % 1000000) << ")"; - e._logger->doLogCore(_timer->getTimestamp(), e._level, e._file.c_str(), + e._logger->doLogCore(*_timer, e._level, e._file.c_str(), e._line, ost.str().c_str(), ost.str().size()); } } @@ -344,16 +347,12 @@ BackingBuffer::toString() const std::ostringstream ost; ost << "Front log cache content:\n"; std::lock_guard<std::mutex> guard(_mutex); - for (LogCacheFront::const_iterator it = _cacheFront.begin(); - it != _cacheFront.end(); ++it) - { - ost << " " << it->toString() << "\n"; + for (const auto & entry : _cacheFront) { + ost << " " << entry.toString() << "\n"; } ost << "Back log cache content:\n"; - for (LogCacheBack::const_iterator it = _cacheBack.begin(); - it != _cacheBack.end(); ++it) - { - ost << " " << it->toString() << "\n"; + for (const auto & entry : _cacheBack) { + ost << " " << entry.toString() << "\n"; } return ost.str(); } @@ -366,12 +365,12 @@ BufferedLogger::setMaxCacheSize(uint32_t size) { void BufferedLogger::setMaxEntryAge(uint64_t seconds) { - _backing->_maxEntryAge = seconds * 1000000; + _backing->_maxEntryAge = std::chrono::seconds(seconds); } void -BufferedLogger::setCountFactor(uint64_t factor) { - _backing->_countFactor = factor * 1000000; +BufferedLogger::setCountFactor(uint64_t seconds) { + _backing->_countFactor = std::chrono::seconds(seconds); } /** Set a fake timer to use for log messages. Used in unit testing. */ diff --git a/vespalog/src/vespa/log/bufferedlogger.h b/vespalog/src/vespa/log/bufferedlogger.h index 8baa32445a5..f3aee2fb3f6 100644 --- a/vespalog/src/vespa/log/bufferedlogger.h +++ b/vespalog/src/vespa/log/bufferedlogger.h @@ -167,10 +167,10 @@ class BackingBuffer; class BufferedLogger { BackingBuffer *_backing; - BufferedLogger(const BufferedLogger & buf); - BufferedLogger & operator = (const BufferedLogger & buf); public: + BufferedLogger(const BufferedLogger & buf) = delete; + BufferedLogger & operator = (const BufferedLogger & buf) = delete; BufferedLogger(); ~BufferedLogger(); @@ -179,7 +179,7 @@ public: // of the default settings for applications void setMaxCacheSize(uint32_t size); void setMaxEntryAge(uint64_t seconds); - void setCountFactor(uint64_t factor); + void setCountFactor(uint64_t seconds); void doLog(Logger&, Logger::LogLevel level, const char *file, int line, const std::string& token, @@ -188,9 +188,6 @@ public: /** Empty buffer and write all log entries in it. */ void flush(); - /** Gives all current content of log buffer. Useful for debugging. */ - std::string toString() const; - /** Set a fake timer to use for log messages. Used in unit testing. */ void setTimer(std::unique_ptr<Timer> timer); diff --git a/vespalog/src/vespa/log/component.cpp b/vespalog/src/vespa/log/component.cpp index 009a69ad0c5..36b1d15e457 100644 --- a/vespalog/src/vespa/log/component.cpp +++ b/vespalog/src/vespa/log/component.cpp @@ -9,6 +9,7 @@ LOG_SETUP_INDIRECT(".log.control", "$Id$"); #include "component.h" #include "control-file.h" #include "internal.h" +#include <cstring> namespace ns_log { diff --git a/vespalog/src/vespa/log/internal.h b/vespalog/src/vespa/log/internal.h index c25c7cc44b6..7e6bf16f39f 100644 --- a/vespalog/src/vespa/log/internal.h +++ b/vespalog/src/vespa/log/internal.h @@ -3,6 +3,7 @@ #include <string> #include <cstdlib> +#include <chrono> namespace ns_log { @@ -20,4 +21,24 @@ public: [[nodiscard]] const char *what() const { return _what.c_str(); } }; +using system_time = std::chrono::system_clock::time_point; +using duration = std::chrono::nanoseconds; + +constexpr int64_t +count_s(duration d) noexcept { + return std::chrono::duration_cast<std::chrono::seconds>(d).count(); +} + +constexpr int64_t +count_us(duration d) noexcept { + return std::chrono::duration_cast<std::chrono::microseconds>(d).count(); +} + +// XXX this is way too complicated, must be some simpler way to do this +/** Timer class used to retrieve timestamp, such that we can override in test */ +struct Timer { + virtual ~Timer() = default; + virtual system_time getTimestamp() const noexcept; +}; + } // end namespace ns_log diff --git a/vespalog/src/vespa/log/llparser.cpp b/vespalog/src/vespa/log/llparser.cpp index 1585b9fde33..063220e50ef 100644 --- a/vespalog/src/vespa/log/llparser.cpp +++ b/vespalog/src/vespa/log/llparser.cpp @@ -4,6 +4,7 @@ #include "llparser.h" #include "internal.h" #include <cstdlib> +#include <cstring> #include <unistd.h> #include <sys/time.h> #include <cassert> diff --git a/vespalog/src/vespa/log/log-target-file.cpp b/vespalog/src/vespa/log/log-target-file.cpp index 4337d6b5bbb..87a9810e3c7 100644 --- a/vespalog/src/vespa/log/log-target-file.cpp +++ b/vespalog/src/vespa/log/log-target-file.cpp @@ -1,15 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "log.h" +LOG_SETUP(".log"); +#include "log-target-file.h" +#include "internal.h" + #include <unistd.h> #include <cstring> #include <fcntl.h> #include <cerrno> #include <cassert> -#include "log.h" -LOG_SETUP(".log"); -#include "log-target-file.h" -#include "internal.h" - namespace ns_log { #ifndef O_LARGEFILE diff --git a/vespalog/src/vespa/log/log.cpp b/vespalog/src/vespa/log/log.cpp index 7430815122d..73cbeef08aa 100644 --- a/vespalog/src/vespa/log/log.cpp +++ b/vespalog/src/vespa/log/log.cpp @@ -6,7 +6,6 @@ LOG_SETUP_INDIRECT(".log", "$Id$"); #undef LOG #define LOG LOG_INDIRECT -#include "lock.h" #include "log-target.h" #include "internal.h" #include "control-file.h" @@ -15,18 +14,16 @@ LOG_SETUP_INDIRECT(".log", "$Id$"); #include <vespa/defaults.h> #include <cassert> #include <cstdarg> +#include <cstring> +#include <cinttypes> #include <unistd.h> #include <sys/time.h> namespace ns_log { -uint64_t Timer::getTimestamp() const { - struct timeval tv; - gettimeofday(&tv, nullptr); - uint64_t timestamp = tv.tv_sec; - timestamp *= 1000000; - timestamp += tv.tv_usec; - return timestamp; +system_time +Timer::getTimestamp() const noexcept { + return std::chrono::system_clock::now(); } LogTarget *Logger::_target = 0; @@ -135,7 +132,7 @@ Logger::ensureHostname() Logger::Logger(const char *name, const char *rcsId) : _logLevels(ControlFile::defaultLevels()), - _timer(new Timer()) + _timer(std::make_unique<Timer>()) { _numInstances++; memset(_rcsId, 0, sizeof(_rcsId)); @@ -172,7 +169,6 @@ Logger::Logger(const char *name, const char *rcsId) } } - Logger::~Logger() { _numInstances--; @@ -190,6 +186,10 @@ Logger::~Logger() } } +void +Logger::setTimer(std::unique_ptr<Timer> timer) { + _timer = std::move(timer); +} int Logger::setRcsId(const char *id) @@ -220,8 +220,7 @@ Logger::tryLog(int sizeofPayload, LogLevel level, const char *file, int line, co const int actualSize = vsnprintf(payload, sizeofPayload, fmt, args); if (actualSize < sizeofPayload) { - uint64_t timestamp = _timer->getTimestamp(); - doLogCore(timestamp, level, file, line, payload, actualSize); + doLogCore(*_timer, level, file, line, payload, actualSize); } delete[] payload; return actualSize; @@ -242,9 +241,11 @@ Logger::doLog(LogLevel level, const char *file, int line, const char *fmt, ...) ns_log::BufferedLogger::instance().trimCache(); } -void Logger::doLogCore(uint64_t timestamp, LogLevel level, - const char *file, int line, const char *msg, size_t msgSize) +void +Logger::doLogCore(const Timer & timer, LogLevel level, + const char *file, int line, const char *msg, size_t msgSize) { + system_time timestamp = timer.getTimestamp(); const size_t sizeofEscapedPayload(msgSize*4+1); const size_t sizeofTotalMessage(sizeofEscapedPayload + 1000); auto escapedPayload = std::make_unique<char[]>(sizeofEscapedPayload); @@ -281,23 +282,23 @@ void Logger::doLogCore(uint64_t timestamp, LogLevel level, // found to be too inaccurate. int32_t tid = (fakePid ? -1 : gettid(pthread_self()) % 0xffff); + time_t secs = count_s(timestamp.time_since_epoch()); + uint32_t usecs_part = count_us(timestamp.time_since_epoch()) % 1000000; if (_target->makeHumanReadable()) { - time_t secs = static_cast<time_t>(timestamp / 1000000); struct tm tmbuf; localtime_r(&secs, &tmbuf); char timebuf[100]; strftime(timebuf, 100, "%Y-%m-%d %H:%M:%S", &tmbuf); snprintf(totalMessage.get(), sizeofTotalMessage, "[%s.%06u] %d/%d (%s%s) %s: %s\n", - timebuf, static_cast<unsigned int>(timestamp % 1000000), + timebuf, usecs_part, fakePid ? -1 : getpid(), tid, _prefix, _appendix, levelName(level), msg); } else if (level == debug || level == spam) { snprintf(totalMessage.get(), sizeofTotalMessage, - "%u.%06u\t%s\t%d/%d\t%s\t%s%s\t%s\t%s:%d %s%s\n", - static_cast<unsigned int>(timestamp / 1000000), - static_cast<unsigned int>(timestamp % 1000000), + "%lu.%06u\t%s\t%d/%d\t%s\t%s%s\t%s\t%s:%d %s%s\n", + secs, usecs_part, _hostname, fakePid ? -1 : getpid(), tid, _serviceName, _prefix, _appendix, levelName(level), file, line, @@ -305,9 +306,8 @@ void Logger::doLogCore(uint64_t timestamp, LogLevel level, escapedPayload.get()); } else { snprintf(totalMessage.get(), sizeofTotalMessage, - "%u.%06u\t%s\t%d/%d\t%s\t%s%s\t%s\t%s\n", - static_cast<unsigned int>(timestamp / 1000000), - static_cast<unsigned int>(timestamp % 1000000), + "%lu.%06u\t%s\t%d/%d\t%s\t%s%s\t%s\t%s\n", + secs, usecs_part, _hostname, fakePid ? -1 : getpid(), tid, _serviceName, _prefix, _appendix, levelName(level), escapedPayload.get()); @@ -354,35 +354,20 @@ Logger::doEventStarted(const char *name) void Logger::doEventStopped(const char *name, pid_t pid, int exitCode) { - doLog(event, "", 0, "stopped/1 name=\"%s\" pid=%d exitcode=%d", name, - static_cast<int>(pid), exitCode); -} - -void -Logger::doEventReloading(const char *name) -{ - doLog(event, "", 0, "reloading/1 name=\"%s\"", name); -} - -void -Logger::doEventReloaded(const char *name) -{ - doLog(event, "", 0, "reloaded/1 name=\"%s\"", name); + doLog(event, "", 0, "stopped/1 name=\"%s\" pid=%d exitcode=%d", name, static_cast<int>(pid), exitCode); } void Logger::doEventCrash(const char *name, pid_t pid, int signal) { - doLog(event, "", 0, "crash/1 name=\"%s\" pid=%d signal=\"%s\"", name, pid, - strsignal(signal)); + doLog(event, "", 0, "crash/1 name=\"%s\" pid=%d signal=\"%s\"", name, pid, strsignal(signal)); } void Logger::doEventProgress(const char *name, double value, double total) { if (total > 0) { - doLog(event, "", 0, "progress/1 name=\"%s\" value=%.18g total=%.18g", - name, value, total); + doLog(event, "", 0, "progress/1 name=\"%s\" value=%.18g total=%.18g", name, value, total); } else { doLog(event, "", 0, "progress/1 name=\"%s\" value=%.18g", name, value); } diff --git a/vespalog/src/vespa/log/log.h b/vespalog/src/vespa/log/log.h index 888e24a0f28..857bf4f2b97 100644 --- a/vespalog/src/vespa/log/log.h +++ b/vespalog/src/vespa/log/log.h @@ -2,13 +2,8 @@ #pragma once #include <memory> -#include <cstdint> -#include <sys/types.h> // for pid_t #include <new> // for placement new -#include <cstdlib> // for malloc -#include <cstring> // for memset -#include <cstdarg> // for va_list -#include <cinttypes> +#include <sys/types.h> // for pid_t /** * If this macro is defined, the regular LOG calls will go through the @@ -24,7 +19,7 @@ static ns_log::Logger ns_log_logger(__VA_ARGS__) // NOLINT #define LOG_SETUP_INDIRECT(x, id) \ -static ns_log::Logger *ns_log_indirect_logger=NULL; \ +static ns_log::Logger *ns_log_indirect_logger=nullptr; \ static bool logInitialised = false; \ static const char *logName = x; \ static const char *indirectRcsId = id @@ -34,9 +29,6 @@ static const char *indirectRcsId = id #define LOG_INDIRECT_WOULD_LOG(levelName) \ ns_log_indirect_logger->wants(ns_log::Logger::levelName) -#define LOG_RCSID(x) \ -static int log_dummmy __attribute__((unused)) = ns_log_logger.setRcsId(x) - // Define LOG if not using log buffer. Otherwise log buffer will define them #ifndef VESPA_LOG_USELOGBUFFERFORREGULARLOG #define LOG(level, ...) \ @@ -144,20 +136,7 @@ namespace ns_log { class LogTarget; class ControlFile; - -// XXX this is way too complicated, must be some simpler way to do this -/** Timer class used to retrieve timestamp, such that we can override in test */ -struct Timer { - virtual ~Timer() = default; - virtual uint64_t getTimestamp() const; -}; - -/** Test timer returning just a given time. Used in tests to fake time. */ -struct TestTimer : public Timer { - uint64_t & _time; - TestTimer(uint64_t & timeVar) : _time(timeVar) { } - uint64_t getTimestamp() const override { return _time; } -}; +struct Timer; class Logger { public: @@ -174,9 +153,6 @@ public: static bool fakePid; private: - Logger(const Logger &); - Logger& operator =(const Logger &); - unsigned int *_logLevels; static char _prefix[64]; @@ -206,6 +182,8 @@ private: public: ~Logger(); explicit Logger(const char *name, const char *rcsId = nullptr); + Logger(const Logger &) = delete; + Logger & operator=(const Logger &) = delete; int setRcsId(const char *rcsId); static const char *levelName(LogLevel level); @@ -219,14 +197,12 @@ public: * * @param timestamp Time in microseconds. */ - void doLogCore(uint64_t timestamp, LogLevel level, + void doLogCore(const Timer &, LogLevel level, const char *file, int line, const char *msg, size_t msgSize); void doEventStarting(const char *name); void doEventStopping(const char *name, const char *why); void doEventStarted(const char *name); void doEventStopped(const char *name, pid_t pid, int exitCode); - void doEventReloading(const char *name); - void doEventReloaded(const char *name); void doEventCrash(const char *name, pid_t pid, int signal); void doEventProgress(const char *name, double value, double total = 0); void doEventCount(const char *name, uint64_t value); @@ -234,7 +210,7 @@ public: void doEventState(const char *name, const char *value); // Only for unit testing - void setTimer(std::unique_ptr<Timer> timer) { _timer = std::move(timer); } + void setTimer(std::unique_ptr<Timer> timer); // Only for internal use static LogTarget *getCurrentTarget(); diff --git a/vespamalloc/CMakeLists.txt b/vespamalloc/CMakeLists.txt index af71d8b7d82..a6a33dce7ff 100644 --- a/vespamalloc/CMakeLists.txt +++ b/vespamalloc/CMakeLists.txt @@ -6,7 +6,6 @@ add_definitions(-DPARANOID_LEVEL=0) vespa_define_module( TEST_DEPENDS - fastos vespalib vespalog diff --git a/vespamalloc/src/tests/allocfree/allocfree.cpp b/vespamalloc/src/tests/allocfree/allocfree.cpp index ea6f2105d27..693deb49c55 100644 --- a/vespamalloc/src/tests/allocfree/allocfree.cpp +++ b/vespamalloc/src/tests/allocfree/allocfree.cpp @@ -2,7 +2,6 @@ #include "producerconsumer.h" #include <vespa/vespalib/testkit/testapp.h> #include <map> -#include <thread> #include <vespa/log/log.h> LOG_SETUP("allocfree_test"); @@ -73,7 +72,7 @@ int Test::Main() { } TEST_INIT("allocfree_test"); - FastOS_ThreadPool pool; + vespalib::ThreadPool pool; std::map<int, std::shared_ptr<FreeWorker> > freeWorkers; std::map<int, std::shared_ptr<MallocWorker> > mallocWorkers; @@ -86,22 +85,23 @@ int Test::Main() { mallocFreeWorkers[i] = std::shared_ptr<MallocFreeWorker>(new MallocFreeWorker(200, 16, (i%2) ? true : false)); } - + std::atomic<bool> stop_flag(false); for(std::map<int, std::shared_ptr<FreeWorker> >::iterator it(freeWorkers.begin()), mt(freeWorkers.end()); it != mt; it++) { - ASSERT_TRUE(pool.NewThread(it->second.get(), NULL) != NULL); + it->second->start(pool, stop_flag); } for(std::map<int, std::shared_ptr<MallocWorker> >::iterator it(mallocWorkers.begin()), mt(mallocWorkers.end()); it != mt; it++) { - ASSERT_TRUE(pool.NewThread(it->second.get(), NULL) != NULL); + it->second->start(pool, stop_flag); } for(std::map<int, std::shared_ptr<MallocFreeWorker> >::iterator it(mallocFreeWorkers.begin()), mt(mallocFreeWorkers.end()); it != mt; it++) { - ASSERT_TRUE(pool.NewThread(it->second.get(), NULL) != NULL); + it->second->start(pool, stop_flag); } for (; duration > 0; --duration) { LOG(info, "%d seconds left...", duration); std::this_thread::sleep_for(1s); } - pool.Close(); + stop_flag = true; + pool.join(); size_t numFreeOperations(0); size_t numMallocOperations(0); size_t numSameThreadMallocFreeOperations(0); diff --git a/vespamalloc/src/tests/allocfree/linklist.cpp b/vespamalloc/src/tests/allocfree/linklist.cpp index f3f23d726fd..8ab6eb8de8b 100644 --- a/vespamalloc/src/tests/allocfree/linklist.cpp +++ b/vespamalloc/src/tests/allocfree/linklist.cpp @@ -124,7 +124,7 @@ int Test::Main() { ASSERT_EQUAL(1024ul, sizeof(List)); - FastOS_ThreadPool pool; + vespalib::ThreadPool pool; List::AtomicHeadPtr sharedList(List::HeadPtr(nullptr, 1)); fprintf(stderr, "Start populating list\n"); for (size_t i=0; i < NumBlocks; i++) { @@ -156,18 +156,20 @@ int Test::Main() { LinkInOutAndIn pc1(sharedList, 16, false); LinkInOutAndIn pc2(sharedList, 16, true); - ASSERT_TRUE(pool.NewThread(&c1, nullptr) != nullptr); - ASSERT_TRUE(pool.NewThread(&c2, nullptr) != nullptr); - ASSERT_TRUE(pool.NewThread(&p1, nullptr) != nullptr); - ASSERT_TRUE(pool.NewThread(&p2, nullptr) != nullptr); - ASSERT_TRUE(pool.NewThread(&pc1, nullptr) != nullptr); - ASSERT_TRUE(pool.NewThread(&pc2, nullptr) != nullptr); + std::atomic<bool> stop_flag(false); + c1.start(pool, stop_flag); + c2.start(pool, stop_flag); + p1.start(pool, stop_flag); + p2.start(pool, stop_flag); + pc1.start(pool, stop_flag); + pc2.start(pool, stop_flag); for (; duration > 0; --duration) { LOG(info, "%d seconds left...", duration); std::this_thread::sleep_for(1s); } - pool.Close(); + stop_flag = true; + pool.join(); fprintf(stderr, "Did (%lu + %lu) = %lu linkIn operations\n", c1.operations(), c2.operations(), c1.operations() + c2.operations()); fprintf(stderr, "Did (%lu + %lu) = %lu linkOut operations\n", diff --git a/vespamalloc/src/tests/allocfree/producerconsumer.cpp b/vespamalloc/src/tests/allocfree/producerconsumer.cpp index 7edd1495fde..ab3cfe6dac5 100644 --- a/vespamalloc/src/tests/allocfree/producerconsumer.cpp +++ b/vespamalloc/src/tests/allocfree/producerconsumer.cpp @@ -38,7 +38,7 @@ ProducerConsumer::~ProducerConsumer() } -void Consumer::Run(FastOS_ThreadInterface *, void *) { +void Consumer::run(std::atomic<bool> &) { for (;;) { MemList ml = _queue.dequeue(); if (ml == NULL) { @@ -59,8 +59,8 @@ void Consumer::Run(FastOS_ThreadInterface *, void *) { } } -void Producer::Run(FastOS_ThreadInterface *t, void *) { - while (!t->GetBreakFlag()) { +void Producer::run(std::atomic<bool> &stop_flag) { + while (!stop_flag.load(std::memory_order_relaxed)) { MemList ml = new MemListImpl(); for (uint32_t i = 0; i < _cnt; ++i) { ml->push_back(produce()); @@ -71,8 +71,8 @@ void Producer::Run(FastOS_ThreadInterface *t, void *) { _target.close(); } -void ProducerConsumer::Run(FastOS_ThreadInterface *t, void *) { - while (!t->GetBreakFlag()) { +void ProducerConsumer::run(std::atomic<bool> &stop_flag) { + while (!stop_flag.load(std::memory_order_relaxed)) { MemListImpl ml; for (uint32_t i = 0; i < _cnt; ++i) { ml.push_back(produce()); diff --git a/vespamalloc/src/tests/allocfree/producerconsumer.h b/vespamalloc/src/tests/allocfree/producerconsumer.h index 23fb95facd8..b89e328c628 100644 --- a/vespamalloc/src/tests/allocfree/producerconsumer.h +++ b/vespamalloc/src/tests/allocfree/producerconsumer.h @@ -3,7 +3,8 @@ #include <vector> #include "queue.h" -#include <vespa/fastos/thread.h> +#include <vespa/vespalib/util/thread.h> +#include <atomic> namespace vespalib { @@ -11,7 +12,15 @@ typedef std::vector<void *> MemListImpl; typedef MemListImpl * MemList; typedef vespalib::Queue<MemList> MemQueue; -class Consumer : public FastOS_Runnable { +struct RunWithStopFlag { + virtual void run(std::atomic<bool> &stop_flag) = 0; + void start(vespalib::ThreadPool &pool, std::atomic<bool> &stop_flag) { + pool.start([this,&stop_flag](){run(stop_flag);}); + } + virtual ~RunWithStopFlag() = default; +}; + +class Consumer : public RunWithStopFlag { private: MemQueue _queue; bool _inverse; @@ -22,11 +31,11 @@ public: virtual ~Consumer(); void enqueue(const MemList &mem) { _queue.enqueue(mem); } void close() { _queue.close(); } - void Run(FastOS_ThreadInterface *t, void *) override; + void run(std::atomic<bool> &stop_flag) override; uint64_t operations() const { return _operations; } }; -class Producer : public FastOS_Runnable { +class Producer : public RunWithStopFlag { private: Consumer & _target; uint32_t _cnt; @@ -35,11 +44,11 @@ private: public: Producer(uint32_t cnt, Consumer &target); virtual ~Producer(); - void Run(FastOS_ThreadInterface *t, void *) override; + void run(std::atomic<bool> &stop_flag) override; uint64_t operations() const { return _operations; } }; -class ProducerConsumer : public FastOS_Runnable { +class ProducerConsumer : public RunWithStopFlag { private: uint32_t _cnt; bool _inverse; @@ -50,7 +59,7 @@ private: public: ProducerConsumer(uint32_t cnt, bool inverse); virtual ~ProducerConsumer(); - void Run(FastOS_ThreadInterface *t, void *) override; + void run(std::atomic<bool> &stop_flag) override; uint64_t operationsConsumed() const { return _operationsConsumed; } uint64_t operationsProduced() const { return _operationsProduced; } }; diff --git a/vespamalloc/src/tests/test.cpp b/vespamalloc/src/tests/test.cpp index f413412dff0..b3694a6dc0c 100644 --- a/vespamalloc/src/tests/test.cpp +++ b/vespamalloc/src/tests/test.cpp @@ -1,6 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastos/thread.h> +#include <vespa/vespalib/util/thread.h> #include <cstdlib> #include <cstdio> @@ -24,31 +24,21 @@ void testdd() free(a); } -class Thread : public FastOS_Runnable -{ -private: - void Run(FastOS_ThreadInterface * ti, void * arg) override; -}; +void thread_run(); int main(int, char *[]) { - FastOS_ThreadPool threadPool; + vespalib::ThreadPool threadPool; printf("Main stack(%p)\n", &threadPool); - Thread context; - FastOS_ThreadInterface * th[4]; - for (size_t i=0; i<sizeof(th)/sizeof(th[0]); i++) { - th[i] = threadPool.NewThread(&context); - } - for (size_t i=0; i<sizeof(th)/sizeof(th[0]); i++) { - th[i]->Join(); - delete th[i]; + for (int i = 0; i < 4; ++i) { + threadPool.start([](){thread_run();}); } - + threadPool.join(); return 0; } -void Thread::Run(FastOS_ThreadInterface *, void *) +void thread_run() { char * a = new char [100]; delete [] a; diff --git a/vespamalloc/src/tests/test1/CMakeLists.txt b/vespamalloc/src/tests/test1/CMakeLists.txt index dd7c92a4dac..f3ce9f70f45 100644 --- a/vespamalloc/src/tests/test1/CMakeLists.txt +++ b/vespamalloc/src/tests/test1/CMakeLists.txt @@ -8,12 +8,15 @@ vespa_add_executable(vespamalloc_testatomic_app TEST vespamalloc_util EXTERNAL_DEPENDS ${VESPA_ATOMIC_LIB} + dl ) vespa_add_test(NAME vespamalloc_testatomic_app NO_VALGRIND COMMAND vespamalloc_testatomic_app) vespa_add_executable(vespamalloc_new_test_app TEST SOURCES new_test.cpp + EXTERNAL_DEPENDS + dl ) vespa_add_test(NAME vespamalloc_new_test_app NO_VALGRIND COMMAND vespamalloc_new_test_app) |