diff options
23 files changed, 399 insertions, 222 deletions
diff --git a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java index ddaca6b5b38..798de5ce2cd 100644 --- a/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java +++ b/bundle-plugin/src/main/java/com/yahoo/container/plugin/mojo/GenerateOsgiManifestMojo.java @@ -48,7 +48,7 @@ public class GenerateOsgiManifestMojo extends AbstractGenerateOsgiManifestMojo { private enum BundleType { CORE, // up to container-dev - INTERNAL, // other vespa bundles (need not be set for groupId 'com.yahoo.vespa') + INTERNAL, // other vespa bundles (some bundle will by default be INTERNAL, see isVespaInternalGroupId) USER } @@ -321,6 +321,7 @@ public class GenerateOsgiManifestMojo extends AbstractGenerateOsgiManifestMojo { private boolean isVespaInternalGroupId(String groupId) { return groupId.equals(VESPA_GROUP_ID) + || groupId.equals(VESPA_GROUP_ID + ".cloud") || groupId.equals(VESPA_GROUP_ID + ".hosted") || groupId.equals(VESPA_GROUP_ID + ".hosted.controller"); } diff --git a/client/js/app/yarn.lock b/client/js/app/yarn.lock index 5627c5ab545..4efcf5204ab 100644 --- a/client/js/app/yarn.lock +++ b/client/js/app/yarn.lock @@ -445,9 +445,9 @@ regenerator-runtime "^0.14.0" "@babel/runtime@^7.12.5", "@babel/runtime@^7.18.3": - version "7.23.6" - resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.23.6.tgz#c05e610dc228855dc92ef1b53d07389ed8ab521d" - integrity sha512-zHd0eUrf5GZoOWVCXp6koAKQTfZV07eit6bGPmJgnZdnSAvvZee6zniW2XMF7Cmc4ISOOnPy3QaSiIJGJkVEDQ== + version "7.23.9" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.23.9.tgz#47791a15e4603bb5f905bc0753801cf21d6345f7" + integrity sha512-0CX6F+BI2s9dkUqr08KFrAIZgNFj75rdBU/DjCyYLIaV/quFjkk6T+EJ2LkZHyZTbEV4L5p97mNkUsHl2wLFAw== dependencies: regenerator-runtime "^0.14.0" @@ -492,7 +492,7 @@ debug "^4.1.0" globals "^11.1.0" -"@babel/types@^7.0.0", "@babel/types@^7.20.7", "@babel/types@^7.22.15", "@babel/types@^7.23.0", "@babel/types@^7.23.5": +"@babel/types@^7.0.0", "@babel/types@^7.20.7", "@babel/types@^7.23.0", "@babel/types@^7.23.5": version "7.23.5" resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.5.tgz#48d730a00c95109fa4393352705954d74fb5b602" integrity sha512-ON5kSOJwVO6xXVRTvOI0eOnWe7VdUcIpsovGo9U/Br4Ie4UVFQTboO2cYnDhAGU6Fp+UxSiT+pMft0SMHfuq6w== @@ -510,6 +510,15 @@ "@babel/helper-validator-identifier" "^7.22.15" to-fast-properties "^2.0.0" +"@babel/types@^7.22.15": + version "7.23.9" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.9.tgz#1dd7b59a9a2b5c87f8b41e52770b5ecbf492e002" + integrity sha512-dQjSq/7HaSjRM43FFGnv5keM2HsxpmyV1PfaSVm0nzzjwwTmjOe6J4bC8e3+pTEIgHaHj+1ZlLThRJ2auc/w1Q== + dependencies: + "@babel/helper-string-parser" "^7.23.4" + "@babel/helper-validator-identifier" "^7.22.20" + to-fast-properties "^2.0.0" + "@babel/types@^7.22.17": version "7.22.19" resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.22.19.tgz#7425343253556916e440e662bb221a93ddb75684" @@ -589,9 +598,9 @@ integrity sha512-W2P2c/VRW1/1tLox0mVUalvnWXxavmv/Oum2aPsRcoDJuob75FC3Y8FbpfLwUegRcxINtGUMPq0tFCvYNTBXNA== "@emotion/react@^11": - version "11.11.3" - resolved "https://registry.yarnpkg.com/@emotion/react/-/react-11.11.3.tgz#96b855dc40a2a55f52a72f518a41db4f69c31a25" - integrity sha512-Cnn0kuq4DoONOMcnoVsTOR8E+AdnKFf//6kUWc4LCdnxj31pZWn7rIULd6Y7/Js1PiPHzn7SKCM9vB/jBni8eA== + version "11.11.4" + resolved "https://registry.yarnpkg.com/@emotion/react/-/react-11.11.4.tgz#3a829cac25c1f00e126408fab7f891f00ecc3c1d" + integrity sha512-t8AjMlF0gHpvvxk5mAtCqR4vmxiGHCeJBaQO6gncUSdklELOgtwjerNY2yuJNfwnc6vi16U/+uMF+afIawJ9iw== dependencies: "@babel/runtime" "^7.18.3" "@emotion/babel-plugin" "^11.11.0" @@ -780,10 +789,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@eslint/js@8.56.0": - version "8.56.0" - resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.56.0.tgz#ef20350fec605a7f7035a01764731b2de0f3782b" - integrity sha512-gMsVel9D7f2HLkBma9VbtzZRehRogVRfbr++f06nL2vnCGCNlzOD+/MUov/F4p8myyAHspEhVobgjpX64q5m6A== +"@eslint/js@8.57.0": + version "8.57.0" + resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.57.0.tgz#a5417ae8427873f1dd08b70b3574b453e67b5f7f" + integrity sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g== "@floating-ui/core@^1.4.2": version "1.5.0" @@ -854,13 +863,13 @@ dependencies: prop-types "^15.8.1" -"@humanwhocodes/config-array@^0.11.13": - version "0.11.13" - resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.11.13.tgz#075dc9684f40a531d9b26b0822153c1e832ee297" - integrity sha512-JSBDMiDKSzQVngfRjOdFXgFfklaXI4K9nLF49Auh21lmBWRLIK3+xTErTWD4KU54pb6coM6ESE7Awz/FNU3zgQ== +"@humanwhocodes/config-array@^0.11.14": + version "0.11.14" + resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.11.14.tgz#d78e481a039f7566ecc9660b4ea7fe6b1fec442b" + integrity sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg== dependencies: - "@humanwhocodes/object-schema" "^2.0.1" - debug "^4.1.1" + "@humanwhocodes/object-schema" "^2.0.2" + debug "^4.3.1" minimatch "^3.0.5" "@humanwhocodes/module-importer@^1.0.1": @@ -868,10 +877,10 @@ resolved "https://registry.yarnpkg.com/@humanwhocodes/module-importer/-/module-importer-1.0.1.tgz#af5b2691a22b44be847b0ca81641c5fb6ad0172c" integrity sha512-bxveV4V8v5Yb4ncFTT3rPSgZBOpCkjfK0y4oVVVJwIuDVBRMDXrPyXRL988i5ap9m9bnyEEjWfm5WkBmtffLfA== -"@humanwhocodes/object-schema@^2.0.1": - version "2.0.1" - resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-2.0.1.tgz#e5211452df060fa8522b55c7b3c0c4d1981cb044" - integrity sha512-dvuCeX5fC9dXgJn9t+X5atfmgQAzUOWqS1254Gh0m6i8wKd10ebXkfNKiRK+1GWi/yTvvLDHpoxLr0xxxeslWw== +"@humanwhocodes/object-schema@^2.0.2": + version "2.0.2" + resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-2.0.2.tgz#d9fae00a2d5cb40f92cfe64b47ad749fbc38f917" + integrity sha512-6EwiSjwWYP7pTckG6I5eyFANjPhmPjUX9JRLUSfNPC7FX7zK9gyZAfUEaECL6ALTpGX5AjnBq3C9XmVWPitNpw== "@istanbuljs/load-nyc-config@^1.0.0": version "1.1.0" @@ -1538,9 +1547,9 @@ acorn-jsx@^5.3.2: integrity sha512-rq9s+JNhf0IChjtDXxllJ7g41oZk5SlXtp0LHwyA5cejwn7vKmKp4pPri6YEePv2PU65sAsegbXtIinmDFDXgQ== acorn@^8.9.0: - version "8.11.2" - resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.11.2.tgz#ca0d78b51895be5390a5903c5b3bdcdaf78ae40b" - integrity sha512-nc0Axzp/0FILLEVsm4fNwLCwMttvhEI263QtVPQcbpfZZ3ts0hLsZGOpE6czNlid7CJ9MlyH8reXkpsf3YUY4w== + version "8.11.3" + resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.11.3.tgz#71e0b14e13a4ec160724b38fb7b0f233b1b81d7a" + integrity sha512-Y9rRfJG5jcKOE0CLisYbojUjIrIEE7AGMzA/Sm4BslANhbS+cDMpgBdcPT91oJ7OuJ9hYJBx59RjbhxVnrF8Xg== ajv@^6.12.4: version "6.12.6" @@ -2184,7 +2193,7 @@ debug@^3.2.7: dependencies: ms "^2.1.1" -debug@^4.1.0, debug@^4.1.1, debug@^4.3.2: +debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2: version "4.3.4" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== @@ -2565,15 +2574,15 @@ eslint-visitor-keys@^3.3.0, eslint-visitor-keys@^3.4.1, eslint-visitor-keys@^3.4 integrity sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag== eslint@^8: - version "8.56.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.56.0.tgz#4957ce8da409dc0809f99ab07a1b94832ab74b15" - integrity sha512-Go19xM6T9puCOWntie1/P997aXxFsOi37JIHRWI514Hc6ZnaHGKY9xFhrU65RT6CcBEzZoGG1e6Nq+DT04ZtZQ== + version "8.57.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.57.0.tgz#c786a6fd0e0b68941aaf624596fb987089195668" + integrity sha512-dZ6+mexnaTIbSBZWgou51U6OmzIhYM2VcNdtiTtI7qPNZm35Akpr0f6vtw3w1Kmn5PYo+tZVfh13WrhpS6oLqQ== dependencies: "@eslint-community/eslint-utils" "^4.2.0" "@eslint-community/regexpp" "^4.6.1" "@eslint/eslintrc" "^2.1.4" - "@eslint/js" "8.56.0" - "@humanwhocodes/config-array" "^0.11.13" + "@eslint/js" "8.57.0" + "@humanwhocodes/config-array" "^0.11.14" "@humanwhocodes/module-importer" "^1.0.1" "@nodelib/fs.walk" "^1.2.8" "@ungap/structured-clone" "^1.2.0" @@ -2758,9 +2767,9 @@ fast-levenshtein@^2.0.6: integrity sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw== fastq@^1.6.0: - version "1.16.0" - resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.16.0.tgz#83b9a9375692db77a822df081edb6a9cf6839320" - integrity sha512-ifCoaXsDrsdkWTtiNJX5uzHDsrck5TzfKKDcuFFTIrrc/BS076qgEIfoIy1VeZqViznfKiysPYTh/QeHtnIsYA== + version "1.17.1" + resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.17.1.tgz#2a523f07a4e7b1e81a42b91b8bf2254107753b47" + integrity sha512-sRVD3lWVIXWg6By68ZN7vho9a1pQcN/WBFaAAsDDFzlJjvoGx0P8z7V1t72grFJfJhu3YPZBuu25f7Kaw2jN1w== dependencies: reusify "^1.0.4" @@ -2826,9 +2835,9 @@ flat-cache@^3.0.4: rimraf "^3.0.2" flatted@^3.2.9: - version "3.2.9" - resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.2.9.tgz#7eb4c67ca1ba34232ca9d2d93e9886e611ad7daf" - integrity sha512-36yxDn5H7OFZQla0/jFJmbIKTdZAQHngCedGxiMmpNfEZM0sdEeT+WczLQrjK6D7o2aiyLYDnkw0R3JK0Qv1RQ== + version "3.3.1" + resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.3.1.tgz#21db470729a6734d4997002f439cb308987f567a" + integrity sha512-X8cqMLLie7KsNUDSdzeN8FYK9rEt4Dt67OsG/DNGnYTSDBG4uFAJFBnUeiV+zCVAvwFy56IjM9sH51jVaEhNxw== for-each@^0.3.3: version "0.3.3" @@ -3065,9 +3074,9 @@ has@^1.0.3: integrity sha512-qdSAmqLF6209RFj4VVItywPMbm3vWylknmB3nvNiUIs72xAimcM8nVYxYr7ncvZq5qzk9MKIZR8ijqD/1QuYjQ== hasown@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.0.tgz#f4c513d454a57b7c7e1650778de226b11700546c" - integrity sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA== + version "2.0.1" + resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.1.tgz#26f48f039de2c0f8d3356c223fb8d50253519faa" + integrity sha512-1/th4MHjnwncwXsIW6QMzlvYL9kG5e/CpVvLRZe4XPa8TOUNbCELqmvhDmnkNsAjwaG4+I8gJJL0JBvTTLO9qA== dependencies: function-bind "^1.1.2" @@ -3093,7 +3102,12 @@ husky@^9.0.0: resolved "https://registry.yarnpkg.com/husky/-/husky-9.0.11.tgz#fc91df4c756050de41b3e478b2158b87c1e79af9" integrity sha512-AB6lFlbwwyIqMdHYhwPe+kjOC3Oc5P3nThEoW/AaO2BX3vJDjWPFxYLxokUZOo6RNX20He3AaT8sESs9NJcmEw== -ignore@^5.2.0, ignore@^5.3.0: +ignore@^5.2.0: + version "5.3.1" + resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.3.1.tgz#5073e554cd42c5b33b394375f538b8593e34d4ef" + integrity sha512-5Fytz/IraMjqpwfd34ke28PTVMjZjJG2MPn5t7OE4eUCUNf8BAa7b5WUS9/Qvr6mwOQS7Mk6vdsMno5he+T8Xw== + +ignore@^5.3.0: version "5.3.0" resolved "https://registry.yarnpkg.com/ignore/-/ignore-5.3.0.tgz#67418ae40d34d6999c95ff56016759c718c82f78" integrity sha512-g7dmpshy+gD7mh88OC9NwSGTKoc3kyLAZQRU1mt53Aw/vnvfXnbC+F/7F7QoYVKbV+KNvJx8wArewKy1vXMtlg== @@ -4696,9 +4710,9 @@ reflect.getprototypeof@^1.0.3: which-builtin-type "^1.1.3" regenerator-runtime@^0.14.0: - version "0.14.0" - resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.14.0.tgz#5e19d68eb12d486f797e15a3c6a918f7cec5eb45" - integrity sha512-srw17NI0TUWHuGa5CFGGmhfNIeja30WMBfbslPNhf6JrqQlLN5gcrvig1oqPxiVaXb0oW0XRKtH6Nngs5lKCIA== + version "0.14.1" + resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz#356ade10263f685dda125100cd862c1db895327f" + integrity sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw== regex-not@^1.0.0, regex-not@^1.0.2: version "1.0.2" diff --git a/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java b/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java index 090e06c221f..a381fd3c9a6 100644 --- a/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java +++ b/container-core/src/main/java/com/yahoo/restapi/RestApiImpl.java @@ -172,7 +172,9 @@ class RestApiImpl implements RestApi { log.log(Level.FINE, e, e::getMessage); ExceptionMapperHolder<?> mapper = exceptionMappers.stream() .filter(holder -> holder.type.isAssignableFrom(e.getClass())) - .findFirst().orElseThrow(() -> e); + // Topologically sort children before superclasses, so most the specific match is found by iterating through mappers in order. + .min((a, b) -> (a.type.isAssignableFrom(b.type) ? 1 : 0) + (b.type.isAssignableFrom(a.type) ? -1 : 0)) + .orElseThrow(() -> e); return mapper.toResponse(context, e); } @@ -210,8 +212,6 @@ class RestApiImpl implements RestApi { if (!disableDefaultMappers){ exceptionMappers.addAll(RestApiMappers.DEFAULT_EXCEPTION_MAPPERS); } - // Topologically sort children before superclasses, so most the specific match is found by iterating through mappers in order. - exceptionMappers.sort((a, b) -> (a.type.isAssignableFrom(b.type) ? 1 : 0) + (b.type.isAssignableFrom(a.type) ? -1 : 0)); return exceptionMappers; } diff --git a/container-core/src/main/java/com/yahoo/restapi/RestApiMappers.java b/container-core/src/main/java/com/yahoo/restapi/RestApiMappers.java index 62b46d26ba9..6d785ba58bc 100644 --- a/container-core/src/main/java/com/yahoo/restapi/RestApiMappers.java +++ b/container-core/src/main/java/com/yahoo/restapi/RestApiMappers.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.restapi; +import ai.vespa.json.InvalidJsonException; import ai.vespa.json.Json; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -52,6 +53,7 @@ public class RestApiMappers { (context, entity) -> new JacksonJsonResponse<>(200, entity, context.jacksonJsonMapper(), true))); static List<ExceptionMapperHolder<?>> DEFAULT_EXCEPTION_MAPPERS = List.of( + new ExceptionMapperHolder<>(InvalidJsonException.class, (ctx, e) -> ErrorResponse.badRequest(e.getMessage())), new ExceptionMapperHolder<>(RestApiException.class, (context, exception) -> exception.response())); private RestApiMappers() {} diff --git a/container-core/src/test/java/com/yahoo/restapi/RestApiImplTest.java b/container-core/src/test/java/com/yahoo/restapi/RestApiImplTest.java index d27e04bbd7a..d959344685a 100644 --- a/container-core/src/test/java/com/yahoo/restapi/RestApiImplTest.java +++ b/container-core/src/test/java/com/yahoo/restapi/RestApiImplTest.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.restapi; +import ai.vespa.json.InvalidJsonException; import com.fasterxml.jackson.annotation.JsonProperty; import com.yahoo.container.jdisc.AclMapping; import com.yahoo.container.jdisc.HttpRequestBuilder; @@ -20,6 +21,8 @@ import java.time.Instant; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.MissingFormatWidthException; +import java.util.NoSuchElementException; import java.util.Set; import static com.yahoo.jdisc.http.HttpRequest.Method; @@ -99,6 +102,23 @@ class RestApiImplTest { } @Test + void chooses_most_specific_exception_mapper() { + RestApi restApi = RestApi.builder() + .addRoute(route("/json").get(ctx -> { throw new InvalidJsonException("oops invalid json"); })) + .addRoute(route("/illegal-argument").get(ctx -> { throw new IllegalArgumentException(); })) + .addRoute(route("/bad-format").get(ctx -> { throw new MissingFormatWidthException(""); })) + .addExceptionMapper(IllegalArgumentException.class, (ctx, exception) -> ErrorResponse.badRequest("oops illegal argument")) + .addExceptionMapper(NoSuchElementException.class, (ctx, exception) -> ErrorResponse.badRequest("oops no such element")) + .addExceptionMapper(RuntimeException.class, (ctx, exception) -> ErrorResponse.internalServerError("oops runtime")) + .addExceptionMapper(MissingFormatWidthException.class, (ctx, exception) -> ErrorResponse.internalServerError("oops bad format width")) + .build(); + // Uses default mapper for `InvalidJsonException` since it's more specific than `IllegalArgumentException` + verifyJsonResponse(restApi, Method.GET, "/json", null, 400, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"oops invalid json\"}"); + verifyJsonResponse(restApi, Method.GET, "/illegal-argument", null, 400, "{\"message\":\"oops illegal argument\", \"error-code\":\"BAD_REQUEST\"}"); + verifyJsonResponse(restApi, Method.GET, "/bad-format", null, 500, "{\"message\":\"oops bad format width\", \"error-code\":\"INTERNAL_SERVER_ERROR\"}"); + } + + @Test void method_handler_can_consume_and_produce_json() { RestApi.HandlerWithRequestEntity<TestEntity, TestEntity> handler = (context, requestEntity) -> requestEntity; RestApi restApi = RestApi.builder() diff --git a/container-search/src/main/java/com/yahoo/search/query/properties/RankProfileInputProperties.java b/container-search/src/main/java/com/yahoo/search/query/properties/RankProfileInputProperties.java index 537ffd05463..3ac4178be5f 100644 --- a/container-search/src/main/java/com/yahoo/search/query/properties/RankProfileInputProperties.java +++ b/container-search/src/main/java/com/yahoo/search/query/properties/RankProfileInputProperties.java @@ -48,7 +48,7 @@ public class RankProfileInputProperties extends Properties { value, query.getModel().getLanguage(), context, - this); + query.properties()); } } catch (IllegalArgumentException e) { diff --git a/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java b/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java index 8e7c7276de1..7bbc1801204 100644 --- a/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/searchers/ValidateNearestNeighborTestCase.java @@ -223,19 +223,33 @@ public class ValidateNearestNeighborTestCase { @Test void testWithQueryProfileArgument() { + testWithQueryProfileArgument("foo"); + } + + @Test + void testWithQueryProfileArgumentFromBuiltInProperties() { + testWithQueryProfileArgument("model.queryTree"); + } + + @Test + void testWithQueryProfileArgumentFromBuiltInPropertyAlias() { + testWithQueryProfileArgument("query"); + } + + private void testWithQueryProfileArgument(String argument) { var embedder = new MockEmbedder("test text", Language.UNKNOWN, Tensor.from("tensor<float>(x[3]):[1.0, 2.0, 3.0]")); var registry = new QueryProfileRegistry(); var profile = new QueryProfile("test"); - profile.set("ranking.features.query(qvector)", "embed(@foo)", registry); + profile.set("ranking.features.query(qvector)", "embed(@" + argument + ")", registry); registry.register(profile); var queryString = makeQuery("fvector", "qvector"); var query = new Query.Builder() .setSchemaInfo(createSchemaInfo()) .setQueryProfile(registry.compile().findQueryProfile("test")) .setEmbedder(embedder) - .setRequestMap(Map.of("foo", "test text")) + .setRequestMap(Map.of(argument, "test text")) .build(); setYqlQuery(query, queryString); var result = doSearch(searcher, query); diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index f07754247d5..3635279e4fd 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -65,7 +65,7 @@ <assertj.vespa.version>3.25.3</assertj.vespa.version> <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> - <aws-sdk.vespa.version>1.12.665</aws-sdk.vespa.version> + <aws-sdk.vespa.version>1.12.668</aws-sdk.vespa.version> <athenz.vespa.version>1.11.52</athenz.vespa.version> <!-- Athenz END --> @@ -130,7 +130,7 @@ <prometheus.client.vespa.version>0.16.0</prometheus.client.vespa.version> <plexus-interpolation.vespa.version>1.27</plexus-interpolation.vespa.version> <protobuf.vespa.version>3.25.3</protobuf.vespa.version> - <questdb.vespa.version>7.3.9</questdb.vespa.version> + <questdb.vespa.version>7.3.10</questdb.vespa.version> <spifly.vespa.version>1.3.7</spifly.vespa.version> <spotbugs.vespa.version>4.8.3</spotbugs.vespa.version> <!-- Must match major version in https://github.com/apache/zookeeper/blob/master/pom.xml --> <snappy.vespa.version>1.1.10.5</snappy.vespa.version> diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentReply.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentReply.java index 0690168f298..1704bd3827e 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentReply.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/GetDocumentReply.java @@ -11,9 +11,10 @@ import java.nio.ByteBuffer; */ public class GetDocumentReply extends DocumentAcceptedReply { - private DocumentDeserializer buffer = null; private Document document = null; private long lastModified = 0; + // TODO Vespa 9: remove. Inherently tied to legacy protocol version. + private DocumentDeserializer buffer = null; private LazyDecoder decoder = null; /** diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/PutDocumentMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/PutDocumentMessage.java index 04a55a6fd16..7ce25011ec1 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/PutDocumentMessage.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/PutDocumentMessage.java @@ -14,9 +14,10 @@ import java.util.Arrays; */ public class PutDocumentMessage extends TestAndSetMessage { - private DocumentDeserializer buffer = null; private DocumentPut put = null; private long time = 0; + // TODO Vespa 9: remove. Inherently tied to legacy protocol version. + private DocumentDeserializer buffer = null; private LazyDecoder decoder = null; /** diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/UpdateDocumentMessage.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/UpdateDocumentMessage.java index d395353209f..3fb14664628 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/UpdateDocumentMessage.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/protocol/UpdateDocumentMessage.java @@ -13,10 +13,11 @@ import java.util.Arrays; */ public class UpdateDocumentMessage extends TestAndSetMessage { - private DocumentDeserializer buffer = null; private DocumentUpdate update = null; private long oldTime = 0; private long newTime = 0; + // TODO Vespa 9: remove. Inherently tied to legacy protocol version. + private DocumentDeserializer buffer = null; private LazyDecoder decoder = null; /** diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java index b82d1809085..69ae4fddb63 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporter.java @@ -397,13 +397,16 @@ public class MetricsReporter extends NodeRepositoryMaintainer { private void updateEmptyExclusiveHosts(NodeList nodes) { Instant now = nodeRepository().clock().instant(); - Duration minActivePeriod = Duration.ofMinutes(10); + Instant tenMinutesAgo = now.minus(Duration.ofMinutes(10)); int emptyHosts = nodes.parents().state(State.active) .matching(node -> (node.type() != NodeType.host && node.type().isHost()) || node.exclusiveToApplicationId().isPresent()) .matching(host -> host.history().hasEventBefore(History.Event.Type.activated, - now.minus(minActivePeriod))) + tenMinutesAgo)) .matching(host -> nodes.childrenOf(host).isEmpty()) + .matching(host -> host.hostEmptyAt() + .map(time -> time.isBefore(tenMinutesAgo)) + .orElse(true)) .size(); metric.set(ConfigServerMetrics.NODES_EMPTY_EXCLUSIVE.baseName(), emptyHosts, null); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java index 152f743900b..75ebf376e9f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/MetricsReporterTest.java @@ -360,6 +360,14 @@ public class MetricsReporterTest { metricsReporter.maintain(); assertEquals(hosts.size(), metric.values.get("nodes.emptyExclusive").intValue()); + // Hosts are not considered empty if children were just deallocated + tester.patchNodes(hosts, (host) -> host.withHostEmptyAt(tester.clock().instant())); + metricsReporter.maintain(); + assertEquals(0, metric.values.get("nodes.emptyExclusive").intValue()); + tester.clock().advance(Duration.ofMinutes(10)); + metricsReporter.maintain(); + assertEquals(hosts.size(), metric.values.get("nodes.emptyExclusive").intValue()); + // Deploy application ClusterSpec spec = ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("c1")).vespaVersion("1").build(); Capacity capacity = Capacity.from(new ClusterResources(4, 1, resources)); diff --git a/parent/pom.xml b/parent/pom.xml index 4483b95a58b..b4ca8e775f7 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -317,7 +317,7 @@ --> <groupId>org.openrewrite.maven</groupId> <artifactId>rewrite-maven-plugin</artifactId> - <version>5.23.1</version> + <version>5.23.2</version> <configuration> <activeRecipes> <recipe>org.openrewrite.java.testing.junit5.JUnit5BestPractices</recipe> diff --git a/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp b/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp index 1af9ee6cff7..968e006622f 100644 --- a/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp +++ b/searchlib/src/tests/queryeval/blueprint/blueprint_test.cpp @@ -24,8 +24,8 @@ auto opts = Blueprint::Options::all(); class MyOr : public IntermediateBlueprint { private: - FlowCalc make_flow_calc(InFlow in_flow) const override { - return flow_calc<OrFlow>(in_flow); + AnyFlow my_flow(InFlow in_flow) const override { + return AnyFlow::create<OrFlow>(in_flow); } public: FlowStats calculate_flow_stats(uint32_t) const final { diff --git a/searchlib/src/tests/queryeval/flow/queryeval_flow_test.cpp b/searchlib/src/tests/queryeval/flow/queryeval_flow_test.cpp index be70a037c98..4562f8cb50e 100644 --- a/searchlib/src/tests/queryeval/flow/queryeval_flow_test.cpp +++ b/searchlib/src/tests/queryeval/flow/queryeval_flow_test.cpp @@ -128,33 +128,31 @@ struct ExpectFlow { bool strict; }; +std::vector<FlowStats> make_flow_stats(const std::vector<double> &est_list, size_t n) { + std::vector<FlowStats> result; + for (size_t i = 0; i < n; ++i) { + result.emplace_back(est_list[i], 123.0, 456.0); + } + return result; +} + void verify_flow(auto flow, const std::vector<double> &est_list, const std::vector<ExpectFlow> &expect) { - FlowCalc calc = flow_calc<decltype(flow)>(InFlow(flow.strict(), flow.flow())); AnyFlow any_flow = AnyFlow::create<decltype(flow)>(InFlow(flow.strict(), flow.flow())); ASSERT_EQ(est_list.size() + 1, expect.size()); - for (size_t i = 0; i < expect.size(); ++i) { + for (size_t i = 0; i < est_list.size(); ++i) { EXPECT_EQ(any_flow.flow(), flow.flow()); - EXPECT_EQ(any_flow.estimate(), flow.estimate()); EXPECT_EQ(any_flow.strict(), flow.strict()); EXPECT_DOUBLE_EQ(flow.flow(), expect[i].flow); - EXPECT_DOUBLE_EQ(flow.estimate(), expect[i].est); EXPECT_EQ(flow.strict(), expect[i].strict); - if (i < est_list.size()) { - EXPECT_EQ(calc(est_list[i]), flow.flow()); - flow.add(est_list[i]); - any_flow.add(est_list[i]); - } else { - EXPECT_EQ(calc(0.5), flow.flow()); - } - } -} - -void verify_flow_calc(FlowCalc flow_calc, const std::vector<double> &est_list, const std::vector<double> &expect) { - ASSERT_EQ(est_list.size() + 1, expect.size()); - for (size_t i = 0; i < est_list.size(); ++i) { - EXPECT_DOUBLE_EQ(flow_calc(est_list[i]), expect[i]); + EXPECT_DOUBLE_EQ(flow.estimate_of(make_flow_stats(est_list, i)), expect[i].est); + any_flow.add(est_list[i]); + flow.add(est_list[i]); } - EXPECT_DOUBLE_EQ(flow_calc(0.5), expect.back()); + EXPECT_EQ(any_flow.flow(), flow.flow()); + EXPECT_EQ(any_flow.strict(), flow.strict()); + EXPECT_DOUBLE_EQ(flow.flow(), expect.back().flow); + EXPECT_EQ(flow.strict(), expect.back().strict); + EXPECT_DOUBLE_EQ(flow.estimate_of(make_flow_stats(est_list, est_list.size())), expect.back().est); } TEST(FlowTest, full_and_flow) { @@ -171,9 +169,9 @@ TEST(FlowTest, partial_and_flow) { for (double in: {1.0, 0.5, 0.25}) { verify_flow(AndFlow(in), {0.4, 0.7, 0.2}, {{in, 0.0, false}, - {in*0.4, in*0.4, false}, - {in*0.4*0.7, in*0.4*0.7, false}, - {in*0.4*0.7*0.2, in*0.4*0.7*0.2, false}}); + {in*0.4, 0.4, false}, + {in*0.4*0.7, 0.4*0.7, false}, + {in*0.4*0.7*0.2, 0.4*0.7*0.2, false}}); } } @@ -194,9 +192,9 @@ TEST(FlowTest, partial_or_flow) { for (double in: {1.0, 0.5, 0.25}) { verify_flow(OrFlow(in), {0.4, 0.7, 0.2}, {{in, 0.0, false}, - {in*0.6, 1.0-in*0.6, false}, - {in*0.6*0.3, 1.0-in*0.6*0.3, false}, - {in*0.6*0.3*0.8, 1.0-in*0.6*0.3*0.8, false}}); + {in*0.6, 1.0-0.6, false}, + {in*0.6*0.3, 1.0-0.6*0.3, false}, + {in*0.6*0.3*0.8, 1.0-0.6*0.3*0.8, false}}); } } @@ -214,37 +212,49 @@ TEST(FlowTest, partial_and_not_flow) { for (double in: {1.0, 0.5, 0.25}) { verify_flow(AndNotFlow(in), {0.4, 0.7, 0.2}, {{in, 0.0, false}, - {in*0.4, in*0.4, false}, - {in*0.4*0.3, in*0.4*0.3, false}, - {in*0.4*0.3*0.8, in*0.4*0.3*0.8, false}}); + {in*0.4, 0.4, false}, + {in*0.4*0.3, 0.4*0.3, false}, + {in*0.4*0.3*0.8, 0.4*0.3*0.8, false}}); } } -TEST(FlowTest, full_first_flow_calc) { +TEST(FlowTest, full_rank_flow) { for (bool strict: {false, true}) { - verify_flow_calc(first_flow_calc(strict), - {0.4, 0.7, 0.2}, {1.0, 0.4, 0.4, 0.4}); + verify_flow(RankFlow(strict), {0.4, 0.7, 0.2}, + {{1.0, 0.0, strict}, + {0.0, 0.4, false}, + {0.0, 0.4, false}, + {0.0, 0.4, false}}); } } -TEST(FlowTest, partial_first_flow_calc) { +TEST(FlowTest, partial_rank_flow) { for (double in: {1.0, 0.5, 0.25}) { - verify_flow_calc(first_flow_calc(in), - {0.4, 0.7, 0.2}, {in, in*0.4, in*0.4, in*0.4}); + verify_flow(RankFlow(in), {0.4, 0.7, 0.2}, + {{in, 0.0, false}, + {0.0, 0.4, false}, + {0.0, 0.4, false}, + {0.0, 0.4, false}}); } } -TEST(FlowTest, full_full_flow_calc) { +TEST(FlowTest, full_blender_flow) { for (bool strict: {false, true}) { - verify_flow_calc(full_flow_calc(strict), - {0.4, 0.7, 0.2}, {1.0, 1.0, 1.0, 1.0}); + verify_flow(BlenderFlow(strict), {0.4, 0.7, 0.2}, + {{1.0, 0.0, strict}, + {1.0, 1.0-0.6, strict}, + {1.0, 1.0-0.6*0.3, strict}, + {1.0, 1.0-0.6*0.3*0.8, strict}}); } } -TEST(FlowTest, partial_full_flow_calc) { +TEST(FlowTest, partial_blender_flow) { for (double in: {1.0, 0.5, 0.25}) { - verify_flow_calc(full_flow_calc(in), - {0.4, 0.7, 0.2}, {in, in, in, in}); + verify_flow(BlenderFlow(in), {0.4, 0.7, 0.2}, + {{in, 0.0, false}, + {in, 1.0-0.6, false}, + {in, 1.0-0.6*0.3, false}, + {in, 1.0-0.6*0.3*0.8, false}}); } } @@ -271,6 +281,37 @@ TEST(FlowTest, flow_cost) { EXPECT_DOUBLE_EQ(dual_ordered_cost_of<OrFlow>(data, true), 0.6 + 0.5 + 0.4); EXPECT_DOUBLE_EQ(dual_ordered_cost_of<AndNotFlow>(data, false), 1.1 + 0.4*1.2 + 0.4*0.3*1.3); EXPECT_DOUBLE_EQ(dual_ordered_cost_of<AndNotFlow>(data, true), 0.6 + 0.4*1.2 + 0.4*0.3*1.3); + EXPECT_DOUBLE_EQ(dual_ordered_cost_of<RankFlow>(data, false), 1.1); + EXPECT_DOUBLE_EQ(dual_ordered_cost_of<RankFlow>(data, true), 0.6); + EXPECT_DOUBLE_EQ(dual_ordered_cost_of<BlenderFlow>(data, false), 1.3); + EXPECT_DOUBLE_EQ(dual_ordered_cost_of<BlenderFlow>(data, true), 0.6); +} + +TEST(FlowTest, rank_flow_cost_accumulation_is_first) { + for (bool strict: {false, true}) { + auto flow = AnyFlow::create<RankFlow>(strict); + double cost = 0.0; + flow.update_cost(cost, 5.0); + EXPECT_EQ(cost, 5.0); + flow.add(0.5); // next child + flow.update_cost(cost, 5.0); + EXPECT_EQ(cost, 5.0); + } +} + +TEST(FlowTest, blender_flow_cost_accumulation_is_max) { + for (bool strict: {false, true}) { + auto flow = AnyFlow::create<BlenderFlow>(strict); + double cost = 0.0; + flow.update_cost(cost, 5.0); + EXPECT_EQ(cost, 5.0); + flow.add(0.5); // next child + flow.update_cost(cost, 3.0); + EXPECT_EQ(cost, 5.0); + flow.add(0.5); // next child + flow.update_cost(cost, 7.0); + EXPECT_EQ(cost, 7.0); + } } TEST(FlowTest, optimal_and_flow) { @@ -328,11 +369,11 @@ TEST(FlowTest, optimal_and_not_flow) { double max_cost = 0.0; AndNotFlow::sort(data, strict); EXPECT_EQ(data[0], first); - EXPECT_EQ(ordered_cost_of<AndNotFlow>(data, strict), min_cost); + EXPECT_DOUBLE_EQ(ordered_cost_of<AndNotFlow>(data, strict), min_cost); auto check = [&](const std::vector<FlowStats> &my_data) noexcept { if (my_data[0] == first) { double my_cost = ordered_cost_of<AndNotFlow>(my_data, strict); - EXPECT_LE(min_cost, my_cost); + EXPECT_LE(min_cost, my_cost + 1e-9); max_cost = std::max(max_cost, my_cost); } }; diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp index 5a225328003..15eb4c0b4fb 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.cpp @@ -572,11 +572,11 @@ IntermediateBlueprint::optimize(Blueprint* &self, OptimizePass pass) double IntermediateBlueprint::sort(InFlow in_flow, const Options &opts) { - auto flow_calc = make_flow_calc(in_flow); sort(_children, in_flow.strict(), opts.sort_by_cost()); + auto flow = my_flow(in_flow); for (size_t i = 0; i < _children.size(); ++i) { - double next_rate = flow_calc(_children[i]->estimate()); - _children[i]->sort(InFlow(in_flow.strict() && inheritStrict(i), next_rate), opts); + _children[i]->sort(InFlow(flow.strict(), flow.flow()), opts); + flow.add(_children[i]->estimate()); } // TODO: better cost estimate (due to known in-flow and eagerness) return in_flow.strict() ? strict_cost() : in_flow.rate() * cost(); @@ -646,11 +646,12 @@ IntermediateBlueprint::visitMembers(vespalib::ObjectVisitor &visitor) const void IntermediateBlueprint::fetchPostings(const ExecuteInfo &execInfo) { - FlowCalc flow_calc = make_flow_calc(InFlow(execInfo.is_strict(), execInfo.hit_rate())); + auto flow = my_flow(InFlow(execInfo.is_strict(), execInfo.hit_rate())); for (size_t i = 0; i < _children.size(); ++i) { + double nextHitRate = flow.flow(); Blueprint & child = *_children[i]; - double nextHitRate = flow_calc(child.estimate()); child.fetchPostings(ExecuteInfo::create(execInfo.is_strict() && inheritStrict(i), nextHitRate, execInfo)); + flow.add(child.estimate()); } } diff --git a/searchlib/src/vespa/searchlib/queryeval/blueprint.h b/searchlib/src/vespa/searchlib/queryeval/blueprint.h index 0c08e6aedf5..b19ae699fdc 100644 --- a/searchlib/src/vespa/searchlib/queryeval/blueprint.h +++ b/searchlib/src/vespa/searchlib/queryeval/blueprint.h @@ -386,7 +386,7 @@ private: bool infer_want_global_filter() const; size_t count_termwise_nodes(const UnpackInfo &unpack) const; - virtual FlowCalc make_flow_calc(InFlow in_flow) const = 0; + virtual AnyFlow my_flow(InFlow in_flow) const = 0; protected: // returns an empty collection if children have empty or diff --git a/searchlib/src/vespa/searchlib/queryeval/flow.h b/searchlib/src/vespa/searchlib/queryeval/flow.h index 9dd6d82a491..785a4c5e63c 100644 --- a/searchlib/src/vespa/searchlib/queryeval/flow.h +++ b/searchlib/src/vespa/searchlib/queryeval/flow.h @@ -117,12 +117,31 @@ struct MinOrCost { } }; -template <typename ADAPTER, typename T, typename F> -double estimate_of(ADAPTER adapter, const T &children, F flow) { +template <typename ADAPTER, typename T> +double estimate_of_and(ADAPTER adapter, const T &children) { + double flow = children.empty() ? 0.0 : adapter.estimate(children[0]); + for (size_t i = 1; i < children.size(); ++i) { + flow *= adapter.estimate(children[i]); + } + return flow; +} + +template <typename ADAPTER, typename T> +double estimate_of_or(ADAPTER adapter, const T &children) { + double flow = 1.0; for (const auto &child: children) { - flow.add(adapter.estimate(child)); + flow *= (1.0 - adapter.estimate(child)); + } + return (1.0 - flow); +} + +template <typename ADAPTER, typename T> +double estimate_of_and_not(ADAPTER adapter, const T &children) { + double flow = children.empty() ? 0.0 : adapter.estimate(children[0]); + for (size_t i = 1; i < children.size(); ++i) { + flow *= (1.0 - adapter.estimate(children[i])); } - return flow.estimate(); + return flow; } template <template <typename> typename ORDER, typename ADAPTER, typename T> @@ -179,12 +198,6 @@ size_t select_strict_and_child(ADAPTER adapter, const T &children) { template <typename FLOW> struct FlowMixin { - static double estimate_of(auto adapter, const auto &children) { - return flow::estimate_of(adapter, children, FLOW(false)); - } - static double estimate_of(const auto &children) { - return estimate_of(flow::make_adapter(children), children); - } static double cost_of(auto adapter, const auto &children, bool strict) { auto my_adapter = flow::IndirectAdapter(adapter, children); auto order = flow::make_index(children.size()); @@ -200,25 +213,23 @@ class AndFlow : public FlowMixin<AndFlow> { private: double _flow; bool _strict; - bool _first; public: - AndFlow(InFlow flow) noexcept : _flow(flow.rate()), _strict(flow.strict()), _first(true) {} + AndFlow(InFlow flow) noexcept : _flow(flow.rate()), _strict(flow.strict()) {} void add(double est) noexcept { _flow *= est; - _first = false; - } - double flow() const noexcept { - return _flow; - } - bool strict() const noexcept { - return _strict && _first; - } - double estimate() const noexcept { - return _first ? 0.0 : _flow; + _strict = false; } + double flow() const noexcept { return _flow; } + bool strict() const noexcept { return _strict; } void update_cost(double &total_cost, double child_cost) noexcept { total_cost += child_cost; } + static double estimate_of(auto adapter, const auto &children) { + return flow::estimate_of_and(adapter, children); + } + static double estimate_of(const auto &children) { + return estimate_of(flow::make_adapter(children), children); + } static void sort(auto adapter, auto &children, bool strict) { flow::sort<flow::MinAndCost>(adapter, children); if (strict && children.size() > 1) { @@ -239,25 +250,24 @@ class OrFlow : public FlowMixin<OrFlow>{ private: double _flow; bool _strict; - bool _first; public: - OrFlow(InFlow flow) noexcept : _flow(flow.rate()), _strict(flow.strict()), _first(true) {} + OrFlow(InFlow flow) noexcept : _flow(flow.rate()), _strict(flow.strict()) {} void add(double est) noexcept { - _flow *= (1.0 - est); - _first = false; - } - double flow() const noexcept { - return _strict ? 1.0 : _flow; - } - bool strict() const noexcept { - return _strict; - } - double estimate() const noexcept { - return _first ? 0.0 : (1.0 - _flow); + if (!_strict) { + _flow *= (1.0 - est); + } } + double flow() const noexcept { return _flow; } + bool strict() const noexcept { return _strict; } void update_cost(double &total_cost, double child_cost) noexcept { total_cost += child_cost; } + static double estimate_of(auto adapter, const auto &children) { + return flow::estimate_of_or(adapter, children); + } + static double estimate_of(const auto &children) { + return estimate_of(flow::make_adapter(children), children); + } static void sort(auto adapter, auto &children, bool strict) { if (!strict) { flow::sort<flow::MinOrCost>(adapter, children); @@ -276,21 +286,25 @@ private: public: AndNotFlow(InFlow flow) noexcept : _flow(flow.rate()), _strict(flow.strict()), _first(true) {} void add(double est) noexcept { - _flow *= _first ? est : (1.0 - est); - _first = false; - } - double flow() const noexcept { - return _flow; - } - bool strict() const noexcept { - return _strict && _first; - } - double estimate() const noexcept { - return _first ? 0.0 : _flow; + if (_first) { + _flow *= est; + _strict = false; + _first = false; + } else { + _flow *= (1.0 - est); + } } + double flow() const noexcept { return _flow; } + bool strict() const noexcept { return _strict; } void update_cost(double &total_cost, double child_cost) noexcept { total_cost += child_cost; } + static double estimate_of(auto adapter, const auto &children) { + return flow::estimate_of_and_not(adapter, children); + } + static double estimate_of(const auto &children) { + return estimate_of(flow::make_adapter(children), children); + } static void sort(auto adapter, auto &children, bool) { flow::sort_partial<flow::MinOrCost>(adapter, children, 1); } @@ -299,34 +313,56 @@ public: } }; -using FlowCalc = std::function<double(double)>; - -template <typename FLOW> -FlowCalc flow_calc(InFlow in_flow) { - return [flow=FLOW(in_flow)](double est) mutable noexcept { - double next_flow = flow.flow(); - flow.add(est); - return next_flow; - }; -} - -inline FlowCalc first_flow_calc(InFlow in_flow) { - bool first = true; - double flow = in_flow.rate(); - return [first,flow](double est) mutable noexcept { - double next_flow = flow; - if (first) { - flow *= est; - first = false; - } - return next_flow; - }; -} +class RankFlow : public FlowMixin<RankFlow> { +private: + double _flow; + bool _strict; + bool _first; +public: + RankFlow(InFlow flow) noexcept : _flow(flow.rate()), _strict(flow.strict()), _first(true) {} + void add(double) noexcept { + _flow = 0.0; + _strict = false; + _first = false; + } + double flow() const noexcept { return _flow; } + bool strict() const noexcept { return _strict; } + void update_cost(double &total_cost, double child_cost) noexcept { + if (_first) { + total_cost += child_cost; + } + }; + static double estimate_of(auto adapter, const auto &children) { + return children.empty() ? 0.0 : adapter.estimate(children[0]); + } + static double estimate_of(const auto &children) { + return estimate_of(flow::make_adapter(children), children); + } + static void sort(auto, auto &, bool) {} + static void sort(auto &, bool) {} +}; -inline FlowCalc full_flow_calc(InFlow in_flow) { - double flow = in_flow.rate(); - return [flow](double) noexcept { return flow; }; -} +class BlenderFlow : public FlowMixin<BlenderFlow> { +private: + double _flow; + bool _strict; +public: + BlenderFlow(InFlow flow) noexcept : _flow(flow.rate()), _strict(flow.strict()) {} + void add(double) noexcept {} + double flow() const noexcept { return _flow; } + bool strict() const noexcept { return _strict; } + void update_cost(double &total_cost, double child_cost) noexcept { + total_cost = std::max(total_cost, child_cost); + }; + static double estimate_of(auto adapter, const auto &children) { + return flow::estimate_of_or(adapter, children); + } + static double estimate_of(const auto &children) { + return estimate_of(flow::make_adapter(children), children); + } + static void sort(auto, auto &, bool) {} + static void sort(auto &, bool) {} +}; // type-erased flow wrapper class AnyFlow { @@ -335,7 +371,6 @@ private: virtual void add(double est) noexcept = 0; virtual double flow() const noexcept = 0; virtual bool strict() const noexcept = 0; - virtual double estimate() const noexcept = 0; virtual void update_cost(double &total_cost, double child_cost) noexcept = 0; virtual ~API() = default; }; @@ -345,8 +380,9 @@ private: void add(double est) noexcept override { _flow.add(est); } double flow() const noexcept override { return _flow.flow(); } bool strict() const noexcept override { return _flow.strict(); } - double estimate() const noexcept override { return _flow.estimate(); } - void update_cost(double &total_cost, double child_cost) noexcept override { return _flow.update_cost(total_cost, child_cost); } + void update_cost(double &total_cost, double child_cost) noexcept override { + _flow.update_cost(total_cost, child_cost); + } ~Wrapper() = default; }; alignas(8) char _space[24]; @@ -357,8 +393,7 @@ private: using stored_type = Wrapper<FLOW>; static_assert(alignof(stored_type) <= alignof(_space)); static_assert(sizeof(stored_type) <= sizeof(_space)); - stored_type *obj = ::new (static_cast<void*>(_space)) stored_type(in_flow); - API *upcasted = obj; + API *upcasted = ::new (static_cast<void*>(_space)) stored_type(in_flow); (void) upcasted; assert(static_cast<void*>(upcasted) == static_cast<void*>(_space)); } @@ -374,8 +409,9 @@ public: void add(double est) noexcept { api().add(est); } double flow() const noexcept { return api().flow(); } bool strict() const noexcept { return api().strict(); } - double estimate() const noexcept { return api().estimate(); } - void update_cost(double &total_cost, double child_cost) noexcept { api().update_cost(total_cost, child_cost); } + void update_cost(double &total_cost, double child_cost) noexcept { + api().update_cost(total_cost, child_cost); + } }; } diff --git a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp index 9d0acc50ce5..bc8eaa98541 100644 --- a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.cpp @@ -207,10 +207,10 @@ AndNotBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) co } -FlowCalc -AndNotBlueprint::make_flow_calc(InFlow in_flow) const +AnyFlow +AndNotBlueprint::my_flow(InFlow in_flow) const { - return flow_calc<AndNotFlow>(in_flow); + return AnyFlow::create<AndNotFlow>(in_flow); } //----------------------------------------------------------------------------- @@ -307,10 +307,10 @@ AndBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) const return create_and_filter(get_children(), strict, constraint); } -FlowCalc -AndBlueprint::make_flow_calc(InFlow in_flow) const +AnyFlow +AndBlueprint::my_flow(InFlow in_flow) const { - return flow_calc<AndFlow>(in_flow); + return AnyFlow::create<AndFlow>(in_flow); } //----------------------------------------------------------------------------- @@ -407,10 +407,10 @@ OrBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) const return create_or_filter(get_children(), strict, constraint); } -FlowCalc -OrBlueprint::make_flow_calc(InFlow in_flow) const +AnyFlow +OrBlueprint::my_flow(InFlow in_flow) const { - return flow_calc<OrFlow>(in_flow); + return AnyFlow::create<OrFlow>(in_flow); } uint8_t @@ -425,10 +425,10 @@ OrBlueprint::calculate_cost_tier() const //----------------------------------------------------------------------------- -FlowCalc -WeakAndBlueprint::make_flow_calc(InFlow in_flow) const +AnyFlow +WeakAndBlueprint::my_flow(InFlow in_flow) const { - return flow_calc<OrFlow>(in_flow); + return AnyFlow::create<OrFlow>(in_flow); } WeakAndBlueprint::~WeakAndBlueprint() = default; @@ -502,10 +502,10 @@ WeakAndBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) c //----------------------------------------------------------------------------- -FlowCalc -NearBlueprint::make_flow_calc(InFlow in_flow) const +AnyFlow +NearBlueprint::my_flow(InFlow in_flow) const { - return flow_calc<AndFlow>(in_flow); + return AnyFlow::create<AndFlow>(in_flow); } FlowStats @@ -573,10 +573,10 @@ NearBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) cons //----------------------------------------------------------------------------- -FlowCalc -ONearBlueprint::make_flow_calc(InFlow in_flow) const +AnyFlow +ONearBlueprint::my_flow(InFlow in_flow) const { - return flow_calc<AndFlow>(in_flow); + return AnyFlow::create<AndFlow>(in_flow); } FlowStats @@ -734,18 +734,18 @@ RankBlueprint::createFilterSearch(bool strict, FilterConstraint constraint) cons return create_first_child_filter(get_children(), strict, constraint); } -FlowCalc -RankBlueprint::make_flow_calc(InFlow in_flow) const +AnyFlow +RankBlueprint::my_flow(InFlow in_flow) const { - return first_flow_calc(in_flow); + return AnyFlow::create<RankFlow>(in_flow); } //----------------------------------------------------------------------------- -FlowCalc -SourceBlenderBlueprint::make_flow_calc(InFlow in_flow) const +AnyFlow +SourceBlenderBlueprint::my_flow(InFlow in_flow) const { - return full_flow_calc(in_flow); + return AnyFlow::create<BlenderFlow>(in_flow); } SourceBlenderBlueprint::SourceBlenderBlueprint(const ISourceSelector &selector) noexcept diff --git a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h index 028898d3f47..e0f4be4be06 100644 --- a/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h +++ b/searchlib/src/vespa/searchlib/queryeval/intermediate_blueprints.h @@ -29,7 +29,7 @@ public: SearchIterator::UP createFilterSearch(bool strict, FilterConstraint constraint) const override; private: - FlowCalc make_flow_calc(InFlow in_flow) const override; + AnyFlow my_flow(InFlow in_flow) const override; uint8_t calculate_cost_tier() const override { return (childCnt() > 0) ? get_children()[0]->getState().cost_tier() : State::COST_TIER_NORMAL; } @@ -57,7 +57,7 @@ public: SearchIterator::UP createFilterSearch(bool strict, FilterConstraint constraint) const override; private: - FlowCalc make_flow_calc(InFlow in_flow) const override; + AnyFlow my_flow(InFlow in_flow) const override; }; //----------------------------------------------------------------------------- @@ -82,7 +82,7 @@ public: SearchIterator::UP createFilterSearch(bool strict, FilterConstraint constraint) const override; private: - FlowCalc make_flow_calc(InFlow in_flow) const override; + AnyFlow my_flow(InFlow in_flow) const override; uint8_t calculate_cost_tier() const override; }; @@ -94,7 +94,7 @@ private: uint32_t _n; std::vector<uint32_t> _weights; - FlowCalc make_flow_calc(InFlow in_flow) const override; + AnyFlow my_flow(InFlow in_flow) const override; public: FlowStats calculate_flow_stats(uint32_t docid_limit) const final; HitEstimate combine(const std::vector<HitEstimate> &data) const override; @@ -125,7 +125,7 @@ class NearBlueprint : public IntermediateBlueprint private: uint32_t _window; - FlowCalc make_flow_calc(InFlow in_flow) const override; + AnyFlow my_flow(InFlow in_flow) const override; public: FlowStats calculate_flow_stats(uint32_t docid_limit) const final; HitEstimate combine(const std::vector<HitEstimate> &data) const override; @@ -148,7 +148,7 @@ class ONearBlueprint : public IntermediateBlueprint private: uint32_t _window; - FlowCalc make_flow_calc(InFlow in_flow) const override; + AnyFlow my_flow(InFlow in_flow) const override; public: FlowStats calculate_flow_stats(uint32_t docid_limit) const final; HitEstimate combine(const std::vector<HitEstimate> &data) const override; @@ -186,7 +186,7 @@ public: return (childCnt() > 0) ? get_children()[0]->getState().cost_tier() : State::COST_TIER_NORMAL; } private: - FlowCalc make_flow_calc(InFlow in_flow) const override; + AnyFlow my_flow(InFlow in_flow) const override; }; //----------------------------------------------------------------------------- @@ -196,7 +196,7 @@ class SourceBlenderBlueprint final : public IntermediateBlueprint private: const ISourceSelector &_selector; - FlowCalc make_flow_calc(InFlow in_flow) const override; + AnyFlow my_flow(InFlow in_flow) const override; public: explicit SourceBlenderBlueprint(const ISourceSelector &selector) noexcept; ~SourceBlenderBlueprint() override; diff --git a/vespajlib/src/main/java/ai/vespa/json/Json.java b/vespajlib/src/main/java/ai/vespa/json/Json.java index b88c804c728..da7aae06e8d 100644 --- a/vespajlib/src/main/java/ai/vespa/json/Json.java +++ b/vespajlib/src/main/java/ai/vespa/json/Json.java @@ -8,6 +8,8 @@ import com.yahoo.slime.SlimeUtils; import com.yahoo.slime.Type; import java.math.BigDecimal; +import java.time.Instant; +import java.time.format.DateTimeParseException; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; @@ -90,6 +92,20 @@ public class Json implements Iterable<Json> { return asBool(); } + public Optional<Instant> asOptionalInstant() { return isMissing() ? Optional.empty() : Optional.of(asInstant()); } + public Instant asInstant() { + requireType(Type.STRING); + try { + return Instant.parse(asString()); + } catch (DateTimeParseException e) { + throw new InvalidJsonException("Expected JSON member '%s' to be a valid timestamp: %s".formatted(path, e.getMessage())); + } + } + public Instant asInstant(Instant defaultValue) { + if (isMissing()) return defaultValue; + return asInstant(); + } + public List<Json> toList() { var list = new ArrayList<Json>(length()); forEachEntry(json -> list.add(json)); @@ -184,7 +200,14 @@ public class Json implements Iterable<Json> { public static Builder.Array newArray() { return new Builder.Array(new Slime().setArray()); } public static Builder.Object newObject() { return new Builder.Object(new Slime().setObject()); } - public static Builder.Object existingObject(Cursor cursor) { return new Builder.Object(cursor); } + public static Builder.Object existingSlimeObjectCursor(Cursor cursor) { + if (cursor.type() != Type.OBJECT) throw new InvalidJsonException("Input is not an object"); + return new Builder.Object(cursor); + } + public static Builder.Array existingSlimeArrayCursor(Cursor cursor) { + if (cursor.type() != Type.ARRAY) throw new InvalidJsonException("Input is not an array"); + return new Builder.Array(cursor); + } private Builder(Cursor cursor) { this.cursor = cursor; } @@ -200,6 +223,8 @@ public class Json implements Iterable<Json> { public Builder.Array add(Json json) { SlimeUtils.addValue(json.inspector, cursor.addObject()); return this; } + public Builder.Array add(Json.Builder builder) { return add(builder.build()); } + /** Note: does not return {@code this}! */ public Builder.Array addArray() { return new Array(cursor.addArray()); } /** Note: does not return {@code this}! */ @@ -223,6 +248,9 @@ public class Json implements Iterable<Json> { public Builder.Object set(String field, Json json) { SlimeUtils.setObjectEntry(json.inspector, field, cursor); return this; } + public Builder.Object set(String field, Json.Builder json) { + SlimeUtils.setObjectEntry(json.build().inspector, field, cursor); return this; + } /** Note: does not return {@code this}! */ public Builder.Array setArray(String field) { return new Array(cursor.setArray(field)); } /** Note: does not return {@code this}! */ @@ -233,6 +261,7 @@ public class Json implements Iterable<Json> { public Builder.Object set(String field, double value) { cursor.setDouble(field, value); return this; } public Builder.Object set(String field, boolean value) { cursor.setBool(field, value); return this; } public Builder.Object set(String field, BigDecimal value) { cursor.setString(field, value.toPlainString()); return this; } + public Builder.Object set(String field, Instant timestamp) { cursor.setString(field, timestamp.toString()); return this; } } public Cursor slimeCursor() { return cursor; } diff --git a/vespajlib/src/test/java/ai/vespa/json/JsonTest.java b/vespajlib/src/test/java/ai/vespa/json/JsonTest.java index 293e99227a7..51b64637fd8 100644 --- a/vespajlib/src/test/java/ai/vespa/json/JsonTest.java +++ b/vespajlib/src/test/java/ai/vespa/json/JsonTest.java @@ -23,7 +23,8 @@ class JsonTest { "array": [1, 2, 3], "quux": { "corge": "grault" - } + }, + "timestamp": "2021-06-01T12:00:00Z" } """; var json = Json.of(text); @@ -37,6 +38,7 @@ class JsonTest { assertEquals(8.25D, json.f("floaty").asDouble()); assertEquals(8L, json.f("floaty").asLong()); assertTrue(json.f("bool").asBool()); + assertEquals("2021-06-01T12:00:00Z", json.f("timestamp").asInstant().toString()); // Array member assertEquals(3, json.f("array").length()); @@ -78,6 +80,9 @@ class JsonTest { exception = assertThrows(InvalidJsonException.class, () -> json.f("string").asLong()); assertEquals("Expected JSON member 'string' to be a 'integer' or 'float' but got 'string'", exception.getMessage()); + + exception = assertThrows(InvalidJsonException.class, () -> json.f("string").asInstant()); + assertEquals("Expected JSON member 'string' to be a valid timestamp: Text 'bar' could not be parsed at index 0", exception.getMessage()); } @Test |