diff options
23 files changed, 319 insertions, 149 deletions
diff --git a/Gemfile.lock b/Gemfile.lock index 1962b421269..0134d845e34 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -2,7 +2,7 @@ GEM remote: https://rubygems.org/ specs: Ascii85 (1.1.0) - addressable (2.8.5) + addressable (2.8.6) public_suffix (>= 2.0.2, < 6.0) afm (0.2.2) async (2.6.4) @@ -25,7 +25,7 @@ GEM fiber-annotation (0.2.0) fiber-local (1.0.0) forwardable-extended (2.6.0) - google-protobuf (3.24.3-x86_64-linux) + google-protobuf (3.24.4) hashery (2.1.2) html-proofer (5.0.8) addressable (~> 2.3) @@ -40,7 +40,7 @@ GEM i18n (1.14.1) concurrent-ruby (~> 1.0) io-event (1.3.2) - jekyll (4.3.2) + jekyll (4.3.3) addressable (~> 2.4) colorator (~> 1.0) em-websocket (~> 0.5) @@ -87,7 +87,7 @@ GEM hashery (~> 2.0) ruby-rc4 ttfunk - public_suffix (5.0.3) + public_suffix (5.0.4) racc (1.7.1) rainbow (3.1.1) rake (13.1.0) diff --git a/client/go/internal/admin/envvars/env_vars.go b/client/go/internal/admin/envvars/env_vars.go index 1586292a13d..5e8b5d29b53 100644 --- a/client/go/internal/admin/envvars/env_vars.go +++ b/client/go/internal/admin/envvars/env_vars.go @@ -22,6 +22,7 @@ const ( MALLOC_ARENA_MAX = "MALLOC_ARENA_MAX" NO_VESPAMALLOC_LIST = "NO_VESPAMALLOC_LIST" NUM_PROCESSES_LIMIT = "num_processes_limit" + OPENBLAS_CORETYPE = "OPENBLAS_CORETYPE" PATH = "PATH" PRELOAD = "PRELOAD" ROOT = "ROOT" diff --git a/client/js/app/yarn.lock b/client/js/app/yarn.lock index 17707aef2c8..b4493e1df56 100644 --- a/client/js/app/yarn.lock +++ b/client/js/app/yarn.lock @@ -15,15 +15,7 @@ "@jridgewell/gen-mapping" "^0.3.0" "@jridgewell/trace-mapping" "^0.3.9" -"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.12.13", "@babel/code-frame@^7.22.5": - version "7.22.13" - resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.22.13.tgz#e3c1c099402598483b7a8c46a721d1038803755e" - integrity sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w== - dependencies: - "@babel/highlight" "^7.22.13" - chalk "^2.4.2" - -"@babel/code-frame@^7.22.13", "@babel/code-frame@^7.23.5": +"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.22.13", "@babel/code-frame@^7.23.5": version "7.23.5" resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.23.5.tgz#9009b69a8c602293476ad598ff53e4562e15c244" integrity sha512-CgH3s1a96LipHCmSUmYFPwY7MNx8C3avkq7i4Wl3cfa662ldtUe4VM1TPXX70pfmrlWTb6jLqTYrZyT2ZTJBgA== @@ -31,6 +23,14 @@ "@babel/highlight" "^7.23.4" chalk "^2.4.2" +"@babel/code-frame@^7.12.13", "@babel/code-frame@^7.22.5": + version "7.22.13" + resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.22.13.tgz#e3c1c099402598483b7a8c46a721d1038803755e" + integrity sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w== + dependencies: + "@babel/highlight" "^7.22.13" + chalk "^2.4.2" + "@babel/compat-data@^7.22.9": version "7.23.5" resolved "https://registry.yarnpkg.com/@babel/compat-data/-/compat-data-7.23.5.tgz#ffb878728bb6bdcb6f4510aa51b1be9afb8cfd98" @@ -181,14 +181,7 @@ dependencies: "@babel/types" "^7.22.5" -"@babel/helper-module-imports@^7.16.7": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.22.5.tgz#1a8f4c9f4027d23f520bd76b364d44434a72660c" - integrity sha512-8Dl6+HD/cKifutF5qGd/8ZJi84QeAKh+CEe1sBzz8UayBBGg1dAIJrdHOcOM5b2MpzWL2yuotJTtGjETq0qjXg== - dependencies: - "@babel/types" "^7.22.5" - -"@babel/helper-module-imports@^7.22.15", "@babel/helper-module-imports@^7.22.5": +"@babel/helper-module-imports@^7.16.7", "@babel/helper-module-imports@^7.22.15", "@babel/helper-module-imports@^7.22.5": version "7.22.15" resolved "https://registry.yarnpkg.com/@babel/helper-module-imports/-/helper-module-imports-7.22.15.tgz#16146307acdc40cc00c3b2c647713076464bdbf0" integrity sha512-0pYVBnDKZO2fnSPCrgM/6WMc7eS20Fbok+0r88fp+YtWVLZrp4CkafFGIp+W0VKw4a22sgebPT99y+FDNMdP4w== @@ -452,11 +445,11 @@ regenerator-runtime "^0.14.0" "@babel/runtime@^7.12.5", "@babel/runtime@^7.18.3": - version "7.22.6" - resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.22.6.tgz#57d64b9ae3cff1d67eb067ae117dac087f5bd438" - integrity sha512-wDb5pWm4WDdF6LFUde3Jl8WzPA+3ZbxYqkC6xAXuD3irdEHN1k0NfTRrJD8ZD378SJ61miMLCqIOXYhd8x+AJQ== + version "7.23.6" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.23.6.tgz#c05e610dc228855dc92ef1b53d07389ed8ab521d" + integrity sha512-zHd0eUrf5GZoOWVCXp6koAKQTfZV07eit6bGPmJgnZdnSAvvZee6zniW2XMF7Cmc4ISOOnPy3QaSiIJGJkVEDQ== dependencies: - regenerator-runtime "^0.13.11" + regenerator-runtime "^0.14.0" "@babel/template@^7.22.15", "@babel/template@^7.22.5", "@babel/template@^7.3.3": version "7.22.15" @@ -499,7 +492,7 @@ debug "^4.1.0" globals "^11.1.0" -"@babel/types@^7.0.0", "@babel/types@^7.20.7", "@babel/types@^7.22.15", "@babel/types@^7.22.5", "@babel/types@^7.23.0", "@babel/types@^7.23.5": +"@babel/types@^7.0.0", "@babel/types@^7.20.7", "@babel/types@^7.22.15", "@babel/types@^7.23.0", "@babel/types@^7.23.5": version "7.23.5" resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.5.tgz#48d730a00c95109fa4393352705954d74fb5b602" integrity sha512-ON5kSOJwVO6xXVRTvOI0eOnWe7VdUcIpsovGo9U/Br4Ie4UVFQTboO2cYnDhAGU6Fp+UxSiT+pMft0SMHfuq6w== @@ -526,6 +519,15 @@ "@babel/helper-validator-identifier" "^7.22.19" to-fast-properties "^2.0.0" +"@babel/types@^7.22.5": + version "7.23.6" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.6.tgz#be33fdb151e1f5a56877d704492c240fc71c7ccd" + integrity sha512-+uarb83brBzPKN38NX1MkB6vb6+mwvR6amUulqAE7ccQw1pEl+bCia9TbdG1lsnFP7lZySvUn37CHyXQdfTwzg== + dependencies: + "@babel/helper-string-parser" "^7.23.4" + "@babel/helper-validator-identifier" "^7.22.20" + to-fast-properties "^2.0.0" + "@babel/types@^7.23.3": version "7.23.3" resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.3.tgz#d5ea892c07f2ec371ac704420f4dcdb07b5f9598" @@ -587,23 +589,23 @@ integrity sha512-W2P2c/VRW1/1tLox0mVUalvnWXxavmv/Oum2aPsRcoDJuob75FC3Y8FbpfLwUegRcxINtGUMPq0tFCvYNTBXNA== "@emotion/react@^11": - version "11.11.1" - resolved "https://registry.yarnpkg.com/@emotion/react/-/react-11.11.1.tgz#b2c36afac95b184f73b08da8c214fdf861fa4157" - integrity sha512-5mlW1DquU5HaxjLkfkGN1GA/fvVGdyHURRiX/0FHl2cfIfRxSOfmxEH5YS43edp0OldZrZ+dkBKbngxcNCdZvA== + version "11.11.3" + resolved "https://registry.yarnpkg.com/@emotion/react/-/react-11.11.3.tgz#96b855dc40a2a55f52a72f518a41db4f69c31a25" + integrity sha512-Cnn0kuq4DoONOMcnoVsTOR8E+AdnKFf//6kUWc4LCdnxj31pZWn7rIULd6Y7/Js1PiPHzn7SKCM9vB/jBni8eA== dependencies: "@babel/runtime" "^7.18.3" "@emotion/babel-plugin" "^11.11.0" "@emotion/cache" "^11.11.0" - "@emotion/serialize" "^1.1.2" + "@emotion/serialize" "^1.1.3" "@emotion/use-insertion-effect-with-fallbacks" "^1.0.1" "@emotion/utils" "^1.2.1" "@emotion/weak-memoize" "^0.3.1" hoist-non-react-statics "^3.3.1" -"@emotion/serialize@^1.1.2": - version "1.1.2" - resolved "https://registry.yarnpkg.com/@emotion/serialize/-/serialize-1.1.2.tgz#017a6e4c9b8a803bd576ff3d52a0ea6fa5a62b51" - integrity sha512-zR6a/fkFP4EAcCMQtLOhIgpprZOwNmCldtpaISpvz348+DP4Mz8ZoKaGGCQpbzepNIUWbq4w6hNZkwDyKoS+HA== +"@emotion/serialize@^1.1.2", "@emotion/serialize@^1.1.3": + version "1.1.3" + resolved "https://registry.yarnpkg.com/@emotion/serialize/-/serialize-1.1.3.tgz#84b77bfcfe3b7bb47d326602f640ccfcacd5ffb0" + integrity sha512-iD4D6QVZFDhcbH0RAG1uVu1CwVLMWUkCvAqqlewO/rxf8+87yIBAlt4+AxMiiKPLs5hFc0owNk/sLLAOROw3cA== dependencies: "@emotion/hash" "^0.9.1" "@emotion/memoize" "^0.8.1" @@ -1488,9 +1490,9 @@ integrity sha512-najjVq5KN2vsH2U/xyh2opaSEz6cZMR2SetLIlxlj08nOcmPOemJmUK2o4kUzfLqfrWE0PIrNeE16XhYDd3nqg== "@types/parse-json@^4.0.0": - version "4.0.0" - resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.0.tgz#2f8bb441434d163b35fb8ffdccd7138927ffb8c0" - integrity sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA== + version "4.0.2" + resolved "https://registry.yarnpkg.com/@types/parse-json/-/parse-json-4.0.2.tgz#5950e50960793055845e956c427fc2b0d70c5239" + integrity sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw== "@types/stack-utils@^2.0.0": version "2.0.1" @@ -2208,9 +2210,9 @@ csstype@3.0.9: integrity sha512-rpw6JPxK6Rfg1zLOYCSwle2GFOOsnjmDYDaBwEcwoOg4qlsIVCN789VkBZDJAGi4T07gI4YSutR43t9Zz4Lzuw== csstype@^3.0.2: - version "3.1.2" - resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.1.2.tgz#1d4bf9d572f11c14031f0436e1c10bc1f571f50b" - integrity sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ== + version "3.1.3" + resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.1.3.tgz#d80ff294d114fb0e6ac500fbf85b60137d7eff81" + integrity sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw== debug@^2.2.0, debug@^2.3.3: version "2.6.9" @@ -2564,12 +2566,12 @@ eslint-plugin-import@^2: tsconfig-paths "^3.15.0" eslint-plugin-prettier@^5: - version "5.1.1" - resolved "https://registry.yarnpkg.com/eslint-plugin-prettier/-/eslint-plugin-prettier-5.1.1.tgz#ab7d9823788b557ff7ccdd50a5849d7760cb8bef" - integrity sha512-WQpV3mSmIobb77s4qiCZu3dBrZZ0rj8ckSfBtRrgNK9Wnh2s3eiaxNTWloz1LJ1WtvqZES/PAI7PLvsrGt/CEA== + version "5.1.2" + resolved "https://registry.yarnpkg.com/eslint-plugin-prettier/-/eslint-plugin-prettier-5.1.2.tgz#584c94d4bf31329b2d4cbeb10fd600d17d6de742" + integrity sha512-dhlpWc9vOwohcWmClFcA+HjlvUpuyynYs0Rf+L/P6/0iQE6vlHW9l5bkfzN62/Stm9fbq8ku46qzde76T1xlSg== dependencies: prettier-linter-helpers "^1.0.0" - synckit "^0.8.5" + synckit "^0.8.6" eslint-plugin-react-hooks@^4: version "4.6.0" @@ -3351,13 +3353,6 @@ is-ci@^2.0.0: dependencies: ci-info "^2.0.0" -is-core-module@^2.11.0, is-core-module@^2.9.0: - version "2.13.0" - resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.13.0.tgz#bb52aa6e2cbd49a30c2ba68c42bf3435ba6072db" - integrity sha512-Z7dk6Qo8pOCp3l4tsX2C5ZVas4V+UxwQodwZhLopL91TX8UyyHEXafPcyoeeWuLrwzHcr3igO78wNLwHJHsMCQ== - dependencies: - has "^1.0.3" - is-core-module@^2.13.0, is-core-module@^2.13.1: version "2.13.1" resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.13.1.tgz#ad0d7532c6fea9da1ebdc82742d74525c6273384" @@ -3365,6 +3360,13 @@ is-core-module@^2.13.0, is-core-module@^2.13.1: dependencies: hasown "^2.0.0" +is-core-module@^2.9.0: + version "2.13.0" + resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.13.0.tgz#bb52aa6e2cbd49a30c2ba68c42bf3435ba6072db" + integrity sha512-Z7dk6Qo8pOCp3l4tsX2C5ZVas4V+UxwQodwZhLopL91TX8UyyHEXafPcyoeeWuLrwzHcr3igO78wNLwHJHsMCQ== + dependencies: + has "^1.0.3" + is-data-descriptor@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/is-data-descriptor/-/is-data-descriptor-0.1.4.tgz#0b5ee648388e2c860282e793f1856fec3f301b56" @@ -4902,11 +4904,6 @@ reflect.getprototypeof@^1.0.3: globalthis "^1.0.3" which-builtin-type "^1.1.3" -regenerator-runtime@^0.13.11: - version "0.13.11" - resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz#f6dca3e7ceec20590d07ada785636a90cdca17f9" - integrity sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg== - regenerator-runtime@^0.14.0: version "0.14.0" resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.14.0.tgz#5e19d68eb12d486f797e15a3c6a918f7cec5eb45" @@ -4985,12 +4982,12 @@ resolve.exports@^2.0.0: resolved "https://registry.yarnpkg.com/resolve.exports/-/resolve.exports-2.0.2.tgz#f8c934b8e6a13f539e38b7098e2e36134f01e800" integrity sha512-X2UW6Nw3n/aMgDVy+0rSqgHlv39WZAlZrXCdnbyEiKm17DSqHX4MmQMaST3FbeWR5FTuRcUwYAziZajji0Y7mg== -resolve@^1.19.0: - version "1.22.2" - resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.2.tgz#0ed0943d4e301867955766c9f3e1ae6d01c6845f" - integrity sha512-Sb+mjNHOULsBv818T40qSPeRiuWLyaGMa5ewydRLFimneixmVy2zdivRl+AF6jaYPC8ERxGDmFSiqui6SfPd+g== +resolve@^1.19.0, resolve@^1.22.4: + version "1.22.8" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.8.tgz#b6c87a9f2aa06dfab52e3d70ac8cde321fa5a48d" + integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw== dependencies: - is-core-module "^2.11.0" + is-core-module "^2.13.0" path-parse "^1.0.7" supports-preserve-symlinks-flag "^1.0.0" @@ -5003,15 +5000,6 @@ resolve@^1.20.0: path-parse "^1.0.7" supports-preserve-symlinks-flag "^1.0.0" -resolve@^1.22.4: - version "1.22.8" - resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.8.tgz#b6c87a9f2aa06dfab52e3d70ac8cde321fa5a48d" - integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw== - dependencies: - is-core-module "^2.13.0" - path-parse "^1.0.7" - supports-preserve-symlinks-flag "^1.0.0" - resolve@^2.0.0-next.4: version "2.0.0-next.4" resolved "https://registry.yarnpkg.com/resolve/-/resolve-2.0.0-next.4.tgz#3d37a113d6429f496ec4752d2a2e58efb1fd4660" @@ -5441,7 +5429,7 @@ supports-preserve-symlinks-flag@^1.0.0: resolved "https://registry.yarnpkg.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09" integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w== -synckit@^0.8.5: +synckit@^0.8.6: version "0.8.6" resolved "https://registry.yarnpkg.com/synckit/-/synckit-0.8.6.tgz#b69b7fbce3917c2673cbdc0d87fb324db4a5b409" integrity sha512-laHF2savN6sMeHCjLRkheIU4wo3Zg9Ln5YOjOo7sZ5dVQW8yF5pPE5SIw1dsPhq3TRp1jisKRCdPhfs/1WMqDA== diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/SecretStoreExternalIdRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/SecretStoreExternalIdRetriever.java index 5afb2188fac..c6f9a0268c1 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/SecretStoreExternalIdRetriever.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/SecretStoreExternalIdRetriever.java @@ -4,7 +4,10 @@ package com.yahoo.vespa.config.server.tenant; import com.yahoo.config.model.api.TenantSecretStore; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; +import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; +import com.yahoo.vespa.config.server.http.InvalidApplicationException; + import java.util.List; import java.util.stream.Collectors; @@ -19,10 +22,14 @@ public class SecretStoreExternalIdRetriever { return tenantSecretStores.stream() .map(tenantSecretStore -> { var secretName = secretName(tenant, system, tenantSecretStore.getName()); - String secret = secretStore.getSecret(secretName); - if (secret == null) - throw new RuntimeException("No secret found in secret store for " + secretName); - return tenantSecretStore.withExternalId(secret); + try { + String secret = secretStore.getSecret(secretName); + if (secret == null) + throw new InvalidApplicationException("No secret found in secret store for " + secretName); + return tenantSecretStore.withExternalId(secret); + } catch (SecretNotFoundException e) { + throw new InvalidApplicationException("Could not find externalId for secret store: %s".formatted(tenantSecretStore.getName())); + } }) .toList(); } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStore.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStore.java index bd469cb8f0b..ac97fe0ba05 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStore.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStore.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server; +import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; import java.util.HashMap; @@ -13,7 +14,7 @@ public class MockSecretStore implements SecretStore { public String getSecret(String key) { if(secrets.containsKey(key)) return secrets.get(key).get(0); - throw new RuntimeException("Key not found: " + key); + throw new SecretNotFoundException("Key not found: " + key); } @Override diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/SecretStoreExternalIdRetrieverTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/SecretStoreExternalIdRetrieverTest.java new file mode 100644 index 00000000000..96c7d9e6957 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/SecretStoreExternalIdRetrieverTest.java @@ -0,0 +1,39 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.config.model.api.TenantSecretStore; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.config.server.MockSecretStore; +import com.yahoo.vespa.config.server.http.InvalidApplicationException; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * @author mortent + */ +public class SecretStoreExternalIdRetrieverTest { + private final MockSecretStore secretStore = new MockSecretStore(); + private final TenantName tenantName = TenantName.from("myTenant"); + private final TenantSecretStore tenantSecretStore = new TenantSecretStore("name", "123456789012", "role"); + + @Test + public void fills_external_ids() { + secretStore.put(SecretStoreExternalIdRetriever.secretName(tenantName, SystemName.PublicCd, "name"), "externalId"); + + List<TenantSecretStore> tenantSecretStores = SecretStoreExternalIdRetriever.populateExternalId(secretStore, tenantName, SystemName.PublicCd, List.of(tenantSecretStore)); + assertEquals(1, tenantSecretStores.size()); + assertEquals("externalId", tenantSecretStores.get(0).getExternalId().get()); + } + + @Test + public void reports_application_package_error_when_external_id_not_found() { + InvalidApplicationException exception = assertThrows(InvalidApplicationException.class, () -> SecretStoreExternalIdRetriever.populateExternalId(secretStore, tenantName, SystemName.PublicCd, List.of(tenantSecretStore))); + assertEquals("Could not find externalId for secret store: name", exception.getMessage()); + } +} diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index 59fca668ebb..308b7842c18 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -36,7 +36,7 @@ <error-prone-annotations.vespa.version>2.24.0</error-prone-annotations.vespa.version> <guava.vespa.version>33.0.0-jre</guava.vespa.version> <guice.vespa.version>6.0.0</guice.vespa.version> - <jackson2.vespa.version>2.16.0</jackson2.vespa.version> + <jackson2.vespa.version>2.16.1</jackson2.vespa.version> <jackson-databind.vespa.version>${jackson2.vespa.version}</jackson-databind.vespa.version> <jakarta.inject.vespa.version>2.0.1</jakarta.inject.vespa.version> <javax.inject.vespa.version>1</javax.inject.vespa.version> @@ -66,7 +66,7 @@ <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> <athenz.vespa.version>1.11.48</athenz.vespa.version> - <aws-sdk.vespa.version>1.12.623</aws-sdk.vespa.version> + <aws-sdk.vespa.version>1.12.629</aws-sdk.vespa.version> <!-- Athenz END --> <!-- WARNING: If you change curator version, you also need to update @@ -114,6 +114,7 @@ <junit.vespa.version>5.10.0</junit.vespa.version> <junit.platform.vespa.version>1.10.0</junit.platform.vespa.version> <junit4.vespa.version>4.13.2</junit4.vespa.version> + <logback.vespa.version>1.2.13</logback.vespa.version> <luben.zstd.vespa.version>1.5.5-11</luben.zstd.vespa.version> <lucene.vespa.version>9.9.1</lucene.vespa.version> <maven-archiver.vespa.version>3.6.1</maven-archiver.vespa.version> @@ -129,6 +130,7 @@ <org.json.vespa.version>20231013</org.json.vespa.version> <org.lz4.vespa.version>1.8.0</org.lz4.vespa.version> <prometheus.client.vespa.version>0.16.0</prometheus.client.vespa.version> + <plexus-interpolation.vespa.version>1.27</plexus-interpolation.vespa.version> <protobuf.vespa.version>3.25.1</protobuf.vespa.version> <questdb.vespa.version>7.3.7</questdb.vespa.version> <spifly.vespa.version>1.3.7</spifly.vespa.version> @@ -154,7 +156,7 @@ <maven-antrun-plugin.vespa.version>3.1.0</maven-antrun-plugin.vespa.version> <maven-assembly-plugin.vespa.version>3.6.0</maven-assembly-plugin.vespa.version> <maven-bundle-plugin.vespa.version>5.1.9</maven-bundle-plugin.vespa.version> - <maven-compiler-plugin.vespa.version>3.12.0</maven-compiler-plugin.vespa.version> + <maven-compiler-plugin.vespa.version>3.12.1</maven-compiler-plugin.vespa.version> <maven-core.vespa.version>3.9.6</maven-core.vespa.version> <maven-dependency-plugin.vespa.version>3.6.1</maven-dependency-plugin.vespa.version> <maven-deploy-plugin.vespa.version>3.1.1</maven-deploy-plugin.vespa.version> diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 79b3fb00d71..c1d9a4cdc1a 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -80,7 +80,7 @@ public class Flags { public static final UnboundStringFlag NESSUS_AGENT_GROUP = defineStringFlag( "nessus-agent-group", ":legacy", - List.of("hakonhall"), "2023-11-29", "2023-12-29", + List.of("hakonhall"), "2023-11-29", "2024-02-29", "Link nessusagent to the given group, or run legacy task (\":legacy\"), or disable task (\"\").", "Takes effect after host admin restart", (String value) -> value.equals(":legacy") || // Run legacy task. Is a no-op outside YAHOO cloud. diff --git a/integration/intellij/build.gradle.kts b/integration/intellij/build.gradle.kts index 502c09c58f8..980e8878efc 100644 --- a/integration/intellij/build.gradle.kts +++ b/integration/intellij/build.gradle.kts @@ -5,7 +5,7 @@ import org.jetbrains.grammarkit.tasks.GenerateParserTask plugins { id("java-library") id("org.jetbrains.intellij") version "1.16.1" - id("org.jetbrains.grammarkit") version "2022.3.2" + id("org.jetbrains.grammarkit") version "2022.3.2.1" id("maven-publish") // to deploy the plugin into a Maven repo } diff --git a/maven-plugins/allowed-maven-dependencies.txt b/maven-plugins/allowed-maven-dependencies.txt index 930a5020dc6..85531c74863 100644 --- a/maven-plugins/allowed-maven-dependencies.txt +++ b/maven-plugins/allowed-maven-dependencies.txt @@ -49,7 +49,7 @@ org.codehaus.plexus:plexus-archiver:4.8.0 org.codehaus.plexus:plexus-cipher:2.0 org.codehaus.plexus:plexus-classworlds:2.7.0 org.codehaus.plexus:plexus-component-annotations:2.1.0 -org.codehaus.plexus:plexus-interpolation:1.26 +org.codehaus.plexus:plexus-interpolation:${plexus-interpolation.vespa.version} org.codehaus.plexus:plexus-io:${maven-enforcer-plugin.vespa.version} org.codehaus.plexus:plexus-sec-dispatcher:2.0 org.codehaus.plexus:plexus-utils:${maven-shade-plugin.vespa.version} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java index 52b68708eee..539f3128091 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java @@ -137,7 +137,7 @@ public class NodeRepository extends AbstractComponent implements HealthCheckerPr this.resourcesCalculator = provisionServiceProvider.getHostResourcesCalculator(); this.nodeResourceLimits = new NodeResourceLimits(this); this.nameResolver = nameResolver; - this.osVersions = new OsVersions(this); + this.osVersions = new OsVersions(this, provisionServiceProvider.getHostProvisioner()); this.infrastructureVersions = new InfrastructureVersions(db); this.firmwareChecks = new FirmwareChecks(db, clock); this.containerImages = new ContainerImages(containerImage, tenantContainerImage, tenantGpuContainerImage); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingOsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingOsUpgrader.java index c2d3f511711..5d8296d6f9d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingOsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingOsUpgrader.java @@ -35,7 +35,7 @@ public class DelegatingOsUpgrader extends OsUpgrader { // This upgrader cannot downgrade nodes. We therefore consider only nodes // on a lower version than the target .osVersionIsBefore(target.version()) - .matching(node -> canUpgradeAt(now, node)) + .matching(node -> canUpgradeTo(target.version(), now, node)) .byIncreasingOsVersion() .first(upgradeSlots(target, activeNodes)); if (nodesToUpgrade.size() == 0) return; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java index 436181f99ba..9e931463999 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.os; +import com.yahoo.component.Version; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.flags.IntFlag; import com.yahoo.vespa.flags.PermanentFlags; @@ -10,20 +11,27 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import java.time.Duration; import java.time.Instant; +import java.util.Objects; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; /** - * Interface for an OS upgrader. + * Interface for an OS upgrader. Instances of this are created on-demand because multiple implementations may be used + * within a single zone. This and subclasses should not have any state. * * @author mpolden */ public abstract class OsUpgrader { + private final Logger LOG = Logger.getLogger(OsUpgrader.class.getName()); + private final IntFlag maxActiveUpgrades; final NodeRepository nodeRepository; public OsUpgrader(NodeRepository nodeRepository) { - this.nodeRepository = nodeRepository; + this.nodeRepository = Objects.requireNonNull(nodeRepository); this.maxActiveUpgrades = PermanentFlags.MAX_OS_UPGRADES.bindTo(nodeRepository.flagSource()); } @@ -43,10 +51,18 @@ public abstract class OsUpgrader { return Math.max(0, max - upgrading); } - /** Returns whether node can change version at given instant */ - final boolean canUpgradeAt(Instant instant, Node node) { - return node.status().osVersion().downgrading() || // Fast-track downgrades - node.history().age(instant).compareTo(gracePeriod()) > 0; + /** Returns whether node can upgrade to version at given instant */ + final boolean canUpgradeTo(Version version, Instant instant, Node node) { + Set<Version> versions = nodeRepository.osVersions().availableTo(node, version); + boolean versionAvailable = versions.contains(version); + if (!versionAvailable) { + LOG.log(Level.WARNING, "Want to upgrade host " + node.hostname() + " to OS version " + + version.toFullString() + ", but this version does not exist in " + + node.cloudAccount() + ". Found " + versions.stream().sorted().toList()); + } + return versionAvailable && + (node.status().osVersion().downgrading() || // Fast-track downgrades + node.history().age(instant).compareTo(gracePeriod()) > 0); } /** The duration this leaves new nodes alone before scheduling any upgrade */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java index daed86dc2ab..0bc074cba23 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java @@ -1,8 +1,11 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.os; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; import com.yahoo.component.Version; import com.yahoo.config.provision.Cloud; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.curator.Lock; @@ -10,11 +13,17 @@ import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Status; import com.yahoo.vespa.hosted.provision.persistence.CuratorDb; +import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; +import com.yahoo.yolean.Exceptions; import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import java.util.function.UnaryOperator; +import java.util.logging.Level; import java.util.logging.Logger; /** @@ -30,20 +39,27 @@ import java.util.logging.Logger; */ public class OsVersions { - private static final Logger log = Logger.getLogger(OsVersions.class.getName()); + private static final Logger LOG = Logger.getLogger(OsVersions.class.getName()); private final NodeRepository nodeRepository; private final CuratorDb db; private final Cloud cloud; - - public OsVersions(NodeRepository nodeRepository) { - this(nodeRepository, nodeRepository.zone().cloud()); + private final Optional<HostProvisioner> hostProvisioner; + // Version is queried for each host to upgrade, so we cache the results for a while to avoid excessive + // API calls to the host provisioner + private final Cache<CloudAccount, Set<Version>> availableVersions = CacheBuilder.newBuilder() + .expireAfterWrite(10, TimeUnit.MINUTES) + .build(); + + public OsVersions(NodeRepository nodeRepository, Optional<HostProvisioner> hostProvisioner) { + this(nodeRepository, nodeRepository.zone().cloud(), hostProvisioner); } - OsVersions(NodeRepository nodeRepository, Cloud cloud) { + OsVersions(NodeRepository nodeRepository, Cloud cloud, Optional<HostProvisioner> hostProvisioner) { this.nodeRepository = Objects.requireNonNull(nodeRepository); this.db = nodeRepository.database(); this.cloud = Objects.requireNonNull(cloud); + this.hostProvisioner = Objects.requireNonNull(hostProvisioner); // Read and write all versions to make sure they are stored in the latest version of the serialized format try (var lock = db.lockOsVersionChange()) { @@ -104,11 +120,30 @@ public class OsVersions { + currentTarget.get().version().toFullString()); } - log.info("Set OS target version for " + nodeType + " nodes to " + version.toFullString()); + LOG.info("Set OS target version for " + nodeType + " nodes to " + version.toFullString()); return change.withTarget(version, nodeType); }); } + /** Returns the versions available to given host */ + public Set<Version> availableTo(Node host, Version requestedVersion) { + if (hostProvisioner.isEmpty()) { + return Set.of(requestedVersion); + } + try { + return availableVersions.get(host.cloudAccount(), + () -> hostProvisioner.get().osVersions(host, requestedVersion.getMajor())); + } catch (ExecutionException e) { + LOG.log(Level.WARNING, "Failed to list supported OS versions in " + host.cloudAccount() + ": " + Exceptions.toMessageString(e)); + return Set.of(); + } + } + + /** Invalidate cached versions. For testing purposes */ + void invalidate() { + availableVersions.invalidateAll(); + } + /** Resume or halt upgrade of given node type */ public void resumeUpgradeOf(NodeType nodeType, boolean resume) { require(nodeType); @@ -126,7 +161,11 @@ public class OsVersions { /** Returns whether node can be upgraded now */ public boolean canUpgrade(Node node) { - return chooseUpgrader(node.type(), Optional.empty()).canUpgradeAt(nodeRepository.clock().instant(), node); + Optional<Version> wantedVersion = node.status().osVersion().wanted(); + if (wantedVersion.isEmpty()) { + return false; + } + return chooseUpgrader(node.type(), Optional.empty()).canUpgradeTo(wantedVersion.get(), nodeRepository.clock().instant(), node); } /** Returns the upgrader to use when upgrading given node type to target */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java index 108093d8379..a5565a6accb 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java @@ -71,7 +71,7 @@ public class RebuildingOsUpgrader extends OsUpgrader { List<Node> hostsToRebuild = new ArrayList<>(rebuildLimit); NodeList candidates = hosts.not().rebuilding(softRebuild) .not().onOsVersion(target.version()) - .matching(node -> canUpgradeAt(now, node)) + .matching(node -> canUpgradeTo(target.version(), now, node)) .byIncreasingOsVersion(); for (Node host : candidates) { if (hostsToRebuild.size() == rebuildLimit) break; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java index c1e8f2b6fa4..cb6c7683f23 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java @@ -54,7 +54,7 @@ public class RetiringOsUpgrader extends OsUpgrader { } return nodes.not().deprovisioning() .not().onOsVersion(target.version()) - .matching(node -> canUpgradeAt(instant, node)) + .matching(node -> canUpgradeTo(target.version(), instant, node)) .byIncreasingOsVersion() .first(upgradeSlots(target, nodes.deprovisioning())); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java index 8ef4b6c8bd1..4214f543c60 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; +import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.HostEvent; @@ -11,6 +12,7 @@ import com.yahoo.vespa.hosted.provision.Node; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Consumer; import java.util.function.Predicate; @@ -94,6 +96,9 @@ public interface HostProvisioner { /** Returns whether flavor for given host can be upgraded to a newer generation */ boolean canUpgradeFlavor(Node host, Node child, Predicate<NodeResources> realHostResourcesWithinLimits); + /** Returns all OS versions available to host for the given major version */ + Set<Version> osVersions(Node host, int majorVersion); + /** Updates the given hosts to indicate that they are allocated to the given application. */ default void updateAllocation(Collection<Node> hosts, ApplicationId owner) { } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java index b5bb91af71a..73985075319 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.testutils; +import com.yahoo.component.Version; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; @@ -32,6 +33,7 @@ import java.util.Optional; import java.util.Set; import java.util.function.Consumer; import java.util.function.Predicate; +import java.util.stream.Collectors; import java.util.stream.IntStream; import static com.yahoo.config.provision.NodeType.host; @@ -50,6 +52,7 @@ public class MockHostProvisioner implements HostProvisioner { private final Map<ClusterSpec.Type, Flavor> hostFlavors = new HashMap<>(); private final Set<String> upgradableFlavors = new HashSet<>(); private final Map<Behaviour, Integer> behaviours = new HashMap<>(); + private final Set<Version> osVersions = new HashSet<>(); private int deprovisionedHosts = 0; @@ -146,6 +149,11 @@ public class MockHostProvisioner implements HostProvisioner { return upgradableFlavors.contains(host.flavor().name()); } + @Override + public Set<Version> osVersions(Node host, int majorVersion) { + return osVersions.stream().filter(v -> v.getMajor() == majorVersion).collect(Collectors.toUnmodifiableSet()); + } + /** Returns the hosts that have been provisioned by this */ public List<ProvisionedHost> provisionedHosts() { return Collections.unmodifiableList(provisionedHosts); @@ -214,6 +222,11 @@ public class MockHostProvisioner implements HostProvisioner { return this; } + public MockHostProvisioner addOsVersion(Version version) { + osVersions.add(version); + return this; + } + public boolean compatible(Flavor flavor, NodeResources resources) { NodeResources resourcesToVerify = resources.withMemoryGb(resources.memoryGb() - memoryTaxGb); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java index 0be90dcb888..dcbac44a37f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java @@ -18,6 +18,7 @@ import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.node.OsVersion; import com.yahoo.vespa.hosted.provision.node.Status; import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; +import com.yahoo.vespa.hosted.provision.testutils.MockHostProvisioner; import org.junit.Test; import java.time.Duration; @@ -41,7 +42,7 @@ public class OsVersionsTest { @Test public void upgrade() { - var versions = new OsVersions(tester.nodeRepository()); + var versions = new OsVersions(tester.nodeRepository(), Optional.ofNullable(tester.hostProvisioner())); provisionInfraApplication(10); Supplier<NodeList> hostNodes = () -> tester.nodeRepository().nodes().list().nodeType(NodeType.host); @@ -94,7 +95,7 @@ public class OsVersionsTest { public void max_active_upgrades() { int totalNodes = 20; int maxActiveUpgrades = 5; - var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud()); + var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud(), Optional.ofNullable(tester.hostProvisioner())); setMaxActiveUpgrades(maxActiveUpgrades); provisionInfraApplication(totalNodes); Supplier<NodeList> hostNodes = () -> tester.nodeRepository().nodes().list().state(Node.State.active).hosts(); @@ -140,7 +141,7 @@ public class OsVersionsTest { @Test public void newer_upgrade_aborts_upgrade_to_stale_version() { - var versions = new OsVersions(tester.nodeRepository()); + var versions = new OsVersions(tester.nodeRepository(), Optional.ofNullable(tester.hostProvisioner())); provisionInfraApplication(10); Supplier<NodeList> hostNodes = () -> tester.nodeRepository().nodes().list().hosts(); @@ -160,7 +161,7 @@ public class OsVersionsTest { @Test public void upgrade_and_downgrade_by_retiring() { int maxActiveUpgrades = 2; - var versions = new OsVersions(tester.nodeRepository(), Cloud.builder().dynamicProvisioning(true).build()); + var versions = new OsVersions(tester.nodeRepository(), Cloud.builder().dynamicProvisioning(true).build(), Optional.ofNullable(tester.hostProvisioner())); setMaxActiveUpgrades(maxActiveUpgrades); int hostCount = 10; // Provision hosts and children @@ -229,7 +230,7 @@ public class OsVersionsTest { @Test public void upgrade_by_retiring_everything_at_once() { - var versions = new OsVersions(tester.nodeRepository(), Cloud.builder().dynamicProvisioning(true).build()); + var versions = new OsVersions(tester.nodeRepository(), Cloud.builder().dynamicProvisioning(true).build(), Optional.ofNullable(tester.hostProvisioner())); setMaxActiveUpgrades(Integer.MAX_VALUE); int hostCount = 3; provisionInfraApplication(hostCount, NodeType.host); @@ -253,7 +254,7 @@ public class OsVersionsTest { @Test public void upgrade_by_rebuilding() { - var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud()); + var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud(), Optional.ofNullable(tester.hostProvisioner())); setMaxActiveUpgrades(1); int hostCount = 10; provisionInfraApplication(hostCount + 1); @@ -337,7 +338,8 @@ public class OsVersionsTest { .dynamicProvisioning(true) .name(CloudName.AWS) .account(CloudAccount.from("000000000000")) - .build()); + .build(), + Optional.ofNullable(tester.hostProvisioner())); provisionInfraApplication(hostCount, NodeType.host, NodeResources.StorageType.remote, NodeResources.Architecture.x86_64); Supplier<NodeList> hostNodes = () -> tester.nodeRepository().nodes().list().nodeType(NodeType.host); @@ -382,7 +384,7 @@ public class OsVersionsTest { @Test public void upgrade_by_rebuilding_multiple_host_types() { setMaxActiveUpgrades(1); - var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud()); + var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud(), Optional.ofNullable(tester.hostProvisioner())); int hostCount = 3; provisionInfraApplication(hostCount, NodeType.host); provisionInfraApplication(hostCount, NodeType.confighost); @@ -415,7 +417,7 @@ public class OsVersionsTest { @Test public void upgrade_by_rebuilding_is_limited_by_stateful_clusters() { setMaxActiveUpgrades(3); - var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud()); + var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud(), Optional.ofNullable(tester.hostProvisioner())); int hostCount = 5; ApplicationId app1 = ApplicationId.from("t1", "a1", "i1"); ApplicationId app2 = ApplicationId.from("t2", "a2", "i2"); @@ -493,7 +495,7 @@ public class OsVersionsTest { public void upgrade_by_rebuilding_limits_infrastructure_host() { int hostCount = 3; setMaxActiveUpgrades(hostCount); - var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud()); + var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud(), Optional.ofNullable(tester.hostProvisioner())); provisionInfraApplication(hostCount, NodeType.proxyhost); Supplier<NodeList> hosts = () -> tester.nodeRepository().nodes().list().nodeType(NodeType.proxyhost); @@ -515,6 +517,31 @@ public class OsVersionsTest { } } + @Test + public void skips_unavailable_version() { + MockHostProvisioner hostProvisioner = new MockHostProvisioner(List.of()); + ProvisioningTester tester = new ProvisioningTester.Builder().dynamicProvisioning(true, false).hostProvisioner(hostProvisioner).build(); + OsVersions versions = tester.nodeRepository().osVersions(); + tester.makeReadyHosts(1, new NodeResources(2,4,8,100)); + tester.activateTenantHosts(); + Supplier<Node> host = () -> tester.nodeRepository().nodes().list().nodeType(NodeType.host).first().get(); + tester.clock().advance(Duration.ofDays(1)); + + hostProvisioner.addOsVersion(Version.fromString("7.0")); + Version version0 = Version.fromString("8.0"); + versions.setTarget(NodeType.host, version0, false); + versions.resumeUpgradeOf(NodeType.host, true); + assertTrue("Upgrade is not triggered to unavailable version", host.get().status().osVersion().wanted().isEmpty()); + + // Version becomes available, but is not used until cache expires + hostProvisioner.addOsVersion(version0); + versions.resumeUpgradeOf(NodeType.host, true); + assertTrue(host.get().status().osVersion().wanted().isEmpty()); + versions.invalidate(); + versions.resumeUpgradeOf(NodeType.host, true); + assertEquals("Host upgrade is triggered", version0, host.get().status().osVersion().wanted().get()); + } + private void setMaxActiveUpgrades(int max) { tester.flagSource().withIntFlag(PermanentFlags.MAX_OS_UPGRADES.id(), max); } diff --git a/parent/pom.xml b/parent/pom.xml index 5fd89facdc5..6cc2902136b 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -460,6 +460,16 @@ <version>${huggingface.vespa.version}</version> </dependency> <dependency> + <groupId>ch.qos.logback</groupId> + <artifactId>logback-classic</artifactId> + <version>${logback.vespa.version}</version> + </dependency> + <dependency> + <groupId>ch.qos.logback</groupId> + <artifactId>logback-core</artifactId> + <version>${logback.vespa.version}</version> + </dependency> + <dependency> <groupId>com.amazonaws</groupId> <artifactId>aws-java-sdk-core</artifactId> <version>${aws-sdk.vespa.version}</version> @@ -979,7 +989,7 @@ <dependency> <groupId>org.codehaus.plexus</groupId> <artifactId>plexus-interpolation</artifactId> - <version>1.26</version> + <version>${plexus-interpolation.vespa.version}</version> </dependency> <dependency> <groupId>org.bouncycastle</groupId> diff --git a/searchlib/src/tests/query/streaming_query_test.cpp b/searchlib/src/tests/query/streaming_query_test.cpp index 306456518b7..08705fa837b 100644 --- a/searchlib/src/tests/query/streaming_query_test.cpp +++ b/searchlib/src/tests/query/streaming_query_test.cpp @@ -522,6 +522,7 @@ void assertInt64Range(const std::string &term, bool expAdjusted, int64_t expLow, EXPECT_EQ(expHigh, (int64_t)res.high); } + TEST(StreamingQueryTest, require_that_int8_limits_are_enforced) { //std::numeric_limits<int8_t>::min() -> -128 @@ -607,6 +608,20 @@ TEST(StreamingQueryTest, require_that_we_can_take_floating_point_values_in_range assertInt64Range("[1.7976931348623157E308;-1.7976931348623157E308]", false, std::numeric_limits<int64_t>::max(), std::numeric_limits<int64_t>::min()); } +void assertIllegalRangeQueries(const QueryTermSimple & qt) { + QueryTermSimple::RangeResult<int64_t> ires = qt.getRange<int64_t>(); + EXPECT_EQ(false, ires.valid); + QueryTermSimple::RangeResult<double> fres = qt.getRange<double>(); + EXPECT_EQ(false, fres.valid); +} + +TEST(StreamingQueryTest, require_safe_parsing_of_illegal_ranges) { + // The 2 below are created when naively splitting numeric terms by dot. + // T=A.B => T EQUIV PHRASE(A, B) + assertIllegalRangeQueries(QueryTermSimple("[1", TermType::WORD)); + assertIllegalRangeQueries(QueryTermSimple(".1;2.1]", TermType::WORD)); +} + TEST(StreamingQueryTest, require_that_we_handle_empty_range_as_expected) { assertInt64Range("[1;1]", false, 1, 1); diff --git a/searchlib/src/vespa/searchlib/query/query_term_simple.cpp b/searchlib/src/vespa/searchlib/query/query_term_simple.cpp index b7a1719fb5f..8df7764c183 100644 --- a/searchlib/src/vespa/searchlib/query/query_term_simple.cpp +++ b/searchlib/src/vespa/searchlib/query/query_term_simple.cpp @@ -259,46 +259,53 @@ template <typename T, typename D> bool QueryTermSimple::getAsNumericTerm(T & lower, T & upper, D d) const { - bool valid(empty()); + if (empty()) return false; + size_t sz(_term.size()); - if (sz) { - char *err(nullptr); - T low(lower); - T high(upper); - const char * q = _term.c_str(); - const char first(q[0]); - const char last(q[sz-1]); - q += ((first == '<') || (first == '>') || (first == '[')) ? 1 : 0; - T ll = d.fromstr(q, &err); - valid = isValid() && ((*err == 0) || (*err == ';')); - if (valid) { - if (first == '<' && (*err == 0)) { - high = d.nearestDownwd(ll, lower); - } else if (first == '>' && (*err == 0)) { - low = d.nearestUpward(ll, upper); - } else if ((first == '[') || (first == '<')) { - if (q != err) { - low = (first == '[') ? ll : d.nearestUpward(ll, upper); - } - q = err + 1; - T hh = d.fromstr(q, &err); - bool hasUpperLimit(q != err); - if (*err == ';') { - err = const_cast<char *>(_term.end() - 1); - } - valid = (*err == last) && ((last == ']') || (last == '>')); - if (hasUpperLimit) { - high = (last == ']') ? hh : d.nearestDownwd(hh, lower); - } - } else { - low = high = ll; - } + char *err(nullptr); + T low(lower); + T high(upper); + const char * q = _term.c_str(); + const char first(q[0]); + const char last(q[sz-1]); + bool isRange = (first == '<') || (first == '>') || (first == '['); + q += isRange ? 1 : 0; + T ll = d.fromstr(q, &err); + bool valid = isValid() && ((*err == 0) || (*err == ';')); + if (!valid) return false; + + if (*err == 0) { + if (first == '<') { + high = d.nearestDownwd(ll, lower); + } else if (first == '>') { + low = d.nearestUpward(ll, upper); + } else { + low = high = ll; + valid = ! isRange; } - if (valid) { - lower = low; - upper = high; + } else { + if ((first == '[') || (first == '<')) { + if (q != err) { + low = (first == '[') ? ll : d.nearestUpward(ll, upper); + } + q = err + 1; + T hh = d.fromstr(q, &err); + bool hasUpperLimit(q != err); + if (*err == ';') { + err = const_cast<char *>(_term.end() - 1); + } + valid = (*err == last) && ((last == ']') || (last == '>')); + if (hasUpperLimit) { + high = (last == ']') ? hh : d.nearestDownwd(hh, lower); + } + } else { + valid = false; } } + if (valid) { + lower = low; + upper = high; + } return valid; } diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index 66153cd3fda..ad044ceca01 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -4,8 +4,8 @@ ai.djl.huggingface:tokenizers:${huggingface.vespa.version} ai.djl:api:${huggingface.vespa.version} aopalliance:aopalliance:${aopalliance.vespa.version} backport-util-concurrent:backport-util-concurrent:3.1 -ch.qos.logback:logback-classic:1.2.10 -ch.qos.logback:logback-core:1.2.10 +ch.qos.logback:logback-classic:${logback.vespa.version} +ch.qos.logback:logback-core:${logback.vespa.version} classworlds:classworlds:1.1-alpha-2 com.amazonaws:aws-java-sdk-core:${aws-sdk.vespa.version} com.amazonaws:aws-java-sdk-kms:${aws-sdk.vespa.version} @@ -138,7 +138,7 @@ org.codehaus.plexus:plexus-archiver:4.8.0 org.codehaus.plexus:plexus-classworlds:2.7.0 org.codehaus.plexus:plexus-component-annotations:2.1.0 org.codehaus.plexus:plexus-container-default:1.0-alpha-9-stable-1 -org.codehaus.plexus:plexus-interpolation:1.26 +org.codehaus.plexus:plexus-interpolation:${plexus-interpolation.vespa.version} org.codehaus.plexus:plexus-io:${maven-enforcer-plugin.vespa.version} org.codehaus.plexus:plexus-utils:${maven-shade-plugin.vespa.version} org.eclipse.angus:angus-activation:${jakarta.inject.vespa.version} |