diff options
77 files changed, 490 insertions, 414 deletions
diff --git a/abi-check-plugin/pom.xml b/abi-check-plugin/pom.xml index 7f77dec7523..a0a1c52428d 100644 --- a/abi-check-plugin/pom.xml +++ b/abi-check-plugin/pom.xml @@ -45,7 +45,7 @@ </dependency> <dependency> <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-all</artifactId> + <artifactId>hamcrest</artifactId> <scope>test</scope> </dependency> <dependency> diff --git a/application/pom.xml b/application/pom.xml index 6e5c6426220..bb1ea4b30ee 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -117,11 +117,7 @@ </exclusion> <exclusion> <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-core</artifactId> - </exclusion> - <exclusion> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-library</artifactId> + <artifactId>*</artifactId> </exclusion> </exclusions> </dependency> diff --git a/client/js/app/package.json b/client/js/app/package.json index ec3da2dd941..23ddccbe7e7 100644 --- a/client/js/app/package.json +++ b/client/js/app/package.json @@ -20,9 +20,9 @@ "@fortawesome/free-regular-svg-icons": "^6", "@fortawesome/free-solid-svg-icons": "^6", "@fortawesome/react-fontawesome": "^0", - "@mantine/core": "^5", - "@mantine/hooks": "^5", - "@mantine/notifications": "^5", + "@mantine/core": "^6.0.0", + "@mantine/hooks": "^6.0.0", + "@mantine/notifications": "^6.0.0", "@vitejs/plugin-react": "^4", "esbuild-jest": "^0", "eslint": "^8", diff --git a/client/js/app/yarn.lock b/client/js/app/yarn.lock index 6eaf6b8ee56..df9465d2478 100644 --- a/client/js/app/yarn.lock +++ b/client/js/app/yarn.lock @@ -943,42 +943,43 @@ "@jridgewell/resolve-uri" "^3.1.0" "@jridgewell/sourcemap-codec" "^1.4.14" -"@mantine/core@^5": - version "5.10.5" - resolved "https://registry.yarnpkg.com/@mantine/core/-/core-5.10.5.tgz#071e14dcf8b94a36d0243f1f4b30305ac0074afd" - integrity sha512-F4tqHSEVM9D6/iSqHfPda+Xl5XgSEPHAAkT01Zwzj4Jnbd10qGrlqr/SFUop2CIcuKYnmra9XltUahUPXBC2BQ== +"@mantine/core@^6.0.0": + version "6.0.19" + resolved "https://registry.yarnpkg.com/@mantine/core/-/core-6.0.19.tgz#612413f0e8eb117e6a39068a625c6ccf2ae2ccdd" + integrity sha512-SvMZCOgCc315SIg6hkuLM0ZnBaAac4VFDHZ0BM5LIE4MPJUpe4QOLsg/5RGxOa5s7JRCtu/dawH3/9frvfDrhw== dependencies: "@floating-ui/react" "^0.19.1" - "@mantine/styles" "5.10.5" - "@mantine/utils" "5.10.5" + "@mantine/styles" "6.0.19" + "@mantine/utils" "6.0.19" "@radix-ui/react-scroll-area" "1.0.2" + react-remove-scroll "^2.5.5" react-textarea-autosize "8.3.4" -"@mantine/hooks@^5": - version "5.10.5" - resolved "https://registry.yarnpkg.com/@mantine/hooks/-/hooks-5.10.5.tgz#568586a0fa649be46f057ddc920bf98761017ffb" - integrity sha512-hFQp71QZDfivPzfIUOQZfMKLiOL/Cn2EnzacRlbUr55myteTfzYN8YMt+nzniE/6c4IRopFHEAdbKEtfyQc6kg== +"@mantine/hooks@^6.0.0": + version "6.0.19" + resolved "https://registry.yarnpkg.com/@mantine/hooks/-/hooks-6.0.19.tgz#39f61434304f687d3ba7bf0040c5adf380c7c4b3" + integrity sha512-YkmuB6kmoenU1PVuE8tLBA+6RJIY9hIsGyIQG1yuPAy6SLWNFT8g2T9YvI/psqsUbVIYGaNEXg8zq42xbxnD8Q== -"@mantine/notifications@^5": - version "5.10.5" - resolved "https://registry.yarnpkg.com/@mantine/notifications/-/notifications-5.10.5.tgz#2f3f2d013ce4637e64e935aa5dd8c1df1a7acec0" - integrity sha512-IzTAXA7Zb9DcI94Mv5O2OinhLmI7fvs/VutDw9uCpp6OHtLuF/XN1d262jrsGhMZT0c4nuUsotSLFZF3GWZwXg== +"@mantine/notifications@^6.0.0": + version "6.0.19" + resolved "https://registry.yarnpkg.com/@mantine/notifications/-/notifications-6.0.19.tgz#1aaf82fd4ac5b32faca8baf17475199d3d6ff58e" + integrity sha512-Cr2y8g2nM8bUAP+JYcKdT+a3d+1awUd40EMrDMwb+yUXUSt1amZerYQ7qRuezqvBgiViy/HGnM4WfeF3sfWJRQ== dependencies: - "@mantine/utils" "5.10.5" + "@mantine/utils" "6.0.19" react-transition-group "4.4.2" -"@mantine/styles@5.10.5": - version "5.10.5" - resolved "https://registry.yarnpkg.com/@mantine/styles/-/styles-5.10.5.tgz#ace82a71b4fe3d14ee14638f1735d5680d93d36d" - integrity sha512-0NXk8c/XGzuTUkZc6KceF2NaTCMEu5mHR4ru0x+ttb9DGnLpHuGWduTHjSfr4hl6eAJgedD0zauO+VAhDzO9zA== +"@mantine/styles@6.0.19": + version "6.0.19" + resolved "https://registry.yarnpkg.com/@mantine/styles/-/styles-6.0.19.tgz#7d9a6f2c2a9b345dfd9d12f8fd66af3976d67ab2" + integrity sha512-0tg3Dvv/kxCc1mbQVFhZaIhlSbSbV1F/3xG0NRlP2DF23mw9088o5KaIXGKM6XkXU6OEt/f99nDCUHBk2ixtUg== dependencies: clsx "1.1.1" csstype "3.0.9" -"@mantine/utils@5.10.5": - version "5.10.5" - resolved "https://registry.yarnpkg.com/@mantine/utils/-/utils-5.10.5.tgz#ad620d714e545c6efb7f69d94ce46e3fd2fe01fb" - integrity sha512-FGMq4dGs5HhDAtI0z46uzxzKKPmZ3h5uKUyKg1ZHoFR1mBtcUMbB6FylFmHqKFRWlJ5IXqX9dwmiVrLYUOfTmA== +"@mantine/utils@6.0.19": + version "6.0.19" + resolved "https://registry.yarnpkg.com/@mantine/utils/-/utils-6.0.19.tgz#0197fccc5649259787d5468228139f8815909803" + integrity sha512-duvtnaW1gDR2gnvUqnWhl6DMW7sN0HEWqS8Z/BbwaMi75U+Xp17Q72R9JtiIrxQbzsq+KvH9L9B/pxMVwbLirg== "@nodelib/fs.scandir@2.1.5": version "2.1.5" @@ -1973,6 +1974,11 @@ detect-newline@^3.0.0: resolved "https://registry.yarnpkg.com/detect-newline/-/detect-newline-3.1.0.tgz#576f5dfc63ae1a192ff192d8ad3af6308991b651" integrity sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA== +detect-node-es@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/detect-node-es/-/detect-node-es-1.1.0.tgz#163acdf643330caa0b4cd7c21e7ee7755d6fa493" + integrity sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ== + diff-sequences@^29.6.3: version "29.6.3" resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-29.6.3.tgz#4deaf894d11407c51efc8418012f9e70b84ea921" @@ -2660,6 +2666,11 @@ get-intrinsic@^1.0.2, get-intrinsic@^1.1.1, get-intrinsic@^1.1.3, get-intrinsic@ has-proto "^1.0.1" has-symbols "^1.0.3" +get-nonce@^1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/get-nonce/-/get-nonce-1.0.1.tgz#fdf3f0278073820d2ce9426c18f07481b1e0cdf3" + integrity sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q== + get-package-type@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/get-package-type/-/get-package-type-0.1.0.tgz#8de2d803cff44df3bc6c456e6668b36c3926e11a" @@ -2916,6 +2927,13 @@ internal-slot@^1.0.3, internal-slot@^1.0.5: has "^1.0.3" side-channel "^1.0.4" +invariant@^2.2.4: + version "2.2.4" + resolved "https://registry.yarnpkg.com/invariant/-/invariant-2.2.4.tgz#610f3c92c9359ce1db616e538008d23ff35158e6" + integrity sha512-phJfQVBuaJM5raOpJjSfkiD6BpbCE4Ns//LaXl6wGYtUBY83nWS6Rf9tXm2e8VaK60JEjYldbPif/A2B1C2gNA== + dependencies: + loose-envify "^1.0.0" + is-accessor-descriptor@^0.1.6: version "0.1.6" resolved "https://registry.yarnpkg.com/is-accessor-descriptor/-/is-accessor-descriptor-0.1.6.tgz#a9e12cb3ae8d876727eeef3843f8a0897b5c98d6" @@ -3878,7 +3896,7 @@ lodash@^4: resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c" integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg== -loose-envify@^1.1.0, loose-envify@^1.4.0: +loose-envify@^1.0.0, loose-envify@^1.1.0, loose-envify@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/loose-envify/-/loose-envify-1.4.0.tgz#71ee51fa7be4caec1a63839f7e682d8132d30caf" integrity sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q== @@ -4378,9 +4396,9 @@ prettier-linter-helpers@^1.0.0: fast-diff "^1.1.2" prettier@3: - version "3.0.2" - resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.0.2.tgz#78fcecd6d870551aa5547437cdae39d4701dca5b" - integrity sha512-o2YR9qtniXvwEZlOKbveKfDQVyqxbEIWn48Z8m3ZJjBjcCmUy3xZGIv+7AkaeuaTr6yPXJjwv07ZWlsWbEy1rQ== + version "3.0.3" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.0.3.tgz#432a51f7ba422d1469096c0fdc28e235db8f9643" + integrity sha512-L/4pUDMxcNa8R/EthV08Zt42WBO4h1rarVtK0K+QJG0X187OLo7l699jWw0GKuwzkPQ//jMFA/8Xm6Fh3J/DAg== pretty-format@^29.6.3: version "29.6.3" @@ -4466,6 +4484,25 @@ react-refresh@^0.14.0: resolved "https://registry.yarnpkg.com/react-refresh/-/react-refresh-0.14.0.tgz#4e02825378a5f227079554d4284889354e5f553e" integrity sha512-wViHqhAd8OHeLS/IRMJjTSDHF3U9eWi62F/MledQGPdJGDhodXJ9PBLNGr6WWL7qlH12Mt3TyTpbS+hGXMjCzQ== +react-remove-scroll-bar@^2.3.4: + version "2.3.4" + resolved "https://registry.yarnpkg.com/react-remove-scroll-bar/-/react-remove-scroll-bar-2.3.4.tgz#53e272d7a5cb8242990c7f144c44d8bd8ab5afd9" + integrity sha512-63C4YQBUt0m6ALadE9XV56hV8BgJWDmmTPY758iIJjfQKt2nYwoUrPk0LXRXcB/yIj82T1/Ixfdpdk68LwIB0A== + dependencies: + react-style-singleton "^2.2.1" + tslib "^2.0.0" + +react-remove-scroll@^2.5.5: + version "2.5.6" + resolved "https://registry.yarnpkg.com/react-remove-scroll/-/react-remove-scroll-2.5.6.tgz#7510b8079e9c7eebe00e65a33daaa3aa29a10336" + integrity sha512-bO856ad1uDYLefgArk559IzUNeQ6SWH4QnrevIUjH+GczV56giDfl3h0Idptf2oIKxQmd1p9BN25jleKodTALg== + dependencies: + react-remove-scroll-bar "^2.3.4" + react-style-singleton "^2.2.1" + tslib "^2.1.0" + use-callback-ref "^1.3.0" + use-sidecar "^1.1.2" + react-router-dom@^6: version "6.15.0" resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.15.0.tgz#6da7db61e56797266fbbef0d5e324d6ac443ee40" @@ -4481,6 +4518,15 @@ react-router@6.15.0: dependencies: "@remix-run/router" "1.8.0" +react-style-singleton@^2.2.1: + version "2.2.1" + resolved "https://registry.yarnpkg.com/react-style-singleton/-/react-style-singleton-2.2.1.tgz#f99e420492b2d8f34d38308ff660b60d0b1205b4" + integrity sha512-ZWj0fHEMyWkHzKYUr2Bs/4zU6XLmq9HsgBURm7g5pAVfyn49DgUiNgY2d4lXRlYSiCif9YBGpQleewkcqddc7g== + dependencies: + get-nonce "^1.0.0" + invariant "^2.2.4" + tslib "^2.0.0" + react-textarea-autosize@8.3.4: version "8.3.4" resolved "https://registry.yarnpkg.com/react-textarea-autosize/-/react-textarea-autosize-8.3.4.tgz#270a343de7ad350534141b02c9cb78903e553524" @@ -5092,6 +5138,11 @@ tslib@^2.0.0, tslib@^2.5.0, tslib@^2.6.0: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.0.tgz#b295854684dbda164e181d259a22cd779dcd7bc3" integrity sha512-7At1WUettjcSRHXCyYtTselblcHl9PJFFVKiCAy/bY97+BPZXSQ2wbq0P9s8tK2G7dFQfNnlJnPAiArVBVBsfA== +tslib@^2.1.0: + version "2.6.2" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" + integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== + type-check@^0.4.0, type-check@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1" @@ -5213,6 +5264,13 @@ urix@^0.1.0: resolved "https://registry.yarnpkg.com/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72" integrity sha512-Am1ousAhSLBeB9cG/7k7r2R0zj50uDRlZHPGbazid5s9rlF1F/QKYObEKSIunSjIOkJZqwRRLpvewjEkM7pSqg== +use-callback-ref@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/use-callback-ref/-/use-callback-ref-1.3.0.tgz#772199899b9c9a50526fedc4993fc7fa1f7e32d5" + integrity sha512-3FT9PRuRdbB9HfXhEq35u4oZkvpJ5kuYbpqhCfmiZyReuRgpnhDlbr2ZEnnuS0RrJAPn6l23xjFg9kpDM+Ms7w== + dependencies: + tslib "^2.0.0" + use-composed-ref@^1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/use-composed-ref/-/use-composed-ref-1.3.0.tgz#3d8104db34b7b264030a9d916c5e94fbe280dbda" @@ -5235,6 +5293,14 @@ use-latest@^1.2.1: dependencies: use-isomorphic-layout-effect "^1.1.1" +use-sidecar@^1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/use-sidecar/-/use-sidecar-1.1.2.tgz#2f43126ba2d7d7e117aa5855e5d8f0276dfe73c2" + integrity sha512-epTbsLuzZ7lPClpz2TyryBfztm7m+28DlEv2ZCQ3MDr5ssiwyOwGH/e5F9CkfWjJ1t4clvI58yF822/GUkjjhw== + dependencies: + detect-node-es "^1.1.0" + tslib "^2.0.0" + use@^3.1.0: version "3.1.1" resolved "https://registry.yarnpkg.com/use/-/use-3.1.1.tgz#d50c8cac79a19fbc20f2911f56eb973f4e10070f" diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index a13d094d19c..1b39095bc55 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -185,6 +185,7 @@ <include>org.eclipse.jetty:jetty-servlet:${jetty.vespa.version}:test</include> <include>org.eclipse.jetty:jetty-util:${jetty.vespa.version}:test</include> + <include>org.hamcrest:hamcrest:${hamcrest.vespa.version}:test</include> <include>org.hamcrest:hamcrest-core:${hamcrest.vespa.version}:test</include> <include>org.hdrhistogram:HdrHistogram:${hdrhistogram.vespa.version}:test</include> <include>org.json:json:${org.json.vespa.version}:test</include> diff --git a/clustercontroller-core/pom.xml b/clustercontroller-core/pom.xml index 6fc29fc811c..2f5b95b1ce1 100644 --- a/clustercontroller-core/pom.xml +++ b/clustercontroller-core/pom.xml @@ -43,7 +43,7 @@ </dependency> <dependency> <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-library</artifactId> + <artifactId>hamcrest</artifactId> <scope>test</scope> </dependency> <dependency> diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownTest.java index 0f9fad2c76c..fafdbbaa121 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/GroupAutoTakedownTest.java @@ -18,7 +18,7 @@ import static com.yahoo.vespa.clustercontroller.core.matchers.EventForNode.event import static com.yahoo.vespa.clustercontroller.core.matchers.NodeEventWithDescription.nodeEventWithDescription; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.core.AllOf.allOf; -import static org.hamcrest.core.IsCollectionContaining.hasItem; +import static org.hamcrest.core.IsIterableContaining.hasItem; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/ClusterEventWithDescription.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/ClusterEventWithDescription.java index 9dc2fd97718..baaeeccf12c 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/ClusterEventWithDescription.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/ClusterEventWithDescription.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.clustercontroller.core.matchers; import com.yahoo.vespa.clustercontroller.core.ClusterEvent; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; -import org.hamcrest.Factory; public class ClusterEventWithDescription extends BaseMatcher<ClusterEvent> { private final String expected; @@ -35,7 +34,6 @@ public class ClusterEventWithDescription extends BaseMatcher<ClusterEvent> { description.appendText(String.format("got description '%s'", other.getDescription())); } - @Factory public static ClusterEventWithDescription clusterEventWithDescription(String description) { return new ClusterEventWithDescription(description); } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/EventForNode.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/EventForNode.java index 8ce0b52ee8b..55be4dbd709 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/EventForNode.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/EventForNode.java @@ -5,7 +5,6 @@ import com.yahoo.vdslib.state.Node; import com.yahoo.vespa.clustercontroller.core.NodeEvent; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; -import org.hamcrest.Factory; public class EventForNode extends BaseMatcher<NodeEvent> { private final Node expected; @@ -30,7 +29,6 @@ public class EventForNode extends BaseMatcher<NodeEvent> { description.appendText(String.format("got node %s", other.getNode().getNode())); } - @Factory public static EventForNode eventForNode(Node expected) { return new EventForNode(expected); } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/EventTimeIs.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/EventTimeIs.java index 42a89b301fe..bcc02e1f4db 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/EventTimeIs.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/EventTimeIs.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.clustercontroller.core.matchers; import com.yahoo.vespa.clustercontroller.core.Event; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; -import org.hamcrest.Factory; public class EventTimeIs extends BaseMatcher<Event> { private final long expected; @@ -32,7 +31,6 @@ public class EventTimeIs extends BaseMatcher<Event> { description.appendText(String.format("event time is %d", other.getTimeMs())); } - @Factory public static EventTimeIs eventTimeIs(long time) { return new EventTimeIs(time); } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/EventTypeIs.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/EventTypeIs.java index 59a532763b0..228723e2fef 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/EventTypeIs.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/EventTypeIs.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.clustercontroller.core.matchers; import com.yahoo.vespa.clustercontroller.core.NodeEvent; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; -import org.hamcrest.Factory; public class EventTypeIs extends BaseMatcher<NodeEvent> { private final NodeEvent.Type expected; @@ -32,7 +31,6 @@ public class EventTypeIs extends BaseMatcher<NodeEvent> { description.appendText(String.format("got description '%s'", other.getDescription())); } - @Factory public static EventTypeIs eventTypeIs(NodeEvent.Type type) { return new EventTypeIs(type); } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/HasMetricContext.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/HasMetricContext.java index 92123752465..95a1d9f1a28 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/HasMetricContext.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/HasMetricContext.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.clustercontroller.core.matchers; import com.yahoo.vespa.clustercontroller.utils.util.MetricReporter; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; -import org.hamcrest.Factory; import java.util.Map; import java.util.TreeMap; @@ -64,7 +63,6 @@ public class HasMetricContext extends BaseMatcher<MetricReporter.Context> { return new Dimension(name, value); } - @Factory public static HasMetricContext hasMetricContext(Dimension... dimensions) { return new HasMetricContext(Stream.of(dimensions).collect(Collectors.toMap(dim -> dim.name, dim -> dim.value))); } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/HasStateReasonForNode.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/HasStateReasonForNode.java index 72f51e21c5a..bf87f58692b 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/HasStateReasonForNode.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/HasStateReasonForNode.java @@ -5,7 +5,6 @@ import com.yahoo.vdslib.state.Node; import com.yahoo.vespa.clustercontroller.core.NodeStateReason; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; -import org.hamcrest.Factory; import java.util.Map; @@ -42,7 +41,6 @@ public class HasStateReasonForNode extends BaseMatcher<Map<Node, NodeStateReason } } - @Factory public static HasStateReasonForNode hasStateReasonForNode(Node node, NodeStateReason reason) { return new HasStateReasonForNode(node, reason); } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/NodeEventForBucketSpace.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/NodeEventForBucketSpace.java index e479e031f10..9951bf50f5c 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/NodeEventForBucketSpace.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/NodeEventForBucketSpace.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.clustercontroller.core.matchers; import com.yahoo.vespa.clustercontroller.core.NodeEvent; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; -import org.hamcrest.Factory; import java.util.Optional; @@ -34,12 +33,10 @@ public class NodeEventForBucketSpace extends BaseMatcher<NodeEvent> { description.appendText(String.format("got bucket space '%s'", other.getBucketSpace().orElse("null"))); } - @Factory public static NodeEventForBucketSpace nodeEventForBucketSpace(String bucketSpace) { return new NodeEventForBucketSpace(Optional.of(bucketSpace)); } - @Factory public static NodeEventForBucketSpace nodeEventForBaseline() { return new NodeEventForBucketSpace(Optional.empty()); } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/NodeEventWithDescription.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/NodeEventWithDescription.java index b070fa27817..ca5720797e2 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/NodeEventWithDescription.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/NodeEventWithDescription.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.clustercontroller.core.matchers; import com.yahoo.vespa.clustercontroller.core.NodeEvent; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; -import org.hamcrest.Factory; public class NodeEventWithDescription extends BaseMatcher<NodeEvent> { private final String expected; @@ -35,7 +34,6 @@ public class NodeEventWithDescription extends BaseMatcher<NodeEvent> { description.appendText(String.format("got description '%s'", other.getDescription())); } - @Factory public static NodeEventWithDescription nodeEventWithDescription(String description) { return new NodeEventWithDescription(description); } diff --git a/clustercontroller-reindexer/pom.xml b/clustercontroller-reindexer/pom.xml index c94b5b134fe..67321197607 100644 --- a/clustercontroller-reindexer/pom.xml +++ b/clustercontroller-reindexer/pom.xml @@ -58,11 +58,7 @@ </exclusion> <exclusion> <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-core</artifactId> - </exclusion> - <exclusion> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-library</artifactId> + <artifactId>*</artifactId> </exclusion> </exclusions> </dependency> diff --git a/config-model/pom.xml b/config-model/pom.xml index bd1e068b61a..7c6fb610bae 100644 --- a/config-model/pom.xml +++ b/config-model/pom.xml @@ -32,7 +32,7 @@ </dependency> <dependency> <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-all</artifactId> + <artifactId>hamcrest</artifactId> <scope>test</scope> </dependency> <dependency> diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 80b676159cb..e7692aeee7b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -755,8 +755,13 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { Element modelsElement = XML.getChild(onnxElement, "models"); for (Element modelElement : XML.getChildren(modelsElement, "model") ) { OnnxModel onnxModel = profiles.getOnnxModels().asMap().get(modelElement.getAttribute("name")); - if (onnxModel == null) - continue; // Skip if model is not found + if (onnxModel == null) { + String availableModels = String.join(", ", profiles.getOnnxModels().asMap().keySet()); + context.getDeployState().getDeployLogger().logApplicationPackage(WARNING, + "Model '" + modelElement.getAttribute("name") + "' not found. Available ONNX " + + "models are: " + availableModels + ". Skipping this configuration."); + continue; + } onnxModel.setStatelessExecutionMode(getStringValue(modelElement, "execution-mode", null)); onnxModel.setStatelessInterOpThreads(getIntValue(modelElement, "interop-threads", -1)); onnxModel.setStatelessIntraOpThreads(getIntValue(modelElement, "intraop-threads", -1)); diff --git a/config-provisioning/pom.xml b/config-provisioning/pom.xml index e405f1192b1..9b10fedd79b 100644 --- a/config-provisioning/pom.xml +++ b/config-provisioning/pom.xml @@ -64,11 +64,7 @@ </exclusion> <exclusion> <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-core</artifactId> - </exclusion> - <exclusion> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-library</artifactId> + <artifactId>*</artifactId> </exclusion> </exclusions> </dependency> diff --git a/config/pom.xml b/config/pom.xml index ae80d00923b..9f3ec28b54b 100755 --- a/config/pom.xml +++ b/config/pom.xml @@ -76,7 +76,7 @@ </dependency> <dependency> <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-core</artifactId> + <artifactId>hamcrest</artifactId> <scope>test</scope> </dependency> <dependency> diff --git a/configserver-flags/pom.xml b/configserver-flags/pom.xml index e9c0c68685c..63525805669 100644 --- a/configserver-flags/pom.xml +++ b/configserver-flags/pom.xml @@ -91,11 +91,7 @@ </exclusion> <exclusion> <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-core</artifactId> - </exclusion> - <exclusion> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-library</artifactId> + <artifactId>*</artifactId> </exclusion> </exclusions> </dependency> diff --git a/configserver/pom.xml b/configserver/pom.xml index 7c94464864c..ece1e497d44 100644 --- a/configserver/pom.xml +++ b/configserver/pom.xml @@ -86,7 +86,7 @@ <dependency> <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-all</artifactId> + <artifactId>hamcrest</artifactId> <scope>test</scope> </dependency> <dependency> diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabase.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabase.java index 70b8dc5f51a..38be13894e2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabase.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabase.java @@ -136,10 +136,7 @@ public class ApplicationCuratorDatabase { * Returns Optional.empty() if application not found or no active session exists. */ public Optional<Long> activeSessionOf(ApplicationId id) { - Optional<byte[]> data = curator.getData(applicationPath(id)); - return (data.isEmpty() || data.get().length == 0) - ? Optional.empty() - : data.map(bytes -> Long.parseLong(Utf8.toString(bytes))); + return applicationData(id).flatMap(ApplicationData::activeSession); } /** @@ -147,25 +144,13 @@ public class ApplicationCuratorDatabase { * Returns Optional.empty() if application not found or no application data exists. */ public Optional<ApplicationData> applicationData(ApplicationId id) { - return applicationData(id, false); - } - - /** - * Returns application data for the given application. - * Returns Optional.empty() if application not found or no application data exists. - */ - public Optional<ApplicationData> applicationData(ApplicationId id, boolean readAsJson) { Optional<byte[]> data = curator.getData(applicationPath(id)); if (data.isEmpty() || data.get().length == 0) return Optional.empty(); - if (readAsJson) { - try { - return Optional.of(ApplicationData.fromBytes(data.get())); - } catch (IllegalArgumentException e) { - return applicationDataOldFormat(id, readAsJson); - } - } else { - return applicationDataOldFormat(id, readAsJson); + try { + return Optional.of(ApplicationData.fromBytes(data.get())); + } catch (IllegalArgumentException e) { + return applicationDataOldFormat(id); } } @@ -173,7 +158,7 @@ public class ApplicationCuratorDatabase { * Returns application data for the given application. * Returns Optional.empty() if application not found or no application data exists. */ - public Optional<ApplicationData> applicationDataOldFormat(ApplicationId id, boolean readAsJson) { + public Optional<ApplicationData> applicationDataOldFormat(ApplicationId id) { Optional<byte[]> data = curator.getData(applicationPath(id)); if (data.isEmpty() || data.get().length == 0) return Optional.empty(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationData.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationData.java index 868ea060fba..31e54bd67a0 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationData.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ApplicationData.java @@ -6,6 +6,7 @@ import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import java.io.IOException; +import java.util.Optional; import java.util.OptionalLong; import static com.yahoo.slime.SlimeUtils.optionalLong; @@ -54,9 +55,17 @@ public class ApplicationData { public ApplicationId applicationId() { return applicationId; } - public OptionalLong activeSession() { return activeSession; } + public Optional<Long> activeSession() { + return Optional.of(activeSession) + .filter(OptionalLong::isPresent) + .map(OptionalLong::getAsLong); + } - public OptionalLong lastDeployedSession() { return lastDeployedSession; } + public Optional<Long> lastDeployedSession() { + return Optional.of(lastDeployedSession) + .filter(OptionalLong::isPresent) + .map(OptionalLong::getAsLong); + } @Override public String toString() { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java index 538534d0040..1d1ed1042ee 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java @@ -80,7 +80,6 @@ public class TenantApplications implements RequestHandler, HostValidator { private final String serverId; private final ListFlag<String> incompatibleVersions; private final BooleanFlag writeApplicationDataAsJson; - private final BooleanFlag readApplicationDataAsJson; public TenantApplications(TenantName tenant, Curator curator, StripedExecutor<TenantName> zkWatcherExecutor, ExecutorService zkCacheExecutor, Metrics metrics, ConfigActivationListener configActivationListener, @@ -103,7 +102,6 @@ public class TenantApplications implements RequestHandler, HostValidator { this.serverId = configserverConfig.serverId(); this.incompatibleVersions = PermanentFlags.INCOMPATIBLE_VERSIONS.bindTo(flagSource); this.writeApplicationDataAsJson = Flags.WRITE_APPLICATION_DATA_AS_JSON.bindTo(flagSource); - this.readApplicationDataAsJson = Flags.READ_APPLICATION_DATA_AS_JSON.bindTo(flagSource); } /** The curator backed ZK storage of this. */ @@ -135,7 +133,7 @@ public class TenantApplications implements RequestHandler, HostValidator { * Returns Optional.empty if application not found or no application data exists. */ public Optional<ApplicationData> applicationData(ApplicationId id) { - return database().applicationData(id, readApplicationDataAsJson.value()); + return database().applicationData(id); } public boolean sessionExistsInFileSystem(long sessionId) { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabaseTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabaseTest.java index 8d1b22c94c5..80f705dcbad 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabaseTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/ApplicationCuratorDatabaseTest.java @@ -51,15 +51,8 @@ public class ApplicationCuratorDatabaseTest { assertEquals(Optional.empty(), db.applicationData(id)); // still empty, as no data has been written to node db.createApplication(id, true); - try { - Optional<ApplicationData> applicationData = db.applicationData(id); - fail("Expected exception, got " + applicationData); - } catch (NumberFormatException e) { - // expected - } - // Can be read as json, but no active session or last deployed session - Optional<ApplicationData> applicationData = db.applicationData(id, true); + Optional<ApplicationData> applicationData = db.applicationData(id); assertTrue(applicationData.isPresent()); assertEquals(id, applicationData.get().applicationId()); assertFalse(applicationData.get().activeSession().isPresent()); @@ -74,11 +67,18 @@ public class ApplicationCuratorDatabaseTest { t.commit(); } // Can be read as session id only - applicationData = db.applicationData(id, false); + applicationData = db.applicationData(id); + assertTrue(applicationData.isPresent()); + assertEquals(id, applicationData.get().applicationId()); + assertTrue(applicationData.get().activeSession().isPresent()); + assertEquals(2, applicationData.get().activeSession().get().longValue()); + assertFalse(applicationData.get().lastDeployedSession().isPresent()); + // Can be read as session data as well + applicationData = db.applicationData(id); assertTrue(applicationData.isPresent()); assertEquals(id, applicationData.get().applicationId()); assertTrue(applicationData.get().activeSession().isPresent()); - assertEquals(2, applicationData.get().activeSession().getAsLong()); + assertEquals(2, applicationData.get().activeSession().get().longValue()); assertFalse(applicationData.get().lastDeployedSession().isPresent()); // Prepare session 3, last deployed session is still 2 @@ -86,25 +86,25 @@ public class ApplicationCuratorDatabaseTest { t.commit(); } // Can be read as json, active session is still 2 and last deployed session is 3 - applicationData = db.applicationData(id, true); + applicationData = db.applicationData(id); assertTrue(applicationData.isPresent()); assertEquals(id, applicationData.get().applicationId()); assertTrue(applicationData.get().activeSession().isPresent()); - assertEquals(2, applicationData.get().activeSession().getAsLong()); + assertEquals(2L, applicationData.get().activeSession().get().longValue()); assertTrue(applicationData.get().lastDeployedSession().isPresent()); - assertEquals(3, applicationData.get().lastDeployedSession().getAsLong()); + assertEquals(3, applicationData.get().lastDeployedSession().get().longValue()); try (var t = db.createWriteActiveTransaction(new CuratorTransaction(curator), id, 3, true)) { t.commit(); } // Can be read as json, active session and last deployed session present - applicationData = db.applicationData(id, true); + applicationData = db.applicationData(id); assertTrue(applicationData.isPresent()); assertEquals(id, applicationData.get().applicationId()); assertTrue(applicationData.get().activeSession().isPresent()); - assertEquals(3, applicationData.get().activeSession().getAsLong()); + assertEquals(3, applicationData.get().activeSession().get().longValue()); assertTrue(applicationData.get().lastDeployedSession().isPresent()); - assertEquals(3, applicationData.get().lastDeployedSession().getAsLong()); + assertEquals(3, applicationData.get().lastDeployedSession().get().longValue()); } } diff --git a/container-core/pom.xml b/container-core/pom.xml index eec8b60077b..b3794142d0e 100644 --- a/container-core/pom.xml +++ b/container-core/pom.xml @@ -395,13 +395,8 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.cthul</groupId> - <artifactId>cthul-matchers</artifactId> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-library</artifactId> + <artifactId>hamcrest</artifactId> <scope>test</scope> </dependency> <dependency> diff --git a/container-core/src/test/java/com/yahoo/container/logging/CircularArrayAccessLogKeeperTest.java b/container-core/src/test/java/com/yahoo/container/logging/CircularArrayAccessLogKeeperTest.java index f942ee588be..01bbe0d009c 100644 --- a/container-core/src/test/java/com/yahoo/container/logging/CircularArrayAccessLogKeeperTest.java +++ b/container-core/src/test/java/com/yahoo/container/logging/CircularArrayAccessLogKeeperTest.java @@ -6,7 +6,7 @@ import org.junit.jupiter.api.Test; import static org.hamcrest.CoreMatchers.hasItems; import static org.hamcrest.Matchers.contains; import static org.hamcrest.core.Is.is; -import static org.hamcrest.core.IsCollectionContaining.hasItem; +import static org.hamcrest.core.IsIterableContaining.hasItem; import static org.hamcrest.core.IsNot.not; import static org.hamcrest.MatcherAssert.assertThat; diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerConformanceTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerConformanceTest.java index ae1a6494acd..cbe21d5581b 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerConformanceTest.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerConformanceTest.java @@ -43,13 +43,12 @@ import static com.yahoo.jdisc.Response.Status.NOT_FOUND; import static com.yahoo.jdisc.Response.Status.OK; import static org.apache.http.HttpStatus.SC_INTERNAL_SERVER_ERROR; import static org.apache.http.HttpStatus.SC_NOT_FOUND; -import static org.cthul.matchers.CthulMatchers.containsPattern; -import static org.cthul.matchers.CthulMatchers.matchesPattern; import static org.hamcrest.CoreMatchers.any; import static org.hamcrest.CoreMatchers.anyOf; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.matchesPattern; /** * @author Simon Thoresen Hult @@ -119,8 +118,8 @@ public class HttpServerConformanceTest extends ServerProviderConformanceTest { @Override @Test public void testBindingNotFoundException() throws Throwable { - final Pattern contentPattern = Pattern.compile("No binding for URI 'http://.+/status.html'\\."); - new TestRunner().expect(errorWithReason(is(NOT_FOUND), containsPattern(contentPattern))) + final Pattern contentPattern = Pattern.compile(".*No binding for URI 'http://.+/status.html'\\."); + new TestRunner().expect(errorWithReason(is(NOT_FOUND), matchesPattern(contentPattern))) .execute(); } diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java index 6f9c854be64..97237e1c016 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/HttpServerTest.java @@ -84,13 +84,12 @@ import static com.yahoo.jdisc.http.server.jetty.SimpleHttpClient.ResponseValidat import static com.yahoo.jdisc.http.server.jetty.Utils.createHttp2Client; import static com.yahoo.jdisc.http.server.jetty.Utils.createSslTestDriver; import static com.yahoo.jdisc.http.server.jetty.Utils.generatePrivateKeyAndCertificate; -import static org.cthul.matchers.CthulMatchers.containsPattern; -import static org.cthul.matchers.CthulMatchers.matchesPattern; import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.startsWith; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.matchesPattern; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -143,7 +142,7 @@ public class HttpServerTest { newBindingSetSelector("unknown")); driver.client().get("/status.html") .expectStatusCode(is(NOT_FOUND)) - .expectContent(containsPattern(Pattern.compile( + .expectContent(matchesPattern(Pattern.compile(".*" + Pattern.quote(BindingSetNotFoundException.class.getName()) + ": No binding set named 'unknown'\\.\n\tat .+", Pattern.DOTALL | Pattern.MULTILINE))); diff --git a/container-dependencies-enforcer/pom.xml b/container-dependencies-enforcer/pom.xml index e633df304fa..2706b95cfb1 100644 --- a/container-dependencies-enforcer/pom.xml +++ b/container-dependencies-enforcer/pom.xml @@ -195,6 +195,7 @@ <include>org.eclipse.jetty:jetty-server:${jetty.vespa.version}:test</include> <include>org.eclipse.jetty:jetty-servlet:${jetty.vespa.version}:test</include> <include>org.eclipse.jetty:jetty-util:${jetty.vespa.version}:test</include> + <include>org.hamcrest:hamcrest:${hamcrest.vespa.version}:test</include> <include>org.hamcrest:hamcrest-core:${hamcrest.vespa.version}:test</include> <include>org.hdrhistogram:HdrHistogram:${hdrhistogram.vespa.version}:test</include> <include>org.json:json:${org.json.vespa.version}:test</include> <!-- TODO: Remove on Vespa 9 --> diff --git a/container-disc/pom.xml b/container-disc/pom.xml index 4ec141d2758..65a03b48c59 100644 --- a/container-disc/pom.xml +++ b/container-disc/pom.xml @@ -32,11 +32,7 @@ </exclusion> <exclusion> <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-core</artifactId> - </exclusion> - <exclusion> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-library</artifactId> + <artifactId>*</artifactId> </exclusion> </exclusions> </dependency> diff --git a/container-search/pom.xml b/container-search/pom.xml index cea05826ce0..31ce9a00e44 100644 --- a/container-search/pom.xml +++ b/container-search/pom.xml @@ -157,11 +157,7 @@ </exclusion> <exclusion> <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-core</artifactId> - </exclusion> - <exclusion> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-library</artifactId> + <artifactId>*</artifactId> </exclusion> </exclusions> </dependency> diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java index 62b56e6e8ff..4956698cc2f 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/LoadBalancerTest.java @@ -7,7 +7,6 @@ import com.yahoo.search.dispatch.LoadBalancer.GroupStatus; import com.yahoo.search.dispatch.searchcluster.Group; import com.yahoo.search.dispatch.searchcluster.Node; import org.junit.jupiter.api.Test; -import org.opentest4j.AssertionFailedError; import java.time.Duration; import java.time.Instant; @@ -34,7 +33,7 @@ public class LoadBalancerTest { Optional<Group> grp = lb.takeGroup(null); Group group = grp.orElseThrow(() -> { - throw new AssertionFailedError("Expected a SearchCluster.Group"); + throw new IllegalStateException("Expected a SearchCluster.Group"); }); assertEquals(1, group.nodes().size()); } @@ -47,7 +46,7 @@ public class LoadBalancerTest { Optional<Group> grp = lb.takeGroup(null); Group group = grp.orElseThrow(() -> { - throw new AssertionFailedError("Expected a SearchCluster.Group"); + throw new IllegalStateException("Expected a SearchCluster.Group"); }); assertEquals(1, group.nodes().size()); } diff --git a/container-search/src/test/java/com/yahoo/search/query/SortingTestCase.java b/container-search/src/test/java/com/yahoo/search/query/SortingTestCase.java index 8422196638a..b325bde05d9 100644 --- a/container-search/src/test/java/com/yahoo/search/query/SortingTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/query/SortingTestCase.java @@ -81,8 +81,8 @@ public class SortingTestCase { private void requireThatChineseHasCorrectRules(Collator col) { final int reorderCodes [] = {UScript.HAN}; - assertEquals("14.0.0.0", col.getUCAVersion().toString()); - assertEquals("153.112.40.0", col.getVersion().toString()); + assertEquals("15.0.0.0", col.getUCAVersion().toString()); + assertEquals("153.120.43.8", col.getVersion().toString()); assertEquals(Arrays.toString(reorderCodes), Arrays.toString(col.getReorderCodes())); assertNotEquals("", ((RuleBasedCollator) col).getRules()); diff --git a/controller-server/pom.xml b/controller-server/pom.xml index 64cc89c3321..5e738a2dd4a 100644 --- a/controller-server/pom.xml +++ b/controller-server/pom.xml @@ -184,11 +184,7 @@ </exclusion> <exclusion> <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-core</artifactId> - </exclusion> - <exclusion> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-library</artifactId> + <artifactId>*</artifactId> </exclusion> </exclusions> </dependency> diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index d1ff38a28d0..5ff4513cdc6 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -72,28 +72,30 @@ find zkfacade/src/main/java/org/apache/curator -name package-info.java | \ xargs perl -pi -e 's/major = [0-9]+, minor = [0-9]+, micro = [0-9]+/major = 5, minor = 3, micro = 0/g' --> - <bouncycastle.vespa.version>1.74</bouncycastle.vespa.version> + <bouncycastle.vespa.version>1.76</bouncycastle.vespa.version> <curator.vespa.version>5.4.0</curator.vespa.version> - <commons-codec.vespa.version>1.15</commons-codec.vespa.version> + <commons-codec.vespa.version>1.16.0</commons-codec.vespa.version> + <commons-csv.vespa.version>1.10.0</commons-csv.vespa.version> <commons-exec.vespa.version>1.3</commons-exec.vespa.version> - <commons-io.vespa.version>2.11.0</commons-io.vespa.version> + <commons-io.vespa.version>2.13.0</commons-io.vespa.version> + <commons-lang3.vespa.version>3.13.0</commons-lang3.vespa.version> <commons.math3.vespa.version>3.6.1</commons.math3.vespa.version> <commons-compress.vespa.version>1.23.0</commons-compress.vespa.version> - <eclipse-collections.vespa.version>11.0.0</eclipse-collections.vespa.version> + <eclipse-collections.vespa.version>11.1.0</eclipse-collections.vespa.version> <felix.vespa.version>7.0.5</felix.vespa.version> <felix.log.vespa.version>1.0.1</felix.log.vespa.version> <findbugs.vespa.version>3.0.2</findbugs.vespa.version> <!-- Should be kept in sync with guava --> - <hamcrest.vespa.version>1.3</hamcrest.vespa.version> + <hamcrest.vespa.version>2.2</hamcrest.vespa.version> <hdrhistogram.vespa.version>2.1.12</hdrhistogram.vespa.version> - <icu4j.vespa.version>70.1</icu4j.vespa.version> + <icu4j.vespa.version>73.2</icu4j.vespa.version> <java-jjwt.vespa.version>0.11.5</java-jjwt.vespa.version> - <java-jwt.vespa.version>3.10.0</java-jwt.vespa.version> - <jaxb.runtime.vespa.version>2.3.2</jaxb.runtime.vespa.version> <!-- 2.3.3 has a BROKEN manifest --> + <java-jwt.vespa.version>3.19.4</java-jwt.vespa.version> + <jaxb.runtime.vespa.version>2.3.8</jaxb.runtime.vespa.version> <jersey.vespa.version>2.25</jersey.vespa.version> <jetty.vespa.version>11.0.15</jetty.vespa.version> <jetty-servlet-api.vespa.version>5.0.2</jetty-servlet-api.vespa.version> - <jna.vespa.version>5.11.0</jna.vespa.version> - <joda-time.vespa.version>2.12.2</joda-time.vespa.version> + <jna.vespa.version>5.13.0</jna.vespa.version> + <joda-time.vespa.version>2.12.5</joda-time.vespa.version> <junit.vespa.version>5.8.1</junit.vespa.version> <junit.platform.vespa.version>1.8.1</junit.platform.vespa.version> <junit4.vespa.version>4.13.2</junit4.vespa.version> @@ -108,12 +110,13 @@ <netty-tcnative.vespa.version>2.0.61.Final</netty-tcnative.vespa.version> <onnxruntime.vespa.version>1.15.1</onnxruntime.vespa.version> <opennlp.vespa.version>1.9.3</opennlp.vespa.version> - <opentest4j.vespa.version>1.2.0</opentest4j.vespa.version> - <org.json.vespa.version>20230227</org.json.vespa.version> + <opentest4j.vespa.version>1.3.0</opentest4j.vespa.version> + <org.json.vespa.version>20230618</org.json.vespa.version> <org.lz4.vespa.version>1.8.0</org.lz4.vespa.version> <prometheus.client.vespa.version>0.6.0</prometheus.client.vespa.version> - <protobuf.vespa.version>3.24.0</protobuf.vespa.version> + <protobuf.vespa.version>3.24.2</protobuf.vespa.version> <spifly.vespa.version>1.3.6</spifly.vespa.version> + <snappy.vespa.version>1.1.10.3</snappy.vespa.version> <surefire.vespa.version>3.1.2</surefire.vespa.version> <wiremock.vespa.version>2.35.0</wiremock.vespa.version> <xerces.vespa.version>2.12.2</xerces.vespa.version> diff --git a/flags/pom.xml b/flags/pom.xml index 2c736d04ccb..0bfb02b1f32 100644 --- a/flags/pom.xml +++ b/flags/pom.xml @@ -62,7 +62,7 @@ </dependency> <dependency> <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-library</artifactId> + <artifactId>hamcrest</artifactId> <scope>test</scope> </dependency> <dependency> @@ -77,11 +77,7 @@ </exclusion> <exclusion> <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-core</artifactId> - </exclusion> - <exclusion> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-library</artifactId> + <artifactId>*</artifactId> </exclusion> </exclusions> </dependency> diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index 7be21b8508d..f856ebeb456 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -150,7 +150,8 @@ public class PermanentFlags { public static final UnboundBooleanFlag SEND_LIMITED_METRIC_SET = defineFeatureFlag( "send-limited-metric-set", true, "Whether a limited metric set should be fetched from metrics-proxy (CD systems only)", - "Takes effect on next host admin tick"); + "Takes effect on next host admin tick", + APPLICATION_ID); private static final String VERSION_QUALIFIER_REGEX = "[a-zA-Z0-9_-]+"; private static final Pattern QUALIFIER_PATTERN = Pattern.compile("^" + VERSION_QUALIFIER_REGEX + "$"); diff --git a/hosted-tenant-base/pom.xml b/hosted-tenant-base/pom.xml index e528439c1de..ef1d3978a81 100644 --- a/hosted-tenant-base/pom.xml +++ b/hosted-tenant-base/pom.xml @@ -116,6 +116,13 @@ <artifactId>junit-vintage-engine</artifactId> <version>${junit.vespa.tenant.version}</version> </dependency> + + <!-- Transitive dependency of junit4 --> + <dependency> + <groupId>org.hamcrest</groupId> + <artifactId>hamcrest-core</artifactId> + <version>${hamcrest.vespa.version}</version> + </dependency> </dependencies> </dependencyManagement> @@ -333,8 +340,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-antrun-plugin</artifactId> - <!-- TODO: fix configuration to allow 3.x of maven-antrun-plugin --> - <version>1.8</version> + <version>${maven-antrun-plugin.vespa.version}</version> <executions> <execution> <id>attach-artifact</id> @@ -343,7 +349,7 @@ <goal>run</goal> </goals> <configuration> - <tasks> + <target> <!-- Workaround to copy src/test/application/tests only when its parents exists: Copy in two steps, eliminating the parents in the helper step--> @@ -365,7 +371,7 @@ </copy> <zip destfile="target/application-test.zip" basedir="target/application-test/" /> - </tasks> + </target> </configuration> </execution> </executions> diff --git a/jdisc_core/pom.xml b/jdisc_core/pom.xml index fa9bf05fad3..4471269358a 100644 --- a/jdisc_core/pom.xml +++ b/jdisc_core/pom.xml @@ -57,11 +57,7 @@ </exclusion> <exclusion> <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-core</artifactId> - </exclusion> - <exclusion> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-library</artifactId> + <artifactId>*</artifactId> </exclusion> </exclusions> </dependency> diff --git a/maven-plugins/allowed-maven-dependencies.txt b/maven-plugins/allowed-maven-dependencies.txt index 55a27588c07..3cb617aea87 100644 --- a/maven-plugins/allowed-maven-dependencies.txt +++ b/maven-plugins/allowed-maven-dependencies.txt @@ -12,14 +12,14 @@ com.google.guava:failureaccess:1.0.1 com.google.guava:guava:32.1.1-jre com.google.inject:guice:4.2.3 com.google.j2objc:j2objc-annotations:2.8 -commons-codec:commons-codec:1.15 -commons-io:commons-io:2.11.0 +commons-codec:commons-codec:1.16.0 +commons-io:commons-io:2.13.0 javax.annotation:javax.annotation-api:1.2 javax.inject:javax.inject:1 org.apache-extras.beanshell:bsh:2.0b6 org.apache.commons:commons-collections4:4.4 org.apache.commons:commons-compress:1.23.0 -org.apache.commons:commons-lang3:3.12.0 +org.apache.commons:commons-lang3:3.13.0 org.apache.maven:maven-archiver:3.6.1 org.apache.maven:maven-artifact:3.9.4 org.apache.maven:maven-builder-support:3.9.4 @@ -71,8 +71,8 @@ junit:junit:4.13.2 net.bytebuddy:byte-buddy:1.11.19 net.bytebuddy:byte-buddy-agent:1.11.19 org.apiguardian:apiguardian-api:1.1.2 -org.hamcrest:hamcrest-all:1.3 -org.hamcrest:hamcrest-core:1.3 +org.hamcrest:hamcrest:2.2 +org.hamcrest:hamcrest-core:2.2 org.junit.jupiter:junit-jupiter:5.8.1 org.junit.jupiter:junit-jupiter-api:5.8.1 org.junit.jupiter:junit-jupiter-engine:5.8.1 @@ -81,4 +81,4 @@ org.junit.platform:junit-platform-commons:1.8.1 org.junit.platform:junit-platform-engine:1.8.1 org.mockito:mockito-core:4.0.0 org.objenesis:objenesis:3.2 -org.opentest4j:opentest4j:1.2.0 +org.opentest4j:opentest4j:1.3.0 diff --git a/metrics/src/main/java/ai/vespa/metrics/set/Vespa9VespaMetricSet.java b/metrics/src/main/java/ai/vespa/metrics/set/Vespa9VespaMetricSet.java index b7a78e78c6c..5a495878bb6 100644 --- a/metrics/src/main/java/ai/vespa/metrics/set/Vespa9VespaMetricSet.java +++ b/metrics/src/main/java/ai/vespa/metrics/set/Vespa9VespaMetricSet.java @@ -353,11 +353,9 @@ public class Vespa9VespaMetricSet { // Executors shared between all document dbs addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_PROTON_QUEUESIZE, EnumSet.of(max, sum, count)); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_PROTON_ACCEPTED.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_PROTON_WAKEUPS.rate()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_PROTON_UTILIZATION, EnumSet.of(max, sum, count)); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_FLUSH_QUEUESIZE, EnumSet.of(max, sum, count)); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_FLUSH_ACCEPTED.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_FLUSH_WAKEUPS.rate()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_FLUSH_UTILIZATION, EnumSet.of(max, sum, count)); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_MATCH_QUEUESIZE, EnumSet.of(max, sum, count)); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_MATCH_ACCEPTED.rate()); @@ -365,19 +363,15 @@ public class Vespa9VespaMetricSet { addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_MATCH_UTILIZATION, EnumSet.of(max, sum, count)); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_DOCSUM_QUEUESIZE, EnumSet.of(max, sum, count)); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_DOCSUM_ACCEPTED.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_DOCSUM_WAKEUPS.rate()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_DOCSUM_UTILIZATION, EnumSet.of(max, sum, count)); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_SHARED_QUEUESIZE, EnumSet.of(max, sum, count)); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_SHARED_ACCEPTED.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_SHARED_WAKEUPS.rate()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_SHARED_UTILIZATION, EnumSet.of(max, sum, count)); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_WARMUP_QUEUESIZE, EnumSet.of(max, sum, count)); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_WARMUP_ACCEPTED.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_WARMUP_WAKEUPS.rate()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_WARMUP_UTILIZATION, EnumSet.of(max, sum, count)); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_FIELD_WRITER_QUEUESIZE, EnumSet.of(max, sum, count)); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_FIELD_WRITER_ACCEPTED.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_FIELD_WRITER_WAKEUPS.rate()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_EXECUTOR_FIELD_WRITER_UTILIZATION, EnumSet.of(max, sum, count)); // jobs @@ -394,32 +388,20 @@ public class Vespa9VespaMetricSet { // Threading service (per document db) addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_MASTER_QUEUESIZE, EnumSet.of(max, sum, count)); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_MASTER_ACCEPTED.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_MASTER_WAKEUPS.rate()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_MASTER_UTILIZATION, EnumSet.of(max, sum, count)); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_QUEUESIZE, EnumSet.of(max, sum, count)); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_ACCEPTED.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_WAKEUPS.rate()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_INDEX_UTILIZATION, EnumSet.of(max, sum, count)); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_QUEUESIZE, EnumSet.of(max, sum, count)); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_ACCEPTED.rate()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_WAKEUPS.rate()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_THREADING_SERVICE_SUMMARY_UTILIZATION, EnumSet.of(max, sum, count)); // lid space - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_LID_BLOAT_FACTOR.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_LID_FRAGMENTATION_FACTOR.average()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_LID_LIMIT, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_HIGHEST_USED_LID, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_LID_SPACE_USED_LIDS, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_LID_BLOAT_FACTOR.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_LID_FRAGMENTATION_FACTOR.average()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_LID_LIMIT, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_HIGHEST_USED_LID, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_LID_SPACE_USED_LIDS, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_LID_BLOAT_FACTOR.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_LID_FRAGMENTATION_FACTOR.average()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_LID_LIMIT, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_HIGHEST_USED_LID, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_LID_SPACE_USED_LIDS, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last // bucket move @@ -451,29 +433,21 @@ public class Vespa9VespaMetricSet { // transaction log addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_TRANSACTIONLOG_ENTRIES.average()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_TRANSACTIONLOG_DISK_USAGE.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_TRANSACTIONLOG_REPLAY_TIME, EnumSet.of(max, last)); // TODO: Vespa 9: Remove last + addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_TRANSACTIONLOG_REPLAY_TIME.max()); // document store addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_DOCUMENT_STORE_DISK_USAGE.average()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_DOCUMENT_STORE_DISK_BLOAT.average()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_DOCUMENT_STORE_MAX_BUCKET_SPREAD.average()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_DOCUMENT_STORE_MEMORY_USAGE_ALLOCATED_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_DOCUMENT_STORE_MEMORY_USAGE_USED_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_DOCUMENT_STORE_MEMORY_USAGE_ONHOLD_BYTES.average()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_DOCUMENT_STORE_DISK_USAGE.average()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_DOCUMENT_STORE_DISK_BLOAT.average()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_DOCUMENT_STORE_MAX_BUCKET_SPREAD.average()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_DOCUMENT_STORE_MEMORY_USAGE_ALLOCATED_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_DOCUMENT_STORE_MEMORY_USAGE_USED_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_DOCUMENT_STORE_MEMORY_USAGE_DEAD_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_DOCUMENT_STORE_MEMORY_USAGE_ONHOLD_BYTES.average()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_DOCUMENT_STORE_DISK_USAGE.average()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_DOCUMENT_STORE_DISK_BLOAT.average()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_DOCUMENT_STORE_MAX_BUCKET_SPREAD.average()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_DOCUMENT_STORE_MEMORY_USAGE_ALLOCATED_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_DOCUMENT_STORE_MEMORY_USAGE_USED_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_DOCUMENT_STORE_MEMORY_USAGE_DEAD_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_REMOVED_DOCUMENT_STORE_MEMORY_USAGE_ONHOLD_BYTES.average()); // document store cache addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_DOCUMENT_STORE_CACHE_MEMORY_USAGE.average()); @@ -487,19 +461,10 @@ public class Vespa9VespaMetricSet { // attribute addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_ATTRIBUTE_MEMORY_USAGE_ALLOCATED_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_ATTRIBUTE_MEMORY_USAGE_USED_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_ATTRIBUTE_MEMORY_USAGE_DEAD_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_READY_ATTRIBUTE_MEMORY_USAGE_ONHOLD_BYTES.average()); addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_ATTRIBUTE_MEMORY_USAGE_ALLOCATED_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_ATTRIBUTE_MEMORY_USAGE_USED_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_ATTRIBUTE_MEMORY_USAGE_DEAD_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_NOTREADY_ATTRIBUTE_MEMORY_USAGE_ONHOLD_BYTES.average()); // index addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_INDEX_MEMORY_USAGE_ALLOCATED_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_INDEX_MEMORY_USAGE_USED_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_INDEX_MEMORY_USAGE_DEAD_BYTES.average()); - addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_INDEX_MEMORY_USAGE_ONHOLD_BYTES.average()); // matching addMetric(metrics, SearchNodeMetrics.CONTENT_PROTON_DOCUMENTDB_MATCHING_QUERIES.rate()); @@ -610,10 +575,6 @@ public class Vespa9VespaMetricSet { addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_BUCKETS_TOOMANYCOPIES.average()); addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_BUCKETS.average()); addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_BUCKETS_NOTRUSTED.average()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_BUCKET_REPLICAS_MOVING_OUT.average()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_BUCKET_REPLICAS_COPYING_OUT.average()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_BUCKET_REPLICAS_COPYING_IN.average()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_BUCKET_REPLICAS_SYNCING.average()); addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_MAX_OBSERVED_TIME_SINCE_LAST_GC_SEC.average()); addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_DELETE_BUCKET_DONE_OK.rate()); addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_DELETE_BUCKET_DONE_FAILED.rate()); @@ -621,11 +582,6 @@ public class Vespa9VespaMetricSet { addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_MERGE_BUCKET_DONE_OK.rate()); addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_MERGE_BUCKET_DONE_FAILED.rate()); addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_MERGE_BUCKET_PENDING.average()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_MERGE_BUCKET_BLOCKED.rate()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_MERGE_BUCKET_THROTTLED.rate()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_MERGE_BUCKET_SOURCE_ONLY_COPY_CHANGED.rate()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_MERGE_BUCKET_SOURCE_ONLY_COPY_DELETE_BLOCKED.rate()); - addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_MERGE_BUCKET_SOURCE_ONLY_COPY_DELETE_FAILED.rate()); addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_SPLIT_BUCKET_DONE_OK.rate()); addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_SPLIT_BUCKET_DONE_FAILED.rate()); addMetric(metrics, DistributorMetrics.VDS_IDEALSTATE_SPLIT_BUCKET_PENDING.average()); diff --git a/metrics/src/tests/metricmanagertest.cpp b/metrics/src/tests/metricmanagertest.cpp index 9e6b0f40be3..a6dd141576f 100644 --- a/metrics/src/tests/metricmanagertest.cpp +++ b/metrics/src/tests/metricmanagertest.cpp @@ -462,6 +462,7 @@ TEST_F(MetricManagerTest, test_snapshots) { MetricLockGuard lockGuard(mm.getMetricLock()); mm.registerMetric(lockGuard, mySet.set); + EXPECT_FALSE(mm.any_snapshots_taken(lockGuard)); // well-defined prior to init() } mm.init(ConfigUri("raw:" "consumer[2]\n" @@ -474,6 +475,7 @@ TEST_F(MetricManagerTest, test_snapshots) MetricNameVisitor visitor; { MetricLockGuard lockGuard(mm.getMetricLock()); + EXPECT_FALSE(mm.any_snapshots_taken(lockGuard)); // No snapshots yet mm.visit(lockGuard, mm.getActiveMetrics(lockGuard), visitor, "snapper"); const MetricManager::ConsumerSpec * consumerSpec = mm.getConsumerSpec(lockGuard, "snapper"); EXPECT_EQ(std::string("\n" @@ -506,6 +508,10 @@ TEST_F(MetricManagerTest, test_snapshots) ASSERT_VALUES(mm, 5 * 60s, "2,4,4,1,7,9,1,1,8,2,10"); ASSERT_VALUES(mm, 60 * 60s, ""); ASSERT_VALUES(mm, 0 * 60s, "2,4,4,1,7,9,1,1,8,2,10"); + { + auto guard = mm.getMetricLock(); + EXPECT_TRUE(mm.any_snapshots_taken(guard)); // At least one snapshot has been taken + } // Adding metrics done in second five minute period. Total should // be updated to account for both @@ -567,6 +573,14 @@ TEST_F(MetricManagerTest, test_json_output) "consumer[0].tags[1]\n" "consumer[0].tags[0] snaptest\n")); + { + // No snapshots have been taken yet, so the non-total getMetrics call should return + // the empty string (i.e. no metrics produced). + metrics::StateApiAdapter adapter(mm); + auto json_str = adapter.getMetrics("snapper"); + EXPECT_EQ(json_str, ""); + } + takeSnapshots(mm, 1000); // Adding metrics to have some values in them diff --git a/metrics/src/vespa/metrics/metricmanager.cpp b/metrics/src/vespa/metrics/metricmanager.cpp index e399a23b0b9..fa18ddc383b 100644 --- a/metrics/src/vespa/metrics/metricmanager.cpp +++ b/metrics/src/vespa/metrics/metricmanager.cpp @@ -168,6 +168,11 @@ MetricManager::isInitialized() const { return static_cast<bool>(_configHandle); } +bool +MetricManager::any_snapshots_taken(const MetricLockGuard&) const noexcept { + return (!_snapshots.empty() && _snapshots[0]->current_is_assigned()); +} + void MetricManager::init(const config::ConfigUri & uri, bool startThread) { @@ -795,6 +800,7 @@ MetricManager::takeSnapshots(const MetricLockGuard & guard, system_time timeToPr _activeMetrics.addToSnapshot(firstTarget, false, timeToProcess); _activeMetrics.addToSnapshot(*_totalMetrics, false, timeToProcess); _activeMetrics.reset(timeToProcess); + _snapshots[0]->tag_current_as_assigned(); LOG(debug, "After snapshotting, active metrics goes from %s to %s, and 5 minute metrics goes from %s to %s.", to_string(_activeMetrics.getFromTime()).c_str(), to_string(_activeMetrics.getToTime()).c_str(), to_string(firstTarget.getFromTime()).c_str(), to_string(firstTarget.getToTime()).c_str()); diff --git a/metrics/src/vespa/metrics/metricmanager.h b/metrics/src/vespa/metrics/metricmanager.h index 81d0c581370..cfb3ab2137a 100644 --- a/metrics/src/vespa/metrics/metricmanager.h +++ b/metrics/src/vespa/metrics/metricmanager.h @@ -267,6 +267,8 @@ public: bool isInitialized() const; + [[nodiscard]] bool any_snapshots_taken(const MetricLockGuard&) const noexcept; + private: void takeSnapshots(const MetricLockGuard &, system_time timeToProcess); diff --git a/metrics/src/vespa/metrics/metricsnapshot.cpp b/metrics/src/vespa/metrics/metricsnapshot.cpp index 08ab1be3033..104ad858e43 100644 --- a/metrics/src/vespa/metrics/metricsnapshot.cpp +++ b/metrics/src/vespa/metrics/metricsnapshot.cpp @@ -78,12 +78,15 @@ MetricSnapshotSet::MetricSnapshotSet(const Metric::String& name, system_time::du : _count(count), _builderCount(0), _current(std::make_unique<MetricSnapshot>(name, period, source, snapshotUnsetMetrics)), - _building(count == 1 ? nullptr : new MetricSnapshot(name, period, source, snapshotUnsetMetrics)) + _building(count == 1 ? nullptr : new MetricSnapshot(name, period, source, snapshotUnsetMetrics)), + _current_is_assigned(false) { _current->reset(); if (_building) _building->reset(); } +MetricSnapshotSet::~MetricSnapshotSet() = default; + MetricSnapshot& MetricSnapshotSet::getNextTarget() { diff --git a/metrics/src/vespa/metrics/metricsnapshot.h b/metrics/src/vespa/metrics/metricsnapshot.h index e2cedf018c9..a6a68b43015 100644 --- a/metrics/src/vespa/metrics/metricsnapshot.h +++ b/metrics/src/vespa/metrics/metricsnapshot.h @@ -77,9 +77,11 @@ class MetricSnapshotSet { // building instance. std::unique_ptr<MetricSnapshot> _current; // The last full period std::unique_ptr<MetricSnapshot> _building; // The building period + bool _current_is_assigned; public: MetricSnapshotSet(const Metric::String& name, system_time::duration period, uint32_t count, const MetricSet& source, bool snapshotUnsetMetrics); + ~MetricSnapshotSet(); const Metric::String& getName() const { return _current->getName(); } system_time::duration getPeriod() const { return _current->getPeriod(); } @@ -109,6 +111,13 @@ public: void recreateSnapshot(const MetricSet& metrics, bool copyUnset); void addMemoryUsage(MemoryConsumption&) const; void setFromTime(system_time fromTime); + + [[nodiscard]] bool current_is_assigned() const noexcept { + return _current_is_assigned; + } + void tag_current_as_assigned() noexcept { + _current_is_assigned = true; + } }; } // metrics diff --git a/metrics/src/vespa/metrics/state_api_adapter.cpp b/metrics/src/vespa/metrics/state_api_adapter.cpp index 136ccf6e06a..6c0cb9a6013 100644 --- a/metrics/src/vespa/metrics/state_api_adapter.cpp +++ b/metrics/src/vespa/metrics/state_api_adapter.cpp @@ -11,8 +11,8 @@ StateApiAdapter::getMetrics(const vespalib::string &consumer) { MetricLockGuard guard(_manager.getMetricLock()); auto periods = _manager.getSnapshotPeriods(guard); - if (periods.empty()) { - return ""; // no configuration yet + if (periods.empty() || !_manager.any_snapshots_taken(guard)) { + return ""; // no configuration or snapshots yet } const MetricSnapshot &snapshot(_manager.getMetricSnapshot(guard, periods[0])); vespalib::asciistream json; diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/configmodelview/ImportedMlModels.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/configmodelview/ImportedMlModels.java index 05ff1aba877..f0cd0b01fa5 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/configmodelview/ImportedMlModels.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/configmodelview/ImportedMlModels.java @@ -15,6 +15,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; +import java.util.regex.Pattern; /** * All models imported from the models/ directory in the application package. @@ -86,6 +87,7 @@ public class ImportedMlModels { Arrays.stream(dir.listFiles()).sorted().forEach(child -> { Optional<MlModelImporter> importer = findImporterOf(child, importers); if (importer.isPresent()) { + validateModelPath(child); String name = toName(child); Future<ImportedMlModel> existing = models.get(name); if (existing != null) { @@ -139,4 +141,27 @@ public class ImportedMlModels { return result.substring(0, result.length()-1); } + private static void validateModelPath(File modelFile) { + Pattern nameRegexp = Pattern.compile("[A-Za-z0-9_.]*"); + + Path path = Path.fromString(modelFile.toString()); + if (modelFile.isFile()) + path = stripFileEnding(path); + + boolean afterModels = false; + for (String element : path.elements()) { + if (afterModels) { + if ( ! nameRegexp.matcher(element).matches()) { + throw new IllegalArgumentException("When Vespa imports a model from the 'models' directory, it " + + "uses the directory structure under 'models' to determine the " + + "name of the model. The directory or file name '" + element + "' " + + "is not valid. Please rename this to only contain letters, " + + "numbers or underscores."); + } + } else if (element.equals("models")) { + afterModels = true; + } + } + } + } diff --git a/node-admin/pom.xml b/node-admin/pom.xml index 0f153663e3d..cf45e010d14 100644 --- a/node-admin/pom.xml +++ b/node-admin/pom.xml @@ -142,11 +142,7 @@ </exclusion> <exclusion> <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-core</artifactId> - </exclusion> - <exclusion> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-library</artifactId> + <artifactId>*</artifactId> </exclusion> </exclusions> </dependency> diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java index bbe96272b4b..5d96787214a 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.node.admin.task.util.file; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.jupiter.api.Test; -import org.opentest4j.AssertionFailedError; import java.nio.charset.StandardCharsets; import java.nio.file.FileSystem; @@ -173,7 +172,7 @@ public class UnixPathTest { assertEquals("bar", absolutePath.getFilename()); var pathWithoutSlash = new UnixPath("foo"); - assertRuntimeException(IllegalStateException.class, "Path has no parent directory: 'foo'", () -> pathWithoutSlash.getParent()); + assertRuntimeException(IllegalStateException.class, "Path has no parent directory: 'foo'", pathWithoutSlash::getParent); assertEquals("foo", pathWithoutSlash.getFilename()); var pathWithSlash = new UnixPath("/foo"); @@ -190,7 +189,7 @@ public class UnixPathTest { fail("No exception was thrown"); } catch (RuntimeException e) { if (!baseClass.isInstance(e)) { - throw new AssertionFailedError("Exception class mismatch", baseClass.getName(), e.getClass().getName()); + fail("Exception class mismatch " + baseClass.getName() + " != " + e.getClass().getName()); } assertEquals(message, e.getMessage()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DiskReplacer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DiskReplacer.java index 0e67027f8ca..0c1d6291baa 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DiskReplacer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DiskReplacer.java @@ -8,17 +8,14 @@ import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeMutex; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; +import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner.RebuildResult; import com.yahoo.yolean.Exceptions; -import com.yahoo.yolean.UncheckedInterruptedException; import java.time.Duration; -import java.util.HashMap; -import java.util.Map; +import java.util.List; import java.util.Optional; -import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.Future; import java.util.logging.Level; import java.util.logging.Logger; @@ -43,34 +40,23 @@ public class DiskReplacer extends NodeRepositoryMaintainer { @Override protected double maintain() { NodeList nodes = nodeRepository().nodes().list().rebuilding(true); - int rebuilding = 0; int failures = 0; + List<Node> rebuilding; try (var locked = nodeRepository().nodes().lockAndGetAll(nodes.asList(), Optional.of(Duration.ofSeconds(10)))) { - Map<String, Future<Node>> rebuilt = new HashMap<>(); - for (NodeMutex node : locked.nodes()) { - if (node.node().status().wantToRebuild() && ++rebuilding <= maxBatchSize) { - rebuilt.put(node.node().hostname(), executor.submit(() -> hostProvisioner.replaceRootDisk(node.node()))); - } - } + rebuilding = locked.nodes().stream().map(NodeMutex::node).toList(); + RebuildResult result = hostProvisioner.replaceRootDisk(rebuilding); + + for (Node updated : result.rebuilt()) + if (!updated.status().wantToRebuild()) + nodeRepository().nodes().write(updated, () -> { }); - for (var node : rebuilt.entrySet()) { - try { - Node updated = node.getValue().get(); - if ( ! updated.status().wantToRebuild()) { - nodeRepository().nodes().write(updated, () -> { }); - } - } - catch (ExecutionException e) { - ++failures; - log.log(Level.WARNING, "Failed to rebuild " + node.getKey() + ", will retry in " + - interval() + ": " + Exceptions.toMessageString(e.getCause())); - } - catch (InterruptedException e) { - throw new UncheckedInterruptedException(e, true); - } + for (var entry : result.failed().entrySet()) { + ++failures; + log.log(Level.WARNING, "Failed to rebuild " + entry.getKey() + ", will retry in " + + interval() + ": " + Exceptions.toMessageString(entry.getValue())); } } - return this.asSuccessFactorDeviation(rebuilding, failures); + return this.asSuccessFactorDeviation(rebuilding.size(), failures); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java index 66d1a4e8bc8..630c8670bdf 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java @@ -7,7 +7,9 @@ import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.Node; +import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.function.Consumer; import java.util.function.Predicate; @@ -65,11 +67,13 @@ public interface HostProvisioner { */ void deprovision(Node host); - /** Replace the root (OS) disk of host. Implementations of this are expected to be idempotent. + /** Replace the root (OS) disk of hosts. Implementations of this are expected to be idempotent. * - * @return the updated node object + * @return the node objects for which updates were made */ - Node replaceRootDisk(Node host); + default RebuildResult replaceRootDisk(Collection<Node> hosts) { throw new UnsupportedOperationException(); } + + record RebuildResult(List<Node> rebuilt, Map<Node, Exception> failed) { } /** * Returns the maintenance events scheduled for hosts in this zone, in given cloud accounts. Host events in the diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java index 965611b9a6e..03923853594 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java @@ -21,9 +21,11 @@ import com.yahoo.vespa.hosted.provision.provisioning.ProvisionedHost; import java.time.Instant; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -32,6 +34,8 @@ import java.util.function.Consumer; import java.util.function.Predicate; import java.util.stream.IntStream; +import static com.yahoo.config.provision.NodeType.host; + /** * @author mpolden */ @@ -82,7 +86,7 @@ public class MockHostProvisioner implements HostProvisioner { List<ProvisionedHost> hosts = new ArrayList<>(); for (int index : request.indices()) { - String hostHostname = request.type() == NodeType.host ? "host" + index : request.type().name() + index; + String hostHostname = request.type() == host ? "host" + index : request.type().name() + index; hosts.add(new ProvisionedHost("id-of-" + request.type().name() + index, hostHostname, hostFlavor, @@ -117,13 +121,17 @@ public class MockHostProvisioner implements HostProvisioner { } @Override - public Node replaceRootDisk(Node host) { - if (!host.type().isHost()) throw new IllegalArgumentException(host + " is not a host"); - if (rebuildsCompleted.remove(host.hostname())) { - return host.withWantToRetire(host.status().wantToRetire(), host.status().wantToDeprovision(), - false, false, Agent.system, Instant.ofEpochMilli(123)); + public RebuildResult replaceRootDisk(Collection<Node> hosts) { + List<Node> updated = new ArrayList<>(); + Map<Node, Exception> failed = new LinkedHashMap<>(); + for (Node host : hosts) { + if ( ! host.type().isHost()) failed.put(host, new IllegalArgumentException(host + " is not a host")); + if (rebuildsCompleted.remove(host.hostname())) { + updated.add(host.withWantToRetire(host.status().wantToRetire(), host.status().wantToDeprovision(), + false, false, Agent.system, Instant.ofEpochMilli(123))); + } } - return host; + return new RebuildResult(updated, failed); } @Override @@ -219,7 +227,7 @@ public class MockHostProvisioner implements HostProvisioner { long numAddresses = Math.max(2, Math.round(flavor.resources().bandwidthGbps())); return IntStream.range(1, (int) numAddresses) .mapToObj(i -> { - String hostname = hostType == NodeType.host + String hostname = hostType == host ? "host" + hostIndex + "-" + i : hostType.childNodeType().name() + i; return HostName.of(hostname); diff --git a/orchestrator/pom.xml b/orchestrator/pom.xml index 3ec98af3565..00244124add 100644 --- a/orchestrator/pom.xml +++ b/orchestrator/pom.xml @@ -129,7 +129,7 @@ </dependency> <dependency> <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-all</artifactId> + <artifactId>hamcrest</artifactId> <scope>test</scope> </dependency> <dependency> diff --git a/parent/pom.xml b/parent/pom.xml index 0d3690a7db6..a18cd4a6576 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -612,7 +612,7 @@ <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> - <version>3.12.0</version> + <version>${commons-lang3.vespa.version}</version> </dependency> <dependency> <groupId>io.jsonwebtoken</groupId> @@ -737,7 +737,7 @@ <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-csv</artifactId> - <version>1.8</version> + <version>${commons-csv.vespa.version}</version> </dependency> <dependency> <groupId>org.apache.felix</groupId> @@ -922,12 +922,6 @@ <version>${bouncycastle.vespa.version}</version> </dependency> <dependency> - <groupId>org.cthul</groupId> - <artifactId>cthul-matchers</artifactId> - <version>1.0</version> - <scope>test</scope> <!-- TODO: remove scope from parent pom --> - </dependency> - <dependency> <groupId>org.eclipse.collections</groupId> <artifactId>eclipse-collections</artifactId> <version>${eclipse-collections.vespa.version}</version> @@ -1009,21 +1003,18 @@ </dependency> <dependency> <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-all</artifactId> + <artifactId>hamcrest</artifactId> <version>${hamcrest.vespa.version}</version> - <scope>test</scope> <!-- TODO: remove scope from parent pom --> </dependency> <dependency> <groupId>org.hamcrest</groupId> <artifactId>hamcrest-core</artifactId> <version>${hamcrest.vespa.version}</version> - <scope>test</scope> <!-- TODO: remove scope from parent pom --> </dependency> <dependency> <groupId>org.hamcrest</groupId> <artifactId>hamcrest-library</artifactId> <version>${hamcrest.vespa.version}</version> - <scope>test</scope> <!-- TODO: remove scope from parent pom --> </dependency> <dependency> <groupId>org.hdrhistogram</groupId> @@ -1118,7 +1109,7 @@ <dependency> <groupId>org.xerial.snappy</groupId> <artifactId>snappy-java</artifactId> - <version>1.1.10.1</version> + <version>${snappy.vespa.version}</version> </dependency> <dependency> <groupId>io.dropwizard.metrics</groupId> diff --git a/security-utils/src/main/java/com/yahoo/security/SharedKeyGenerator.java b/security-utils/src/main/java/com/yahoo/security/SharedKeyGenerator.java index 22503292413..5582bd4d106 100644 --- a/security-utils/src/main/java/com/yahoo/security/SharedKeyGenerator.java +++ b/security-utils/src/main/java/com/yahoo/security/SharedKeyGenerator.java @@ -105,7 +105,7 @@ public class SharedKeyGenerator { private static AeadCipher makeAesGcmCipher(SecretSharedKey secretSharedKey, boolean forEncryption) { var aeadParams = new AEADParameters(new KeyParameter(secretSharedKey.secretKey().getEncoded()), AES_GCM_AUTH_TAG_BITS, FIXED_96BIT_IV_FOR_SINGLE_USE_KEY); - var cipher = new GCMBlockCipher(new AESEngine()); + var cipher = GCMBlockCipher.newInstance(AESEngine.newInstance()); cipher.init(forEncryption, aeadParams); return AeadCipher.of(cipher); } diff --git a/storage/src/tests/bucketdb/bucketmanagertest.cpp b/storage/src/tests/bucketdb/bucketmanagertest.cpp index ea3a782d432..e9a9520e8c0 100644 --- a/storage/src/tests/bucketdb/bucketmanagertest.cpp +++ b/storage/src/tests/bucketdb/bucketmanagertest.cpp @@ -22,7 +22,7 @@ #include <vespa/vdslib/state/random.h> #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/clusterstate.h> -#include <vespa/vespalib/io/fileutil.h> +#include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/gtest/gtest.h> #include <future> @@ -518,6 +518,20 @@ TEST_F(BucketManagerTest, metrics_are_tracked_per_bucket_space) { EXPECT_EQ(0, global_m->second->ready_buckets.getLast()); verify_db_memory_metrics_present(global_m->second->bucket_db_metrics); + using namespace vespalib::jsonstream; + vespalib::asciistream ascii; + vespalib::JsonStream jsonStream(ascii, false); + jsonStream << Object() << "values" << Array(); + _manager->report(jsonStream); + jsonStream << End(); + EXPECT_EQ(std::string("{\"values\":[" + "{\"name\":\"vds.datastored.bucket_space.buckets_total\",\"values\":{\"last\":1},\"dimensions\":{\"bucketSpace\":\"global\"}}," + "{\"name\":\"vds.datastored.bucket_space.buckets_total\",\"values\":{\"last\":1},\"dimensions\":{\"bucketSpace\":\"default\"}}," + "{\"name\":\"vds.datastored.alldisks.docs\",\"values\":{\"last\":250}}," + "{\"name\":\"vds.datastored.alldisks.bytes\",\"values\":{\"last\":500}}," + "{\"name\":\"vds.datastored.alldisks.buckets\",\"values\":{\"last\":2}}" + "]"), + std::string(ascii.c_str())); } void diff --git a/storage/src/tests/storageserver/statemanagertest.cpp b/storage/src/tests/storageserver/statemanagertest.cpp index 5764460f5bb..d757a83db01 100644 --- a/storage/src/tests/storageserver/statemanagertest.cpp +++ b/storage/src/tests/storageserver/statemanagertest.cpp @@ -1,6 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/metrics/metricmanager.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/state.h> #include <vespa/vdslib/state/cluster_state_bundle.h> @@ -20,10 +19,9 @@ using namespace ::testing; namespace storage { -struct StateManagerTest : Test { +struct StateManagerTest : Test, NodeStateReporter { std::unique_ptr<TestServiceLayerApp> _node; std::unique_ptr<DummyStorageLink> _upper; - std::unique_ptr<metrics::MetricManager> _metricManager; StateManager* _manager; DummyStorageLink* _lower; @@ -42,6 +40,7 @@ struct StateManagerTest : Test { std::string get_node_info() const { return _manager->getNodeInfo(); } + void report(vespalib::JsonStream &) const override {} void extract_cluster_state_version_from_host_info(uint32_t& version_out); }; @@ -60,11 +59,8 @@ StateManagerTest::SetUp() _node = std::make_unique<TestServiceLayerApp>(NodeIndex(2)); // Clock will increase 1 sec per call. _node->getClock().setAbsoluteTimeInSeconds(1); - _metricManager = std::make_unique<metrics::MetricManager>(); _upper = std::make_unique<DummyStorageLink>(); - _manager = new StateManager(_node->getComponentRegister(), - *_metricManager, - std::make_unique<HostInfo>()); + _manager = new StateManager(_node->getComponentRegister(), std::make_unique<HostInfo>(), *this, false); _lower = new DummyStorageLink(); _upper->push_back(StorageLink::UP(_manager)); _upper->push_back(StorageLink::UP(_lower)); @@ -83,7 +79,6 @@ StateManagerTest::TearDown() { _upper->flush(); _upper.reset(); _node.reset(); - _metricManager.reset(); } void diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp index a3a55ba736e..c8c36f94579 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.cpp +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.cpp @@ -17,6 +17,7 @@ #include <vespa/storageapi/message/state.h> #include <vespa/storageapi/message/bucketsplitting.h> #include <vespa/storageapi/message/stat.h> +#include <vespa/metrics/jsonwriter.h> #include <vespa/document/bucket/fixed_bucket_spaces.h> #include <vespa/vespalib/util/stringfmt.h> #include <ranges> @@ -73,7 +74,8 @@ BucketManager::~BucketManager() closeNextLink(); } -void BucketManager::onClose() +void +BucketManager::onClose() { // Stop internal thread such that we don't send any more messages down. if (_thread) { @@ -155,7 +157,6 @@ struct MetricsUpdater { constexpr Count() noexcept : docs(0), bytes(0), buckets(0), active(0), ready(0) {} }; - Count count; uint32_t lowestUsedBit; @@ -197,21 +198,64 @@ struct MetricsUpdater { } // End of anonymous namespace +namespace { + +void +output(vespalib::JsonStream & json, vespalib::stringref name, uint64_t value, vespalib::stringref bucketSpace) { + using namespace vespalib::jsonstream; + json << Object(); + json << "name" << name; + json << "values" << Object() << "last" << value << End(); + if ( ! bucketSpace.empty()) { + json << "dimensions" << Object(); + json << "bucketSpace" << bucketSpace; + json << End(); + } + json << End(); +} + +void +output(vespalib::JsonStream & json, vespalib::stringref name, uint64_t value) { + output(json, name, value, ""); +} + +MetricsUpdater +getMetrics(const StorBucketDatabase & db) { + MetricsUpdater m; + auto guard = db.acquire_read_guard(); + guard->for_each(std::ref(m)); + return m; +} + +} + +void +BucketManager::report(vespalib::JsonStream & json) const { + MetricsUpdater total; + for (const auto& space : _component.getBucketSpaceRepo()) { + MetricsUpdater m = getMetrics(space.second->bucketDatabase()); + output(json, "vds.datastored.bucket_space.buckets_total", m.count.buckets, + document::FixedBucketSpaces::to_string(space.first)); + total.add(m); + } + const auto & src = total.count; + output(json, "vds.datastored.alldisks.docs", src.docs); + output(json, "vds.datastored.alldisks.bytes", src.bytes); + output(json, "vds.datastored.alldisks.buckets", src.buckets); +} + StorBucketDatabase::Entry BucketManager::getBucketInfo(const document::Bucket &bucket) const { - StorBucketDatabase::WrappedEntry entry(_component.getBucketDatabase(bucket.getBucketSpace()).get(bucket.getBucketId(), "BucketManager::getBucketInfo")); - return *entry; + return *_component.getBucketDatabase(bucket.getBucketSpace()).get(bucket.getBucketId(), "BucketManager::getBucketInfo"); } void -BucketManager::updateMetrics() +BucketManager::updateMetrics() const { MetricsUpdater total; for (const auto& space : _component.getBucketSpaceRepo()) { - MetricsUpdater m; - auto guard = space.second->bucketDatabase().acquire_read_guard(); - guard->for_each(std::ref(m)); + MetricsUpdater m = getMetrics(space.second->bucketDatabase()); total.add(m); auto bm = _metrics->bucket_spaces.find(space.first); assert(bm != _metrics->bucket_spaces.end()); @@ -231,14 +275,17 @@ BucketManager::updateMetrics() update_bucket_db_memory_usage_metrics(); } -void BucketManager::update_bucket_db_memory_usage_metrics() { + +void +BucketManager::update_bucket_db_memory_usage_metrics() const { for (const auto& space : _component.getBucketSpaceRepo()) { auto bm = _metrics->bucket_spaces.find(space.first); bm->second->bucket_db_metrics.memory_usage.update(space.second->bucketDatabase().detailed_memory_usage()); } } -void BucketManager::updateMinUsedBits() +void +BucketManager::updateMinUsedBits() { MetricsUpdater m; _component.getBucketSpaceRepo().for_each_bucket(std::ref(m)); diff --git a/storage/src/vespa/storage/bucketdb/bucketmanager.h b/storage/src/vespa/storage/bucketdb/bucketmanager.h index 547ff87dfc0..eea5719ad3b 100644 --- a/storage/src/vespa/storage/bucketdb/bucketmanager.h +++ b/storage/src/vespa/storage/bucketdb/bucketmanager.h @@ -13,6 +13,7 @@ #include <vespa/storage/bucketdb/config-stor-bucketdb.h> #include <vespa/storage/common/servicelayercomponent.h> #include <vespa/storage/common/storagelinkqueued.h> +#include <vespa/storage/common/nodestateupdater.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storageframework/generic/metric/metricupdatehook.h> #include <vespa/storageframework/generic/status/statusreporter.h> @@ -28,6 +29,7 @@ namespace storage { class BucketManager : public StorageLink, public framework::StatusReporter, + public NodeStateReporter, private framework::Runnable, private framework::MetricUpdateHook { @@ -37,7 +39,9 @@ public: using BucketInfoRequestMap = std::unordered_map<document::BucketSpace, BucketInfoRequestList, document::BucketSpace::hash>; private: - config::ConfigUri _configUri; + using ReplyQueue = std::vector<api::StorageReply::SP>; + using ConflictingBuckets = std::unordered_set<document::BucketId, document::BucketId::hash>; + config::ConfigUri _configUri; BucketInfoRequestMap _bucketInfoRequests; /** @@ -49,25 +53,23 @@ private: /** * Lock kept for access to 3 values below concerning cluster state. */ - std::mutex _clusterStateLock; + std::mutex _clusterStateLock; + mutable std::mutex _queueProcessingLock; - mutable std::mutex _queueProcessingLock; - using ReplyQueue = std::vector<api::StorageReply::SP>; - using ConflictingBuckets = std::unordered_set<document::BucketId, document::BucketId::hash>; - ReplyQueue _queuedReplies; - ConflictingBuckets _conflictingBuckets; + ReplyQueue _queuedReplies; + ConflictingBuckets _conflictingBuckets; // The most current cluster state versions that we've observed on the way _down_ // through the chain, i.e. prior to being enabled on the node. - uint32_t _last_cluster_state_version_initiated; + uint32_t _last_cluster_state_version_initiated; // The most current cluster state we've observed on the way _up_ through the // chain, i.e. after being enabled on the node. - uint32_t _last_cluster_state_version_completed; - bool _doneInitialized; - size_t _requestsCurrentlyProcessing; - ServiceLayerComponent _component; + uint32_t _last_cluster_state_version_completed; + bool _doneInitialized; + size_t _requestsCurrentlyProcessing; + ServiceLayerComponent _component; std::shared_ptr<BucketManagerMetrics> _metrics; - std::unique_ptr<framework::Thread> _thread; - std::chrono::milliseconds _simulated_processing_delay; + std::unique_ptr<framework::Thread> _thread; + std::chrono::milliseconds _simulated_processing_delay; class ScopedQueueDispatchGuard { BucketManager& _mgr; @@ -97,7 +99,7 @@ public: void force_db_sweep_and_metric_update() { updateMetrics(); } bool onUp(const std::shared_ptr<api::StorageMessage>&) override; - + void report(vespalib::JsonStream &writer) const override; private: friend struct BucketManagerTest; @@ -112,9 +114,9 @@ private: void onDoneInit() override { _doneInitialized = true; } void onClose() override; - void updateMetrics(); + void updateMetrics() const; void updateMetrics(const MetricLockGuard &) override { updateMetrics(); } - void update_bucket_db_memory_usage_metrics(); + void update_bucket_db_memory_usage_metrics() const; void updateMinUsedBits(); bool onRequestBucketInfo(const std::shared_ptr<api::RequestBucketInfoCommand>&) override; diff --git a/storage/src/vespa/storage/common/nodestateupdater.h b/storage/src/vespa/storage/common/nodestateupdater.h index 842828a1b89..3d32b9e4b4b 100644 --- a/storage/src/vespa/storage/common/nodestateupdater.h +++ b/storage/src/vespa/storage/common/nodestateupdater.h @@ -25,7 +25,9 @@ #include <string> #include <vespa/vdslib/state/nodestate.h> +#include "vespa/vespalib/util/jsonstream.h" +namespace metrics { class JsonWriter; } namespace storage { namespace lib { class ClusterStateBundle; } @@ -77,6 +79,12 @@ struct NodeStateUpdater { virtual void request_almost_immediate_node_state_replies() = 0; }; +class NodeStateReporter { +public: + virtual ~NodeStateReporter() = default; + virtual void report(vespalib::JsonStream &writer) const = 0; +}; + } // storage diff --git a/storage/src/vespa/storage/storageserver/distributornode.cpp b/storage/src/vespa/storage/storageserver/distributornode.cpp index 45abd34e131..ab80381f5d4 100644 --- a/storage/src/vespa/storage/storageserver/distributornode.cpp +++ b/storage/src/vespa/storage/storageserver/distributornode.cpp @@ -39,7 +39,7 @@ DistributorNode::DistributorNode( set_storage_chain_builder(std::move(storage_chain_builder)); } try { - initialize(); + initialize(*this); } catch (const vespalib::Exception & e) { shutdownDistributor(); throw; diff --git a/storage/src/vespa/storage/storageserver/distributornode.h b/storage/src/vespa/storage/storageserver/distributornode.h index 7224abfa59c..5d61c86d48a 100644 --- a/storage/src/vespa/storage/storageserver/distributornode.h +++ b/storage/src/vespa/storage/storageserver/distributornode.h @@ -10,6 +10,7 @@ #include "distributornodecontext.h" #include "storagenode.h" +#include "vespa/vespalib/util/jsonstream.h" #include <vespa/storage/common/distributorcomponent.h> #include <vespa/storageframework/generic/thread/tickingthread.h> #include <mutex> @@ -22,7 +23,8 @@ class IStorageChainBuilder; class DistributorNode : public StorageNode, - private UniqueTimeCalculator + private UniqueTimeCalculator, + private NodeStateReporter { framework::TickingThreadPool::UP _threadPool; std::unique_ptr<distributor::DistributorStripePool> _stripe_pool; @@ -58,8 +60,9 @@ public: void handleConfigChange(vespa::config::content::core::StorVisitordispatcherConfig&); private: - void initializeNodeSpecific() override; + void report(vespalib::JsonStream &) const override { /* no-op */ } void perform_post_chain_creation_init_steps() override { /* no-op */ } + void initializeNodeSpecific() override; void createChain(IStorageChainBuilder &builder) override; api::Timestamp generate_unique_timestamp() override; diff --git a/storage/src/vespa/storage/storageserver/servicelayernode.cpp b/storage/src/vespa/storage/storageserver/servicelayernode.cpp index b76e2cca02a..b5bed68259a 100644 --- a/storage/src/vespa/storage/storageserver/servicelayernode.cpp +++ b/storage/src/vespa/storage/storageserver/servicelayernode.cpp @@ -39,6 +39,14 @@ ServiceLayerNode::ServiceLayerNode(const config::ConfigUri & configUri, ServiceL { } +void +ServiceLayerNode::report(vespalib::JsonStream &writer) const +{ + if (_bucket_manager) { + _bucket_manager->report(writer); + } +} + void ServiceLayerNode::init() { assert( ! _init_has_been_called); @@ -50,7 +58,7 @@ void ServiceLayerNode::init() } try{ - initialize(); + initialize(*this); } catch (spi::HandledException& e) { requestShutdown("Failed to initialize: " + e.getMessage()); throw; @@ -164,9 +172,9 @@ ServiceLayerNode::createChain(IStorageChainBuilder &builder) auto bucket_manager = std::make_unique<BucketManager>(_configUri, _context.getComponentRegister()); _bucket_manager = bucket_manager.get(); builder.add(std::move(bucket_manager)); - builder.add(std::make_unique<VisitorManager>(_configUri, _context.getComponentRegister(), static_cast<VisitorMessageSessionFactory &>(*this), _externalVisitors)); - builder.add(std::make_unique<ModifiedBucketChecker>( - _context.getComponentRegister(), _persistenceProvider, _configUri)); + builder.add(std::make_unique<VisitorManager>(_configUri, _context.getComponentRegister(), + static_cast<VisitorMessageSessionFactory &>(*this), _externalVisitors)); + builder.add(std::make_unique<ModifiedBucketChecker>(_context.getComponentRegister(), _persistenceProvider, _configUri)); auto state_manager = releaseStateManager(); auto filstor_manager = std::make_unique<FileStorManager>(_configUri, _persistenceProvider, _context.getComponentRegister(), getDoneInitializeHandler(), state_manager->getHostInfo()); diff --git a/storage/src/vespa/storage/storageserver/servicelayernode.h b/storage/src/vespa/storage/storageserver/servicelayernode.h index 4b719a0330b..e308c020856 100644 --- a/storage/src/vespa/storage/storageserver/servicelayernode.h +++ b/storage/src/vespa/storage/storageserver/servicelayernode.h @@ -11,8 +11,10 @@ #include "applicationgenerationfetcher.h" #include "servicelayernodecontext.h" #include "storagenode.h" +#include "vespa/vespalib/util/jsonstream.h" #include <vespa/storage/visiting/visitormessagesessionfactory.h> #include <vespa/storage/common/visitorfactory.h> +#include <vespa/storage/common/nodestateupdater.h> namespace storage { @@ -23,18 +25,19 @@ class FileStorManager; class ServiceLayerNode : public StorageNode, - private VisitorMessageSessionFactory + private VisitorMessageSessionFactory, + private NodeStateReporter { - ServiceLayerNodeContext& _context; - spi::PersistenceProvider& _persistenceProvider; - VisitorFactory::Map _externalVisitors; + ServiceLayerNodeContext & _context; + spi::PersistenceProvider & _persistenceProvider; + VisitorFactory::Map _externalVisitors; // FIXME: Should probably use the fetcher in StorageNode - std::unique_ptr<config::ConfigFetcher> _configFetcher; - BucketManager* _bucket_manager; - FileStorManager* _fileStorManager; - bool _init_has_been_called; + std::unique_ptr<config::ConfigFetcher> _configFetcher; + BucketManager * _bucket_manager; + FileStorManager * _fileStorManager; + bool _init_has_been_called; public: using UP = std::unique_ptr<ServiceLayerNode>; @@ -55,6 +58,7 @@ public: ResumeGuard pause() override; private: + void report(vespalib::JsonStream &writer) const override; void subscribeToConfigs() override; void initializeNodeSpecific() override; void perform_post_chain_creation_init_steps() override; diff --git a/storage/src/vespa/storage/storageserver/statemanager.cpp b/storage/src/vespa/storage/storageserver/statemanager.cpp index 76a8cabb515..49957d1d321 100644 --- a/storage/src/vespa/storage/storageserver/statemanager.cpp +++ b/storage/src/vespa/storage/storageserver/statemanager.cpp @@ -49,13 +49,13 @@ StateManager::StateManagerMetrics::~StateManagerMetrics() = default; using lib::ClusterStateBundle; StateManager::StateManager(StorageComponentRegister& compReg, - metrics::MetricManager& metricManager, std::unique_ptr<HostInfo> hostInfo, + const NodeStateReporter & reporter, bool testMode) : StorageLink("State manager"), framework::HtmlStatusReporter("systemstate", "Node and system state"), _component(compReg, "statemanager"), - _metricManager(metricManager), + _nodeStateReporter(reporter), _metrics(std::make_unique<StateManagerMetrics>()), _stateLock(), _stateCond(), @@ -572,17 +572,10 @@ StateManager::getNodeInfo() const vespalib::JsonStream stream(json, true); stream << Object(); { // Print metrics - stream << "metrics"; try { - metrics::MetricLockGuard lock(_metricManager.getMetricLock()); - auto periods(_metricManager.getSnapshotPeriods(lock)); - if (!periods.empty()) { - const metrics::MetricSnapshot& snapshot(_metricManager.getMetricSnapshot(lock, periods[0])); - metrics::JsonWriter metricJsonWriter(stream); - _metricManager.visit(lock, snapshot, metricJsonWriter, "fleetcontroller"); - } else { - stream << Object() << "error" << "no snapshot periods" << End(); - } + stream << "metrics" << Object() << "values" << Array(); + _nodeStateReporter.report(stream); + stream << End() << End(); } catch (vespalib::Exception& e) { stream << Object() << "error" << e.getMessage() << End(); } diff --git a/storage/src/vespa/storage/storageserver/statemanager.h b/storage/src/vespa/storage/storageserver/statemanager.h index 3b1291b1c3f..a69675adb1b 100644 --- a/storage/src/vespa/storage/storageserver/statemanager.h +++ b/storage/src/vespa/storage/storageserver/statemanager.h @@ -49,7 +49,7 @@ class StateManager : public NodeStateUpdater, struct StateManagerMetrics; StorageComponent _component; - metrics::MetricManager& _metricManager; + const NodeStateReporter & _nodeStateReporter; std::unique_ptr<StateManagerMetrics> _metrics; mutable std::mutex _stateLock; std::condition_variable _stateCond; @@ -80,8 +80,8 @@ class StateManager : public NodeStateUpdater, std::atomic<bool> _requested_almost_immediate_node_state_replies; public: - explicit StateManager(StorageComponentRegister&, metrics::MetricManager&, - std::unique_ptr<HostInfo>, bool testMode = false); + explicit StateManager(StorageComponentRegister&, std::unique_ptr<HostInfo>, + const NodeStateReporter & reporter, bool testMode); ~StateManager() override; void onOpen() override; @@ -110,7 +110,6 @@ public: private: struct ExternalStateLock; - friend struct ExternalStateLock; friend struct StateManagerTest; void notifyStateListeners(); diff --git a/storage/src/vespa/storage/storageserver/storagenode.cpp b/storage/src/vespa/storage/storageserver/storagenode.cpp index f0f410d3076..99a879e19db 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.cpp +++ b/storage/src/vespa/storage/storageserver/storagenode.cpp @@ -127,7 +127,7 @@ StorageNode::subscribeToConfigs() } void -StorageNode::initialize() +StorageNode::initialize(const NodeStateReporter & nodeStateReporter) { // Avoid racing with concurrent reconfigurations before we've set up the entire // node component stack. @@ -164,8 +164,8 @@ StorageNode::initialize() // dead lock detector too, but not before open() _stateManager = std::make_unique<StateManager>( _context.getComponentRegister(), - _context.getComponentRegister().getMetricManager(), std::move(_hostInfo), + nodeStateReporter, _singleThreadedDebugMode); _context.getComponentRegister().setNodeStateUpdater(*_stateManager); @@ -176,11 +176,11 @@ StorageNode::initialize() initializeNodeSpecific(); - _statusMetrics = std::make_unique<StatusMetricConsumer>( - _context.getComponentRegister(), _context.getComponentRegister().getMetricManager()); - _stateReporter = std::make_unique<StateReporter>( - _context.getComponentRegister(), _context.getComponentRegister().getMetricManager(), - _generationFetcher); + _statusMetrics = std::make_unique<StatusMetricConsumer>(_context.getComponentRegister(), + _context.getComponentRegister().getMetricManager()); + _stateReporter = std::make_unique<StateReporter>(_context.getComponentRegister(), + _context.getComponentRegister().getMetricManager(), + _generationFetcher); // Start deadlock detector _deadLockDetector = std::make_unique<DeadLockDetector>(_context.getComponentRegister()); diff --git a/storage/src/vespa/storage/storageserver/storagenode.h b/storage/src/vespa/storage/storageserver/storagenode.h index 344181adad5..5a521d7c66c 100644 --- a/storage/src/vespa/storage/storageserver/storagenode.h +++ b/storage/src/vespa/storage/storageserver/storagenode.h @@ -43,6 +43,7 @@ class StatusMetricConsumer; class StatusWebServer; class StorageComponent; class StorageLink; +class NodeStateReporter; struct DeadLockDetector; struct StorageMetricSet; struct StorageNodeContext; @@ -172,7 +173,7 @@ protected: */ std::unique_ptr<StateManager> releaseStateManager(); - void initialize(); + void initialize(const NodeStateReporter & reporter); virtual void subscribeToConfigs(); virtual void initializeNodeSpecific() = 0; virtual void perform_post_chain_creation_init_steps() = 0; diff --git a/testutil/pom.xml b/testutil/pom.xml index f3bc9cb2212..7f57b6defee 100644 --- a/testutil/pom.xml +++ b/testutil/pom.xml @@ -13,7 +13,7 @@ <packaging>jar</packaging> <version>8-SNAPSHOT</version> <name>${project.artifactId}</name> - <description>Library of useful Hamcrest matchers.</description> + <description>Library of useful test utilities.</description> <dependencies> <dependency> <groupId>com.google.guava</groupId> @@ -34,12 +34,7 @@ </dependency> <dependency> <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-core</artifactId> - <scope>compile</scope> - </dependency> - <dependency> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-library</artifactId> + <artifactId>hamcrest</artifactId> <scope>compile</scope> </dependency> <dependency> diff --git a/testutil/src/main/java/com/yahoo/test/LinePatternMatcher.java b/testutil/src/main/java/com/yahoo/test/LinePatternMatcher.java index 855a0bc851e..c870c0b29b4 100644 --- a/testutil/src/main/java/com/yahoo/test/LinePatternMatcher.java +++ b/testutil/src/main/java/com/yahoo/test/LinePatternMatcher.java @@ -3,7 +3,6 @@ package com.yahoo.test; import org.hamcrest.BaseMatcher; import org.hamcrest.Description; -import org.hamcrest.Factory; import org.hamcrest.Matcher; /** @@ -35,8 +34,7 @@ public class LinePatternMatcher extends BaseMatcher<String> { return false; } - @Factory - public static <T> Matcher<String> containsLineWithPattern(String pattern) { + public static Matcher<String> containsLineWithPattern(String pattern) { return new LinePatternMatcher(pattern); } diff --git a/vespa-athenz/pom.xml b/vespa-athenz/pom.xml index 66b369f00fe..55fd25f8b99 100644 --- a/vespa-athenz/pom.xml +++ b/vespa-athenz/pom.xml @@ -118,11 +118,7 @@ </exclusion> <exclusion> <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-core</artifactId> - </exclusion> - <exclusion> - <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-library</artifactId> + <artifactId>*</artifactId> </exclusion> </exclusions> </dependency> @@ -158,6 +154,10 @@ <artifactId>jakarta.activation-api</artifactId> </exclusion> <exclusion> + <groupId>com.sun.activation</groupId> + <artifactId>jakarta.activation</artifactId> + </exclusion> + <exclusion> <groupId>jakarta.xml.bind</groupId> <artifactId>jakarta.xml.bind-api</artifactId> </exclusion> diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index 8b0c76bf973..9dc1b0d2fda 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -13,7 +13,7 @@ com.amazonaws:aws-java-sdk-core:1.12.460 com.amazonaws:aws-java-sdk-ssm:1.12.460 com.amazonaws:aws-java-sdk-sts:1.12.460 com.amazonaws:jmespath-java:1.12.460 -com.auth0:java-jwt:3.10.0 +com.auth0:java-jwt:3.19.4 com.fasterxml.jackson.core:jackson-annotations:2.15.2 com.fasterxml.jackson.core:jackson-core:2.15.2 com.fasterxml.jackson.core:jackson-databind:2.15.2 @@ -38,15 +38,14 @@ com.google.http-client:google-http-client-apache-v2:1.43.3 com.google.http-client:google-http-client-gson:1.42.3 com.google.inject:guice:4.2.3:no_aop com.google.j2objc:j2objc-annotations:2.8 -com.google.protobuf:protobuf-java:3.24.0 -com.ibm.icu:icu4j:70.1 +com.google.protobuf:protobuf-java:3.24.2 +com.ibm.icu:icu4j:73.2 com.intellij:annotations:9.0.4 com.microsoft.onnxruntime:onnxruntime:1.15.1 com.sun.activation:javax.activation:1.2.0 -com.sun.istack:istack-commons-runtime:3.0.8 +com.sun.istack:istack-commons-runtime:3.0.12 com.sun.xml.bind:jaxb-core:2.3.0 com.sun.xml.bind:jaxb-impl:2.3.0 -com.sun.xml.fastinfoset:FastInfoset:1.2.16 com.thaiopensource:jing:20091111 com.yahoo.athenz:athenz-auth-core:1.11.34 com.yahoo.athenz:athenz-client-common:1.11.34 @@ -55,9 +54,9 @@ com.yahoo.athenz:athenz-zpe-java-client:1.11.34 com.yahoo.athenz:athenz-zts-core:1.11.34 com.yahoo.rdl:rdl-java:1.5.4 commons-cli:commons-cli:1.5.0 -commons-codec:commons-codec:1.15 +commons-codec:commons-codec:1.16.0 commons-fileupload:commons-fileupload:1.5 -commons-io:commons-io:2.11.0 +commons-io:commons-io:2.13.0 commons-logging:commons-logging:1.2 io.airlift:airline:0.9 io.dropwizard.metrics:metrics-core:3.2.5 @@ -86,17 +85,17 @@ javax.servlet:javax.servlet-api:3.1.0 javax.validation:validation-api:1.1.0.Final javax.ws.rs:javax.ws.rs-api:2.0.1 javax.xml.bind:jaxb-api:2.3.0 -joda-time:joda-time:2.12.2 +joda-time:joda-time:2.12.5 junit:junit:4.13.2 -net.java.dev.jna:jna:5.11.0 +net.java.dev.jna:jna:5.13.0 net.openhft:zero-allocation-hashing:0.16 org.antlr:antlr-runtime:3.5.3 org.antlr:antlr4-runtime:4.13.0 org.apache.aries.spifly:org.apache.aries.spifly.dynamic.bundle:1.3.6 org.apache.commons:commons-compress:1.23.0 -org.apache.commons:commons-csv:1.8 +org.apache.commons:commons-csv:1.10.0 org.apache.commons:commons-exec:1.3 -org.apache.commons:commons-lang3:3.12.0 +org.apache.commons:commons-lang3:3.13.0 org.apache.commons:commons-math3:3.6.1 org.apache.curator:curator-client:5.4.0 org.apache.curator:curator-framework:5.4.0 @@ -133,9 +132,9 @@ org.apache.zookeeper:zookeeper:3.8.1 org.apache.zookeeper:zookeeper-jute:3.8.0 org.apache.zookeeper:zookeeper-jute:3.8.1 org.apiguardian:apiguardian-api:1.1.2 -org.bouncycastle:bcpkix-jdk18on:1.74 -org.bouncycastle:bcprov-jdk18on:1.74 -org.bouncycastle:bcutil-jdk18on:1.74 +org.bouncycastle:bcpkix-jdk18on:1.76 +org.bouncycastle:bcprov-jdk18on:1.76 +org.bouncycastle:bcutil-jdk18on:1.76 org.codehaus.plexus:plexus-archiver:4.8.0 org.codehaus.plexus:plexus-classworlds:2.7.0 org.codehaus.plexus:plexus-component-annotations:1.5.5 @@ -143,8 +142,8 @@ org.codehaus.plexus:plexus-container-default:1.0-alpha-9-stable-1 org.codehaus.plexus:plexus-interpolation:1.26 org.codehaus.plexus:plexus-io:3.4.1 org.codehaus.plexus:plexus-utils:3.5.1 -org.eclipse.collections:eclipse-collections:11.0.0 -org.eclipse.collections:eclipse-collections-api:11.0.0 +org.eclipse.collections:eclipse-collections:11.1.0 +org.eclipse.collections:eclipse-collections-api:11.1.0 org.eclipse.jetty:jetty-alpn-client:11.0.15 org.eclipse.jetty:jetty-alpn-java-client:11.0.15 org.eclipse.jetty:jetty-alpn-java-server:11.0.15 @@ -172,8 +171,8 @@ org.glassfish.hk2:hk2-utils:2.5.0-b30 org.glassfish.hk2:osgi-resource-locator:1.0.1 org.glassfish.hk2.external:aopalliance-repackaged:2.5.0-b30 org.glassfish.hk2.external:javax.inject:2.5.0-b30 -org.glassfish.jaxb:jaxb-runtime:2.3.2 -org.glassfish.jaxb:txw2:2.3.2 +org.glassfish.jaxb:jaxb-runtime:2.3.8 +org.glassfish.jaxb:txw2:2.3.8 org.glassfish.jersey.bundles.repackaged:jersey-guava:2.25 org.glassfish.jersey.core:jersey-client:2.25 org.glassfish.jersey.core:jersey-common:2.25 @@ -182,20 +181,21 @@ org.glassfish.jersey.ext:jersey-entity-filtering:2.25 org.glassfish.jersey.ext:jersey-proxy-client:2.25 org.glassfish.jersey.media:jersey-media-json-jackson:2.25 org.glassfish.jersey.media:jersey-media-multipart:2.25 +org.hamcrest:hamcrest:2.2 +org.hamcrest:hamcrest-core:2.2 org.hdrhistogram:HdrHistogram:2.1.12 org.iq80.snappy:snappy:0.4 org.javassist:javassist:3.20.0-GA -org.json:json:20230227 +org.json:json:20230618 org.junit.jupiter:junit-jupiter-api:5.8.1 org.junit.jupiter:junit-jupiter-engine:5.8.1 org.junit.platform:junit-platform-commons:1.8.1 org.junit.platform:junit-platform-engine:1.8.1 org.junit.platform:junit-platform-launcher:1.8.1 org.jvnet.mimepull:mimepull:1.9.6 -org.jvnet.staxex:stax-ex:1.8.1 org.kohsuke:libpam4j:1.11 org.lz4:lz4-java:1.8.0 -org.opentest4j:opentest4j:1.2.0 +org.opentest4j:opentest4j:1.3.0 org.osgi:org.osgi.compendium:4.1.0 org.osgi:org.osgi.core:4.1.0 org.ow2.asm:asm:9.3 @@ -210,7 +210,7 @@ org.slf4j:slf4j-api:1.7.32 org.slf4j:slf4j-jdk14:1.7.32 org.slf4j:slf4j-simple:1.7.32 org.tukaani:xz:1.9 -org.xerial.snappy:snappy-java:1.1.10.1 +org.xerial.snappy:snappy-java:1.1.10.3 software.amazon.ion:ion-java:1.0.2 xerces:xercesImpl:2.12.2 xml-apis:xml-apis:1.4.01 @@ -226,10 +226,6 @@ net.bytebuddy:byte-buddy-agent:1.11.19 org.apache.curator:curator-test:5.4.0 org.assertj:assertj-core:3.11.1 org.checkerframework:checker-qual:3.30.0 -org.cthul:cthul-matchers:1.0 -org.hamcrest:hamcrest-all:1.3 -org.hamcrest:hamcrest-core:1.3 -org.hamcrest:hamcrest-library:1.3 org.junit.jupiter:junit-jupiter:5.8.1 org.junit.jupiter:junit-jupiter-params:5.8.1 org.junit.vintage:junit-vintage-engine:5.8.1 diff --git a/vespajlib/pom.xml b/vespajlib/pom.xml index 5ba7f2d2ff2..d3c9831c596 100644 --- a/vespajlib/pom.xml +++ b/vespajlib/pom.xml @@ -67,7 +67,7 @@ <!-- test scope --> <dependency> <groupId>org.hamcrest</groupId> - <artifactId>hamcrest-library</artifactId> + <artifactId>hamcrest</artifactId> <scope>test</scope> </dependency> <dependency> diff --git a/vespalib/src/vespa/vespalib/util/jsonstream.cpp b/vespalib/src/vespa/vespalib/util/jsonstream.cpp index e8814ee95f6..86bb8a1a8d1 100644 --- a/vespalib/src/vespa/vespalib/util/jsonstream.cpp +++ b/vespalib/src/vespa/vespalib/util/jsonstream.cpp @@ -39,7 +39,7 @@ JsonStream::JsonStream(asciistream& as, bool createIndents) push({State::ROOT}); } -JsonStream::~JsonStream() {} +JsonStream::~JsonStream() = default; JsonStream& JsonStream::operator<<(stringref value) |