diff options
160 files changed, 1729 insertions, 1477 deletions
diff --git a/application/pom.xml b/application/pom.xml index bb1ea4b30ee..bbb4b171676 100644 --- a/application/pom.xml +++ b/application/pom.xml @@ -87,6 +87,12 @@ <dependency> <groupId>org.apache.felix</groupId> <artifactId>org.apache.felix.log</artifactId> + <exclusions> + <exclusion> + <groupId>org.osgi</groupId> + <artifactId>*</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.apache.opennlp</groupId> @@ -104,6 +110,10 @@ <groupId>org.apache.commons</groupId> <artifactId>commons-math3</artifactId> </dependency> + <dependency> + <groupId>jakarta.inject</groupId> + <artifactId>jakarta.inject-api</artifactId> + </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/client/go/go.mod b/client/go/go.mod index d797017a810..1a48b1d97a0 100644 --- a/client/go/go.mod +++ b/client/go/go.mod @@ -16,7 +16,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 github.com/zalando/go-keyring v0.2.3 - golang.org/x/net v0.14.0 + golang.org/x/net v0.15.0 golang.org/x/sys v0.12.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -30,7 +30,7 @@ require ( github.com/kr/pretty v0.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - golang.org/x/term v0.11.0 // indirect - golang.org/x/text v0.12.0 // indirect + golang.org/x/term v0.12.0 // indirect + golang.org/x/text v0.13.0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect ) diff --git a/client/go/go.sum b/client/go/go.sum index 4bea3accfae..1676eda6400 100644 --- a/client/go/go.sum +++ b/client/go/go.sum @@ -66,6 +66,8 @@ golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -80,10 +82,14 @@ golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/client/js/app/yarn.lock b/client/js/app/yarn.lock index df9465d2478..706ecbcc97e 100644 --- a/client/js/app/yarn.lock +++ b/client/js/app/yarn.lock @@ -307,7 +307,14 @@ dependencies: "@babel/helper-plugin-utils" "^7.22.5" -"@babel/runtime@^7.10.2", "@babel/runtime@^7.12.5", "@babel/runtime@^7.13.10", "@babel/runtime@^7.18.3", "@babel/runtime@^7.5.5", "@babel/runtime@^7.8.7": +"@babel/runtime@^7.10.2", "@babel/runtime@^7.13.10", "@babel/runtime@^7.5.5", "@babel/runtime@^7.8.7": + version "7.22.15" + resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.22.15.tgz#38f46494ccf6cf020bd4eed7124b425e83e523b8" + integrity sha512-T0O+aa+4w0u06iNmapipJXMV4HoUir03hpx3/YqXXhu9xim3w+dVphjFWl1OH8NbZHw5Lbm9k45drDkgq2VNNA== + dependencies: + regenerator-runtime "^0.14.0" + +"@babel/runtime@^7.12.5", "@babel/runtime@^7.18.3": version "7.22.6" resolved "https://registry.yarnpkg.com/@babel/runtime/-/runtime-7.22.6.tgz#57d64b9ae3cff1d67eb067ae117dac087f5bd438" integrity sha512-wDb5pWm4WDdF6LFUde3Jl8WzPA+3ZbxYqkC6xAXuD3irdEHN1k0NfTRrJD8ZD378SJ61miMLCqIOXYhd8x+AJQ== @@ -591,17 +598,20 @@ resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.48.0.tgz#642633964e217905436033a2bd08bf322849b7fb" integrity sha512-ZSjtmelB7IJfWD2Fvb7+Z+ChTIKWq6kjda95fLcQKNS5aheVHn4IkfgRQE3sIIzTcSLwLcLZUD9UBt+V7+h+Pw== -"@floating-ui/core@^1.3.1": - version "1.3.1" - resolved "https://registry.yarnpkg.com/@floating-ui/core/-/core-1.3.1.tgz#4d795b649cc3b1cbb760d191c80dcb4353c9a366" - integrity sha512-Bu+AMaXNjrpjh41znzHqaz3r2Nr8hHuHZT6V2LBKMhyMl0FgKA62PNYbqnfgmzOhoWZj70Zecisbo4H1rotP5g== +"@floating-ui/core@^1.4.1": + version "1.4.1" + resolved "https://registry.yarnpkg.com/@floating-ui/core/-/core-1.4.1.tgz#0d633f4b76052668afb932492ac452f7ebe97f17" + integrity sha512-jk3WqquEJRlcyu7997NtR5PibI+y5bi+LS3hPmguVClypenMsCY3CBa3LAQnozRCtCrYWSEtAdiskpamuJRFOQ== + dependencies: + "@floating-ui/utils" "^0.1.1" "@floating-ui/dom@^1.2.1": - version "1.4.5" - resolved "https://registry.yarnpkg.com/@floating-ui/dom/-/dom-1.4.5.tgz#336dfb9870c98b471ff5802002982e489b8bd1c5" - integrity sha512-96KnRWkRnuBSSFbj0sFGwwOUd8EkiecINVl0O9wiZlZ64EkpyAOG3Xc2vKKNJmru0Z7RqWNymA+6b8OZqjgyyw== + version "1.5.1" + resolved "https://registry.yarnpkg.com/@floating-ui/dom/-/dom-1.5.1.tgz#88b70defd002fe851f17b4a25efb2d3c04d7a8d7" + integrity sha512-KwvVcPSXg6mQygvA1TjbN/gh///36kKtllIF8SUm0qpFj8+rvYrpvlYdL1JoA71SHpDqgSSdGOSoQ0Mp3uY5aw== dependencies: - "@floating-ui/core" "^1.3.1" + "@floating-ui/core" "^1.4.1" + "@floating-ui/utils" "^0.1.1" "@floating-ui/react-dom@^1.3.0": version "1.3.0" @@ -619,6 +629,11 @@ aria-hidden "^1.1.3" tabbable "^6.0.1" +"@floating-ui/utils@^0.1.1": + version "0.1.1" + resolved "https://registry.yarnpkg.com/@floating-ui/utils/-/utils-0.1.1.tgz#1a5b1959a528e374e8037c4396c3e825d6cf4a83" + integrity sha512-m0G6wlnhm/AX0H12IOWtK8gASEMffnX08RtKkCgTdHb9JpHKGloI7icFfLg9ZmQeavcvR0PKmzxClyuFPSjKWw== + "@fortawesome/fontawesome-common-types@6.4.2": version "6.4.2" resolved "https://registry.yarnpkg.com/@fortawesome/fontawesome-common-types/-/fontawesome-common-types-6.4.2.tgz#1766039cad33f8ad87f9467b98e0d18fbc8f01c5" @@ -944,42 +959,42 @@ "@jridgewell/sourcemap-codec" "^1.4.14" "@mantine/core@^6.0.0": - version "6.0.19" - resolved "https://registry.yarnpkg.com/@mantine/core/-/core-6.0.19.tgz#612413f0e8eb117e6a39068a625c6ccf2ae2ccdd" - integrity sha512-SvMZCOgCc315SIg6hkuLM0ZnBaAac4VFDHZ0BM5LIE4MPJUpe4QOLsg/5RGxOa5s7JRCtu/dawH3/9frvfDrhw== + version "6.0.20" + resolved "https://registry.yarnpkg.com/@mantine/core/-/core-6.0.20.tgz#31a4aa3e71c9a365d93d2a82a7c66091fca4e567" + integrity sha512-v/51klCP3nRZIIv0tl8zvOHpksbEK6+AXr0PcQzLnV0KY9xEbV6mh6mEM/w6ygigXRMTQ3oSK85fX1JYg9xC6Q== dependencies: "@floating-ui/react" "^0.19.1" - "@mantine/styles" "6.0.19" - "@mantine/utils" "6.0.19" + "@mantine/styles" "6.0.20" + "@mantine/utils" "6.0.20" "@radix-ui/react-scroll-area" "1.0.2" react-remove-scroll "^2.5.5" react-textarea-autosize "8.3.4" "@mantine/hooks@^6.0.0": - version "6.0.19" - resolved "https://registry.yarnpkg.com/@mantine/hooks/-/hooks-6.0.19.tgz#39f61434304f687d3ba7bf0040c5adf380c7c4b3" - integrity sha512-YkmuB6kmoenU1PVuE8tLBA+6RJIY9hIsGyIQG1yuPAy6SLWNFT8g2T9YvI/psqsUbVIYGaNEXg8zq42xbxnD8Q== + version "6.0.20" + resolved "https://registry.yarnpkg.com/@mantine/hooks/-/hooks-6.0.20.tgz#b52b385ff3a92caba0711c6015899a112efe66fa" + integrity sha512-Sys2qr6KwyNjAWgzm94F9rGG94l709G69pO3iofQXwzKX/fZushk1NMIt5g9De9F+qXm+67wa0DpA0QWLNLxlg== "@mantine/notifications@^6.0.0": - version "6.0.19" - resolved "https://registry.yarnpkg.com/@mantine/notifications/-/notifications-6.0.19.tgz#1aaf82fd4ac5b32faca8baf17475199d3d6ff58e" - integrity sha512-Cr2y8g2nM8bUAP+JYcKdT+a3d+1awUd40EMrDMwb+yUXUSt1amZerYQ7qRuezqvBgiViy/HGnM4WfeF3sfWJRQ== + version "6.0.20" + resolved "https://registry.yarnpkg.com/@mantine/notifications/-/notifications-6.0.20.tgz#08f7c89a994c29e2022620e1cd8eaad948a6584c" + integrity sha512-cEBmuu5G3Pz1dUBtFTfcT56jK8QGd87/TQKpYn4vtiEnwSqVAJ+KwIv30CBl8z1T9/LGDpNhcFtho8H2tkGOxQ== dependencies: - "@mantine/utils" "6.0.19" + "@mantine/utils" "6.0.20" react-transition-group "4.4.2" -"@mantine/styles@6.0.19": - version "6.0.19" - resolved "https://registry.yarnpkg.com/@mantine/styles/-/styles-6.0.19.tgz#7d9a6f2c2a9b345dfd9d12f8fd66af3976d67ab2" - integrity sha512-0tg3Dvv/kxCc1mbQVFhZaIhlSbSbV1F/3xG0NRlP2DF23mw9088o5KaIXGKM6XkXU6OEt/f99nDCUHBk2ixtUg== +"@mantine/styles@6.0.20": + version "6.0.20" + resolved "https://registry.yarnpkg.com/@mantine/styles/-/styles-6.0.20.tgz#0648e8eda28513c8688666219bcd51bac5f4818c" + integrity sha512-Ft0kVwr299Cots5/z3EFXsFpZBAGqC1AlvYQ4XvDRSfHH12QfFAl3zKz1UCFSfk25TAd9cgEm5PFEo/DRxbN/w== dependencies: clsx "1.1.1" csstype "3.0.9" -"@mantine/utils@6.0.19": - version "6.0.19" - resolved "https://registry.yarnpkg.com/@mantine/utils/-/utils-6.0.19.tgz#0197fccc5649259787d5468228139f8815909803" - integrity sha512-duvtnaW1gDR2gnvUqnWhl6DMW7sN0HEWqS8Z/BbwaMi75U+Xp17Q72R9JtiIrxQbzsq+KvH9L9B/pxMVwbLirg== +"@mantine/utils@6.0.20": + version "6.0.20" + resolved "https://registry.yarnpkg.com/@mantine/utils/-/utils-6.0.20.tgz#c331040c62050edf3a513f3cfb702f3170597c10" + integrity sha512-9vsSskPtFyfPW2DOad6brLl9x4MnZtlyhszyoZOyMgXf/vdQbThKhL6PpqUbmlalwQpnCObbkEwVydugeh2dyQ== "@nodelib/fs.scandir@2.1.5": version "2.1.5" @@ -4570,6 +4585,11 @@ regenerator-runtime@^0.13.11: resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz#f6dca3e7ceec20590d07ada785636a90cdca17f9" integrity sha512-kY1AZVr2Ra+t+piVaJ4gxaFaReZVH40AKNo7UCX6W+dEwBo/2oZJzqfuN1qLq1oL45o56cPaTXELwrTh8Fpggg== +regenerator-runtime@^0.14.0: + version "0.14.0" + resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.14.0.tgz#5e19d68eb12d486f797e15a3c6a918f7cec5eb45" + integrity sha512-srw17NI0TUWHuGa5CFGGmhfNIeja30WMBfbslPNhf6JrqQlLN5gcrvig1oqPxiVaXb0oW0XRKtH6Nngs5lKCIA== + regex-not@^1.0.0, regex-not@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/regex-not/-/regex-not-1.0.2.tgz#1f4ece27e00b0b65e0247a6810e6a85d83a5752c" @@ -5133,16 +5153,16 @@ tsconfig-paths@^3.14.2: minimist "^1.2.6" strip-bom "^3.0.0" -tslib@^2.0.0, tslib@^2.5.0, tslib@^2.6.0: - version "2.6.0" - resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.0.tgz#b295854684dbda164e181d259a22cd779dcd7bc3" - integrity sha512-7At1WUettjcSRHXCyYtTselblcHl9PJFFVKiCAy/bY97+BPZXSQ2wbq0P9s8tK2G7dFQfNnlJnPAiArVBVBsfA== - -tslib@^2.1.0: +tslib@^2.0.0, tslib@^2.1.0: version "2.6.2" resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== +tslib@^2.5.0, tslib@^2.6.0: + version "2.6.0" + resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.0.tgz#b295854684dbda164e181d259a22cd779dcd7bc3" + integrity sha512-7At1WUettjcSRHXCyYtTselblcHl9PJFFVKiCAy/bY97+BPZXSQ2wbq0P9s8tK2G7dFQfNnlJnPAiArVBVBsfA== + type-check@^0.4.0, type-check@~0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.4.0.tgz#07b8203bfa7056c0657050e3ccd2c37730bab8f1" diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index 3a5098b8b4e..c3c1f10697a 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -21,6 +21,7 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-enforcer-plugin</artifactId> + <version>${maven-enforcer-plugin.vespa.version}</version> <dependencies> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -53,7 +54,7 @@ <include>com.google.guava:failureaccess:[1.0.1, 2):provided</include> <include>com.google.j2objc:j2objc-annotations:[2.8, 3):provided</include> - <include>com.google.inject:guice:jar:no_aop:${guice.vespa.version}:provided</include> + <include>com.google.inject:guice:jar:${guice.vespa.version}:provided</include> <include>com.sun.activation:javax.activation:[1.2.0, 2):provided</include> <include>com.sun.xml.bind:jaxb-core:${jaxb-core.vespa.version}:provided</include> <include>com.sun.xml.bind:jaxb-impl:${jaxb-impl.vespa.version}:provided</include> @@ -66,7 +67,6 @@ <include>org.slf4j:log4j-over-slf4j:${slf4j.vespa.version}:provided</include> <include>org.slf4j:slf4j-api:${slf4j.vespa.version}:provided</include> <include>org.slf4j:slf4j-jdk14:${slf4j.vespa.version}:provided</include> - <include>xml-apis:xml-apis:${xml-apis.vespa.version}:provided</include> <!-- Vespa provided dependencies --> <include>com.yahoo.vespa:annotations:*:provided</include> @@ -149,6 +149,7 @@ <include>io.prometheus:simpleclient_tracer_common:${prometheus.client.vespa.version}:test</include> <include>io.prometheus:simpleclient_tracer_otel:${prometheus.client.vespa.version}:test</include> <include>io.prometheus:simpleclient_tracer_otel_agent:${prometheus.client.vespa.version}:test</include> + <include>jakarta.inject:jakarta.inject-api:${jakarta.inject.vespa.version}:test</include> <include>junit:junit:${junit4.vespa.version}:test</include> <include>net.java.dev.jna:jna:${jna.vespa.version}:test</include> <include>net.openhft:zero-allocation-hashing:jar:${zero-allocation-hashing.vespa.version}:test</include> @@ -192,15 +193,13 @@ <include>org.hamcrest:hamcrest-core:${hamcrest.vespa.version}:test</include> <include>org.hdrhistogram:HdrHistogram:${hdrhistogram.vespa.version}:test</include> <include>org.json:json:${org.json.vespa.version}:test</include> - <include>org.junit.jupiter:junit-jupiter-api:${junit.vespa.version}:test</include> - <include>org.junit.jupiter:junit-jupiter-engine:${junit.vespa.version}:test</include> - <include>org.junit.platform:junit-platform-commons:${junit.platform.vespa.version}:test</include> - <include>org.junit.platform:junit-platform-engine:${junit.platform.vespa.version}:test</include> - <include>org.junit.vintage:junit-vintage-engine:${junit.vespa.version}:test</include> + <include>org.junit.jupiter:junit-jupiter-api:${junit.vespa.tenant.version}:test</include> + <include>org.junit.jupiter:junit-jupiter-engine:${junit.vespa.tenant.version}:test</include> + <include>org.junit.platform:junit-platform-commons:${junit.platform.vespa.tenant.version}:test</include> + <include>org.junit.platform:junit-platform-engine:${junit.platform.vespa.tenant.version}:test</include> + <include>org.junit.vintage:junit-vintage-engine:${junit.vespa.tenant.version}:test</include> <include>org.lz4:lz4-java:${org.lz4.vespa.version}:test</include> <include>org.opentest4j:opentest4j:${opentest4j.vespa.version}:test</include> - <include>org.osgi:org.osgi.compendium:[4.1.0, 5):test</include> - <include>org.osgi:org.osgi.core:[4.1.0, 5):test</include> <include>xerces:xercesImpl:${xerces.vespa.version}:test</include> </allowed> </enforceDependencies> diff --git a/config-lib/src/main/java/com/yahoo/config/BooleanNode.java b/config-lib/src/main/java/com/yahoo/config/BooleanNode.java index d7ca5b55b55..cb91912bd5a 100644 --- a/config-lib/src/main/java/com/yahoo/config/BooleanNode.java +++ b/config-lib/src/main/java/com/yahoo/config/BooleanNode.java @@ -5,6 +5,7 @@ package com.yahoo.config; * The BooleanNode class represents a boolean in a configuration tree. */ public class BooleanNode extends LeafNode<Boolean> { + public BooleanNode() { } diff --git a/config-lib/src/main/java/com/yahoo/config/ChangesRequiringRestart.java b/config-lib/src/main/java/com/yahoo/config/ChangesRequiringRestart.java index 3eeb637a991..3cd7c5541a9 100644 --- a/config-lib/src/main/java/com/yahoo/config/ChangesRequiringRestart.java +++ b/config-lib/src/main/java/com/yahoo/config/ChangesRequiringRestart.java @@ -63,8 +63,8 @@ public class ChangesRequiringRestart { } } - private ArrayList<ReportLine> report = new ArrayList<>(); - private String componentName; + private final ArrayList<ReportLine> report = new ArrayList<>(); + private final String componentName; public ChangesRequiringRestart(String componentName) { this.componentName = componentName; @@ -105,14 +105,14 @@ public class ChangesRequiringRestart { int commonElements = Math.min(from.size(), to.size()); for (int i = 0; i < commonElements; ++i) { ChangesRequiringRestart childReport = func.getChangesRequiringRestart(from.get(i), to.get(i)); - String prefix = childReport.componentName + "[" + Integer.toString(i) + "]"; + String prefix = childReport.componentName + "[" + i + "]"; mergeChanges(prefix, childReport); } for (int i = commonElements; i < from.size(); ++i) { - report.add(new ReportLine(name + "[" + Integer.toString(i) + "]", from.get(i), null, comment)); + report.add(new ReportLine(name + "[" + i + "]", from.get(i), null, comment)); } for (int i = commonElements; i < to.size(); ++i) { - report.add(new ReportLine(name + "[" + Integer.toString(i) + "]", null, to.get(i), comment)); + report.add(new ReportLine(name + "[" + i + "]", null, to.get(i), comment)); } } return this; diff --git a/config-lib/src/main/java/com/yahoo/config/DoubleNode.java b/config-lib/src/main/java/com/yahoo/config/DoubleNode.java index 5be0c2dc4c8..c90284f9e2e 100644 --- a/config-lib/src/main/java/com/yahoo/config/DoubleNode.java +++ b/config-lib/src/main/java/com/yahoo/config/DoubleNode.java @@ -5,6 +5,7 @@ package com.yahoo.config; * The DoubleNode class represents a double in a configuration tree. */ public class DoubleNode extends LeafNode<Double> { + public DoubleNode() { } diff --git a/config-lib/src/main/java/com/yahoo/config/EnumNode.java b/config-lib/src/main/java/com/yahoo/config/EnumNode.java index b3969dcfd8c..5427e6dc057 100644 --- a/config-lib/src/main/java/com/yahoo/config/EnumNode.java +++ b/config-lib/src/main/java/com/yahoo/config/EnumNode.java @@ -5,6 +5,7 @@ package com.yahoo.config; * The EnumNode class is a superclass for Enumerations in a configuration tree. */ public abstract class EnumNode<ENUM extends Enum<?>> extends LeafNode<ENUM> { + public EnumNode() { } diff --git a/config-lib/src/main/java/com/yahoo/config/FileNode.java b/config-lib/src/main/java/com/yahoo/config/FileNode.java index a7c1ebb1488..170c1e438a9 100644 --- a/config-lib/src/main/java/com/yahoo/config/FileNode.java +++ b/config-lib/src/main/java/com/yahoo/config/FileNode.java @@ -36,5 +36,4 @@ public class FileNode extends LeafNode<FileReference> { return true; } - } diff --git a/config-lib/src/main/java/com/yahoo/config/InnerNode.java b/config-lib/src/main/java/com/yahoo/config/InnerNode.java index 610221094d2..ac60ee9a6e9 100644 --- a/config-lib/src/main/java/com/yahoo/config/InnerNode.java +++ b/config-lib/src/main/java/com/yahoo/config/InnerNode.java @@ -7,6 +7,7 @@ import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Iterator; import java.util.Map; +import java.util.Objects; /** * Superclass for all inner nodes in a {@link ConfigInstance}. @@ -44,7 +45,7 @@ public abstract class InnerNode extends Node { /** * Overrides {@link Node#postInitialize(String)}. - * Perform post initialization on this nodes children. + * Perform post initialization on this node's children. * * @param configId The config id of this instance. */ @@ -65,7 +66,7 @@ public abstract class InnerNode extends Node { return false; /* This implementation requires getChildren() to return elements in order. - Hence we should make it final. Or make equals independent of order. */ + Hence, we should make it final. Or make equals independent of order. */ Collection<Object> children = getChildren().values(); Collection<Object> otherChildren = ((InnerNode)other).getChildren().values(); @@ -74,7 +75,7 @@ public abstract class InnerNode extends Node { while(e1.hasNext() && e2.hasNext()) { Object o1 = e1.next(); Object o2 = e2.next(); - if (!(o1 == null ? o2 == null : o1.equals(o2))) + if (!(Objects.equals(o1, o2))) return false; } return !(e1.hasNext() || e2.hasNext()); @@ -135,7 +136,6 @@ public abstract class InnerNode extends Node { ret.put(name + "{" + entry.getKey() + "}", entry.getValue()); } - private static void addNodeVectorEntries(Map<String, Node> ret, String name, NodeVector<?> vector) { for (int j = 0; j < vector.length(); j++) ret.put(name + "[" + j + "]", (Node) vector.get(j)); diff --git a/config-lib/src/main/java/com/yahoo/config/InnerNodeVector.java b/config-lib/src/main/java/com/yahoo/config/InnerNodeVector.java index 8bf9b4c1520..30a7580c6dd 100644 --- a/config-lib/src/main/java/com/yahoo/config/InnerNodeVector.java +++ b/config-lib/src/main/java/com/yahoo/config/InnerNodeVector.java @@ -5,7 +5,6 @@ import java.util.List; /** * @author gjoranv - * @since 5.1.4 */ public class InnerNodeVector<NODE extends InnerNode> extends NodeVector<NODE> { diff --git a/config-lib/src/main/java/com/yahoo/config/LeafNodeVector.java b/config-lib/src/main/java/com/yahoo/config/LeafNodeVector.java index ddbed4258dc..a4fea95088d 100644 --- a/config-lib/src/main/java/com/yahoo/config/LeafNodeVector.java +++ b/config-lib/src/main/java/com/yahoo/config/LeafNodeVector.java @@ -57,7 +57,7 @@ public class LeafNodeVector<REAL, NODE extends LeafNode<REAL>> extends NodeVecto // TODO: Try to eliminate the need for this method when we have moved FileAcquirer to the config library // It is needed now because the builder has a list of String, while REAL=FileReference. public static LeafNodeVector<FileReference, FileNode> createFileNodeVector(Collection<String> values) { - List<FileReference> fileReferences = new ArrayList<FileReference>(); + List<FileReference> fileReferences = new ArrayList<>(); for (String s : values) fileReferences.add(new FileReference(ReferenceNode.stripQuotes(s))); diff --git a/config-lib/src/main/java/com/yahoo/config/LongNode.java b/config-lib/src/main/java/com/yahoo/config/LongNode.java index 02b6a5f7654..ac6ae457087 100755 --- a/config-lib/src/main/java/com/yahoo/config/LongNode.java +++ b/config-lib/src/main/java/com/yahoo/config/LongNode.java @@ -48,4 +48,5 @@ public class LongNode extends LeafNode<Long> { void serialize(Serializer serializer) { serializer.serialize(value); } + } diff --git a/config-lib/src/main/java/com/yahoo/config/ModelReference.java b/config-lib/src/main/java/com/yahoo/config/ModelReference.java index fbf34c7ff23..38ddf753bd6 100644 --- a/config-lib/src/main/java/com/yahoo/config/ModelReference.java +++ b/config-lib/src/main/java/com/yahoo/config/ModelReference.java @@ -67,8 +67,8 @@ public class ModelReference { public String toString() { if (resolved != null) return resolved.toString(); return modelId.orElse("\"\"") + " " + - url.map(v -> v.value()).orElse("\"\"") + " " + - path.map(v -> v.value()).orElse("\"\""); + url.map(UrlReference::value).orElse("\"\"") + " " + + path.map(FileReference::value).orElse("\"\""); } /** diff --git a/config-lib/src/main/java/com/yahoo/config/Node.java b/config-lib/src/main/java/com/yahoo/config/Node.java index a07409b19e7..188a9ecc623 100644 --- a/config-lib/src/main/java/com/yahoo/config/Node.java +++ b/config-lib/src/main/java/com/yahoo/config/Node.java @@ -9,16 +9,16 @@ package com.yahoo.config; public abstract class Node { /** - * Postinitialize this node. Any node needing to process its values depending on the config + * Post-initialize this node. Any node needing to process its values depending on the config * id should override this method. * * @param configId the configId of the ConfigInstance that owns (or is) this node */ - public void postInitialize(String configId) { return; } + public void postInitialize(String configId) {} /** * This method is meant for internal use in the configuration system. - * Overrides Object.clone(), and is overriden by LeafNode.clone(). + * Overrides Object.clone(), and is overridden by LeafNode.clone(). * * @return a new instance similar to this object. */ diff --git a/config-lib/src/main/java/com/yahoo/config/NodeVector.java b/config-lib/src/main/java/com/yahoo/config/NodeVector.java index 3d07de944fd..5ab1f97ecde 100644 --- a/config-lib/src/main/java/com/yahoo/config/NodeVector.java +++ b/config-lib/src/main/java/com/yahoo/config/NodeVector.java @@ -12,7 +12,7 @@ import java.util.*; */ public abstract class NodeVector<NODE> implements java.util.List<NODE> { - protected final ArrayList<NODE> vector = new ArrayList<NODE>(); + protected final ArrayList<NODE> vector = new ArrayList<>(); /** * Returns the number of elements in this NodeVector. @@ -24,7 +24,6 @@ public abstract class NodeVector<NODE> implements java.util.List<NODE> { return size(); } - @SuppressWarnings("serial") public static class ReadOnlyException extends RuntimeException { } @@ -88,7 +87,6 @@ public abstract class NodeVector<NODE> implements java.util.List<NODE> { return vector.hashCode(); } - @SuppressWarnings("unchecked") public NODE get(int index) { return vector.get(index); } diff --git a/config-lib/src/main/java/com/yahoo/config/Serializer.java b/config-lib/src/main/java/com/yahoo/config/Serializer.java index 12c45c789a4..5e2d35cb2af 100644 --- a/config-lib/src/main/java/com/yahoo/config/Serializer.java +++ b/config-lib/src/main/java/com/yahoo/config/Serializer.java @@ -3,7 +3,6 @@ package com.yahoo.config; /** * @author Ulf Lilleengen -* @since 5.1 */ public interface Serializer { Serializer createInner(String name); @@ -28,4 +27,5 @@ public interface Serializer { void serialize(long value); void serialize(int value); void serialize(String value); + } diff --git a/config-lib/src/main/java/com/yahoo/config/StringNode.java b/config-lib/src/main/java/com/yahoo/config/StringNode.java index 2e4f0df07af..0a87a09ddc6 100644 --- a/config-lib/src/main/java/com/yahoo/config/StringNode.java +++ b/config-lib/src/main/java/com/yahoo/config/StringNode.java @@ -29,10 +29,10 @@ public class StringNode extends LeafNode<String> { /** * Returns the value of this string. Same as {@link #getValue()} * since the value of this node is a String (but implementations - * in other {@link LeafNode} sub-classes differ). + * in other {@link LeafNode} subclasses differ). * * @return the string representation of this StringNode, or null if - * the vlaue is explicitly set to null + * the value is explicitly set to null */ public String value() { return value; @@ -63,30 +63,18 @@ public class StringNode extends LeafNode<String> { throw new IllegalArgumentException("Parse error" + string); } switch (sb.charAt(i)) { - case'n': - sb.setCharAt(i, '\n'); - break; - case'r': - sb.setCharAt(i, '\r'); - break; - case't': - sb.setCharAt(i, '\t'); - break; - case'f': - sb.setCharAt(i, '\f'); - break; - case'x': + case 'n' -> sb.setCharAt(i, '\n'); + case 'r' -> sb.setCharAt(i, '\r'); + case 't' -> sb.setCharAt(i, '\t'); + case 'f' -> sb.setCharAt(i, '\f'); + case 'x' -> { if (i + 2 >= sb.length()) { - throw new IllegalArgumentException - ("Could not parse hex value " + string); + throw new IllegalArgumentException("Could not parse hex value " + string); } - sb.setCharAt(i, (char) Integer.parseInt - (sb.substring(i + 1, i + 3), 16)); + sb.setCharAt(i, (char) Integer.parseInt(sb.substring(i + 1, i + 3), 16)); sb.delete(i + 1, i + 3); - break; - case'\\': - sb.setCharAt(i, '\\'); - break; + } + case '\\' -> sb.setCharAt(i, '\\'); } } } @@ -101,7 +89,7 @@ public class StringNode extends LeafNode<String> { } /** - * Sets the value of this string from a the string representation + * Sets the value of this string from the string representation * of this value in the (escaped) input configuration. The value * supplied to this method needs un-escaping and will be * un-escaped. diff --git a/config-lib/src/main/java/com/yahoo/config/text/StringUtilities.java b/config-lib/src/main/java/com/yahoo/config/text/StringUtilities.java index ca0bc1758e3..5a1a570e913 100644 --- a/config-lib/src/main/java/com/yahoo/config/text/StringUtilities.java +++ b/config-lib/src/main/java/com/yahoo/config/text/StringUtilities.java @@ -20,9 +20,9 @@ public class StringUtilities { private static class ReplacementCharacters { - public byte[] needEscape = new byte[256]; - public byte[] replacement1 = new byte[256]; - public byte[] replacement2 = new byte[256]; + public final byte[] needEscape = new byte[256]; + public final byte[] replacement1 = new byte[256]; + public final byte[] replacement2 = new byte[256]; public ReplacementCharacters() { for (int i=0; i<256; ++i) { diff --git a/config-lib/src/test/java/com/yahoo/config/ConfigInstanceBuilderTest.java b/config-lib/src/test/java/com/yahoo/config/ConfigInstanceBuilderTest.java index 76871aaca42..15e35682980 100644 --- a/config-lib/src/test/java/com/yahoo/config/ConfigInstanceBuilderTest.java +++ b/config-lib/src/test/java/com/yahoo/config/ConfigInstanceBuilderTest.java @@ -15,7 +15,6 @@ import java.lang.reflect.Method; import java.util.Arrays; import java.util.HashMap; import java.util.Map; -import java.util.Optional; import static com.yahoo.foo.StructtypesConfig.Simple.Gender.Enum.FEMALE; import static com.yahoo.test.FunctionTestConfig.BasicStruct; @@ -257,9 +256,8 @@ public class ConfigInstanceBuilderTest { assertEquals(configId, config.refwithdef()); assertEquals("etc", config.fileVal().value()); assertEquals(1, config.boolarr().size()); - assertEquals(1, config.boolarr().size()); // new api with accessor for a List of the original Java type - assertFalse(config.boolarr().get(0)); // new List api - assertFalse(config.boolarr(0)); // short-hand + assertFalse(config.boolarr().get(0)); + assertFalse(config.boolarr(0)); // shorthand assertEquals(0, config.intarr().size()); assertEquals(2, config.longarr().size()); assertEquals(Long.MAX_VALUE, config.longarr(0)); @@ -267,7 +265,7 @@ public class ConfigInstanceBuilderTest { assertEquals(2, config.doublearr().size()); assertEquals(1, config.stringarr().size()); assertEquals(1, config.enumarr().size()); - assertEquals(FunctionTestConfig.Enumarr.VALUES, config.enumarr().get(0)); // new List api, don't have to call value() + assertEquals(FunctionTestConfig.Enumarr.VALUES, config.enumarr().get(0)); assertEquals(3, config.refarr().size()); assertEquals(1, config.fileArr().size()); assertEquals(configId, config.refarr(0)); @@ -283,10 +281,10 @@ public class ConfigInstanceBuilderTest { assertEquals("basicFoo", config.basicStruct().foo()); assertEquals(3, config.basicStruct().bar()); // new List api assertEquals(2, config.basicStruct().intArr().size()); - assertEquals(310, config.basicStruct().intArr().get(0).intValue()); // new List api - assertEquals(311, config.basicStruct().intArr().get(1).intValue()); // new List api - assertEquals(310, config.basicStruct().intArr(0)); // short-hand - assertEquals("inner0", config.rootStruct().inner0().name()); // new List api + assertEquals(310, config.basicStruct().intArr().get(0).intValue()); + assertEquals(311, config.basicStruct().intArr().get(1).intValue()); + assertEquals(310, config.basicStruct().intArr(0)); // shorthand + assertEquals("inner0", config.rootStruct().inner0().name()); assertEquals(11, config.rootStruct().inner0().index()); assertEquals("inner1", config.rootStruct().inner1().name()); assertEquals(12, config.rootStruct().inner1().index()); @@ -297,8 +295,8 @@ public class ConfigInstanceBuilderTest { assertEquals("blue a=\"escaped\"", config.rootStruct().innerArr(1).stringVal()); assertEquals(2, config.myarray().size()); // new List api - assertEquals(configId, config.myarray().get(0).refval()); // new List api - assertEquals(configId, config.myarray(0).refval()); // short-hand + assertEquals(configId, config.myarray().get(0).refval()); + assertEquals(configId, config.myarray(0).refval()); // shorthand assertEquals("file0", config.myarray(0).fileVal().value()); assertEquals(1, config.myarray(0).myStruct().a()); assertEquals(2, config.myarray(0).myStruct().b()); diff --git a/config-lib/src/test/java/com/yahoo/config/ConfigInstanceEqualsTest.java b/config-lib/src/test/java/com/yahoo/config/ConfigInstanceEqualsTest.java index db1509fba93..4a16eebfeb9 100644 --- a/config-lib/src/test/java/com/yahoo/config/ConfigInstanceEqualsTest.java +++ b/config-lib/src/test/java/com/yahoo/config/ConfigInstanceEqualsTest.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config; -import com.yahoo.test.AppConfig; import com.yahoo.test.FunctionTestConfig; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -48,11 +47,6 @@ public class ConfigInstanceEqualsTest { } @Test - void require_false_for_different_subclass() { - assertNotEquals(config1, new AppConfig(new AppConfig.Builder())); - } - - @Test void require_false_for_different_scalars_at_root_node() { assertNotEquals(config1, new FunctionTestConfig(newBuilder().bool_val(true))); assertNotEquals(config1, new FunctionTestConfig(newBuilder().int_val(0))); diff --git a/config-lib/src/test/java/com/yahoo/config/EnumNodeTest.java b/config-lib/src/test/java/com/yahoo/config/EnumNodeTest.java index d53db9fa9c7..da82ec69a62 100644 --- a/config-lib/src/test/java/com/yahoo/config/EnumNodeTest.java +++ b/config-lib/src/test/java/com/yahoo/config/EnumNodeTest.java @@ -23,7 +23,7 @@ public class EnumNodeTest { try { value = Enum.valueOf(name); return true; - } catch (IllegalArgumentException e) { + } catch (IllegalArgumentException ignored) { } return false; } diff --git a/config-lib/src/test/java/com/yahoo/config/NodeVectorTest.java b/config-lib/src/test/java/com/yahoo/config/NodeVectorTest.java index fb751931062..b5080f76605 100644 --- a/config-lib/src/test/java/com/yahoo/config/NodeVectorTest.java +++ b/config-lib/src/test/java/com/yahoo/config/NodeVectorTest.java @@ -4,6 +4,7 @@ package com.yahoo.config; import org.junit.jupiter.api.Test; import java.util.Arrays; +import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -17,72 +18,52 @@ public class NodeVectorTest { @Test void require_that_add_throws_exception() { - assertThrows(NodeVector.ReadOnlyException.class, () -> { - new TestNodeVector("foo").add(barNode()); - }); + assertThrows(NodeVector.ReadOnlyException.class, () -> new TestNodeVector("foo").add(barNode())); } @Test void require_that_addindex_throws_exception() { - assertThrows(NodeVector.ReadOnlyException.class, () -> { - new TestNodeVector("foo").add(0, barNode()); - }); + assertThrows(NodeVector.ReadOnlyException.class, () -> new TestNodeVector("foo").add(0, barNode())); } @Test void require_that_addAll_throws_exception() { - assertThrows(NodeVector.ReadOnlyException.class, () -> { - new TestNodeVector("foo").addAll(Arrays.asList(barNode())); - }); + assertThrows(NodeVector.ReadOnlyException.class, () -> new TestNodeVector("foo").add(barNode())); } @Test void require_that_addAllindex_throws_exception() { - assertThrows(NodeVector.ReadOnlyException.class, () -> { - new TestNodeVector("foo").addAll(0, Arrays.asList(barNode())); - }); + assertThrows(NodeVector.ReadOnlyException.class, () -> new TestNodeVector("foo").addAll(0, List.of(barNode()))); } @Test void require_that_clear_throws_exception() { - assertThrows(NodeVector.ReadOnlyException.class, () -> { - new TestNodeVector("foo").clear(); - }); + assertThrows(NodeVector.ReadOnlyException.class, () -> new TestNodeVector("foo").clear()); } @Test void require_that_remove_index_throws_exception() { - assertThrows(NodeVector.ReadOnlyException.class, () -> { - new TestNodeVector("foo").remove(0); - }); + assertThrows(NodeVector.ReadOnlyException.class, () -> new TestNodeVector("foo").remove(0)); } @Test void require_that_remove_object_throws_exception() { - assertThrows(NodeVector.ReadOnlyException.class, () -> { - new TestNodeVector("foo").remove(null); - }); + assertThrows(NodeVector.ReadOnlyException.class, () -> new TestNodeVector("foo").remove(null)); } @Test void require_that_removeAll_throws_exception() { - assertThrows(NodeVector.ReadOnlyException.class, () -> { - new TestNodeVector("foo").removeAll(null); - }); + assertThrows(NodeVector.ReadOnlyException.class, () -> new TestNodeVector("foo").removeAll(null)); } @Test void require_that_retainAll_throws_exception() { - assertThrows(NodeVector.ReadOnlyException.class, () -> { - new TestNodeVector("foo").retainAll(null); - }); + assertThrows(NodeVector.ReadOnlyException.class, () -> new TestNodeVector("foo").retainAll(null)); } @Test void require_that_set_throws_exception() { - assertThrows(NodeVector.ReadOnlyException.class, () -> { - new TestNodeVector("foo").set(0, null); - }); + assertThrows(NodeVector.ReadOnlyException.class, () -> new TestNodeVector("foo").set(0, null)); } @Test @@ -91,7 +72,7 @@ public class NodeVectorTest { TestNodeVector v = new TestNodeVector(val.getValue()); assertTrue(v.contains(val)); assertFalse(v.contains(barNode())); - assertTrue(v.containsAll(Arrays.asList(val))); + assertTrue(v.contains(val)); assertFalse(v.containsAll(Arrays.asList(val, barNode()))); } diff --git a/config-lib/src/test/java/com/yahoo/config/StringNodeTest.java b/config-lib/src/test/java/com/yahoo/config/StringNodeTest.java index 3470c96bbcc..7211f8ad776 100644 --- a/config-lib/src/test/java/com/yahoo/config/StringNodeTest.java +++ b/config-lib/src/test/java/com/yahoo/config/StringNodeTest.java @@ -23,9 +23,7 @@ public class StringNodeTest { @Test void testUnescapedQuotedStringExceptions() { - assertThrows(IllegalArgumentException.class, () -> { - StringNode.unescapeQuotedString("foo\\"); - }); + assertThrows(IllegalArgumentException.class, () -> StringNode.unescapeQuotedString("foo\\")); } @Test diff --git a/config-lib/src/test/java/com/yahoo/config/codegen/NamespaceAndPackageTest.java b/config-lib/src/test/java/com/yahoo/config/codegen/NamespaceAndPackageTest.java index 39e5f64e5e2..8d15832b73a 100644 --- a/config-lib/src/test/java/com/yahoo/config/codegen/NamespaceAndPackageTest.java +++ b/config-lib/src/test/java/com/yahoo/config/codegen/NamespaceAndPackageTest.java @@ -12,8 +12,8 @@ import static org.junit.jupiter.api.Assertions.assertEquals; * @author gjoranv */ public class NamespaceAndPackageTest { - private static String NAMESPACE = "my.namespace"; - private static String PACKAGE = "com.github.myproject"; + private static final String NAMESPACE = "my.namespace"; + private static final String PACKAGE = "com.github.myproject"; @Test void namespace_is_set_from_def_file() { diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java index b7969267328..69749ee6f96 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ApplicationClusterEndpoint.java @@ -18,21 +18,6 @@ import java.util.stream.Stream; * @author mortent */ public class ApplicationClusterEndpoint { - @Override - public String toString() { - return "ApplicationClusterEndpoint{" + - "dnsName=" + dnsName + - ", scope=" + scope + - ", routingMethod=" + routingMethod + - ", weight=" + weight + - ", hostNames=" + hostNames + - ", clusterId='" + clusterId + "'" + - '}'; - } - - public enum Scope {application, global, zone} - - public enum RoutingMethod {shared, sharedLayer4, exclusive} private final DnsName dnsName; private final Scope scope; @@ -40,14 +25,16 @@ public class ApplicationClusterEndpoint { private final int weight; private final List<String> hostNames; private final String clusterId; + private final AuthMethod authMethod; - private ApplicationClusterEndpoint(DnsName dnsName, Scope scope, RoutingMethod routingMethod, int weight, List<String> hostNames, String clusterId) { - this.dnsName = dnsName; - this.scope = scope; - this.routingMethod = routingMethod; + private ApplicationClusterEndpoint(DnsName dnsName, Scope scope, RoutingMethod routingMethod, int weight, List<String> hostNames, String clusterId, AuthMethod authMethod) { + this.dnsName = Objects.requireNonNull(dnsName); + this.scope = Objects.requireNonNull(scope); + this.routingMethod = Objects.requireNonNull(routingMethod); this.weight = weight; - this.hostNames = List.copyOf(hostNames); - this.clusterId = clusterId; + this.hostNames = List.copyOf(Objects.requireNonNull(hostNames)); + this.clusterId = Objects.requireNonNull(clusterId); + this.authMethod = Objects.requireNonNull(authMethod); } public DnsName dnsName() { @@ -74,18 +61,42 @@ public class ApplicationClusterEndpoint { return clusterId; } + public AuthMethod authMethod() { + return authMethod; + } + + @Override + public String toString() { + return "ApplicationClusterEndpoint{" + + "dnsName=" + dnsName + + ", scope=" + scope + + ", routingMethod=" + routingMethod + + ", weight=" + weight + + ", hostNames=" + hostNames + + ", clusterId='" + clusterId + '\'' + + ", authMethod=" + authMethod + + '}'; + } + public static Builder builder() { return new Builder(); } + public enum Scope { application, global, zone } + + public enum RoutingMethod { shared, sharedLayer4, exclusive } + + public enum AuthMethod { mtls, token } + public static class Builder { private DnsName dnsName; private Scope scope; private RoutingMethod routingMethod; - private int weigth = 1; + private int weight = 1; private List<String> hosts; private String clusterId; + private AuthMethod authMethod = AuthMethod.mtls; // TODO(mpolden): For compatibility with older config-models. Remove when < 8.221 is gone public Builder dnsName(DnsName name) { this.dnsName = name; @@ -118,7 +129,7 @@ public class ApplicationClusterEndpoint { } public Builder weight(int weigth) { - this.weigth = weigth; + this.weight = weigth; return this; } @@ -132,9 +143,15 @@ public class ApplicationClusterEndpoint { return this; } + public Builder authMethod(AuthMethod authMethod) { + this.authMethod = authMethod; + return this; + } + public ApplicationClusterEndpoint build() { - return new ApplicationClusterEndpoint(dnsName, scope, routingMethod, weigth, hosts, clusterId); + return new ApplicationClusterEndpoint(dnsName, scope, routingMethod, weight, hosts, clusterId, authMethod); } + } public static class DnsName implements Comparable<DnsName> { diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ContainerEndpoint.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ContainerEndpoint.java index 78da750fb5b..de06ddd549a 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ContainerEndpoint.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ContainerEndpoint.java @@ -3,9 +3,7 @@ package com.yahoo.config.model.api; import java.util.List; import java.util.Objects; -import java.util.Optional; import java.util.OptionalInt; -import java.util.OptionalLong; /** * ContainerEndpoint tracks the service names that a Container Cluster should be @@ -21,6 +19,7 @@ public class ContainerEndpoint { private final List<String> names; private final OptionalInt weight; private final ApplicationClusterEndpoint.RoutingMethod routingMethod; + private final ApplicationClusterEndpoint.AuthMethod authMethod; public ContainerEndpoint(String clusterId, ApplicationClusterEndpoint.Scope scope, List<String> names) { this(clusterId, scope, names, OptionalInt.empty()); @@ -31,11 +30,16 @@ public class ContainerEndpoint { } public ContainerEndpoint(String clusterId, ApplicationClusterEndpoint.Scope scope, List<String> names, OptionalInt weight, ApplicationClusterEndpoint.RoutingMethod routingMethod) { + this(clusterId, scope, names, weight, routingMethod, ApplicationClusterEndpoint.AuthMethod.mtls); + } + + public ContainerEndpoint(String clusterId, ApplicationClusterEndpoint.Scope scope, List<String> names, OptionalInt weight, ApplicationClusterEndpoint.RoutingMethod routingMethod, ApplicationClusterEndpoint.AuthMethod authMethod) { this.clusterId = Objects.requireNonNull(clusterId); this.scope = Objects.requireNonNull(scope); this.names = List.copyOf(Objects.requireNonNull(names)); - this.weight = weight; - this.routingMethod = routingMethod; + this.weight = Objects.requireNonNull(weight); + this.routingMethod = Objects.requireNonNull(routingMethod); + this.authMethod = Objects.requireNonNull(authMethod); } public String clusterId() { @@ -58,6 +62,10 @@ public class ContainerEndpoint { return routingMethod; } + public ApplicationClusterEndpoint.AuthMethod authMethod() { + return authMethod; + } + @Override public boolean equals(Object o) { if (this == o) return true; @@ -67,16 +75,17 @@ public class ContainerEndpoint { Objects.equals(scope, that.scope) && Objects.equals(names, that.names) && Objects.equals(weight, that.weight) && - Objects.equals(routingMethod, that.routingMethod); + Objects.equals(routingMethod, that.routingMethod) && + Objects.equals(authMethod, that.authMethod); } @Override public int hashCode() { - return Objects.hash(clusterId, names, scope, weight, routingMethod); + return Objects.hash(clusterId, names, scope, weight, routingMethod, authMethod); } @Override public String toString() { - return String.format("container endpoint %s -> %s [scope=%s, weight=%s, routingMetod=%s]", clusterId, names, scope, weight, routingMethod); + return String.format("container endpoint %s -> %s [scope=%s, weight=%s, routingMethod=%s, authMethod=%s]", clusterId, names, scope, weight, routingMethod, authMethod); } } diff --git a/config-model-fat/pom.xml b/config-model-fat/pom.xml index adf511faabe..55592897447 100644 --- a/config-model-fat/pom.xml +++ b/config-model-fat/pom.xml @@ -98,16 +98,10 @@ javax.security.auth.callback, javax.security.auth.x500, javax.security.auth, - javax.xml.datatype, - javax.xml.namespace, - javax.xml.parsers, - javax.xml.transform, - javax.xml.xpath, + javax.xml.*, <!-- expands to all packages in Java module java.xml --> org.bouncycastle.*, <!-- expands to all BC packages by Felix plugin --> - org.w3c.dom.bootstrap, - org.w3c.dom.ls, - org.w3c.dom, - org.xml.sax, + org.w3c.dom.*, <!-- expands to all packages in Java module java.xml --> + org.xml.sax.*, <!-- expands to all packages in Java module java.xml --> <!-- TODO: The fat bundle becomes more brittle for each package added below. Use interfaces in model-api instead. --> com.yahoo.vespa.config, com.yahoo.vespa.config.buildergen, @@ -195,7 +189,7 @@ <i>com.google.errorprone:error_prone_annotations:*:*</i> <i>com.google.guava:failureaccess:*:*</i> <i>com.google.guava:guava:*:*</i> - <i>com.google.inject:guice:jar:no_aop:*:*</i> + <i>com.google.inject:guice:jar:*:*</i> <i>com.google.j2objc:j2objc-annotations:*:*</i> <i>com.google.protobuf:protobuf-java:*:*</i> <i>com.sun.activation:javax.activation:*:*</i> @@ -207,6 +201,7 @@ <i>io.prometheus:simpleclient_tracer_common:*:*</i> <i>io.prometheus:simpleclient_tracer_otel:*:*</i> <i>io.prometheus:simpleclient_tracer_otel_agent:*:*</i> + <i>jakarta.inject:jakarta.inject-api:*:*</i> <i>javax.inject:javax.inject:*:*</i> <i>net.openhft:zero-allocation-hashing:*:*</i> <i>org.antlr:antlr-runtime:*:*</i> @@ -219,7 +214,6 @@ <i>org.slf4j:slf4j-api:*:*</i> <i>org.slf4j:slf4j-jdk14:*:*</i> <i>xerces:xercesImpl:*:*</i> - <i>xml-apis:xml-apis:*:*</i> </allowed> </enforceDependencies> </rules> diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java index 04faff688f8..e3c7693c608 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java @@ -7,7 +7,6 @@ import com.yahoo.config.ConfigInstance; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AnyConfigProducer; -import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.io.IOUtils; import com.yahoo.log.InvalidLogFormatException; import com.yahoo.log.LogMessage; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 584207caeac..07983c7c85a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -39,6 +39,7 @@ import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.configserver.ConfigserverCluster; import com.yahoo.vespa.model.utils.FileSender; + import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -48,7 +49,6 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -import static com.yahoo.config.model.api.ApplicationClusterEndpoint.RoutingMethod.sharedLayer4; import static com.yahoo.vespa.model.container.docproc.DocprocChains.DOCUMENT_TYPE_MANAGER_CLASS; /** @@ -98,7 +98,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat private Integer memoryPercentage = null; - private List<ApplicationClusterEndpoint> endpointList = List.of(); + private List<ApplicationClusterEndpoint> endpoints = List.of(); public ApplicationContainerCluster(TreeConfigProducer<?> parent, String configSubId, String clusterId, DeployState deployState) { super(parent, configSubId, clusterId, deployState, true, 10); @@ -132,7 +132,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat super.doPrepare(deployState); addAndSendApplicationBundles(deployState); sendUserConfiguredFiles(deployState); - createEndpointList(deployState); + createEndpoints(deployState); } private void addAndSendApplicationBundles(DeployState deployState) { @@ -198,7 +198,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat } /** Create list of endpoints, these will be consumed later by LbServicesProducer */ - private void createEndpointList(DeployState deployState) { + private void createEndpoints(DeployState deployState) { if (!deployState.isHosted()) return; if (deployState.getProperties().applicationId().instance().isTester()) return; List<ApplicationClusterEndpoint> endpoints = new ArrayList<>(); @@ -224,25 +224,26 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat .dnsName(l4Name) .hosts(hosts) .clusterId(getName()) + .authMethod(ApplicationClusterEndpoint.AuthMethod.mtls) .build()); } } // Include all endpoints provided by controller endpointsFromController.stream() - .filter(ce -> ce.clusterId().equals(getName())) - .filter(ce -> ce.routingMethod() == sharedLayer4) - .forEach(ce -> ce.names().forEach( - name -> endpoints.add(ApplicationClusterEndpoint.builder() - .scope(ce.scope()) - .weight(Long.valueOf(ce.weight().orElse(1)).intValue()) // Default to weight=1 if not set - .routingMethod(ce.routingMethod()) - .dnsName(ApplicationClusterEndpoint.DnsName.from(name)) - .hosts(hosts) - .clusterId(getName()) - .build()) - )); - endpointList = List.copyOf(endpoints); + .filter(ce -> ce.clusterId().equals(getName())) + .forEach(ce -> ce.names().forEach( + name -> endpoints.add(ApplicationClusterEndpoint.builder() + .scope(ce.scope()) + .weight(ce.weight().orElse(1)) // Default to weight=1 if not set + .routingMethod(ce.routingMethod()) + .dnsName(ApplicationClusterEndpoint.DnsName.from(name)) + .hosts(hosts) + .clusterId(getName()) + .authMethod(ce.authMethod()) + .build()) + )); + this.endpoints = List.copyOf(endpoints); } @Override @@ -364,7 +365,7 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat @Override public List<ApplicationClusterEndpoint> endpoints() { - return endpointList; + return endpoints; } @Override diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 2562e1e3124..894fc55c014 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -439,7 +439,6 @@ public class ContainerClusterTest { cluster.doPrepare(state); List<ApplicationClusterEndpoint> endpoints = cluster.endpoints(); - assertNames(List.of(), endpoints.stream().filter(e -> e.routingMethod() == shared).toList()); assertNames(expectedSharedL4Names, endpoints.stream().filter(e -> e.routingMethod() == sharedLayer4).toList()); List<ContainerEndpoint> endpointsWithWeight = diff --git a/configserver-flags/pom.xml b/configserver-flags/pom.xml index 02824f2e6e3..02395fc3559 100644 --- a/configserver-flags/pom.xml +++ b/configserver-flags/pom.xml @@ -58,7 +58,7 @@ <groupId>com.google.inject</groupId> <artifactId>guice</artifactId> <scope>provided</scope> - <classifier>no_aop</classifier> + <exclusions> <exclusion> <groupId>junit</groupId> diff --git a/configserver/pom.xml b/configserver/pom.xml index c750e4fe0f6..95ab31155ce 100644 --- a/configserver/pom.xml +++ b/configserver/pom.xml @@ -12,78 +12,6 @@ <packaging>container-plugin</packaging> <version>8-SNAPSHOT</version> <dependencies> - - <!-- BEGIN Jersey deps. - TODO: Vespa > 8, and provision-controller does not import any Jersey related packages: - Remove, and remove all package-info.java files for jersey/jackson packages. --> - - <dependency> - <groupId>com.fasterxml.jackson.jaxrs</groupId> - <artifactId>jackson-jaxrs-json-provider</artifactId> - <version>${jackson2.vespa.version}</version> - <exclusions> - <exclusion> - <!-- Conflicts with javax.activation:javax.activation-api:1.2.0, which is "exported" via jdisc_core. --> - <groupId>jakarta.activation</groupId> - <artifactId>jakarta.activation-api</artifactId> - </exclusion> - <exclusion> - <!-- Conflicts with javax.xml.bind:jaxb-api:2.3, which is "exported" via jdisc_core.--> - <groupId>jakarta.xml.bind</groupId> - <artifactId>jakarta.xml.bind-api</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>javax.ws.rs</groupId> - <artifactId>javax.ws.rs-api</artifactId> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>org.glassfish.jersey.core</groupId> - <artifactId>jersey-client</artifactId> - <version>${jersey.vespa.version}</version> - </dependency> - <dependency> - <groupId>org.glassfish.jersey.core</groupId> - <artifactId>jersey-server</artifactId> - <version>${jersey.vespa.version}</version> - <exclusions> - <exclusion> - <groupId>org.glassfish.jersey.media</groupId> - <artifactId>jersey-media-jaxb</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.glassfish.jersey.ext</groupId> - <artifactId>jersey-proxy-client</artifactId> - <version>${jersey.vespa.version}</version> - </dependency> - <dependency> - <groupId>org.glassfish.jersey.media</groupId> - <artifactId>jersey-media-json-jackson</artifactId> - <version>${jersey.vespa.version}</version> - <exclusions> - <!-- Prevent embedding deps provided by jdisc --> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-annotations</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.glassfish.jersey.media</groupId> - <artifactId>jersey-media-multipart</artifactId> - <version>${jersey.vespa.version}</version> - </dependency> - - <!-- END Jersey deps --> - <dependency> <groupId>org.hamcrest</groupId> <artifactId>hamcrest</artifactId> @@ -288,11 +216,7 @@ <artifactId>bundle-plugin</artifactId> <extensions>true</extensions> <configuration> - <!-- TODO: Vespa > 8: remove importPackage when the jackson-jaxrs-json-provider bundle is no longer installed in jdisc --> - <!-- added to ensure using the same class as orchestrator, core-dump-reporter, provision-controller and controller-clients --> - <importPackage>com.fasterxml.jackson.jaxrs.json;version="[2.12.6,3)"</importPackage> - <allowEmbeddedArtifacts>com.fasterxml.jackson.core:jackson-annotations, com.fasterxml.jackson.core:jackson-core, - com.yahoo.vespa:airlift-zstd</allowEmbeddedArtifacts> + <allowEmbeddedArtifacts>com.yahoo.vespa:airlift-zstd</allowEmbeddedArtifacts> </configuration> </plugin> <plugin> diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java index b813d56b345..33f47bbc154 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java @@ -11,6 +11,7 @@ import com.yahoo.slime.SlimeUtils; import java.util.ArrayList; import java.util.List; +import java.util.OptionalInt; /** * Contains all methods for de-/serializing ContainerEndpoints to/from JSON. @@ -32,48 +33,48 @@ public class ContainerEndpointSerializer { private static final String namesField = "names"; private static final String weightField = "weight"; private static final String routingMethodField = "routingMethod"; + private static final String authMethodField = "authMethod"; private ContainerEndpointSerializer() {} public static ContainerEndpoint endpointFromSlime(Inspector inspector) { - final var clusterId = inspector.field(clusterIdField).asString(); - final var scope = inspector.field(scopeField).asString(); - final var namesInspector = inspector.field(namesField); - final var weight = SlimeUtils.optionalInteger(inspector.field(weightField)); + String clusterId = inspector.field(clusterIdField).asString(); + String scope = inspector.field(scopeField).asString(); + Inspector namesInspector = inspector.field(namesField); + OptionalInt weight = SlimeUtils.optionalInteger(inspector.field(weightField)); // assign default routingmethod. Remove when 7.507 is latest version - // Cannot be used before all endpoints are assigned explicit routingmethod (from controller) - final var routingMethod = SlimeUtils.optionalString(inspector.field(routingMethodField)).orElse(ApplicationClusterEndpoint.RoutingMethod.sharedLayer4.name()); + // Cannot be used before all endpoints are assigned explicit routing method (from controller) + ApplicationClusterEndpoint.RoutingMethod routingMethod = SlimeUtils.optionalString(inspector.field(routingMethodField)) + .map(ContainerEndpointSerializer::routingMethodFrom) + .orElse(ApplicationClusterEndpoint.RoutingMethod.sharedLayer4); + ApplicationClusterEndpoint.AuthMethod authMethod = SlimeUtils.optionalString(inspector.field(authMethodField)) + .map(ContainerEndpointSerializer::authMethodFrom) + .orElse(ApplicationClusterEndpoint.AuthMethod.mtls); if (clusterId.isEmpty()) { throw new IllegalStateException("'clusterId' missing on serialized ContainerEndpoint"); } - if (scope.isEmpty()) { throw new IllegalStateException("'scope' missing on serialized ContainerEndpoint"); } - - if (! namesInspector.valid()) { + if (!namesInspector.valid()) { throw new IllegalStateException("'names' missing on serialized ContainerEndpoint"); } - if(routingMethod.isEmpty()) { - throw new IllegalStateException("'routingMethod' missing on serialized ContainerEndpoint"); - } - - final var names = new ArrayList<String>(); + List<String> names = new ArrayList<>(); namesInspector.traverse((ArrayTraverser) (idx, nameInspector) -> { final var containerName = nameInspector.asString(); names.add(containerName); }); - return new ContainerEndpoint(clusterId, ApplicationClusterEndpoint.Scope.valueOf(scope), names, weight, - ApplicationClusterEndpoint.RoutingMethod.valueOf(routingMethod)); + return new ContainerEndpoint(clusterId, scopeFrom(scope), names, weight, routingMethod, authMethod); } public static List<ContainerEndpoint> endpointListFromSlime(Slime slime) { final var inspector = slime.get(); return endpointListFromSlime(inspector); } + public static List<ContainerEndpoint> endpointListFromSlime(Inspector inspector) { final var endpoints = new ArrayList<ContainerEndpoint>(); @@ -88,11 +89,12 @@ public class ContainerEndpointSerializer { public static void endpointToSlime(Cursor cursor, ContainerEndpoint endpoint) { cursor.setString(clusterIdField, endpoint.clusterId()); - cursor.setString(scopeField, endpoint.scope().name()); + cursor.setString(scopeField, asString(endpoint.scope())); endpoint.weight().ifPresent(w -> cursor.setLong(weightField, w)); final var namesInspector = cursor.setArray(namesField); endpoint.names().forEach(namesInspector::addString); - cursor.setString(routingMethodField, endpoint.routingMethod().name()); + cursor.setString(routingMethodField, asString(endpoint.routingMethod())); + cursor.setString(authMethodField, asString(endpoint.authMethod())); } public static Slime endpointListToSlime(List<ContainerEndpoint> endpoints) { @@ -107,4 +109,53 @@ public class ContainerEndpointSerializer { return slime; } + private static ApplicationClusterEndpoint.RoutingMethod routingMethodFrom(String s) { + return switch (s) { + case "shared" -> ApplicationClusterEndpoint.RoutingMethod.shared; + case "sharedLayer4" -> ApplicationClusterEndpoint.RoutingMethod.sharedLayer4; + case "exclusive" -> ApplicationClusterEndpoint.RoutingMethod.exclusive; + default -> throw new IllegalArgumentException("Unknown routing method '" + s + "'"); + }; + } + + private static ApplicationClusterEndpoint.AuthMethod authMethodFrom(String s) { + return switch (s) { + case "mtls" -> ApplicationClusterEndpoint.AuthMethod.mtls; + case "token" -> ApplicationClusterEndpoint.AuthMethod.token; + default -> throw new IllegalArgumentException("Unknown auth method '" + s + "'"); + }; + } + + private static ApplicationClusterEndpoint.Scope scopeFrom(String s) { + return switch (s) { + case "global" -> ApplicationClusterEndpoint.Scope.global; + case "application" -> ApplicationClusterEndpoint.Scope.application; + case "zone" -> ApplicationClusterEndpoint.Scope.zone; + default -> throw new IllegalArgumentException("Unknown scope '" + s + "'"); + }; + } + + private static String asString(ApplicationClusterEndpoint.RoutingMethod routingMethod) { + return switch (routingMethod) { + case shared -> "shared"; + case sharedLayer4 -> "sharedLayer4"; + case exclusive -> "exclusive"; + }; + } + + private static String asString(ApplicationClusterEndpoint.AuthMethod authMethod) { + return switch (authMethod) { + case mtls -> "mtls"; + case token -> "token"; + }; + } + + private static String asString(ApplicationClusterEndpoint.Scope scope) { + return switch (scope) { + case global -> "global"; + case application -> "application"; + case zone -> "zone"; + }; + } + } diff --git a/configserver/src/main/java/org/glassfish/jersey/client/package-info.java b/configserver/src/main/java/org/glassfish/jersey/client/package-info.java deleted file mode 100644 index 151d9dbe952..00000000000 --- a/configserver/src/main/java/org/glassfish/jersey/client/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -@ExportPackage(version = @Version(major = 2, minor = 25)) -package org.glassfish.jersey.client; - -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; diff --git a/configserver/src/main/java/org/glassfish/jersey/client/proxy/package-info.java b/configserver/src/main/java/org/glassfish/jersey/client/proxy/package-info.java deleted file mode 100644 index 9ac5941eb3d..00000000000 --- a/configserver/src/main/java/org/glassfish/jersey/client/proxy/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -@ExportPackage(version = @Version(major = 2, minor = 25)) -package org.glassfish.jersey.client.proxy; - -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; diff --git a/configserver/src/main/java/org/glassfish/jersey/logging/package-info.java b/configserver/src/main/java/org/glassfish/jersey/logging/package-info.java deleted file mode 100644 index 6d9049e3c43..00000000000 --- a/configserver/src/main/java/org/glassfish/jersey/logging/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -@ExportPackage(version = @Version(major = 2, minor = 25)) -package org.glassfish.jersey.logging; - -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; diff --git a/configserver/src/main/java/org/glassfish/jersey/media/multipart/file/package-info.java b/configserver/src/main/java/org/glassfish/jersey/media/multipart/file/package-info.java deleted file mode 100644 index 06248ca88c1..00000000000 --- a/configserver/src/main/java/org/glassfish/jersey/media/multipart/file/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -@ExportPackage(version = @Version(major = 2, minor = 25)) -package org.glassfish.jersey.media.multipart.file; - -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; diff --git a/configserver/src/main/java/org/glassfish/jersey/media/multipart/package-info.java b/configserver/src/main/java/org/glassfish/jersey/media/multipart/package-info.java deleted file mode 100644 index 1faa237a7ea..00000000000 --- a/configserver/src/main/java/org/glassfish/jersey/media/multipart/package-info.java +++ /dev/null @@ -1,5 +0,0 @@ -@ExportPackage(version = @Version(major = 2, minor = 25)) -package org.glassfish.jersey.media.multipart; - -import com.yahoo.osgi.annotation.ExportPackage; -import com.yahoo.osgi.annotation.Version; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializerTest.java index c8f31697c5e..5e7fe64f998 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializerTest.java @@ -8,7 +8,6 @@ import org.junit.Test; import java.util.List; import java.util.OptionalInt; -import java.util.OptionalLong; import static org.junit.Assert.assertEquals; @@ -46,11 +45,9 @@ public class ContainerEndpointSerializerTest { @Test public void writeReadEndpoints() { - final var endpoints = List.of(new ContainerEndpoint("foo", ApplicationClusterEndpoint.Scope.global, List.of("a", "b"), OptionalInt.of(3), ApplicationClusterEndpoint.RoutingMethod.shared)); - final var serialized = ContainerEndpointSerializer.endpointListToSlime(endpoints); - final var deserialized = ContainerEndpointSerializer.endpointListFromSlime(serialized); - - assertEquals(endpoints, deserialized); + List<ContainerEndpoint> endpoints = List.of(new ContainerEndpoint("foo", ApplicationClusterEndpoint.Scope.global, List.of("a", "b"), OptionalInt.of(3), + ApplicationClusterEndpoint.RoutingMethod.shared, ApplicationClusterEndpoint.AuthMethod.token)); + assertEquals(endpoints, ContainerEndpointSerializer.endpointListFromSlime(ContainerEndpointSerializer.endpointListToSlime(endpoints))); } } diff --git a/container-core/pom.xml b/container-core/pom.xml index 8c30dad7e22..549b3ad5953 100644 --- a/container-core/pom.xml +++ b/container-core/pom.xml @@ -290,7 +290,7 @@ <dependency> <groupId>com.google.inject</groupId> <artifactId>guice</artifactId> - <classifier>no_aop</classifier> + <scope>provided</scope> </dependency> <dependency> diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java index 2a382d22a68..e4775fba46f 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/MetricDefinitions.java @@ -14,7 +14,6 @@ class MetricDefinitions { static final String METHOD_DIMENSION = "httpMethod"; static final String SCHEME_DIMENSION = "scheme"; static final String REQUEST_TYPE_DIMENSION = "requestType"; - static final String CLIENT_IP_DIMENSION = "clientIp"; static final String CLIENT_AUTHENTICATED_DIMENSION = "clientAuthenticated"; static final String REQUEST_SERVER_NAME_DIMENSION = "requestServerName"; static final String FILTER_CHAIN_ID_DIMENSION = "chainId"; diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListener.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListener.java index d96989830ab..817a99bb57f 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListener.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListener.java @@ -7,11 +7,8 @@ import org.eclipse.jetty.io.ssl.SslHandshakeListener; import javax.net.ssl.SSLHandshakeException; import java.util.HashMap; import java.util.Map; -import java.util.Optional; -import java.util.function.Predicate; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.regex.Pattern; /** * A {@link SslHandshakeListener} that reports metrics for SSL handshake failures. @@ -45,8 +42,6 @@ class SslHandshakeFailedListener implements SslHandshakeListener { Map<String, Object> dimensions = new HashMap<>(); dimensions.put(MetricDefinitions.NAME_DIMENSION, connectorName); dimensions.put(MetricDefinitions.PORT_DIMENSION, listenPort); - Optional.ofNullable(event.getSSLEngine().getPeerHost()) - .ifPresent(clientIp -> dimensions.put(MetricDefinitions.CLIENT_IP_DIMENSION, clientIp)); return Map.copyOf(dimensions); } } diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListenerTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListenerTest.java index 3858d6b9e4e..ad34cc5f024 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListenerTest.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/SslHandshakeFailedListenerTest.java @@ -25,7 +25,7 @@ public class SslHandshakeFailedListenerTest { @Test void includes_client_ip_dimension_present_when_peer_available() { listener.handshakeFailed(handshakeEvent(true), new SSLHandshakeException("Empty server certificate chain")); - verify(metrics).createContext(eq(Map.of("clientIp", "127.0.0.1", "serverName", "connector", "serverPort", 1234))); + verify(metrics).createContext(eq(Map.of("serverName", "connector", "serverPort", 1234))); } @Test diff --git a/container-dependencies-enforcer/pom.xml b/container-dependencies-enforcer/pom.xml index 885be3b6ee6..fa1212a5b12 100644 --- a/container-dependencies-enforcer/pom.xml +++ b/container-dependencies-enforcer/pom.xml @@ -73,7 +73,7 @@ <include>com.google.guava:failureaccess:[1.0.1, 2):provided</include> <include>com.google.j2objc:j2objc-annotations:[2.8, 3):provided</include> - <include>com.google.inject:guice:jar:no_aop:${guice.vespa.version}:provided</include> + <include>com.google.inject:guice:jar:${guice.vespa.version}:provided</include> <include>com.sun.activation:javax.activation:[1.2.0, 2):provided</include> <include>com.sun.xml.bind:jaxb-core:${jaxb-core.vespa.version}:provided</include> <include>com.sun.xml.bind:jaxb-impl:${jaxb-impl.vespa.version}:provided</include> @@ -86,7 +86,6 @@ <include>org.slf4j:log4j-over-slf4j:${slf4j.vespa.version}:provided</include> <include>org.slf4j:slf4j-api:${slf4j.vespa.version}:provided</include> <include>org.slf4j:slf4j-jdk14:${slf4j.vespa.version}:provided</include> - <include>xml-apis:xml-apis:${xml-apis.vespa.version}:provided</include> <!-- Vespa provided dependencies --> <include>com.yahoo.vespa:annotations:*:provided</include> @@ -163,6 +162,7 @@ <include>io.prometheus:simpleclient_tracer_common:${prometheus.client.vespa.version}:test</include> <include>io.prometheus:simpleclient_tracer_otel:${prometheus.client.vespa.version}:test</include> <include>io.prometheus:simpleclient_tracer_otel_agent:${prometheus.client.vespa.version}:test</include> + <include>jakarta.inject:jakarta.inject-api:${jakarta.inject.vespa.version}:test</include> <include>junit:junit:${junit4.vespa.version}:test</include> <include>net.java.dev.jna:jna:${jna.vespa.version}:test</include> <include>net.openhft:zero-allocation-hashing:jar:${zero-allocation-hashing.vespa.version}:test</include> @@ -203,8 +203,6 @@ <include>org.hdrhistogram:HdrHistogram:${hdrhistogram.vespa.version}:test</include> <include>org.json:json:${org.json.vespa.version}:test</include> <!-- TODO: Remove on Vespa 9 --> <include>org.lz4:lz4-java:${org.lz4.vespa.version}:test</include> - <include>org.osgi:org.osgi.compendium:[4.1.0, 5):test</include> - <include>org.osgi:org.osgi.core:[4.1.0, 5):test</include> <include>xerces:xercesImpl:${xerces.vespa.version}:test</include> </allowed> </enforceDependencies> diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index 8d3ae6aca0e..cf4d0bbe851 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -83,12 +83,6 @@ <version>${guice.vespa.version}</version> </dependency> <dependency> - <groupId>com.google.inject</groupId> - <artifactId>guice</artifactId> - <version>${guice.vespa.version}</version> - <classifier>no_aop</classifier> - </dependency> - <dependency> <groupId>commons-logging</groupId> <artifactId>commons-logging</artifactId> <version>${commons-logging.vespa.version}</version> @@ -159,11 +153,6 @@ <artifactId>slf4j-jdk14</artifactId> <version>${slf4j.vespa.version}</version> </dependency> - <dependency> - <groupId>xml-apis</groupId> - <artifactId>xml-apis</artifactId> - <version>${xml-apis.vespa.version}</version> - </dependency> </dependencies> </dependencyManagement> diff --git a/container-dev/pom.xml b/container-dev/pom.xml index 96aa120d42f..76ed8b1e3d4 100644 --- a/container-dev/pom.xml +++ b/container-dev/pom.xml @@ -55,7 +55,12 @@ <dependency> <groupId>com.google.inject</groupId> <artifactId>guice</artifactId> - <classifier>no_aop</classifier> + <exclusions> + <exclusion> + <groupId>jakarta.inject</groupId> + <artifactId>jakarta.inject-api</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>net.java.dev.jna</groupId> diff --git a/container-messagebus/pom.xml b/container-messagebus/pom.xml index 2ad5633b7dc..38a2c8e2b78 100644 --- a/container-messagebus/pom.xml +++ b/container-messagebus/pom.xml @@ -19,7 +19,7 @@ <dependency> <groupId>com.google.inject</groupId> <artifactId>guice</artifactId> - <classifier>no_aop</classifier> + <scope>provided</scope> </dependency> <dependency> diff --git a/container-search-and-docproc/pom.xml b/container-search-and-docproc/pom.xml index 71d547ecacd..3137fd449a4 100644 --- a/container-search-and-docproc/pom.xml +++ b/container-search-and-docproc/pom.xml @@ -101,7 +101,7 @@ <dependency> <groupId>com.google.inject</groupId> <artifactId>guice</artifactId> - <classifier>no_aop</classifier> + <scope>provided</scope> </dependency> <dependency> diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index cdb660f294a..57d455a02f0 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -2169,13 +2169,14 @@ ], "methods" : [ "public void <init>(com.yahoo.search.cluster.NodeManager, boolean)", + "public synchronized void reconfigure(java.util.Collection)", "public void start()", "public com.yahoo.search.cluster.MonitorConfiguration getConfiguration()", "public boolean isClosed()", "public void add(java.lang.Object, boolean)", "public synchronized void failed(java.lang.Object, com.yahoo.search.result.ErrorMessage)", "public synchronized void responded(java.lang.Object)", - "public void ping(java.util.concurrent.Executor)", + "public synchronized void ping(java.util.concurrent.Executor)", "public java.util.Iterator nodeMonitorIterator()", "public java.util.List nodeMonitors()", "public void shutdown()" diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java index 332bf4ea2c4..d81f9079a02 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterMonitor.java @@ -3,16 +3,22 @@ package com.yahoo.search.cluster; import com.yahoo.concurrent.ThreadFactoryFactory; import com.yahoo.search.result.ErrorMessage; +import com.yahoo.yolean.UncheckedInterruptedException; +import java.util.Collection; import java.util.Collections; import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; import java.util.logging.Level; import java.util.logging.Logger; @@ -36,7 +42,16 @@ public class ClusterMonitor<T> { private final AtomicBoolean closed = new AtomicBoolean(false); /** A map from Node to corresponding MonitoredNode */ - private final Map<T, TrafficNodeMonitor<T>> nodeMonitors = Collections.synchronizedMap(new java.util.LinkedHashMap<>()); + private final Map<T, TrafficNodeMonitor<T>> nodeMonitors = Collections.synchronizedMap(new LinkedHashMap<>()); + + // Used during reconfiguration to ensure async RPC calls are complete. + private final Set<T> nodesToRemove = new LinkedHashSet<>(); + + // Used during reconfiguration to ensure all nodes have data. + private final Set<T> nodesToUpdate = new LinkedHashSet<>(); + + // Used for reconfiguration, and during shutdown. + private boolean skipNextWait = false; public ClusterMonitor(NodeManager<T> manager, boolean startPingThread) { nodeManager = manager; @@ -46,6 +61,22 @@ public class ClusterMonitor<T> { } } + /** Updates the monitored set of nodes, and waits for 1. data on new nodes, and 2. RPC completion of removed nodes. */ + public synchronized void reconfigure(Collection<T> nodes) { + if ( ! monitorThread.isAlive()) throw new IllegalStateException("monitor thread must be alive for reconfiguration"); + + nodesToUpdate.addAll(nodes); + nodesToRemove.addAll(nodeMonitors.keySet()); + nodesToRemove.removeAll(nodes); + for (T node : nodes) if ( ! nodeMonitors.containsKey(node)) add(node, true); + + synchronized (nodeManager) { skipNextWait = true; nodeManager.notifyAll(); } + try { while ( ! nodesToRemove.isEmpty() || ! nodesToUpdate.isEmpty()) wait(1); } + catch (InterruptedException e) { throw new UncheckedInterruptedException(e, true); } + + nodeManager.pingIterationCompleted(); + } + public void start() { if ( ! monitorThread.isAlive()) { monitorThread.start(); @@ -74,30 +105,48 @@ public class ClusterMonitor<T> { /** Called from ClusterSearcher/NodeManager when a node failed */ public synchronized void failed(T node, ErrorMessage error) { - if (closed.get()) return; // Do not touch state if close has started. - TrafficNodeMonitor<T> monitor = nodeMonitors.get(node); - Boolean wasWorking = monitor.isKnownWorking(); - monitor.failed(error); - if (wasWorking != monitor.isKnownWorking()) - nodeManager.failed(node); + updateMonitoredNode(node, monitor -> monitor.failed(error), nodeManager::failed); } /** Called when a node responded */ public synchronized void responded(T node) { - if (closed.get()) return; // Do not touch state if close has started. + updateMonitoredNode(node, TrafficNodeMonitor::responded, nodeManager::working); + } + + private void updateMonitoredNode(T node, Consumer<TrafficNodeMonitor<T>> monitorUpdate, Consumer<T> nodeUpdate) { TrafficNodeMonitor<T> monitor = nodeMonitors.get(node); - Boolean wasWorking = monitor.isKnownWorking(); - monitor.responded(); - if (wasWorking != monitor.isKnownWorking()) - nodeManager.working(node); + + // Don't touch state during shutdown. + if (closed.get()) monitor = null; + + // Node was removed during reconfiguration, and should no longer be monitored. + if (nodesToRemove.remove(node)) { + nodeMonitors.remove(node); + monitor = null; + } + + // Update monitor state only when it actually changes. + if (monitor != null) { + Boolean wasWorking = monitor.isKnownWorking(); + monitorUpdate.accept(monitor); + if (wasWorking != monitor.isKnownWorking()) + nodeUpdate.accept(node); + } + + // If the node was added in a recent reconfiguration, we now have its required data. + nodesToUpdate.remove(node); } /** * Ping all nodes which needs pinging to discover state changes */ - public void ping(Executor executor) { + public synchronized void ping(Executor executor) { for (var monitor : nodeMonitors()) { if (closed.get()) return; // Do nothing to change state if close has started. + if (nodesToRemove.remove(monitor.getNode())) { + nodeMonitors.remove(monitor.getNode()); + continue; + } nodeManager.ping(this, monitor.getNode(), executor); } nodeManager.pingIterationCompleted(); @@ -120,6 +169,7 @@ public class ClusterMonitor<T> { nodeMonitors.clear(); } synchronized (nodeManager) { + skipNextWait = true; nodeManager.notifyAll(); } try { @@ -148,7 +198,9 @@ public class ClusterMonitor<T> { log.finest("Activating ping"); ping(pingExecutor); synchronized (nodeManager) { - nodeManager.wait(configuration.getCheckInterval()); + if ( ! skipNextWait) + nodeManager.wait(configuration.getCheckInterval()); + skipNextWait = false; } } catch (Throwable e) { diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java index eca0c8058a1..43d0e08886d 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/Dispatcher.java @@ -60,20 +60,19 @@ public class Dispatcher extends AbstractComponent { private final DispatchConfig dispatchConfig; private final RpcConnectionPool rpcResourcePool; private final SearchCluster searchCluster; + private final ClusterMonitor<Node> clusterMonitor; private volatile VolatileItems volatileItems; private static class VolatileItems { final LoadBalancer loadBalancer; final InvokerFactory invokerFactory; - final ClusterMonitor<Node> clusterMonitor; final AtomicInteger inflight = new AtomicInteger(1); // Initial reference. Runnable cleanup = () -> { }; - VolatileItems(LoadBalancer loadBalancer, InvokerFactory invokerFactory, ClusterMonitor<Node> clusterMonitor) { + VolatileItems(LoadBalancer loadBalancer, InvokerFactory invokerFactory) { this.loadBalancer = loadBalancer; this.invokerFactory = invokerFactory; - this.clusterMonitor = clusterMonitor; } private void countDown() { @@ -128,7 +127,7 @@ public class Dispatcher extends AbstractComponent { Dispatcher(DispatchConfig dispatchConfig, RpcConnectionPool rpcConnectionPool, SearchCluster searchCluster, InvokerFactoryFactory invokerFactories) { this(dispatchConfig, rpcConnectionPool, searchCluster, new ClusterMonitor<>(searchCluster, false), invokerFactories); - this.volatileItems.clusterMonitor.start(); // Populate nodes to monitor before starting it. + this.clusterMonitor.start(); // Populate nodes to monitor before starting it. } Dispatcher(DispatchConfig dispatchConfig, RpcConnectionPool rpcConnectionPool, @@ -137,7 +136,8 @@ public class Dispatcher extends AbstractComponent { this.rpcResourcePool = rpcConnectionPool; this.searchCluster = searchCluster; this.invokerFactories = invokerFactories; - this.volatileItems = update(clusterMonitor); + this.clusterMonitor = clusterMonitor; + this.volatileItems = update(); searchCluster.addMonitoring(clusterMonitor); } @@ -171,7 +171,7 @@ public class Dispatcher extends AbstractComponent { * 3. The load balancer is owned by the volatile snapshot, and is swapped atomically with it; * it is used internally by the dispatcher to select search nodes for queries, and is discarded with its snapshot. * 4. The cluster monitor is a subordinate to the search cluster, and does whatever that tells it to, at any time; - * it is technically owned by the volatile snapshot, but mostly to show it is swapped together with that. + * it is technically owned by the dispatcher, but in updated by the search cluster, when that is updated. * 5. The search cluster is owned by the dispatcher, and is updated on node set changes; * its responsibility is to keep track of the state of the backend, and to provide a view of it to the dispatcher, * as well as keep the container vip status updated accordingly; it should therefore preserve as much as possible @@ -192,21 +192,16 @@ public class Dispatcher extends AbstractComponent { }; // Update the nodes the search cluster keeps track of, and what nodes are monitored. - ClusterMonitor<Node> newMonitor = searchCluster.updateNodes(toNodes(searchCluster.name(), nodesConfig), dispatchConfig.minActivedocsPercentage()); + searchCluster.updateNodes(toNodes(searchCluster.name(), nodesConfig), clusterMonitor, dispatchConfig.minActivedocsPercentage()); // Update the snapshot to use the new nodes set in the search cluster; the RPC pool is ready for this. - this.volatileItems = update(newMonitor); - - // Wait for the old cluster monitor to die; it may be pinging nodes we want to shut down RPC connections to. - items.get().clusterMonitor.shutdown(); + this.volatileItems = update(); } // Close the old snapshot, which may trigger the RPC cleanup now, or when the last invoker is closed, by a search thread. } - private VolatileItems update(ClusterMonitor<Node> clusterMonitor) { - var items = new VolatileItems(new LoadBalancer(searchCluster.groupList().groups(), toLoadBalancerPolicy(dispatchConfig.distributionPolicy())), - invokerFactories.create(rpcResourcePool, searchCluster.groupList(), dispatchConfig), - clusterMonitor); - return items; + private VolatileItems update() { + return new VolatileItems(new LoadBalancer(searchCluster.groupList().groups(), toLoadBalancerPolicy(dispatchConfig.distributionPolicy())), + invokerFactories.create(rpcResourcePool, searchCluster.groupList(), dispatchConfig)); } private void initialWarmup(double warmupTime) { @@ -255,7 +250,7 @@ public class Dispatcher extends AbstractComponent { @Override public void deconstruct() { // The clustermonitor must be shutdown first as it uses the invokerfactory through the searchCluster. - volatileItems.clusterMonitor.shutdown(); + clusterMonitor.shutdown(); if (rpcResourcePool != null) { rpcResourcePool.close(); } diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java index 59b4637a627..99c607b489a 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/searchcluster/SearchCluster.java @@ -6,7 +6,6 @@ import com.yahoo.net.HostName; import com.yahoo.prelude.Pong; import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.cluster.NodeManager; -import com.yahoo.yolean.UncheckedInterruptedException; import java.util.ArrayList; import java.util.Collection; @@ -32,7 +31,8 @@ public class SearchCluster implements NodeManager<Node> { private final String clusterId; private final VipStatus vipStatus; private final PingFactory pingFactory; - private volatile SearchGroupsImpl groups; + private volatile SearchGroupsImpl groups; // Groups in this cluster + private volatile SearchGroupsImpl monitoredGroups; // Same as groups, except during reconfiguration. private volatile long nextLogTime = 0; /** @@ -54,6 +54,7 @@ public class SearchCluster implements NodeManager<Node> { this.clusterId = clusterId; this.vipStatus = vipStatus; this.pingFactory = pingFactory; + this.monitoredGroups = groups; this.groups = groups; this.localCorpusDispatchTarget = findLocalCorpusDispatchTarget(HostName.getLocalhost(), groups); } @@ -62,7 +63,7 @@ public class SearchCluster implements NodeManager<Node> { public String name() { return clusterId; } /** Sets the new nodes to monitor to be the new nodes, but keep any existing node instances which equal the new ones. */ - public ClusterMonitor<Node> updateNodes(Collection<Node> newNodes, double minActivedocsPercentage) { + public void updateNodes(Collection<Node> newNodes, ClusterMonitor<Node> monitor, double minActivedocsPercentage) { List<Node> currentNodes = new ArrayList<>(newNodes); List<Node> addedNodes = new ArrayList<>(); Map<Node, Node> retainedNodes = groups.nodes().stream().collect(toMap(node -> node, node -> node)); @@ -72,19 +73,14 @@ public class SearchCluster implements NodeManager<Node> { else addedNodes.add(currentNodes.get(i)); } SearchGroupsImpl groups = toGroups(currentNodes, minActivedocsPercentage); - ClusterMonitor<Node> monitor = new ClusterMonitor<>(this, false); - for (Node node : groups.nodes()) monitor.add(node, true); - monitor.start(); - try { while (addedNodes.stream().anyMatch(node -> node.isWorking() == null)) { Thread.sleep(1); } } - catch (InterruptedException e) { throw new UncheckedInterruptedException(e, true); } - pingIterationCompleted(groups); this.localCorpusDispatchTarget = findLocalCorpusDispatchTarget(HostName.getLocalhost(), groups); + this.monitoredGroups = groups; + monitor.reconfigure(groups.nodes()); this.groups = groups; - return monitor; } public void addMonitoring(ClusterMonitor<Node> clusterMonitor) { - for (Node node : groups.nodes()) clusterMonitor.add(node, true); + for (Node node : monitoredGroups.nodes()) clusterMonitor.add(node, true); } private static Node findLocalCorpusDispatchTarget(String selfHostname, SearchGroups groups) { @@ -197,15 +193,15 @@ public class SearchCluster implements NodeManager<Node> { } public boolean hasInformationAboutAllNodes() { - return groups().stream().allMatch(group -> group.nodes().stream().allMatch(node -> node.isWorking() != null)); + return monitoredGroups.nodes().stream().allMatch(node -> node.isWorking() != null); } long nonWorkingNodeCount() { - return groups().stream().flatMap(group -> group.nodes().stream()).filter(node -> node.isWorking() == Boolean.FALSE).count(); + return monitoredGroups.nodes().stream().filter(node -> node.isWorking() == Boolean.FALSE).count(); } private boolean hasWorkingNodes() { - return groups().stream().anyMatch(group -> group.nodes().stream().anyMatch(node -> node.isWorking() != Boolean.FALSE)); + return monitoredGroups.nodes().stream().anyMatch(node -> node.isWorking() != Boolean.FALSE); } private boolean usesLocalCorpusIn(Node node) { @@ -251,7 +247,7 @@ public class SearchCluster implements NodeManager<Node> { */ @Override public void pingIterationCompleted() { - pingIterationCompleted(groups); + pingIterationCompleted(monitoredGroups); } private void pingIterationCompleted(SearchGroupsImpl groups) { diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java index 3397638b950..d0f1f46d6ea 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/DispatcherTest.java @@ -21,6 +21,7 @@ import com.yahoo.search.dispatch.searchcluster.SearchGroups; import com.yahoo.search.searchchain.Execution; import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.config.search.DispatchNodesConfig; +import com.yahoo.yolean.UncheckedInterruptedException; import org.junit.jupiter.api.Test; import java.io.IOException; @@ -36,6 +37,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.Phaser; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -181,7 +183,11 @@ public class DispatcherTest { pingPhasers.put(1, new Phaser(2)); pingPhasers.put(2, new Phaser(2)); + AtomicBoolean doPing = new AtomicBoolean(); + PingFactory pingFactory = (node, monitor, pongHandler) -> () -> { + try { while ( ! doPing.getAndSet(false)) { monitor.wait(1); } } // Need to avoid hogging monitor lock while waiting for phaser. + catch (InterruptedException e) { throw new UncheckedInterruptedException(e, true); } pingPhasers.get(node.key()).arriveAndAwaitAdvance(); pongHandler.handle(new Pong(2, 2)); pingPhasers.get(node.key()).arriveAndAwaitAdvance(); @@ -255,8 +261,8 @@ public class DispatcherTest { Dispatcher dispatcher = new Dispatcher(dispatchConfig, rpcPool, cluster, invokerFactories); ExecutorService executor = Executors.newFixedThreadPool(1); - // Set two groups with a single node each. The first cluster-monitor has nothing to do, and is shut down immediately. - // There are also no invokers, so the whole reconfiguration completes once the new cluster monitor has seen all nodes. + // Set two groups with a single node each. + // There are no invokers, so the whole reconfiguration completes once the cluster monitor has seen all the new nodes. Future<?> reconfiguration = executor.submit(() -> { dispatcher.updateWithNewConfig(new DispatchNodesConfig.Builder() .node(new DispatchNodesConfig.Node.Builder().key(0).group(0).port(123).host("host0")) @@ -265,8 +271,10 @@ public class DispatcherTest { }); // Let pings return, to allow the search cluster to reconfigure. + doPing.set(true); pingPhasers.get(0).arriveAndAwaitAdvance(); pingPhasers.get(0).arriveAndAwaitAdvance(); + doPing.set(true); pingPhasers.get(1).arriveAndAwaitAdvance(); pingPhasers.get(1).arriveAndAwaitAdvance(); // We need to wait for the cluster to have at least one group, lest dispatch will fail below. @@ -287,9 +295,10 @@ public class DispatcherTest { search1.search(new Query(), null); // Wait for the current cluster monitor to be mid-ping-round. + doPing.set(true); pingPhasers.get(0).arriveAndAwaitAdvance(); - // Then reconfigure the dispatcher with new nodes, replacing node0 with node2. + // Reconfigure the dispatcher with new nodes, removing node0 and adding node2. reconfiguration = executor.submit(() -> { dispatcher.updateWithNewConfig(new DispatchNodesConfig.Builder() .node(new DispatchNodesConfig.Node.Builder().key(2).group(0).port(123).host("host2")) @@ -297,16 +306,23 @@ public class DispatcherTest { .build()); }); // Reconfiguration starts, but groups are only updated once the search cluster has knowledge about all of them. + pingPhasers.get(0).arriveAndAwaitAdvance(); // Ping for node to remove completes. + doPing.set(true); + pingPhasers.get(1).arriveAndAwaitAdvance(); // Ping for node to keep completes. pingPhasers.get(1).arriveAndAwaitAdvance(); + // New round of pings starts, with nodes 1 and 2. + doPing.set(true); pingPhasers.get(1).arriveAndAwaitAdvance(); - pingPhasers.get(2).arriveAndAwaitAdvance(); + pingPhasers.get(1).arriveAndAwaitAdvance(); + // Cluster has not yet updated its group reference. assertEquals(1, cluster.group(0).workingNodes()); // Node0 is still working. assertSame(node0, cluster.group(0).nodes().get(0)); + + doPing.set(true); + pingPhasers.get(2).arriveAndAwaitAdvance(); pingPhasers.get(2).arriveAndAwaitAdvance(); - // Old cluster monitor is waiting for that ping to complete before it can shut down, and let reconfiguration complete. - pingPhasers.get(0).arriveAndAwaitAdvance(); reconfiguration.get(); Node node2 = cluster.group(0).nodes().get(0); assertNotSame(node0, node2); diff --git a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java index f6a1ca5cae3..dd0980322c4 100644 --- a/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java +++ b/container-search/src/test/java/com/yahoo/search/dispatch/searchcluster/SearchClusterTest.java @@ -6,12 +6,14 @@ import com.yahoo.container.handler.ClustersStatus; import com.yahoo.container.handler.VipStatus; import com.yahoo.net.HostName; import com.yahoo.prelude.Pong; +import com.yahoo.search.cluster.BaseNodeMonitor; import com.yahoo.search.cluster.ClusterMonitor; import com.yahoo.search.result.ErrorMessage; import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -23,7 +25,10 @@ import java.util.stream.IntStream; import static java.util.function.Function.identity; import static java.util.stream.Collectors.toMap; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author baldersheim @@ -153,6 +158,18 @@ public class SearchClusterTest { } @Test + void requireThatVipStatusWorksWhenReconfiguredFromZeroNodes() { + try (State test = new State("test", 2, "a", "b")) { + test.clusterMonitor.start(); + test.searchCluster.updateNodes(List.of(), test.clusterMonitor, 100.0); + assertEquals(Set.of(), test.searchCluster.groupList().nodes()); + + test.searchCluster.updateNodes(List.of(new Node("test", 0, "a", 0), new Node("test", 1, "b", 0)), test.clusterMonitor, 100.0); + assertTrue(test.vipStatus.isInRotation()); + } + } + + @Test void requireThatVipStatusIsDefaultDownButComesUpAfterPinging() { try (State test = new State("cluster.1", 2, "a", "b")) { assertTrue(test.searchCluster.localCorpusDispatchTarget().isEmpty()); @@ -384,6 +401,7 @@ public class SearchClusterTest { @Test void requireThatPreciselyTheRetainedNodesAreKeptWhenNodesAreUpdated() { try (State state = new State("query", 2, IntStream.range(0, 6).mapToObj(i -> "node-" + i).toList())) { + state.clusterMonitor.start(); List<Node> referenceNodes = List.of(new Node("test", 0, "node-0", 0), new Node("test", 1, "node-1", 0), new Node("test", 0, "node-2", 1), @@ -392,6 +410,7 @@ public class SearchClusterTest { new Node("test", 1, "node-5", 2)); SearchGroups oldGroups = state.searchCluster.groupList(); assertEquals(Set.copyOf(referenceNodes), oldGroups.nodes()); + List<BaseNodeMonitor<Node>> oldMonitors = state.clusterMonitor.nodeMonitors(); List<Node> updatedNodes = List.of(new Node("test", 0, "node-1", 0), // Swap node-0 and node-1 new Node("test", 1, "node-0", 0), // Swap node-1 and node-0 @@ -399,7 +418,7 @@ public class SearchClusterTest { new Node("test", 1, "node-3", 1), new Node("test", 0, "node-2", 2), // Swap node-4 and node-2 new Node("test", 1, "node-6", 2)); // Replace node-6 - state.searchCluster.updateNodes(updatedNodes, 100.0); + state.searchCluster.updateNodes(updatedNodes, state.clusterMonitor, 100.0); SearchGroups newGroups = state.searchCluster.groupList(); assertEquals(Set.copyOf(updatedNodes), newGroups.nodes()); @@ -417,6 +436,13 @@ public class SearchClusterTest { for (Node node : updatedNodes) assertEquals(pathIndexWithinGroup[node.group()]++, newNodesByIdentity.get(node).pathIndex(), "search path index within group should follow updated node order for: " + node); + + // Precisely the one retained node keeps its monitor through reconfiguration. + Set<BaseNodeMonitor<Node>> retainedMonitors = new HashSet<>(state.clusterMonitor.nodeMonitors()); + assertEquals(6, retainedMonitors.size()); + retainedMonitors.retainAll(oldMonitors); + assertEquals(1, retainedMonitors.size()); + assertSame(oldNodesByIdentity.get(updatedNodes.get(3)), retainedMonitors.iterator().next().getNode()); } } diff --git a/container-test/pom.xml b/container-test/pom.xml index a22d0b59ace..9c6fe8025a2 100644 --- a/container-test/pom.xml +++ b/container-test/pom.xml @@ -115,6 +115,11 @@ <groupId>org.lz4</groupId> <artifactId>lz4-java</artifactId> </dependency> + <dependency> + <groupId>jakarta.inject</groupId> + <artifactId>jakarta.inject-api</artifactId> + <version>${jakarta.inject.vespa.version}</version> + </dependency> <!-- START JETTY embedded jars --> <dependency> diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java index 7303320a4f7..058d3c8fb07 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/ApplicationVersion.java @@ -169,12 +169,12 @@ public class ApplicationVersion implements Comparable<ApplicationVersion> { return new ApplicationVersion(id, source, authorEmail, compileVersion, allowedMajor, buildTime, sourceUrl, commit, bundleHash, obsoleteAt, hasPackage, true, description, submittedAt, risk); } - /** Whether we still have the package for this revision. */ + /** Whether we have chosen to skip this version. */ public boolean shouldSkip() { return shouldSkip; } - /** Whether this revision should be deployed. */ + /** Whether this revision can be deployed. */ public boolean isDeployable() { return hasPackage && ! shouldSkip; } diff --git a/controller-server/pom.xml b/controller-server/pom.xml index 3c3c93735e7..0fcd55eb7d3 100644 --- a/controller-server/pom.xml +++ b/controller-server/pom.xml @@ -91,7 +91,7 @@ <dependency> <groupId>com.google.inject</groupId> <artifactId>guice</artifactId> - <classifier>no_aop</classifier> + <scope>provided</scope> </dependency> diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java index bc4dc74afff..b1cadcc341c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java @@ -108,17 +108,29 @@ public class EndpointCertificates { } private EndpointCertificate assignFromPool(Instance instance, ZoneId zone) { + // For deployments to manually deployed environments: use per instance certificate + // For all other environments (apply in order): + // * Use per instance certificate if it exists and is assigned a randomized id + // * Use per application certificate if it exits and is assigned a randomized id + // * Assign from pool + + Optional<AssignedCertificate> perInstanceAssignedCertificate = curator.readAssignedCertificate(TenantAndApplicationId.from(instance.id()), Optional.of(instance.name())); + if (perInstanceAssignedCertificate.isPresent() && perInstanceAssignedCertificate.get().certificate().randomizedId().isPresent()) { + return updateLastRequested(perInstanceAssignedCertificate.get()).certificate(); + } else if (! zone.environment().isManuallyDeployed()){ + TenantAndApplicationId application = TenantAndApplicationId.from(instance.id()); + Optional<AssignedCertificate> perApplicationAssignedCertificate = curator.readAssignedCertificate(TenantAndApplicationId.from(instance.id()), Optional.empty()); + if (perApplicationAssignedCertificate.isPresent() && perApplicationAssignedCertificate.get().certificate().randomizedId().isPresent()) { + return updateLastRequested(perApplicationAssignedCertificate.get()).certificate(); + } + } + + // For new applications which is assigned from pool we follow these rules: // Assign certificate per instance only in manually deployed environments. In other environments, we share the // certificate because application endpoints can span instances Optional<InstanceName> instanceName = zone.environment().isManuallyDeployed() ? Optional.of(instance.name()) : Optional.empty(); TenantAndApplicationId application = TenantAndApplicationId.from(instance.id()); - // Re-use existing certificate if it contains a randomized ID - Optional<AssignedCertificate> assignedCertificate = curator.readAssignedCertificate(application, instanceName); - if (assignedCertificate.isPresent() && assignedCertificate.get().certificate().randomizedId().isPresent()) { - AssignedCertificate updated = assignedCertificate.get().with(assignedCertificate.get().certificate().withLastRequested(clock.instant().getEpochSecond())); - curator.writeAssignedCertificate(updated); - return updated.certificate(); - } + try (Mutex lock = controller.curator().lockCertificatePool()) { Optional<UnassignedCertificate> candidate = curator.readUnassignedCertificates().stream() .filter(pc -> pc.state() == State.ready) @@ -136,6 +148,12 @@ public class EndpointCertificates { } } + AssignedCertificate updateLastRequested(AssignedCertificate assignedCertificate) { + AssignedCertificate updated = assignedCertificate.with(assignedCertificate.certificate().withLastRequested(clock.instant().getEpochSecond())); + curator.writeAssignedCertificate(updated); + return updated; + } + private Optional<EndpointCertificate> getOrProvision(Instance instance, ZoneId zone, DeploymentSpec deploymentSpec) { if (controller.routing().randomizedEndpointsEnabled(instance.id())) { return Optional.of(assignFromPool(instance, zone)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 03489222922..5ccb59e3ebc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -591,6 +591,7 @@ public class JobController { application = application.withProjectId(projectId == -1 ? OptionalLong.empty() : OptionalLong.of(projectId)); application = application.withRevisions(revisions -> revisions.with(version.get())); application = withPrunedPackages(application, version.get().id()); + version.set(application.get().revisions().get(version.get().id())); validate(id, submission); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index e3f3d0e7929..c95112268c5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -281,6 +281,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { .filter(c -> c.instance().isPresent()) .filter(c -> c.certificate().randomizedId().isEmpty()) .filter(c -> assignRandomizedId.with(FetchVector.Dimension.APPLICATION_ID, c.application().instance(c.instance().get()).serializedForm()).value()) + .limit(5) .forEach(c -> assignRandomizedId(c.application(), c.instance().get())); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index a0e8b1c5610..30f3f74c400 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -1979,9 +1979,10 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { response.setString("region", deploymentId.zoneId().region().value()); addAvailabilityZone(response, deployment.zone()); var application = controller.applications().requireApplication(TenantAndApplicationId.from(deploymentId.applicationId())); - boolean legacyEndpoints = request.getBooleanProperty("includeLegacyEndpoints"); + boolean includeAllEndpoints = request.getBooleanProperty("includeAllEndpoints") || + request.getBooleanProperty("includeLegacyEndpoints"); var endpointArray = response.setArray("endpoints"); - for (var endpoint : endpointsOf(deploymentId, application, legacyEndpoints)) { + for (var endpoint : endpointsOf(deploymentId, application, includeAllEndpoints)) { toSlime(endpoint, endpointArray.addObject()); } response.setString("clusters", withPath(toPath(deploymentId) + "/clusters", request.getUri()).toString()); @@ -2056,20 +2057,19 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { metrics.instant().ifPresent(instant -> metricsObject.setLong("lastUpdated", instant.toEpochMilli())); } - private EndpointList endpointsOf(DeploymentId deploymentId, Application application, boolean legacyEndpoints) { - EndpointList zoneEndpoints = controller.routing().readEndpointsOf(deploymentId).scope(Endpoint.Scope.zone); - if (!legacyEndpoints) { - zoneEndpoints = zoneEndpoints.not().legacy().direct(); - } + private EndpointList endpointsOf(DeploymentId deploymentId, Application application, boolean includeHidden) { + EndpointList zoneEndpoints = controller.routing().readEndpointsOf(deploymentId).direct(); EndpointList declaredEndpoints = controller.routing().readDeclaredEndpointsOf(application).targets(deploymentId); - if (!legacyEndpoints) { - declaredEndpoints = declaredEndpoints.not().legacy().direct(); - } EndpointList endpoints = zoneEndpoints.and(declaredEndpoints); - // If the application has any generated endpoints, we show only those EndpointList generatedEndpoints = endpoints.generated(); - if (!generatedEndpoints.isEmpty()) { - endpoints = generatedEndpoints; + if (!includeHidden) { + // If we have generated endpoints, hide non-generated + if (!generatedEndpoints.isEmpty()) { + endpoints = endpoints.generated(); + } + // Hide legacy and weighted endpoints + endpoints = endpoints.not().legacy() + .not().scope(Endpoint.Scope.weighted); } return endpoints; } @@ -2969,6 +2969,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { object.setString("diskSpeed", valueOf(resources.diskSpeed())); object.setString("storageType", valueOf(resources.storageType())); object.setString("architecture", valueOf(resources.architecture())); + object.setLong("gpuCount", resources.gpuResources().count()); + object.setDouble("gpuMemoryGb", resources.gpuResources().memoryGb()); } // A tenant has different content when in a list ... antipattern, but not solvable before application/v5 diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index d290e52034f..982e81d92b1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -221,7 +221,10 @@ class JobControllerApiHandlerHelper { Slime slime = new Slime(); Cursor root = slime.setObject(); ApplicationVersion submitted = jobController.submit(id, submission, projectId); - root.setString("message", "application " + submitted); + String skipped = submitted.shouldSkip() + ? "; only applying deployment spec changes, as this build is otherwise equal to the previous" + : ""; + root.setString("message", "application " + submitted + skipped); root.setLong("build", submitted.buildNumber()); return new SlimeJsonResponse(slime); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java index f151b90d760..1cb43453918 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java @@ -28,6 +28,7 @@ import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.integration.SecretStoreMock; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; +import com.yahoo.vespa.hosted.controller.maintenance.EndpointCertificateMaintainer; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -35,6 +36,7 @@ import org.junit.jupiter.api.Test; import javax.security.auth.x500.X500Principal; import java.security.KeyPair; import java.security.cert.X509Certificate; +import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; @@ -326,6 +328,42 @@ public class EndpointCertificatesTest { } } + @Test + void reuse_per_instance_certificate_if_assigned_random_id() { + // Initial certificate is requested directly from provider + Optional<EndpointCertificate> certFromProvider = endpointCertificates.get(instance, prodZone, DeploymentSpec.empty); + assertTrue(certFromProvider.isPresent()); + assertFalse(certFromProvider.get().randomizedId().isPresent()); + + // Simulate endpoint certificate maintainer to assign random id + TenantAndApplicationId tenantAndApplicationId = TenantAndApplicationId.from(instance.id()); + Optional<InstanceName> instanceName = Optional.of(instance.name()); + Optional<AssignedCertificate> assignedCertificate = tester.controller().curator().readAssignedCertificate(tenantAndApplicationId, instanceName); + assertTrue(assignedCertificate.isPresent()); + String assignedRandomId = "randomid"; + AssignedCertificate updated = assignedCertificate.get().with(assignedCertificate.get().certificate().withRandomizedId(assignedRandomId)); + tester.controller().curator().writeAssignedCertificate(updated); + + // Pooled certificates become available + tester.flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); + + // Create 1 cert in pool + String certId = "pool-cert-1"; + addCertificateToPool(certId, UnassignedCertificate.State.ready); + + // Request cert for app + Optional<EndpointCertificate> cert = endpointCertificates.get(instance, prodZone, DeploymentSpec.empty); + assertEquals(assignedRandomId, cert.get().randomizedId().get()); + + // Pooled cert remains unassigned + List<String> unassignedCertificateIds = tester.curator().readUnassignedCertificates().stream() + .map(UnassignedCertificate::certificate) + .map(EndpointCertificate::randomizedId) + .map(Optional::get) + .toList(); + assertEquals(List.of(certId), unassignedCertificateIds); + } + private void addCertificateToPool(String id, UnassignedCertificate.State state) { EndpointCertificate cert = new EndpointCertificate(testKeyName, testCertName, 1, 0, "request-id", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index d3386a3a2ad..ab70dfd6073 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -919,10 +919,11 @@ public class ApplicationApiTest extends ControllerContainerTest { // Fifth attempt has the right content hash in a header, and succeeds. MultiPartStreamer streamer = createApplicationSubmissionData(packageWithService, 123); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/submit", POST) - .screwdriverIdentity(SCREWDRIVER_ID) - .header("X-Content-Hash", Base64.getEncoder().encodeToString(Signatures.sha256Digest(streamer::data))) - .data(streamer), - "{\"message\":\"application build 3, source revision of repository 'repository1', branch 'master' with commit 'commit1', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\",\"build\":3}"); + .screwdriverIdentity(SCREWDRIVER_ID) + .header("X-Content-Hash", Base64.getEncoder().encodeToString(Signatures.sha256Digest(streamer::data))) + .data(streamer), + """ + {"message":"application build 3, source revision of repository 'repository1', branch 'master' with commit 'commit1', by a@b, built against 6.1 at 1970-01-01T00:00:01Z; only applying deployment spec changes, as this build is otherwise equal to the previous","build":3}"""); // Sixth attempt has a multi-instance deployment spec, and is accepted. ApplicationPackage multiInstanceSpec = new ApplicationPackageBuilder() @@ -1808,7 +1809,8 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/submit/", POST) .data(createApplicationSubmissionData(applicationPackageDefault, SCREWDRIVER_ID.value())) .screwdriverIdentity(SCREWDRIVER_ID), - "{\"message\":\"application build 1, source revision of repository 'repository1', branch 'master' with commit 'commit1', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\",\"build\":1}", + """ + {"message":"application build 1, source revision of repository 'repository1', branch 'master' with commit 'commit1', by a@b, built against 6.1 at 1970-01-01T00:00:01Z","build":1}""", 200); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json index 1040e809362..1214d3f73f4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-clusters.json @@ -12,7 +12,9 @@ "bandwidthGbps": 1.0, "diskSpeed": "slow", "storageType": "remote", - "architecture": "any" + "architecture": "any", + "gpuCount": 0, + "gpuMemoryGb": 0.0 }, "cost": 0.11 }, @@ -26,7 +28,9 @@ "bandwidthGbps": 1.0, "diskSpeed": "slow", "storageType": "remote", - "architecture": "any" + "architecture": "any", + "gpuCount": 0, + "gpuMemoryGb": 0.0 }, "cost": 0.43 }, @@ -43,7 +47,9 @@ "bandwidthGbps": 1.0, "diskSpeed": "slow", "storageType": "remote", - "architecture": "any" + "architecture": "any", + "gpuCount": 0, + "gpuMemoryGb": 0.0 }, "cost": 0.22 }, @@ -60,7 +66,9 @@ "bandwidthGbps": 1.0, "diskSpeed": "slow", "storageType": "remote", - "architecture": "any" + "architecture": "any", + "gpuCount": 0, + "gpuMemoryGb": 0.0 }, "cost": 0.29 }, @@ -103,7 +111,9 @@ "bandwidthGbps": 0.0, "diskSpeed": "fast", "storageType": "any", - "architecture": "any" + "architecture": "any", + "gpuCount": 0, + "gpuMemoryGb": 0.0 }, "cost": 0.0 }, @@ -117,7 +127,9 @@ "bandwidthGbps": 1.0, "diskSpeed": "slow", "storageType": "remote", - "architecture": "any" + "architecture": "any", + "gpuCount": 0, + "gpuMemoryGb": 0.0 }, "cost": 0.22 }, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json index ffcb9ab6dc8..8abb1197d03 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json @@ -13,6 +13,8 @@ "diskSpeed": "slow", "storageType": "remote", "architecture": "any", + "gpuCount": 0, + "gpuMemoryGb": 0.0, "clusterId": "default", "clusterType": "container", "down": false, diff --git a/dependency-versions/pom.xml b/dependency-versions/pom.xml index 0e779267371..5ac932d0ee2 100644 --- a/dependency-versions/pom.xml +++ b/dependency-versions/pom.xml @@ -36,9 +36,10 @@ <commons-logging.vespa.version>1.2</commons-logging.vespa.version> <!-- This version is exported by jdisc via jcl-over-slf4j. --> <error-prone-annotations.vespa.version>2.21.1</error-prone-annotations.vespa.version> <guava.vespa.version>32.1.2-jre</guava.vespa.version> - <guice.vespa.version>4.2.3</guice.vespa.version> + <guice.vespa.version>6.0.0</guice.vespa.version> <jackson2.vespa.version>2.15.2</jackson2.vespa.version> <jackson-databind.vespa.version>2.15.2</jackson-databind.vespa.version> + <jakarta.inject.vespa.version>2.0.1</jakarta.inject.vespa.version> <javax.inject.vespa.version>1</javax.inject.vespa.version> <javax.servlet-api.vespa.version>3.1.0</javax.servlet-api.vespa.version> <javax.ws.rs-api.vespa.version>2.1.1</javax.ws.rs-api.vespa.version> @@ -46,7 +47,6 @@ <jaxb-core.vespa.version>2.3.0.1</jaxb-core.vespa.version> <jaxb-impl.vespa.version>2.3.0</jaxb-impl.vespa.version> <slf4j.vespa.version>1.7.36</slf4j.vespa.version> - <xml-apis.vespa.version>1.4.01</xml-apis.vespa.version> <!-- END Dependencies available from the Jdisc container --> @@ -55,7 +55,7 @@ <airline.vespa.version>0.9</airline.vespa.version> <antlr.vespa.version>3.5.3</antlr.vespa.version> - <antlr4.vespa.version>4.13.0</antlr4.vespa.version> + <antlr4.vespa.version>4.13.1</antlr4.vespa.version> <apache.httpclient.vespa.version>4.5.14</apache.httpclient.vespa.version> <apache.httpcore.vespa.version>4.4.16</apache.httpcore.vespa.version> <apache.httpclient5.vespa.version>5.2.1</apache.httpclient5.vespa.version> @@ -89,7 +89,7 @@ <dropwizard.metrics.vespa.version>4.2.19</dropwizard.metrics.vespa.version> <eclipse-collections.vespa.version>11.1.0</eclipse-collections.vespa.version> <felix.vespa.version>7.0.5</felix.vespa.version> - <felix.log.vespa.version>1.0.1</felix.log.vespa.version> + <felix.log.vespa.version>1.3.0</felix.log.vespa.version> <findbugs.vespa.version>3.0.2</findbugs.vespa.version> <!-- Should be kept in sync with guava --> <hamcrest.vespa.version>2.2</hamcrest.vespa.version> <hdrhistogram.vespa.version>2.1.12</hdrhistogram.vespa.version> @@ -97,14 +97,13 @@ <java-jjwt.vespa.version>0.11.5</java-jjwt.vespa.version> <java-jwt.vespa.version>4.4.0</java-jwt.vespa.version> <jaxb.runtime.vespa.version>4.0.3</jaxb.runtime.vespa.version> - <jersey.vespa.version>2.40</jersey.vespa.version> <jetty.vespa.version>11.0.16</jetty.vespa.version> <jetty-servlet-api.vespa.version>5.0.2</jetty-servlet-api.vespa.version> <jimfs.vespa.version>1.3.0</jimfs.vespa.version> <jna.vespa.version>5.13.0</jna.vespa.version> <joda-time.vespa.version>2.12.5</joda-time.vespa.version> - <junit.vespa.version>5.8.1</junit.vespa.version> - <junit.platform.vespa.version>1.8.1</junit.platform.vespa.version> + <junit.vespa.version>5.10.0</junit.vespa.version> + <junit.platform.vespa.version>1.10.0</junit.platform.vespa.version> <junit4.vespa.version>4.13.2</junit4.vespa.version> <luben.zstd.vespa.version>1.5.5-5</luben.zstd.vespa.version> <lucene.vespa.version>9.7.0</lucene.vespa.version> @@ -126,11 +125,15 @@ <spifly.vespa.version>1.3.6</spifly.vespa.version> <snappy.vespa.version>1.1.10.3</snappy.vespa.version> <surefire.vespa.version>3.1.2</surefire.vespa.version> - <wiremock.vespa.version>3.0.1</wiremock.vespa.version> + <wiremock.vespa.version>3.0.2</wiremock.vespa.version> <xerces.vespa.version>2.12.2</xerces.vespa.version> <zero-allocation-hashing.vespa.version>0.16</zero-allocation-hashing.vespa.version> <zookeeper.client.vespa.version>3.8.0</zookeeper.client.vespa.version> + <!-- Versions used by tenant parent pom and testing framework --> + <!-- CAUTION: upgrading junit for tenants poms may break testing frameworks --> + <junit.vespa.tenant.version>5.8.1</junit.vespa.tenant.version> + <junit.platform.vespa.tenant.version>1.8.1</junit.platform.vespa.tenant.version> <!-- Maven plugins --> <clover-maven-plugin.vespa.version>4.4.1</clover-maven-plugin.vespa.version> diff --git a/document/pom.xml b/document/pom.xml index 5db432d2447..2b13a4ace4f 100644 --- a/document/pom.xml +++ b/document/pom.xml @@ -65,7 +65,7 @@ <dependency> <groupId>com.google.inject</groupId> <artifactId>guice</artifactId> - <classifier>no_aop</classifier> + <scope>provided</scope> </dependency> <dependency> diff --git a/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp b/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp index a3aa7cbb32f..03db333d582 100644 --- a/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp +++ b/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp @@ -9,6 +9,7 @@ #include <vespa/vespalib/util/require.h> #include <vespa/vespalib/util/guard.h> #include <vespa/vespalib/util/stringfmt.h> +#include <charconv> using vespalib::make_string_short::fmt; @@ -20,7 +21,12 @@ using vespalib::FilePointer; using namespace vespalib::eval; using namespace vespalib::eval::test; -struct MyError { +struct MyError : public std::exception { + explicit MyError(vespalib::stringref m) : + std::exception(), + msg(m) + {} + const char * what() const noexcept override { return msg.c_str(); } vespalib::string msg; }; @@ -47,17 +53,42 @@ void extract(const vespalib::string &str, const vespalib::string &prefix, vespal dst = str.substr(pos); } } +struct MemoryUsage { + size_t size; + size_t rss; +}; -void report_memory_usage(const vespalib::string &desc) { - vespalib::string vm_size = "unknown"; - vespalib::string vm_rss = "unknown"; - vespalib::string line; +static const vespalib::string UNKNOWN = "unknown"; + +size_t convert(const vespalib::string & s) { + if (s == UNKNOWN) return 0; + size_t v(0); + size_t end = s.find("kB"); + auto [ptr,ec] = std::from_chars(s.data(), s.data()+std::min(s.size(), end), v, 10); + if (ec != std::errc()) { + throw std::runtime_error(fmt("Bad format : '%s' at '%s'", s.c_str(), ptr)); + } + if (end == vespalib::string::npos) { + throw std::runtime_error(fmt("Bad format : %s", s.c_str())); + } + return v * 1024; +} + +MemoryUsage extract_memory_usage() { + vespalib::string vm_size = UNKNOWN; + vespalib::string vm_rss = UNKNOWN; FilePointer file(fopen("/proc/self/status", "r")); + vespalib::string line; while (read_line(file, line)) { extract(line, "VmSize:", vm_size); extract(line, "VmRSS:", vm_rss); } - fprintf(stderr, "vm_size: %s, vm_rss: %s (%s)\n", vm_size.c_str(), vm_rss.c_str(), desc.c_str()); + return {convert(vm_size), convert(vm_rss)}; +} + +void report_memory_usage(const vespalib::string &desc) { + MemoryUsage vm = extract_memory_usage(); + fprintf(stderr, "vm_size: %zu kB, vm_rss: %zu kB (%s)\n", vm.size/1024, vm.rss/1024, desc.c_str()); } struct Options { @@ -118,7 +149,7 @@ void dump_wire_info(const Onnx::WireInfo &wire) { struct MakeInputType { Options &opts; std::map<vespalib::string,int> symbolic_sizes; - MakeInputType(Options &opts_in) : opts(opts_in), symbolic_sizes() {} + explicit MakeInputType(Options &opts_in) : opts(opts_in), symbolic_sizes() {} ValueType operator()(const Onnx::TensorInfo &info) { int d = 0; std::vector<ValueType::Dimension> dim_list; @@ -229,30 +260,32 @@ int probe_types() { if (!JsonFormat::decode(std_in, params)) { throw MyError{"invalid json"}; } + MemoryUsage vm_before = extract_memory_usage(); Slime result; auto &root = result.setObject(); auto &types = root.setObject("outputs"); - Onnx model(params["model"].asString().make_string(), Onnx::Optimize::DISABLE); + Onnx model(params["model"].asString().make_string(), Onnx::Optimize::ENABLE); Onnx::WirePlanner planner; - for (size_t i = 0; i < model.inputs().size(); ++i) { - auto spec = params["inputs"][model.inputs()[i].name].asString().make_string(); + for (const auto & input : model.inputs()) { + auto spec = params["inputs"][input.name].asString().make_string(); auto input_type = ValueType::from_spec(spec); if (input_type.is_error()) { - if (!params["inputs"][model.inputs()[i].name].valid()) { - throw MyError{fmt("missing type for model input '%s'", - model.inputs()[i].name.c_str())}; + if (!params["inputs"][input.name].valid()) { + throw MyError(fmt("missing type for model input '%s'", input.name.c_str())); } else { - throw MyError{fmt("invalid type for model input '%s': '%s'", - model.inputs()[i].name.c_str(), spec.c_str())}; + throw MyError(fmt("invalid type for model input '%s': '%s'",input.name.c_str(), spec.c_str())); } } - bind_input(planner, model.inputs()[i], input_type); + bind_input(planner, input, input_type); } planner.prepare_output_types(model); for (const auto &output: model.outputs()) { auto output_type = make_output(planner, output); types.setString(output.name, output_type.to_spec()); } + MemoryUsage vm_after = extract_memory_usage(); + root.setLong("vm_size", vm_after.size - vm_before.size); + root.setLong("vm_rss", vm_after.rss - vm_before.rss); write_compact(result, std_out); return 0; } diff --git a/eval/src/tests/instruction/sparse_join_reduce_plan/sparse_join_reduce_plan_test.cpp b/eval/src/tests/instruction/sparse_join_reduce_plan/sparse_join_reduce_plan_test.cpp index e101487ff59..cfc2277278f 100644 --- a/eval/src/tests/instruction/sparse_join_reduce_plan/sparse_join_reduce_plan_test.cpp +++ b/eval/src/tests/instruction/sparse_join_reduce_plan/sparse_join_reduce_plan_test.cpp @@ -40,7 +40,9 @@ struct Event { res_addr.push_back(make_handle(label)); } } - auto operator<=>(const Event &rhs) const = default; + bool operator==(const Event& rhs) const noexcept { + return lhs_idx == rhs.lhs_idx && rhs_idx == rhs.rhs_idx && res_addr == rhs.res_addr; + } }; struct Trace { @@ -55,7 +57,9 @@ struct Trace { events.emplace_back(lhs_idx, rhs_idx, res_addr); return *this; } - auto operator<=>(const Trace &rhs) const = default; + bool operator==(const Trace& rhs) const noexcept { + return estimate == rhs.estimate && events == rhs.events; + } }; std::ostream & diff --git a/eval/src/tests/instruction/universal_dot_product/universal_dot_product_test.cpp b/eval/src/tests/instruction/universal_dot_product/universal_dot_product_test.cpp index 4040ec9ea57..6c0726dab37 100644 --- a/eval/src/tests/instruction/universal_dot_product/universal_dot_product_test.cpp +++ b/eval/src/tests/instruction/universal_dot_product/universal_dot_product_test.cpp @@ -204,7 +204,7 @@ void benchmark(const vespalib::string &expr, std::vector<Optimize> list) { for (size_t i = 0; i < ctf_meta.steps.size(); ++i) { auto name = strip_ns(ctf_meta.steps[i].class_name); if (name.find("Inject") > name.size() && name.find("ConstValue") > name.size()) { - fprintf(stderr, " %s: %zu ns\n", name.c_str(), count_ns(min_time[i])); + fprintf(stderr, " %s: %zu ns\n", name.c_str(), (size_t)count_ns(min_time[i])); fprintf(stderr, " +-- %s\n", strip_ns(ctf_meta.steps[i].symbol_name).c_str()); } } diff --git a/eval/src/vespa/eval/onnx/onnx_wrapper.cpp b/eval/src/vespa/eval/onnx/onnx_wrapper.cpp index 8f9450c2660..89d88dcc32c 100644 --- a/eval/src/vespa/eval/onnx/onnx_wrapper.cpp +++ b/eval/src/vespa/eval/onnx/onnx_wrapper.cpp @@ -8,10 +8,6 @@ #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/typify.h> #include <vespa/vespalib/util/classname.h> -#include <assert.h> -#include <cmath> -#include <stdlib.h> -#include <stdio.h> #include <type_traits> #include <vespa/log/log.h> @@ -171,15 +167,15 @@ private: public: OnnxString(const OnnxString &rhs) = delete; OnnxString &operator=(const OnnxString &rhs) = delete; - OnnxString(OnnxString &&rhs) = default; - OnnxString &operator=(OnnxString &&rhs) = default; + OnnxString(OnnxString &&rhs) noexcept = default; + OnnxString &operator=(OnnxString &&rhs) noexcept = default; const char *get() const { return _str.get(); } ~OnnxString() = default; static OnnxString get_input_name(const Ort::Session &session, size_t idx) { - return OnnxString(session.GetInputNameAllocated(idx, _alloc)); + return {session.GetInputNameAllocated(idx, _alloc)}; } static OnnxString get_output_name(const Ort::Session &session, size_t idx) { - return OnnxString(session.GetOutputNameAllocated(idx, _alloc)); + return {session.GetOutputNameAllocated(idx, _alloc)}; } }; Ort::AllocatorWithDefaultOptions OnnxString::_alloc; @@ -216,7 +212,7 @@ Onnx::TensorType get_type_of(const Ort::Value &value) { throw Ort::Exception("[onnx wrapper] actual value has unknown dimension size", ORT_FAIL); } } - return Onnx::TensorType(make_element_type(element_type), shape); + return {make_element_type(element_type), shape}; } std::vector<int64_t> extract_sizes(const ValueType &type) { @@ -306,7 +302,7 @@ Onnx::WirePlanner::do_model_probe(const Onnx &model) result_values.emplace_back(nullptr); } Ort::RunOptions run_opts(nullptr); - Ort::Session &session = const_cast<Ort::Session&>(model._session); + auto &session = const_cast<Ort::Session&>(model._session); session.Run(run_opts, model._input_name_refs.data(), param_values.data(), param_values.size(), model._output_name_refs.data(), result_values.data(), result_values.size()); @@ -554,7 +550,7 @@ Onnx::EvalContext::EvalContext(const Onnx &model, const WireInfo &wire_info) const auto &vespa = _wire_info.vespa_inputs[i]; const auto &onnx = _wire_info.onnx_inputs[i]; if (is_same_type(vespa.cell_type(), onnx.elements)) { - _param_values.push_back(Ort::Value(nullptr)); + _param_values.emplace_back(nullptr); _param_binders.push_back(SelectAdaptParam()(vespa.cell_type())); } else { _param_values.push_back(CreateOnnxTensor()(onnx, _alloc)); @@ -587,7 +583,7 @@ Onnx::EvalContext::bind_param(size_t i, const Value ¶m) void Onnx::EvalContext::eval() { - Ort::Session &session = const_cast<Ort::Session&>(_model._session); + auto &session = const_cast<Ort::Session&>(_model._session); Ort::RunOptions run_opts(nullptr); session.Run(run_opts, _model._input_name_refs.data(), _param_values.data(), _param_values.size(), diff --git a/flags/pom.xml b/flags/pom.xml index 0bfb02b1f32..816e5416cec 100644 --- a/flags/pom.xml +++ b/flags/pom.xml @@ -51,7 +51,7 @@ <dependency> <groupId>com.google.inject</groupId> <artifactId>guice</artifactId> - <classifier>no_aop</classifier> + <scope>provided</scope> </dependency> diff --git a/hosted-tenant-base/pom.xml b/hosted-tenant-base/pom.xml index ef1d3978a81..33f47cf262f 100644 --- a/hosted-tenant-base/pom.xml +++ b/hosted-tenant-base/pom.xml @@ -43,8 +43,8 @@ <bundle-plugin.failOnWarnings>false</bundle-plugin.failOnWarnings> <target_jdk_version>17</target_jdk_version> - <!-- NOTE: this must not be overriden by users, and must be in sync with junit version specified in 'tenant-cd-api' --> - <junit.vespa.tenant.version>5.8.1</junit.vespa.tenant.version> + <!-- NOTE: junit version must not be overridden by users --> + <test.categories>!integration</test.categories> <!-- To allow specialized base pom to include additional "test provided" dependencies --> diff --git a/jdisc-security-filters/pom.xml b/jdisc-security-filters/pom.xml index 3440f9089d7..8a456d06a40 100644 --- a/jdisc-security-filters/pom.xml +++ b/jdisc-security-filters/pom.xml @@ -69,6 +69,11 @@ <version>${project.version}</version> <scope>test</scope> </dependency> + <dependency> + <groupId>jakarta.inject</groupId> + <artifactId>jakarta.inject-api</artifactId> + <scope>test</scope> + </dependency> </dependencies> <build> diff --git a/jdisc_core/pom.xml b/jdisc_core/pom.xml index 4471269358a..9bc27cd4e77 100644 --- a/jdisc_core/pom.xml +++ b/jdisc_core/pom.xml @@ -36,11 +36,6 @@ <!-- jaxb end --> <dependency> - <!-- Newer version than the one in rt.jar, including the ElementTraversal class needed by Xerces (Aug 2015, still valid Sep 2017) --> - <groupId>xml-apis</groupId> - <artifactId>xml-apis</artifactId> - </dependency> - <dependency> <groupId>org.mockito</groupId> <artifactId>mockito-core</artifactId> <scope>test</scope> @@ -87,7 +82,6 @@ <dependency> <groupId>com.google.inject</groupId> <artifactId>guice</artifactId> - <classifier>no_aop</classifier> <!-- Non-AOP version required for Java 8 compatibility --> </dependency> <dependency> <groupId>org.apache.felix</groupId> @@ -105,11 +99,7 @@ <exclusions> <exclusion> <groupId>org.osgi</groupId> - <artifactId>org.osgi.compendium</artifactId> - </exclusion> - <exclusion> - <groupId>org.osgi</groupId> - <artifactId>org.osgi.core</artifactId> + <artifactId>*</artifactId> </exclusion> </exclusions> </dependency> @@ -217,7 +207,7 @@ <!-- WARNING: Removing jars from the list below usually requires a new major Vespa version. --> <!-- NOTE: This list must be kept in sync with ExportPackagesIT.java --> <argument>__REPLACE_VERSION__${project.build.directory}/dependency/guava.jar</argument> - <argument>${project.build.directory}/dependency/guice-no_aop.jar</argument> + <argument>${project.build.directory}/dependency/guice.jar</argument> <argument>${project.build.directory}/dependency/slf4j-api.jar</argument> <argument>${project.build.directory}/dependency/slf4j-jdk14.jar</argument> <argument>${project.build.directory}/dependency/jcl-over-slf4j.jar</argument> diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ConsoleLogFormatter.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ConsoleLogFormatter.java index 080f3d3f74b..f7f53d304df 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ConsoleLogFormatter.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ConsoleLogFormatter.java @@ -4,7 +4,7 @@ package com.yahoo.jdisc.core; import org.osgi.framework.Bundle; import org.osgi.framework.ServiceReference; import org.osgi.service.log.LogEntry; -import org.osgi.service.log.LogService; +import org.osgi.service.log.LogLevel; import java.io.PrintWriter; import java.io.StringWriter; @@ -86,17 +86,17 @@ class ConsoleLogFormatter { } private StringBuilder formatLevel(LogEntry entry, StringBuilder out) { - switch (entry.getLevel()) { - case LogService.LOG_ERROR: + switch (entry.getLogLevel()) { + case ERROR: out.append("error"); break; - case LogService.LOG_WARNING: + case WARN: out.append("warning"); break; - case LogService.LOG_INFO: + case INFO: out.append("info"); break; - case LogService.LOG_DEBUG: + case DEBUG: out.append("debug"); break; default: @@ -117,7 +117,7 @@ class ConsoleLogFormatter { private StringBuilder formatException(LogEntry entry, StringBuilder out) { Throwable t = entry.getException(); if (t != null) { - if (entry.getLevel() == LogService.LOG_INFO) { + if (entry.getLogLevel() == LogLevel.INFO) { out.append(": "); String msg = t.getMessage(); if (msg != null) { diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ConsoleLogListener.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ConsoleLogListener.java index 2cfa604109b..1d872bbcb64 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ConsoleLogListener.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ConsoleLogListener.java @@ -3,62 +3,64 @@ package com.yahoo.jdisc.core; import com.yahoo.net.HostName; import org.osgi.service.log.LogEntry; +import org.osgi.service.log.LogLevel; import org.osgi.service.log.LogListener; import java.io.PrintStream; +import java.util.Optional; /** * @author Vikas Panwar */ class ConsoleLogListener implements LogListener { - public static final int DEFAULT_LOG_LEVEL = Integer.MAX_VALUE; + public static final LogLevel DEFAULT_LOG_LEVEL = LogLevel.TRACE; private final ConsoleLogFormatter formatter; private final PrintStream out; - private final int maxLevel; + private final LogLevel maxLevel; ConsoleLogListener(PrintStream out, String serviceName, String logLevel) { this.out = out; this.formatter = new ConsoleLogFormatter(getHostname(), getProcessId(), serviceName); - this.maxLevel = parseLogLevel(logLevel); + this.maxLevel = parseLogLevel(logLevel).orElse(null); } @Override public void logged(LogEntry entry) { - if (entry.getLevel() > maxLevel) { + if (maxLevel == null || !maxLevel.implies(entry.getLogLevel())) { return; } out.println(formatter.formatEntry(entry)); } - public static int parseLogLevel(String logLevel) { + public static Optional<LogLevel> parseLogLevel(String logLevel) { if (logLevel == null || logLevel.isEmpty()) { - return DEFAULT_LOG_LEVEL; + return Optional.of(DEFAULT_LOG_LEVEL); } if (logLevel.equalsIgnoreCase("OFF")) { - return Integer.MIN_VALUE; + return Optional.empty(); } if (logLevel.equalsIgnoreCase("ERROR")) { - return 1; + return Optional.of(LogLevel.ERROR); } if (logLevel.equalsIgnoreCase("WARNING")) { - return 2; + return Optional.of(LogLevel.WARN); } if (logLevel.equalsIgnoreCase("INFO")) { - return 3; + return Optional.of(LogLevel.INFO); } if (logLevel.equalsIgnoreCase("DEBUG")) { - return 4; + return Optional.of(LogLevel.DEBUG); } if (logLevel.equalsIgnoreCase("ALL")) { - return Integer.MAX_VALUE; + return Optional.of(LogLevel.TRACE); } try { - return Integer.valueOf(logLevel); - } catch (NumberFormatException e) { + return Optional.of(LogLevel.values()[Integer.parseInt(logLevel)]); + } catch (NumberFormatException | IndexOutOfBoundsException e) { // fall through } - return DEFAULT_LOG_LEVEL; + return Optional.of(DEFAULT_LOG_LEVEL); } public static ConsoleLogListener newInstance() { diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/OsgiLogHandler.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/OsgiLogHandler.java index 48fdb2a0293..989bc26dd85 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/OsgiLogHandler.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/OsgiLogHandler.java @@ -4,6 +4,7 @@ package com.yahoo.jdisc.core; import com.google.common.collect.ImmutableMap; import org.osgi.framework.Bundle; import org.osgi.framework.ServiceReference; +import org.osgi.service.log.LogLevel; import org.osgi.service.log.LogService; import java.util.Dictionary; @@ -45,8 +46,9 @@ class OsgiLogHandler extends Handler { } @Override + @SuppressWarnings("deprecation") public void publish(LogRecord record) { - logService.log(new LogRecordReference(record), toServiceLevel(record.getLevel()), record.getMessage(), + logService.log(new LogRecordReference(record), toServiceLevel(record.getLevel()).ordinal(), record.getMessage(), record.getThrown()); } @@ -60,22 +62,22 @@ class OsgiLogHandler extends Handler { // empty } - public static int toServiceLevel(Level level) { + public static LogLevel toServiceLevel(Level level) { int val = level.intValue(); if (val >= Level.SEVERE.intValue()) { - return LogService.LOG_ERROR; + return LogLevel.ERROR; } if (val >= Level.WARNING.intValue()) { - return LogService.LOG_WARNING; + return LogLevel.WARN; } if (val >= Level.INFO.intValue()) { - return LogService.LOG_INFO; + return LogLevel.INFO; } // Level.CONFIG // Level.FINE // Level.FINER // Level.FINEST - return LogService.LOG_DEBUG; + return LogLevel.DEBUG; } private static <T> Map<String, T> createDictionary(T[] in) { diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/OsgiLogManager.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/OsgiLogManager.java deleted file mode 100644 index 0530d63fe7a..00000000000 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/OsgiLogManager.java +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.core; - -import org.osgi.framework.BundleContext; -import org.osgi.framework.ServiceReference; -import org.osgi.service.log.LogService; -import org.osgi.util.tracker.ServiceTracker; -import org.osgi.util.tracker.ServiceTrackerCustomizer; - -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.logging.Handler; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** - * TODO: unused, remove (not public api) - * - * @author Simon Thoresen Hult - */ -class OsgiLogManager implements LogService { - - private static final Object globalLock = new Object(); - private final CopyOnWriteArrayList<LogService> services = new CopyOnWriteArrayList<>(); - private final boolean configureLogLevel; - private ServiceTracker<LogService,LogService> tracker; - - OsgiLogManager(boolean configureLogLevel) { - this.configureLogLevel = configureLogLevel; - } - - public void install(BundleContext osgiContext) { - if (tracker != null) { - throw new IllegalStateException("OsgiLogManager already installed."); - } - tracker = new ServiceTracker<>(osgiContext, LogService.class, new ServiceTrackerCustomizer<>() { - - @Override - public LogService addingService(ServiceReference<LogService> reference) { - LogService service = osgiContext.getService(reference); - services.add(service); - return service; - } - - @Override - public void modifiedService(ServiceReference<LogService> reference, LogService service) { - - } - - @Override - public void removedService(ServiceReference<LogService> reference, LogService service) { - services.remove(service); - } - }); - tracker.open(); - synchronized (globalLock) { - Logger root = Logger.getLogger(""); - if (configureLogLevel) { - root.setLevel(Level.ALL); - } - for (Handler handler : root.getHandlers()) { - root.removeHandler(handler); - } - root.addHandler(new OsgiLogHandler(this)); - } - } - - public boolean uninstall() { - if (tracker == null) { - return false; - } - tracker.close(); // implicitly clears the services array - tracker = null; - return true; - } - - @Override - public void log(int level, String message) { - log(null, level, message, null); - } - - @Override - public void log(int level, String message, Throwable throwable) { - log(null, level, message, throwable); - } - - @SuppressWarnings("rawtypes") - @Override - public void log(ServiceReference serviceRef, int level, String message) { - log(serviceRef, level, message, null); - } - - @SuppressWarnings("rawtypes") - @Override - public void log(ServiceReference serviceRef, int level, String message, Throwable throwable) { - for (LogService obj : services) { - obj.log(serviceRef, level, message, throwable); - } - } - - public static OsgiLogManager newInstance() { - return new OsgiLogManager(System.getProperty("java.util.logging.config.file") == null); - } -} diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/core/ConsoleLogFormatterTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ConsoleLogFormatterTestCase.java index 64130ddc125..4376ccb6c7e 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/core/ConsoleLogFormatterTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ConsoleLogFormatterTestCase.java @@ -6,7 +6,7 @@ import org.mockito.Mockito; import org.osgi.framework.Bundle; import org.osgi.framework.ServiceReference; import org.osgi.service.log.LogEntry; -import org.osgi.service.log.LogService; +import org.osgi.service.log.LogLevel; import java.io.PrintWriter; import java.io.StringWriter; @@ -22,14 +22,14 @@ import static org.junit.jupiter.api.Assertions.assertEquals; public class ConsoleLogFormatterTestCase { private static final ConsoleLogFormatter SIMPLE_FORMATTER = new ConsoleLogFormatter(null, null, null); - private static final LogEntry SIMPLE_ENTRY = new MyEntry(0, 0, null); + private static final LogEntry SIMPLE_ENTRY = new MyEntry(0, LogLevel.AUDIT, null); // TODO: Should (at least) use ConsoleLogFormatter.ABSENCE_REPLACEMENT instead of literal '-'. See ticket 7128315. @Test void requireThatMillisecondsArePadded() { for (int i = 0; i < 10000; ++i) { - LogEntry entry = new MyEntry(i, 0, null); + LogEntry entry = new MyEntry(i, LogLevel.AUDIT, null); Instant instant = Instant.ofEpochMilli(i); assertEquals(String.format("%d.%06d\t-\t-\t-\t-\tunknown\t", instant.getEpochSecond(), instant.getNano() / 1000), SIMPLE_FORMATTER.formatEntry(entry)); @@ -70,7 +70,7 @@ public class ConsoleLogFormatterTestCase { @Test void requireThatProcessIdIncludesThreadIdWhenAvailable() { - LogEntry entry = new MyEntry(0, 0, null).putProperty("THREAD_ID", "threadId"); + LogEntry entry = new MyEntry(0, LogLevel.AUDIT, null).putProperty("THREAD_ID", "threadId"); assertEquals("0.000000\t-\tprocessId/threadId\t-\t-\tunknown\t", new ConsoleLogFormatter(null, "processId", null).formatEntry(entry)); } @@ -93,7 +93,7 @@ public class ConsoleLogFormatterTestCase { @Test void requireThatBundleNameIsIncluded() { - LogEntry entry = new MyEntry(0, 0, null).setBundleSymbolicName("bundleName"); + LogEntry entry = new MyEntry(0, LogLevel.AUDIT, null).setBundleSymbolicName("bundleName"); assertEquals("0.000000\t-\t-\t-\tbundleName\tunknown\t", SIMPLE_FORMATTER.formatEntry(entry)); } @@ -106,7 +106,7 @@ public class ConsoleLogFormatterTestCase { @Test void requireThatLoggerNameIsIncluded() { - LogEntry entry = new MyEntry(0, 0, null).putProperty("LOGGER_NAME", "loggerName"); + LogEntry entry = new MyEntry(0, LogLevel.AUDIT, null).putProperty("LOGGER_NAME", "loggerName"); assertEquals("0.000000\t-\t-\t-\t/loggerName\tunknown\t", SIMPLE_FORMATTER.formatEntry(entry)); } @@ -119,7 +119,7 @@ public class ConsoleLogFormatterTestCase { @Test void requireThatBundleAndLoggerNameIsCombined() { - LogEntry entry = new MyEntry(0, 0, null).setBundleSymbolicName("bundleName") + LogEntry entry = new MyEntry(0, LogLevel.AUDIT, null).setBundleSymbolicName("bundleName") .putProperty("LOGGER_NAME", "loggerName"); assertEquals("0.000000\t-\t-\t-\tbundleName/loggerName\tunknown\t", SIMPLE_FORMATTER.formatEntry(entry)); @@ -129,34 +129,32 @@ public class ConsoleLogFormatterTestCase { void requireThatLevelNameIsIncluded() { ConsoleLogFormatter formatter = SIMPLE_FORMATTER; assertEquals("0.000000\t-\t-\t-\t-\terror\t", - formatter.formatEntry(new MyEntry(0, LogService.LOG_ERROR, null))); + formatter.formatEntry(new MyEntry(0, LogLevel.ERROR, null))); assertEquals("0.000000\t-\t-\t-\t-\twarning\t", - formatter.formatEntry(new MyEntry(0, LogService.LOG_WARNING, null))); + formatter.formatEntry(new MyEntry(0, LogLevel.WARN, null))); assertEquals("0.000000\t-\t-\t-\t-\tinfo\t", - formatter.formatEntry(new MyEntry(0, LogService.LOG_INFO, null))); + formatter.formatEntry(new MyEntry(0, LogLevel.INFO, null))); assertEquals("0.000000\t-\t-\t-\t-\tdebug\t", - formatter.formatEntry(new MyEntry(0, LogService.LOG_DEBUG, null))); - assertEquals("0.000000\t-\t-\t-\t-\tunknown\t", - formatter.formatEntry(new MyEntry(0, 69, null))); + formatter.formatEntry(new MyEntry(0, LogLevel.DEBUG, null))); } @Test void requireThatMessageIsIncluded() { - LogEntry entry = new MyEntry(0, 0, "message"); + LogEntry entry = new MyEntry(0, LogLevel.AUDIT, "message"); assertEquals("0.000000\t-\t-\t-\t-\tunknown\tmessage", SIMPLE_FORMATTER.formatEntry(entry)); } @Test void requireThatMessageIsOptional() { - LogEntry entry = new MyEntry(0, 0, null); + LogEntry entry = new MyEntry(0, LogLevel.AUDIT, null); assertEquals("0.000000\t-\t-\t-\t-\tunknown\t", SIMPLE_FORMATTER.formatEntry(entry)); } @Test void requireThatMessageIsEscaped() { - LogEntry entry = new MyEntry(0, 0, "\\\n\r\t"); + LogEntry entry = new MyEntry(0, LogLevel.AUDIT, "\\\n\r\t"); assertEquals("0.000000\t-\t-\t-\t-\tunknown\t\\\\\\n\\r\\t", SIMPLE_FORMATTER.formatEntry(entry)); } @@ -164,7 +162,7 @@ public class ConsoleLogFormatterTestCase { @Test void requireThatExceptionIsIncluded() { Throwable t = new Throwable(); - LogEntry entry = new MyEntry(0, 0, null).setException(t); + LogEntry entry = new MyEntry(0, LogLevel.AUDIT, null).setException(t); assertEquals("0.000000\t-\t-\t-\t-\tunknown\t\\n" + formatThrowable(t), SIMPLE_FORMATTER.formatEntry(entry)); } @@ -172,7 +170,7 @@ public class ConsoleLogFormatterTestCase { @Test void requireThatExceptionIsEscaped() { Throwable t = new Throwable("\\\n\r\t"); - LogEntry entry = new MyEntry(0, 0, null).setException(t); + LogEntry entry = new MyEntry(0, LogLevel.AUDIT, null).setException(t); assertEquals("0.000000\t-\t-\t-\t-\tunknown\t\\n" + formatThrowable(t), SIMPLE_FORMATTER.formatEntry(entry)); } @@ -180,7 +178,7 @@ public class ConsoleLogFormatterTestCase { @Test void requireThatExceptionIsSimplifiedForInfoEntries() { Throwable t = new Throwable("exception"); - LogEntry entry = new MyEntry(0, LogService.LOG_INFO, "entry").setException(t); + LogEntry entry = new MyEntry(0, LogLevel.INFO, "entry").setException(t); assertEquals("0.000000\t-\t-\t-\t-\tinfo\tentry: exception", SIMPLE_FORMATTER.formatEntry(entry)); } @@ -188,7 +186,7 @@ public class ConsoleLogFormatterTestCase { @Test void requireThatSimplifiedExceptionIsEscaped() { Throwable t = new Throwable("\\\n\r\t"); - LogEntry entry = new MyEntry(0, LogService.LOG_INFO, "entry").setException(t); + LogEntry entry = new MyEntry(0, LogLevel.INFO, "entry").setException(t); assertEquals("0.000000\t-\t-\t-\t-\tinfo\tentry: \\\\\\n\\r\\t", SIMPLE_FORMATTER.formatEntry(entry)); } @@ -196,7 +194,7 @@ public class ConsoleLogFormatterTestCase { @Test void requireThatSimplifiedExceptionMessageIsOptional() { Throwable t = new Throwable(); - LogEntry entry = new MyEntry(0, LogService.LOG_INFO, "entry").setException(t); + LogEntry entry = new MyEntry(0, LogLevel.INFO, "entry").setException(t); assertEquals("0.000000\t-\t-\t-\t-\tinfo\tentry: java.lang.Throwable", SIMPLE_FORMATTER.formatEntry(entry)); } @@ -210,13 +208,13 @@ public class ConsoleLogFormatterTestCase { private static class MyEntry implements LogEntry { final String message; - final int level; + final LogLevel level; final long time; Bundle bundle = null; ServiceReference<?> serviceReference = null; Throwable exception; - MyEntry(long time, int level, String message) { + MyEntry(long time, LogLevel level, String message) { this.message = message; this.level = level; this.time = time; @@ -244,9 +242,15 @@ public class ConsoleLogFormatterTestCase { return time; } - @Override + @Override public LogLevel getLogLevel() { return level; } + @Override public String getLoggerName() { return null; } + @Override public long getSequence() { return 0; } + @Override public String getThreadInfo() { return null; } + @Override public StackTraceElement getLocation() { return null; } + + @Override @SuppressWarnings("deprecation") public int getLevel() { - return level; + return level.ordinal(); } @Override diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/core/ConsoleLogListenerTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ConsoleLogListenerTestCase.java index 0efefc21a2f..88d73f32550 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/core/ConsoleLogListenerTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ConsoleLogListenerTestCase.java @@ -5,11 +5,12 @@ import org.junit.jupiter.api.Test; import org.osgi.framework.Bundle; import org.osgi.framework.ServiceReference; import org.osgi.service.log.LogEntry; +import org.osgi.service.log.LogLevel; import org.osgi.service.log.LogListener; -import org.osgi.service.log.LogService; import java.io.ByteArrayOutputStream; import java.io.PrintStream; +import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -23,42 +24,35 @@ public class ConsoleLogListenerTestCase { @Test void requireThatLogLevelParserKnowsOsgiLogLevels() { - assertEquals(LogService.LOG_ERROR, ConsoleLogListener.parseLogLevel("ERROR")); - assertEquals(LogService.LOG_WARNING, ConsoleLogListener.parseLogLevel("WARNING")); - assertEquals(LogService.LOG_INFO, ConsoleLogListener.parseLogLevel("INFO")); - assertEquals(LogService.LOG_DEBUG, ConsoleLogListener.parseLogLevel("DEBUG")); + assertEquals(LogLevel.ERROR, ConsoleLogListener.parseLogLevel("ERROR").orElseThrow()); + assertEquals(LogLevel.WARN, ConsoleLogListener.parseLogLevel("WARNING").orElseThrow()); + assertEquals(LogLevel.INFO, ConsoleLogListener.parseLogLevel("INFO").orElseThrow()); + assertEquals(LogLevel.DEBUG, ConsoleLogListener.parseLogLevel("DEBUG").orElseThrow()); } @Test void requireThatLogLevelParserKnowsOff() { - assertEquals(Integer.MIN_VALUE, ConsoleLogListener.parseLogLevel("OFF")); + assertEquals(Optional.empty(), ConsoleLogListener.parseLogLevel("OFF")); } @Test void requireThatLogLevelParserKnowsAll() { - assertEquals(Integer.MAX_VALUE, ConsoleLogListener.parseLogLevel("ALL")); - } - - @Test - void requireThatLogLevelParserKnowsIntegers() { - for (int i = -69; i < 69; ++i) { - assertEquals(i, ConsoleLogListener.parseLogLevel(String.valueOf(i))); - } + assertEquals(LogLevel.TRACE, ConsoleLogListener.parseLogLevel("ALL").orElseThrow()); } @Test void requireThatLogLevelParserErrorsReturnDefault() { - assertEquals(ConsoleLogListener.DEFAULT_LOG_LEVEL, ConsoleLogListener.parseLogLevel(null)); - assertEquals(ConsoleLogListener.DEFAULT_LOG_LEVEL, ConsoleLogListener.parseLogLevel("")); - assertEquals(ConsoleLogListener.DEFAULT_LOG_LEVEL, ConsoleLogListener.parseLogLevel("foo")); + assertEquals(ConsoleLogListener.DEFAULT_LOG_LEVEL, ConsoleLogListener.parseLogLevel(null).orElseThrow()); + assertEquals(ConsoleLogListener.DEFAULT_LOG_LEVEL, ConsoleLogListener.parseLogLevel("").orElseThrow()); + assertEquals(ConsoleLogListener.DEFAULT_LOG_LEVEL, ConsoleLogListener.parseLogLevel("foo").orElseThrow()); } @Test void requireThatLogEntryWithLevelAboveThresholdIsNotOutput() { ByteArrayOutputStream out = new ByteArrayOutputStream(); LogListener listener = new ConsoleLogListener(new PrintStream(out), null, "5"); - for (int i = 0; i < 10; ++i) { - listener.logged(new MyEntry(0, i, "message")); + for (LogLevel l : LogLevel.values()) { + listener.logged(new MyEntry(0, l, "message")); } // TODO: Should use ConsoleLogFormatter.ABSENCE_REPLACEMENT instead of literal '-'. See ticket 7128315. assertEquals("0.000000\t" + HOSTNAME + "\t" + PROCESS_ID + "\t-\t-\tunknown\tmessage\n" + @@ -73,10 +67,10 @@ public class ConsoleLogListenerTestCase { private static class MyEntry implements LogEntry { final String message; - final int level; + final LogLevel level; final long time; - MyEntry(long time, int level, String message) { + MyEntry(long time, LogLevel level, String message) { this.message = message; this.level = level; this.time = time; @@ -87,9 +81,15 @@ public class ConsoleLogListenerTestCase { return time; } - @Override + @Override public LogLevel getLogLevel() { return level; } + @Override public String getLoggerName() { return null; } + @Override public long getSequence() { return 0; } + @Override public String getThreadInfo() { return null; } + @Override public StackTraceElement getLocation() { return null; } + + @Override @SuppressWarnings("deprecation") public int getLevel() { - return level; + return level.ordinal(); } @Override diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/core/ExportPackagesIT.java b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ExportPackagesIT.java index b5a9e19bb2a..2d0b9ba9651 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/core/ExportPackagesIT.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ExportPackagesIT.java @@ -36,7 +36,7 @@ public class ExportPackagesIT { // This list must be kept in sync with the list in the export-packages execution in pom.xml. private static final List<String> RE_EXPORTED_BUNDLES = Stream.of( "guava.jar", - "guice-no_aop.jar", + "guice.jar", "slf4j-api.jar", "slf4j-jdk14.jar", "jcl-over-slf4j.jar", diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/core/OsgiLogHandlerTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/core/OsgiLogHandlerTestCase.java index 626cae67c41..f5a86b63ae5 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/core/OsgiLogHandlerTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/core/OsgiLogHandlerTestCase.java @@ -2,7 +2,9 @@ package com.yahoo.jdisc.core; import org.junit.jupiter.api.Test; +import org.osgi.framework.Bundle; import org.osgi.framework.ServiceReference; +import org.osgi.service.log.LogLevel; import org.osgi.service.log.LogService; import java.time.Instant; @@ -30,40 +32,40 @@ public class OsgiLogHandlerTestCase { Logger log = newLogger(logService); log.log(Level.INFO, "foo"); - assertEquals(OsgiLogHandler.toServiceLevel(Level.INFO), logService.lastLevel); + assertEquals(OsgiLogHandler.toServiceLevel(Level.INFO), logService.lastLevel()); assertEquals("foo", logService.lastMessage); assertNull(logService.lastThrowable); Throwable t = new Throwable(); log.log(Level.SEVERE, "bar", t); - assertEquals(OsgiLogHandler.toServiceLevel(Level.SEVERE), logService.lastLevel); + assertEquals(OsgiLogHandler.toServiceLevel(Level.SEVERE), logService.lastLevel()); assertEquals("bar", logService.lastMessage); assertEquals(t, logService.lastThrowable); } @Test void requireThatStadardLogLevelsAreConverted() { - assertLogLevel(LogService.LOG_ERROR, Level.SEVERE); - assertLogLevel(LogService.LOG_WARNING, Level.WARNING); - assertLogLevel(LogService.LOG_INFO, Level.INFO); - assertLogLevel(LogService.LOG_DEBUG, Level.CONFIG); - assertLogLevel(LogService.LOG_DEBUG, Level.FINE); - assertLogLevel(LogService.LOG_DEBUG, Level.FINER); - assertLogLevel(LogService.LOG_DEBUG, Level.FINEST); + assertLogLevel(LogLevel.ERROR, Level.SEVERE); + assertLogLevel(LogLevel.WARN, Level.WARNING); + assertLogLevel(LogLevel.INFO, Level.INFO); + assertLogLevel(LogLevel.DEBUG, Level.CONFIG); + assertLogLevel(LogLevel.DEBUG, Level.FINE); + assertLogLevel(LogLevel.DEBUG, Level.FINER); + assertLogLevel(LogLevel.DEBUG, Level.FINEST); } @Test void requireThatCustomLogLevelsAreConverted() { for (int i = Level.ALL.intValue() - 69; i < Level.OFF.intValue() + 69; ++i) { - int expectedLevel; + LogLevel expectedLevel; if (i >= Level.SEVERE.intValue()) { - expectedLevel = LogService.LOG_ERROR; + expectedLevel = LogLevel.ERROR; } else if (i >= Level.WARNING.intValue()) { - expectedLevel = LogService.LOG_WARNING; + expectedLevel = LogLevel.WARN; } else if (i >= Level.INFO.intValue()) { - expectedLevel = LogService.LOG_INFO; + expectedLevel = LogLevel.INFO; } else { - expectedLevel = LogService.LOG_DEBUG; + expectedLevel = LogLevel.DEBUG; } assertLogLevel(expectedLevel, new MyLogLevel(i)); } @@ -120,11 +122,11 @@ public class OsgiLogHandlerTestCase { assertNull(ref.getProperty("unknown")); } - private static void assertLogLevel(int expectedLevel, Level level) { + private static void assertLogLevel(LogLevel expectedLevel, Level level) { MyLogService logService = new MyLogService(); Logger log = newLogger(logService); log.log(level, "message"); - assertEquals(expectedLevel, logService.lastLevel); + assertEquals(expectedLevel, logService.lastLevel()); } @SuppressWarnings("unchecked") @@ -154,28 +156,36 @@ public class OsgiLogHandlerTestCase { String lastMessage; Throwable lastThrowable; - @Override + LogLevel lastLevel() { return LogLevel.values()[lastLevel]; } + + @Override @SuppressWarnings("deprecation") public void log(int level, String message) { log(null, level, message, null); } - @Override + @Override @SuppressWarnings("deprecation") public void log(int level, String message, Throwable throwable) { log(null, level, message, throwable); } - @Override + @Override @SuppressWarnings("deprecation") public void log(ServiceReference serviceReference, int level, String message) { log(serviceReference, level, message, null); } - @Override + @Override @SuppressWarnings("deprecation") public void log(ServiceReference serviceReference, int level, String message, Throwable throwable) { lastServiceReference = serviceReference; lastLevel = level; lastMessage = message; lastThrowable = throwable; } + + @Override public org.osgi.service.log.Logger getLogger(String s) { return null; } + @Override public org.osgi.service.log.Logger getLogger(Class<?> aClass) { return null; } + @Override public <L extends org.osgi.service.log.Logger> L getLogger(String s, Class<L> aClass) { return null; } + @Override public <L extends org.osgi.service.log.Logger> L getLogger(Class<?> aClass, Class<L> aClass1) { return null; } + @Override public <L extends org.osgi.service.log.Logger> L getLogger(Bundle bundle, String s, Class<L> aClass) { return null; } } private static class MyResourceBundle extends ResourceBundle { diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/core/OsgiLogManagerTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/core/OsgiLogManagerTestCase.java deleted file mode 100644 index 7b5af97ca13..00000000000 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/core/OsgiLogManagerTestCase.java +++ /dev/null @@ -1,185 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.core; - -import com.yahoo.jdisc.test.TestDriver; -import org.junit.jupiter.api.Test; -import org.mockito.Mockito; -import org.osgi.framework.BundleContext; -import org.osgi.framework.BundleException; -import org.osgi.framework.ServiceReference; -import org.osgi.framework.ServiceRegistration; -import org.osgi.service.log.LogService; - -import java.util.logging.Level; -import java.util.logging.Logger; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertSame; - - -/** - * @author Simon Thoresen Hult - */ -public class OsgiLogManagerTestCase { - - @Test - void requireThatAllLogMethodsAreImplemented() throws BundleException { - FelixFramework felix = TestDriver.newOsgiFramework(); - felix.start(); - - BundleContext ctx = felix.bundleContext(); - OsgiLogManager manager = new OsgiLogManager(true); - manager.install(ctx); - MyLogService service = new MyLogService(); - ctx.registerService(LogService.class.getName(), service, null); - - manager.log(2, "a"); - assertLast(service, null, 2, "a", null); - - Throwable t1 = new Throwable(); - manager.log(4, "b", t1); - assertLast(service, null, 4, "b", t1); - - ServiceReference<?> ref1 = Mockito.mock(ServiceReference.class); - manager.log(ref1, 8, "c"); - assertLast(service, ref1, 8, "c", null); - - ServiceReference<?> ref2 = Mockito.mock(ServiceReference.class); - Throwable t2 = new Throwable(); - manager.log(ref2, 16, "d", t2); - assertLast(service, ref2, 16, "d", t2); - - manager.uninstall(); - felix.stop(); - } - - @Test - void requireThatLogManagerWritesToAllRegisteredLogServices() throws BundleException { - FelixFramework felix = TestDriver.newOsgiFramework(); - felix.start(); - - BundleContext ctx = felix.bundleContext(); - MyLogService foo = new MyLogService(); - ServiceRegistration<LogService> fooReg = ctx.registerService(LogService.class, foo, null); - - OsgiLogManager manager = new OsgiLogManager(true); - manager.install(ctx); - - ServiceReference<?> ref1 = Mockito.mock(ServiceReference.class); - Throwable t1 = new Throwable(); - manager.log(ref1, 2, "a", t1); - assertLast(foo, ref1, 2, "a", t1); - - MyLogService bar = new MyLogService(); - ServiceRegistration<LogService> barReg = ctx.registerService(LogService.class, bar, null); - - ServiceReference<?> ref2 = Mockito.mock(ServiceReference.class); - Throwable t2 = new Throwable(); - manager.log(ref2, 4, "b", t2); - assertLast(foo, ref2, 4, "b", t2); - assertLast(bar, ref2, 4, "b", t2); - - MyLogService baz = new MyLogService(); - ServiceRegistration<LogService> bazReg = ctx.registerService(LogService.class, baz, null); - - ServiceReference<?> ref3 = Mockito.mock(ServiceReference.class); - Throwable t3 = new Throwable(); - manager.log(ref3, 8, "c", t3); - assertLast(foo, ref3, 8, "c", t3); - assertLast(bar, ref3, 8, "c", t3); - assertLast(baz, ref3, 8, "c", t3); - - fooReg.unregister(); - - ServiceReference<?> ref4 = Mockito.mock(ServiceReference.class); - Throwable t4 = new Throwable(); - manager.log(ref4, 16, "d", t4); - assertLast(foo, ref3, 8, "c", t3); - assertLast(bar, ref4, 16, "d", t4); - assertLast(baz, ref4, 16, "d", t4); - - barReg.unregister(); - - ServiceReference<?> ref5 = Mockito.mock(ServiceReference.class); - Throwable t5 = new Throwable(); - manager.log(ref5, 32, "e", t5); - assertLast(foo, ref3, 8, "c", t3); - assertLast(bar, ref4, 16, "d", t4); - assertLast(baz, ref5, 32, "e", t5); - - bazReg.unregister(); - - ServiceReference<?> ref6 = Mockito.mock(ServiceReference.class); - Throwable t6 = new Throwable(); - manager.log(ref6, 64, "f", t6); - assertLast(foo, ref3, 8, "c", t3); - assertLast(bar, ref4, 16, "d", t4); - assertLast(baz, ref5, 32, "e", t5); - - manager.uninstall(); - felix.stop(); - } - - @Test - void requireThatRootLoggerModificationCanBeDisabled() throws BundleException { - Logger logger = Logger.getLogger(""); - logger.setLevel(Level.WARNING); - - new OsgiLogManager(false).install(Mockito.mock(BundleContext.class)); - assertEquals(Level.WARNING, logger.getLevel()); - - new OsgiLogManager(true).install(Mockito.mock(BundleContext.class)); - assertEquals(Level.ALL, logger.getLevel()); - } - - @Test - void requireThatRootLoggerLevelIsModifiedIfNoLoggerConfigIsGiven() { - Logger logger = Logger.getLogger(""); - logger.setLevel(Level.WARNING); - - OsgiLogManager.newInstance().install(Mockito.mock(BundleContext.class)); - - assertNull(System.getProperty("java.util.logging.config.file")); - assertEquals(Level.ALL, logger.getLevel()); - } - - private static void assertLast(MyLogService service, ServiceReference<?> ref, int level, String message, Throwable t) { - assertSame(ref, service.lastServiceReference); - assertEquals(level, service.lastLevel); - assertEquals(message, service.lastMessage); - assertSame(t, service.lastThrowable); - } - - @SuppressWarnings("rawtypes") - private static class MyLogService implements LogService { - - ServiceReference lastServiceReference; - int lastLevel; - String lastMessage; - Throwable lastThrowable; - - @Override - public void log(int level, String message) { - log(null, level, message, null); - } - - @Override - public void log(int level, String message, Throwable throwable) { - log(null, level, message, throwable); - } - - @Override - public void log(ServiceReference serviceReference, int level, String message) { - log(serviceReference, level, message, null); - } - - @Override - public void log(ServiceReference serviceReference, int level, String message, Throwable throwable) { - lastServiceReference = serviceReference; - lastLevel = level; - lastMessage = message; - lastThrowable = throwable; - } - } -} diff --git a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/OsgiLogManagerIntegrationTest.java b/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/OsgiLogManagerIntegrationTest.java deleted file mode 100644 index 629bef6ded3..00000000000 --- a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/OsgiLogManagerIntegrationTest.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.jdisc.core; - -import org.junit.Test; -import org.mockito.Mockito; -import org.osgi.framework.BundleContext; - -import java.util.logging.Level; -import java.util.logging.Logger; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; - - -/** - * @author Simon Thoresen Hult - */ -public class OsgiLogManagerIntegrationTest { - - @Test - public void requireThatRootLoggerLevelIsNotModifiedIfLoggerConfigIsGiven() { - Logger logger = Logger.getLogger(""); - logger.setLevel(Level.WARNING); - - OsgiLogManager.newInstance().install(Mockito.mock(BundleContext.class)); - - assertNotNull(System.getProperty("java.util.logging.config.file")); - assertEquals(Level.WARNING, logger.getLevel()); - } -} diff --git a/linguistics-components/pom.xml b/linguistics-components/pom.xml index 19bb244b5d9..68b0437ac3f 100644 --- a/linguistics-components/pom.xml +++ b/linguistics-components/pom.xml @@ -87,7 +87,7 @@ <groupId>com.google.inject</groupId> <artifactId>guice</artifactId> <scope>provided</scope> - <classifier>no_aop</classifier> + </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/linguistics/pom.xml b/linguistics/pom.xml index 5db3302b597..8813af8b981 100644 --- a/linguistics/pom.xml +++ b/linguistics/pom.xml @@ -56,7 +56,7 @@ <groupId>com.google.inject</groupId> <artifactId>guice</artifactId> <scope>provided</scope> - <classifier>no_aop</classifier> + </dependency> </dependencies> <build> diff --git a/lucene-linguistics/pom.xml b/lucene-linguistics/pom.xml index 929d33a0736..18f2b1a8574 100644 --- a/lucene-linguistics/pom.xml +++ b/lucene-linguistics/pom.xml @@ -63,7 +63,7 @@ <dependency> <groupId>com.google.inject</groupId> <artifactId>guice</artifactId> - <classifier>no_aop</classifier> + <scope>provided</scope> </dependency> <dependency> diff --git a/maven-plugins/allowed-maven-dependencies.txt b/maven-plugins/allowed-maven-dependencies.txt index 319d6874f40..4b47810ea74 100644 --- a/maven-plugins/allowed-maven-dependencies.txt +++ b/maven-plugins/allowed-maven-dependencies.txt @@ -10,10 +10,11 @@ com.github.luben:zstd-jni:1.5.5-5 com.google.errorprone:error_prone_annotations:2.21.1 com.google.guava:failureaccess:1.0.1 com.google.guava:guava:32.1.2-jre -com.google.inject:guice:4.2.3 +com.google.inject:guice:6.0.0 com.google.j2objc:j2objc-annotations:2.8 commons-codec:commons-codec:1.16.0 commons-io:commons-io:2.13.0 +jakarta.inject:jakarta.inject-api:2.0.1 javax.annotation:javax.annotation-api:1.2 javax.inject:javax.inject:1 org.apache-extras.beanshell:bsh:2.0b6 @@ -72,12 +73,12 @@ net.bytebuddy:byte-buddy-agent:1.14.7 org.apiguardian:apiguardian-api:1.1.2 org.hamcrest:hamcrest:2.2 org.hamcrest:hamcrest-core:2.2 -org.junit.jupiter:junit-jupiter:5.8.1 -org.junit.jupiter:junit-jupiter-api:5.8.1 -org.junit.jupiter:junit-jupiter-engine:5.8.1 -org.junit.jupiter:junit-jupiter-params:5.8.1 -org.junit.platform:junit-platform-commons:1.8.1 -org.junit.platform:junit-platform-engine:1.8.1 +org.junit.jupiter:junit-jupiter:5.10.0 +org.junit.jupiter:junit-jupiter-api:5.10.0 +org.junit.jupiter:junit-jupiter-engine:5.10.0 +org.junit.jupiter:junit-jupiter-params:5.10.0 +org.junit.platform:junit-platform-commons:1.10.0 +org.junit.platform:junit-platform-engine:1.10.0 org.mockito:mockito-core:5.5.0 org.objenesis:objenesis:3.3 org.opentest4j:opentest4j:1.3.0 diff --git a/metrics/src/main/java/ai/vespa/metrics/GridLogMetrics.java b/metrics/src/main/java/ai/vespa/metrics/GridLogMetrics.java new file mode 100644 index 00000000000..d7589b0ec4c --- /dev/null +++ b/metrics/src/main/java/ai/vespa/metrics/GridLogMetrics.java @@ -0,0 +1,39 @@ +package ai.vespa.metrics; + +/** + * @author yngveaasheim + */ +public enum GridLogMetrics implements VespaMetrics { + + RECEIVED("gridlog.received", Unit.ITEM, "Entries requested to send"), + SENT("gridlog.sent", Unit.ITEM, "Entries successfully sent"), + NOT_SENT("gridlog.not_sent", Unit.ITEM, "Entries not sent, due to error or pruned by sampling"), + REJECTED("gridlog.rejected", Unit.ITEM, "Entries not sent due to send queue being full"), + SIZE("gridlog.size", Unit.BYTE, "Size of sent entries"), + PAYLOAD_TIME("gridlog.payload_time", Unit.SECOND, "Time spent building payload"), + BUILD_TIME("gridlog.build_time", Unit.SECOND, "Time spent building entries"), + SEND_TIME("gridlog.send_time", Unit.SECOND, "Total time spend in worker thread"); + + private final String name; + private final Unit unit; + private final String description; + + GridLogMetrics(String name, Unit unit, String description) { + this.name = name; + this.unit = unit; + this.description = description; + } + + public String baseName() { + return name; + } + + public Unit unit() { + return unit; + } + + public String description() { + return description; + } + +} diff --git a/metrics/src/main/java/ai/vespa/metrics/SentinelMetrics.java b/metrics/src/main/java/ai/vespa/metrics/SentinelMetrics.java index 35ecbae85d8..4e9d84011b3 100644 --- a/metrics/src/main/java/ai/vespa/metrics/SentinelMetrics.java +++ b/metrics/src/main/java/ai/vespa/metrics/SentinelMetrics.java @@ -1,7 +1,7 @@ package ai.vespa.metrics; /** - * @author yngve + * @author yngveaasheim */ public enum SentinelMetrics implements VespaMetrics { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java index 48e50b935a4..ad150cfc389 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/QuestMetricsDb.java @@ -84,13 +84,27 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { System.setProperty("out", logConfig); this.dataDir = dataDir; - engine = new CairoEngine(new DefaultCairoConfiguration(dataDir)); + engine = createEngine(dataDir); sqlCompilerPool = new ConcurrentResourcePool<>(() -> SqlCompilerFactoryImpl.INSTANCE.getInstance(engine())); nodeTable = new Table(dataDir, "metrics"); clusterTable = new Table(dataDir, "clusterMetrics"); ensureTablesExist(); } + private static CairoEngine createEngine(String dataDir) { + try { + return new CairoEngine(new DefaultCairoConfiguration(dataDir)); + } + catch (CairoException e) { + if (e.getMessage().contains("partitions are not ordered")) { // Happens when migrating 6.7 -> 7.3.1 + repairTables(dataDir, e, "metrics", "clusterMetrics"); + return new CairoEngine(new DefaultCairoConfiguration(dataDir)); + } + throw new IllegalStateException("Could not create Quest db in " + dataDir, e); + } + + } + private CairoEngine engine() { if (closed.get()) throw new IllegalStateException("Attempted to access QuestDb after calling close"); @@ -336,9 +350,6 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { SqlCompiler sqlCompiler = sqlCompilerPool.alloc(); try { return sqlCompiler.compile(sql, context); - } catch (SqlException e) { - log.log(Level.WARNING, "Could not execute SQL statement '" + sql + "'"); - throw e; } finally { sqlCompilerPool.free(sqlCompiler); } @@ -360,6 +371,19 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { .with(AllowAllSecurityContext.INSTANCE, null); } + private static void repairTables(String dataDir, Exception e, String ... tableNames) { + log.log(Level.WARNING, "QuestDb seems corrupted, wiping data and starting over", e); + for (String name : tableNames) + repairTable(dataDir, name); + } + + private static void repairTable(String dataDir, String name) { + var dir = new File(dataDir, name); + IOUtils.createDirectory(dir.getPath()); + IOUtils.recursiveDeleteDir(dir); + IOUtils.createDirectory(dir.getPath()); + } + /** A questDb table */ private class Table { @@ -375,6 +399,7 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { // https://stackoverflow.com/questions/67785629/what-does-max-txn-txn-inflight-limit-reached-in-questdb-and-how-to-i-avoid-it new File(dir + "/_txn_scoreboard").delete(); } + private TableToken token() { return engine().getTableTokenIfExists(name); } boolean exists() { @@ -406,8 +431,7 @@ public class QuestMetricsDb extends AbstractComponent implements MetricsDb { */ private void repair(Exception e) { log.log(Level.WARNING, "QuestDb seems corrupted, wiping data and starting over", e); - IOUtils.recursiveDeleteDir(dir); - IOUtils.createDirectory(dir.getPath()); + repairTable(dataDir, name); ensureTablesExist(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java index 74f2225adb5..a5aa7f42bc4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancer.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.provision.lb; import com.yahoo.vespa.applicationmodel.InfrastructureApplication; import com.yahoo.vespa.hosted.provision.maintenance.LoadBalancerExpirer; -import com.yahoo.vespa.service.duper.ConfigServerApplication; import java.time.Instant; import java.util.Objects; @@ -69,8 +68,8 @@ public class LoadBalancer { } /** Returns a copy of this with instance set to given instance */ - public LoadBalancer with(Optional<LoadBalancerInstance> instance) { - return new LoadBalancer(id, instance, state, changedAt); + public LoadBalancer with(LoadBalancerInstance instance) { + return new LoadBalancer(id, Optional.of(instance), state, changedAt); } public enum State { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java index 91a10a1d08e..6b084704474 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/ApplicationMaintainer.java @@ -80,7 +80,7 @@ public abstract class ApplicationMaintainer extends NodeRepositoryMaintainer { if ( ! canDeployNow(application)) return; // redeployment is no longer needed log.log(Level.INFO, () -> application + " will be redeployed" + (reason == null || reason.isBlank() ? "" : " due to " + reason) + - ", last activated " + activationTime(application)); + ", last deployed " + deployTime(application) + " and last activated " + activationTime(application)); deployment.activate(); } finally { pendingDeployments.remove(application); @@ -92,6 +92,11 @@ public abstract class ApplicationMaintainer extends NodeRepositoryMaintainer { return deployer.activationTime(application).orElse(Instant.EPOCH); } + /** Returns the last time application was deployed. Epoch is returned if the application has never been deployed. */ + protected final Instant deployTime(ApplicationId application) { + return deployer.deployTime(application).orElse(Instant.EPOCH); + } + @Override public void shutdown() { super.shutdown(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index baa2e596b36..895fdd522e7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -116,7 +116,7 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { new LoadBalancerSpec(lb.id().application(), lb.id().cluster(), reals, lb.instance().get().settings(), lb.instance().get().cloudAccount()), true); - db.writeLoadBalancer(lb.with(Optional.of(instance)), lb.state()); + db.writeLoadBalancer(lb.with(instance), lb.state()); } catch (Exception e) { failed.add(lb.id()); lastException.set(e); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index 5506fdf8ea3..c414c70f315 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -207,14 +207,11 @@ public class LoadBalancerProvisioner { throw new LoadBalancerServiceException("Could not (re)configure " + id + " due to change in load balancer visibility. The operation will be retried on next deployment"); } LoadBalancerInstance instance = provisionInstance(id, loadBalancer, zoneEndpoint, cloudAccount); - newLoadBalancer = newLoadBalancer.with(Optional.of(instance)); - } - catch (LoadBalancerServiceException e) { + newLoadBalancer = newLoadBalancer.with(instance); + } catch (LoadBalancerServiceException e) { log.log(Level.WARNING, "Failed to provision load balancer", e); - newLoadBalancer = newLoadBalancer.with(Optional.empty()); throw e; - } - finally { + } finally { db.writeLoadBalancer(newLoadBalancer, fromState); } } @@ -233,21 +230,19 @@ public class LoadBalancerProvisioner { try { LoadBalancerInstance instance = configureInstance(id, nodes, loadBalancer.get(), settings, loadBalancer.get().instance().get().cloudAccount()); - db.writeLoadBalancers(List.of(loadBalancer.get().with(Optional.of(instance)).with(State.active, now)), - loadBalancer.get().state(), transaction.nested()); - } - catch (LoadBalancerServiceException e) { - db.writeLoadBalancers(List.of(loadBalancer.get().with(Optional.empty())), + db.writeLoadBalancers(List.of(loadBalancer.get().with(instance).with(State.active, now)), loadBalancer.get().state(), transaction.nested()); + } catch (LoadBalancerServiceException e) { + db.writeLoadBalancers(List.of(loadBalancer.get()), loadBalancer.get().state(), transaction.nested()); throw e; } } /** Provision a load balancer instance, if necessary */ private LoadBalancerInstance provisionInstance(LoadBalancerId id, - Optional<LoadBalancer> currentLoadBalancer, - ZoneEndpoint zoneEndpoint, - CloudAccount cloudAccount) { + Optional<LoadBalancer> currentLoadBalancer, + ZoneEndpoint zoneEndpoint, + CloudAccount cloudAccount) { Set<Real> reals = currentLoadBalancer.flatMap(LoadBalancer::instance) .map(LoadBalancerInstance::reals) .orElse(Set.of()); // Targeted reals are changed on activation. diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index a67a513550a..ac13ee992c2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -116,8 +116,11 @@ public class NodeRepositoryProvisioner implements Provisioner { nodeRepository.nodeResourceLimits().ensureWithinAdvertisedLimits("Min", requested.minResources().nodeResources(), application, cluster); nodeRepository.nodeResourceLimits().ensureWithinAdvertisedLimits("Max", requested.maxResources().nodeResources(), application, cluster); - if ( ! requested.minResources().nodeResources().gpuResources().equals(requested.maxResources().nodeResources().gpuResources())) - throw new IllegalArgumentException(requested + " is invalid: Gpu capacity cannot have ranges"); + if (!requested.minResources().nodeResources().gpuResources().equals(requested.maxResources().nodeResources().gpuResources())) + throw new IllegalArgumentException(requested + " is invalid: GPU capacity cannot have ranges"); + + if (!requested.minResources().nodeResources().gpuResources().isZero() && !zone.system().isPublic()) + throw new IllegalArgumentException(requested + " is invalid: GPUs are not supported in " + zone); logInsufficientDiskResources(cluster, requested, logger); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java index 5539bb0cb6e..38b8836188b 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java @@ -9,7 +9,6 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; -import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeResources.Architecture; @@ -502,10 +501,16 @@ public class DynamicProvisioningTest { public void gpu_host() { List<Flavor> flavors = List.of(new Flavor("gpu", new NodeResources(4, 16, 125, 10, fast, local, Architecture.x86_64, new NodeResources.GpuResources(1, 16)))); - ProvisioningTester tester = new ProvisioningTester.Builder().dynamicProvisioning(true, false) - .flavors(flavors) + ProvisioningTester tester = new ProvisioningTester.Builder().flavors(flavors) .hostProvisioner(new MockHostProvisioner(flavors)) .nameResolver(nameResolver) + .zone(new Zone(Cloud.builder() + .dynamicProvisioning(true) + .allowHostSharing(false) + .build(), + SystemName.Public, + Environment.defaultEnvironment(), + RegionName.defaultName())) .build(); NodeResources resources = new NodeResources(4, 16, 125, 0.3, NodeResources.DiskSpeed.any, NodeResources.StorageType.any, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index 53054ba3f24..2b36bacf1b1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -300,6 +300,17 @@ public class LoadBalancerProvisionerTest { loadBalancers = lbs.get(); assertSame(LoadBalancer.State.active, loadBalancers.get(0).state()); assertTrue("Load balancer has instance", loadBalancers.get(0).instance().isPresent()); + + // Reconfiguration of load balancer fails on next prepare, but instance is preserved + tester.loadBalancerService().throwOnCreate(true); + ZoneEndpoint settings = new ZoneEndpoint(true, true, List.of(new AllowedUrn(AccessType.awsPrivateLink, "alice"), new AllowedUrn(AccessType.gcpServiceConnect, "bob"))); + try { + prepare(app1, clusterRequest(ClusterSpec.Type.container, cluster, Optional.empty(), settings)); + fail("Expected exception"); + } catch (LoadBalancerServiceException ignored) { + } + assertSame(LoadBalancer.State.active, loadBalancers.get(0).state()); + assertTrue("Load balancer has instance", loadBalancers.get(0).instance().isPresent()); } @Test diff --git a/parent/pom.xml b/parent/pom.xml index eeacc667b22..b1ea1e0dab9 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -1042,6 +1042,11 @@ <version>${junit.vespa.version}</version> </dependency> <dependency> + <groupId>org.junit.platform</groupId> + <artifactId>junit-platform-launcher</artifactId> + <version>${junit.platform.vespa.version}</version> + </dependency> + <dependency> <groupId>org.junit.vintage</groupId> <artifactId>junit-vintage-engine</artifactId> <version>${junit.vespa.version}</version> @@ -1120,6 +1125,12 @@ <groupId>xerces</groupId> <artifactId>xercesImpl</artifactId> <version>${xerces.vespa.version}</version> + <exclusions> + <exclusion> + <groupId>xml-apis</groupId> + <artifactId>xml-apis</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <!-- TODO: Remove on Vespa 9 --> <groupId>org.json</groupId> @@ -1146,7 +1157,11 @@ <artifactId>google-auth-library-oauth2-http</artifactId> <version>1.19.0</version> </dependency> - + <dependency> + <groupId>jakarta.inject</groupId> + <artifactId>jakarta.inject-api</artifactId> + <version>${jakarta.inject.vespa.version}</version> + </dependency> </dependencies> </dependencyManagement> diff --git a/provided-dependencies/pom.xml b/provided-dependencies/pom.xml index 09d76265466..8bf84956a12 100755 --- a/provided-dependencies/pom.xml +++ b/provided-dependencies/pom.xml @@ -49,7 +49,7 @@ <dependency> <groupId>com.google.inject</groupId> <artifactId>guice</artifactId> - <classifier>no_aop</classifier> + </dependency> <!-- Dependencies used by container-core --> diff --git a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp index d80604919de..28899b40408 100644 --- a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp +++ b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp @@ -164,8 +164,8 @@ VerifyRankSetup::~VerifyRankSetup() = default; bool VerifyRankSetup::verify(const search::index::Schema &schema, - const search::fef::Properties &props, - const IRankingAssetsRepo &repo) + const search::fef::Properties &props, + const IRankingAssetsRepo &repo) { proton::matching::IndexEnvironment indexEnv(0, schema, props, repo); search::fef::BlueprintFactory factory; @@ -195,12 +195,12 @@ VerifyRankSetup::verify(const search::index::Schema &schema, bool VerifyRankSetup::verifyConfig(const VerifyRanksetupConfig &myCfg, - const RankProfilesConfig &rankCfg, - const IndexschemaConfig &schemaCfg, - const AttributesConfig &attributeCfg, - const RankingConstantsConfig &constantsCfg, - const RankingExpressionsConfig &expressionsCfg, - const OnnxModelsConfig &modelsCfg) + const RankProfilesConfig &rankCfg, + const IndexschemaConfig &schemaCfg, + const AttributesConfig &attributeCfg, + const RankingConstantsConfig &constantsCfg, + const RankingExpressionsConfig &expressionsCfg, + const OnnxModelsConfig &modelsCfg) { bool ok = true; search::index::Schema schema; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp index 6244bdbea33..6f34fc512aa 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.cpp @@ -2,25 +2,31 @@ #include "attribute_vector_explorer.h" #include "attribute_executor.h" +#include <vespa/searchcommon/attribute/config.h> +#include <vespa/searchlib/attribute/attributevector.h> +#include <vespa/searchlib/attribute/distance_metric_utils.h> #include <vespa/searchlib/attribute/i_enum_store.h> #include <vespa/searchlib/attribute/i_enum_store_dictionary.h> -#include <vespa/searchlib/attribute/multi_value_mapping.h> -#include <vespa/searchlib/attribute/attributevector.h> #include <vespa/searchlib/attribute/ipostinglistattributebase.h> -#include <vespa/searchlib/util/state_explorer_utils.h> +#include <vespa/searchlib/attribute/multi_value_mapping.h> #include <vespa/searchlib/tensor/i_tensor_attribute.h> +#include <vespa/searchlib/util/state_explorer_utils.h> #include <vespa/vespalib/data/slime/cursor.h> -using search::attribute::Status; using search::AddressSpaceUsage; using search::AttributeVector; using search::IEnumStore; using search::StateExplorerUtils; -using vespalib::AddressSpace; -using vespalib::MemoryUsage; -using search::attribute::MultiValueMappingBase; +using search::attribute::BasicType; +using search::attribute::CollectionType; +using search::attribute::Config; +using search::attribute::DistanceMetricUtils; using search::attribute::IAttributeVector; using search::attribute::IPostingListAttributeBase; +using search::attribute::MultiValueMappingBase; +using search::attribute::Status; +using vespalib::AddressSpace; +using vespalib::MemoryUsage; using namespace vespalib::slime; namespace proton { @@ -97,6 +103,39 @@ convertPostingBaseToSlime(const IPostingListAttributeBase &postingBase, Cursor & convertMemoryUsageToSlime(postingBase.getMemoryUsage(), object.setObject("memoryUsage")); } +vespalib::string +type_to_string(const Config& cfg) +{ + if (cfg.basicType().type() == BasicType::TENSOR) { + return cfg.tensorType().to_spec(); + } + if (cfg.collectionType().type() == CollectionType::SINGLE) { + return cfg.basicType().asString(); + } + return vespalib::string(cfg.collectionType().asString()) + + "<" + vespalib::string(cfg.basicType().asString()) + ">"; +} + +void +convert_config_to_slime(const Config& cfg, bool full, Cursor& object) +{ + object.setString("type", type_to_string(cfg)); + object.setBool("fast_search", cfg.fastSearch()); + object.setBool("filter", cfg.getIsFilter()); + object.setBool("paged", cfg.paged()); + if (full) { + if (cfg.basicType().type() == BasicType::TENSOR) { + object.setString("distance_metric", DistanceMetricUtils::to_string(cfg.distance_metric())); + } + if (cfg.hnsw_index_params().has_value()) { + const auto& hnsw_cfg = cfg.hnsw_index_params().value(); + auto& hnsw = object.setObject("hnsw"); + hnsw.setLong("max_links_per_node", hnsw_cfg.max_links_per_node()); + hnsw.setLong("neighbors_to_explore_at_insert", hnsw_cfg.neighbors_to_explore_at_insert()); + } + } +} + } AttributeVectorExplorer::AttributeVectorExplorer(std::unique_ptr<AttributeExecutor> executor) @@ -117,6 +156,7 @@ AttributeVectorExplorer::get_state_helper(const AttributeVector& attr, const ves const Status &status = attr.getStatus(); Cursor &object = inserter.insertObject(); if (full) { + convert_config_to_slime(attr.getConfig(), full, object.setObject("config")); StateExplorerUtils::status_to_slime(status, object.setObject("status")); convertGenerationToSlime(attr, object.setObject("generation")); convertAddressSpaceUsageToSlime(attr.getAddressSpaceUsage(), object.setObject("addressSpaceUsage")); @@ -144,13 +184,9 @@ AttributeVectorExplorer::get_state_helper(const AttributeVector& attr, const ves object.setLong("committedDocIdLimit", attr.getCommittedDocIdLimit()); object.setLong("createSerialNum", attr.getCreateSerialNum()); } else { - object.setLong("numDocs", status.getNumDocs()); - object.setLong("lastSerialNum", status.getLastSyncToken()); - object.setLong("allocatedMemory", status.getAllocated()); - object.setLong("usedMemory", status.getUsed()); - object.setLong("onHoldMemory", status.getOnHold()); - object.setLong("committedDocIdLimit", attr.getCommittedDocIdLimit()); + convert_config_to_slime(attr.getConfig(), full, object); + object.setLong("allocated_bytes", status.getAllocated()); } } -} // namespace proton +} diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer_explorer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer_explorer.cpp index 0e26dcdfa21..eed5f62de6b 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_writer_explorer.cpp @@ -2,13 +2,9 @@ #include "attribute_writer.h" #include "attribute_writer_explorer.h" -#include <vespa/searchcommon/attribute/config.h> #include <vespa/searchlib/attribute/attributevector.h> #include <vespa/vespalib/data/slime/cursor.h> -using search::attribute::BasicType; -using search::attribute::CollectionType; -using search::attribute::Config; using vespalib::slime::Cursor; using vespalib::slime::Inserter; @@ -23,28 +19,13 @@ AttributeWriterExplorer::~AttributeWriterExplorer() = default; namespace { -vespalib::string -type_to_string(const Config& cfg) -{ - if (cfg.basicType().type() == BasicType::TENSOR) { - return cfg.tensorType().to_spec(); - } - if (cfg.collectionType().type() == CollectionType::SINGLE) { - return cfg.basicType().asString(); - } - return vespalib::string(cfg.collectionType().asString()) + - "<" + vespalib::string(cfg.basicType().asString()) + ">"; -} - void convert_to_slime(const AttributeWriter::WriteContext& context, Cursor& object) { object.setLong("executor_id", context.getExecutorId().getId()); - Cursor& fields = object.setArray("attribute_fields"); + Cursor& fields = object.setArray("fields"); for (const auto& field : context.getFields()) { - Cursor& f = fields.addObject(); - f.setString("name", field.getAttribute().getName()); - f.setString("type", type_to_string(field.getAttribute().getConfig())); + fields.addString(field.getAttribute().getName()); } } diff --git a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h index 3283bfb68eb..2157e6a49ec 100644 --- a/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h +++ b/searchcore/src/vespa/searchcore/proton/index/memoryindexwrapper.h @@ -94,6 +94,10 @@ public: _index.pruneRemovedFields(schema); } void flushToDisk(const vespalib::string &flushDir, uint32_t docIdLimit, SerialNum serialNum) override; + + void insert_write_context_state(vespalib::slime::Cursor& object) const override { + _index.insert_write_context_state(object); + } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt index f5544ed1b15..2f2fb1ded97 100644 --- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt @@ -42,14 +42,14 @@ vespa_add_library(searchcore_server STATIC executor_thread_service.cpp executor_threading_service_explorer.cpp executorthreadingservice.cpp - fast_access_document_retriever.cpp fast_access_doc_subdb.cpp fast_access_doc_subdb_configurer.cpp + fast_access_document_retriever.cpp fast_access_feed_view.cpp + feed_handler_stats.cpp feedhandler.cpp feedstate.cpp feedstates.cpp - feed_handler_stats.cpp fileconfigmanager.cpp flushhandlerproxy.cpp forcecommitcontext.cpp @@ -100,9 +100,10 @@ vespa_add_library(searchcore_server STATIC searchcontext.cpp searchhandlerproxy.cpp searchview.cpp - simpleflush.cpp + sequenced_task_executor_explorer.cpp shared_threading_service.cpp shared_threading_service_config.cpp + simpleflush.cpp storeonlydocsubdb.cpp storeonlyfeedview.cpp summaryadapter.cpp diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_thread_pools_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/proton_thread_pools_explorer.cpp index 7e94825d3f2..54fb38cd5d0 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_thread_pools_explorer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton_thread_pools_explorer.cpp @@ -2,6 +2,7 @@ #include "proton_thread_pools_explorer.h" #include "executor_explorer_utils.h" +#include "sequenced_task_executor_explorer.h" #include <vespa/vespalib/data/slime/cursor.h> #include <vespa/vespalib/util/threadexecutor.h> @@ -17,7 +18,7 @@ ProtonThreadPoolsExplorer::ProtonThreadPoolsExplorer(const ThreadExecutor* share const ThreadExecutor* flush, const ThreadExecutor* proton, const ThreadExecutor* warmup, - const vespalib::ISequencedTaskExecutor* field_writer) + vespalib::ISequencedTaskExecutor* field_writer) : _shared(shared), _match(match), _docsum(docsum), @@ -39,8 +40,24 @@ ProtonThreadPoolsExplorer::get_state(const vespalib::slime::Inserter& inserter, convert_executor_to_slime(_flush, object.setObject("flush")); convert_executor_to_slime(_proton, object.setObject("proton")); convert_executor_to_slime(_warmup, object.setObject("warmup")); - convert_executor_to_slime(_field_writer, object.setObject("field_writer")); } } +const vespalib::string FIELD_WRITER = "field_writer"; + +std::vector<vespalib::string> +ProtonThreadPoolsExplorer::get_children_names() const +{ + return {FIELD_WRITER}; +} + +std::unique_ptr<vespalib::StateExplorer> +ProtonThreadPoolsExplorer::get_child(vespalib::stringref name) const +{ + if (name == FIELD_WRITER) { + return std::make_unique<SequencedTaskExecutorExplorer>(_field_writer); + } + return {}; +} + } diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_thread_pools_explorer.h b/searchcore/src/vespa/searchcore/proton/server/proton_thread_pools_explorer.h index 2891309fb89..269c548bf1e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_thread_pools_explorer.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton_thread_pools_explorer.h @@ -22,7 +22,7 @@ private: const vespalib::ThreadExecutor* _flush; const vespalib::ThreadExecutor* _proton; const vespalib::ThreadExecutor* _warmup; - const vespalib::ISequencedTaskExecutor* _field_writer; + vespalib::ISequencedTaskExecutor* _field_writer; public: ProtonThreadPoolsExplorer(const vespalib::ThreadExecutor* shared, @@ -31,9 +31,11 @@ public: const vespalib::ThreadExecutor* flush, const vespalib::ThreadExecutor* proton, const vespalib::ThreadExecutor* warmup, - const vespalib::ISequencedTaskExecutor* field_writer); + vespalib::ISequencedTaskExecutor* field_writer); void get_state(const vespalib::slime::Inserter& inserter, bool full) const override; + std::vector<vespalib::string> get_children_names() const override; + std::unique_ptr<vespalib::StateExplorer> get_child(vespalib::stringref name) const override; }; } diff --git a/searchcore/src/vespa/searchcore/proton/server/sequenced_task_executor_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/sequenced_task_executor_explorer.cpp new file mode 100644 index 00000000000..6651cd80da2 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/sequenced_task_executor_explorer.cpp @@ -0,0 +1,64 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "sequenced_task_executor_explorer.h" +#include "executor_explorer_utils.h" +#include <vespa/vespalib/data/slime/cursor.h> +#include <vespa/vespalib/util/sequencedtaskexecutor.h> + +using vespalib::ISequencedTaskExecutor; +using vespalib::SequencedTaskExecutor; +using vespalib::slime::Cursor; + +namespace proton { + +using explorer::convert_executor_to_slime; + +SequencedTaskExecutorExplorer::SequencedTaskExecutorExplorer(vespalib::ISequencedTaskExecutor *executor) + : _executor(executor) +{ +} + +namespace { + +void +convert_raw_executor_stats_to_slime(ISequencedTaskExecutor* executor, Cursor& array) +{ + if (executor == nullptr) { + return; + } + auto* seq = dynamic_cast<SequencedTaskExecutor*>(executor); + if (!seq) { + return; + } + auto raw_stats = seq->get_raw_stats(); + for (size_t executor_id = 0; executor_id < raw_stats.size(); ++executor_id) { + const auto& stats = raw_stats[executor_id]; + auto& obj = array.addObject(); + obj.setLong("executor_id", executor_id); + obj.setDouble("saturation", stats.get_saturation()); + obj.setDouble("utilization", stats.getUtil()); + obj.setLong("accepted_tasks", stats.acceptedTasks); + obj.setLong("rejected_tasks", stats.rejectedTasks); + obj.setLong("wakeups", stats.wakeupCount); + auto& qs = obj.setObject("queue_size"); + qs.setLong("min", stats.queueSize.min()); + qs.setLong("max", stats.queueSize.max()); + qs.setLong("count", stats.queueSize.count()); + qs.setLong("total", stats.queueSize.total()); + qs.setDouble("average", stats.queueSize.average()); + } +} + +} + +void +SequencedTaskExecutorExplorer::get_state(const vespalib::slime::Inserter& inserter, bool full) const +{ + auto& object = inserter.insertObject(); + convert_executor_to_slime(_executor, object); + if (full) { + convert_raw_executor_stats_to_slime(_executor, object.setArray("executors")); + } +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/server/sequenced_task_executor_explorer.h b/searchcore/src/vespa/searchcore/proton/server/sequenced_task_executor_explorer.h new file mode 100644 index 00000000000..1d9327cf02d --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/sequenced_task_executor_explorer.h @@ -0,0 +1,28 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/net/http/state_explorer.h> + +namespace vespalib { +class ISequencedTaskExecutor; +} + +namespace proton { + +/** + * Class used to explore a SequencedTaskExecutor. + */ +class SequencedTaskExecutorExplorer : public vespalib::StateExplorer { +private: + // This is non-const in order to call get_raw_stats(). + vespalib::ISequencedTaskExecutor* _executor; + +public: + SequencedTaskExecutorExplorer(vespalib::ISequencedTaskExecutor* executor); + + void get_state(const vespalib::slime::Inserter& inserter, bool full) const override; +}; + +} + diff --git a/searchcore/src/vespa/searchcorespi/index/imemoryindex.h b/searchcore/src/vespa/searchcorespi/index/imemoryindex.h index 1b370dff262..130042bc048 100644 --- a/searchcore/src/vespa/searchcorespi/index/imemoryindex.h +++ b/searchcore/src/vespa/searchcorespi/index/imemoryindex.h @@ -7,8 +7,9 @@ #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/memoryusage.h> -namespace vespalib { class IDestructorCallback; } namespace document { class Document; } +namespace vespalib { class IDestructorCallback; } +namespace vespalib::slime { struct Cursor; } namespace searchcorespi::index { /** @@ -78,6 +79,8 @@ struct IMemoryIndex : public searchcorespi::IndexSearchable { virtual void pruneRemovedFields(const search::index::Schema &schema) = 0; virtual search::index::Schema::SP getPrunedSchema() const = 0; + + virtual void insert_write_context_state(vespalib::slime::Cursor& object) const = 0; }; } diff --git a/searchcore/src/vespa/searchcorespi/index/index_manager_explorer.cpp b/searchcore/src/vespa/searchcorespi/index/index_manager_explorer.cpp index 855f3c69bc9..1634937f094 100644 --- a/searchcore/src/vespa/searchcorespi/index/index_manager_explorer.cpp +++ b/searchcore/src/vespa/searchcorespi/index/index_manager_explorer.cpp @@ -2,7 +2,8 @@ #include "index_manager_explorer.h" #include "index_manager_stats.h" - +#include <vespa/searchcorespi/index/imemoryindex.h> +#include <vespa/searchcorespi/index/indexsearchablevisitor.h> #include <vespa/vespalib/data/slime/cursor.h> using vespalib::slime::Cursor; @@ -45,8 +46,23 @@ insertMemoryIndex(Cursor &arrayCursor, const MemoryIndexStats &memoryIndex) insertMemoryUsage(memoryIndexCursor, sstats.memoryUsage()); } -} +class WriteContextInserter : public IndexSearchableVisitor { +private: + Cursor& _object; + bool _has_inserted; + +public: + WriteContextInserter(Cursor& object) : _object(object), _has_inserted(false) {} + void visit(const index::IDiskIndex&) override {} + void visit(const index::IMemoryIndex& index) override { + if (!_has_inserted) { + index.insert_write_context_state(_object); + _has_inserted = true; + } + } +}; +} IndexManagerExplorer::IndexManagerExplorer(IIndexManager::SP mgr) : _mgr(std::move(mgr)) @@ -68,6 +84,9 @@ IndexManagerExplorer::get_state(const Inserter &inserter, bool full) const for (const auto &memoryIndex : stats.getMemoryIndexes()) { insertMemoryIndex(memoryIndexArrayCursor, memoryIndex); } + auto& write_contexts = object.setObject("write_contexts"); + WriteContextInserter visitor(write_contexts); + _mgr->getSearchable()->accept(visitor); } } diff --git a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp index e3b9cf9702d..3547bf6c9a8 100644 --- a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp +++ b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp @@ -8,24 +8,25 @@ #include <vespa/searchlib/fef/matchdatalayout.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/searchlib/index/i_field_length_inspector.h> -#include <vespa/searchlib/test/doc_builder.h> -#include <vespa/searchlib/test/schema_builder.h> -#include <vespa/searchlib/test/string_field_builder.h> #include <vespa/searchlib/memoryindex/memory_index.h> #include <vespa/searchlib/query/tree/simplequery.h> +#include <vespa/searchlib/queryeval/blueprint.h> #include <vespa/searchlib/queryeval/booleanmatchiteratorwrapper.h> #include <vespa/searchlib/queryeval/fake_requestcontext.h> #include <vespa/searchlib/queryeval/fake_searchable.h> #include <vespa/searchlib/queryeval/leaf_blueprints.h> #include <vespa/searchlib/queryeval/searchiterator.h> -#include <vespa/searchlib/queryeval/simpleresult.h> #include <vespa/searchlib/queryeval/simple_phrase_blueprint.h> -#include <vespa/searchlib/queryeval/blueprint.h> +#include <vespa/searchlib/queryeval/simpleresult.h> +#include <vespa/searchlib/test/doc_builder.h> +#include <vespa/searchlib/test/schema_builder.h> +#include <vespa/searchlib/test/string_field_builder.h> +#include <vespa/vespalib/data/slime/slime.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/util/sequencedtaskexecutor.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/stringfmt.h> #include <vespa/vespalib/util/threadstackexecutor.h> -#include <vespa/vespalib/gtest/gtest.h> #include <vespa/log/log.h> LOG_SETUP("memory_index_test"); @@ -36,7 +37,6 @@ using document::FieldValue; using search::ScheduleTaskCallback; using search::index::FieldLengthInfo; using search::index::IFieldLengthInspector; -using vespalib::makeLambdaTask; using search::query::Node; using search::query::SimplePhrase; using search::query::SimpleStringTerm; @@ -45,6 +45,11 @@ using search::test::SchemaBuilder; using search::test::StringFieldBuilder; using vespalib::ISequencedTaskExecutor; using vespalib::SequencedTaskExecutor; +using vespalib::Slime; +using vespalib::makeLambdaTask; +using vespalib::slime::JsonFormat; +using vespalib::slime::SlimeInserter; + using namespace search::fef; using namespace search::index; using namespace search::memoryindex; @@ -542,4 +547,18 @@ TEST(MemoryIndexTest, field_length_info_can_be_retrieved_per_field) EXPECT_EQ(0, index.index.get_field_length_info("na").get_num_samples()); } +TEST(MemoryIndexTest, write_context_state_as_slime) +{ + Index index(MySetup().field(title).field(body)); + Slime act; + SlimeInserter inserter(act); + index.index.insert_write_context_state(inserter.insertObject()); + Slime exp; + JsonFormat::decode("{\"invert\": [{\"executor_id\": 0, \"fields\": [\"body\"]}," + "{\"executor_id\": 1, \"fields\": [\"title\"]}]," + "\"push\": [{\"executor_id\": 0, \"fields\": [\"body\"]}," + "{\"executor_id\": 1, \"fields\": [\"title\"]}]}", exp); + EXPECT_EQ(exp, act); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchlib/src/vespa/searchcommon/attribute/CMakeLists.txt b/searchlib/src/vespa/searchcommon/attribute/CMakeLists.txt index 704fe238ed5..f2161196c32 100644 --- a/searchlib/src/vespa/searchcommon/attribute/CMakeLists.txt +++ b/searchlib/src/vespa/searchcommon/attribute/CMakeLists.txt @@ -9,3 +9,6 @@ vespa_add_library(searchcommon_searchcommon_attribute OBJECT status.cpp DEPENDS ) + +file(GLOB HEADERS *.h) +install(FILES ${HEADERS} DESTINATION include/vespa/searchcommon/attribute) diff --git a/searchlib/src/vespa/searchcommon/common/CMakeLists.txt b/searchlib/src/vespa/searchcommon/common/CMakeLists.txt index 6cc02ae7884..67adea7ea96 100644 --- a/searchlib/src/vespa/searchcommon/common/CMakeLists.txt +++ b/searchlib/src/vespa/searchcommon/common/CMakeLists.txt @@ -8,3 +8,6 @@ vespa_add_library(searchcommon_searchcommon_common OBJECT schemaconfigurer.cpp DEPENDS ) + +file(GLOB HEADERS *.h) +install(FILES ${HEADERS} DESTINATION include/vespa/searchcommon/common) diff --git a/searchlib/src/vespa/searchlib/common/bitvectorcache.h b/searchlib/src/vespa/searchlib/common/bitvectorcache.h index bb8f019c128..f4cced0afa4 100644 --- a/searchlib/src/vespa/searchlib/common/bitvectorcache.h +++ b/searchlib/src/vespa/searchlib/common/bitvectorcache.h @@ -14,10 +14,10 @@ public: class Iterator { public: using UP = std::unique_ptr<Iterator>; - virtual ~Iterator() { } + virtual ~Iterator() = default; virtual int32_t getNext() = 0; }; - virtual ~PopulateInterface() { } + virtual ~PopulateInterface() = default; virtual Iterator::UP lookup(uint64_t key) const = 0; }; @@ -30,7 +30,7 @@ public: using CountVector = CondensedBitVector::CountVector; using GenerationHolder = vespalib::GenerationHolder; - BitVectorCache(GenerationHolder &genHolder); + explicit BitVectorCache(GenerationHolder &genHolder); ~BitVectorCache(); void computeCountVector(KeySet & keys, CountVector & v) const; KeySet lookupCachedSet(const KeyAndCountSet & keys); @@ -44,27 +44,38 @@ public: private: class KeyMeta { public: - KeyMeta() : - _lookupCount(0), - _bitCount(0), - _chunkId(-1), - _chunkIndex(0) + KeyMeta() noexcept + : _lookupCount(0), + _bitCount(0), + _chunkId(-1), + _chunkIndex(0) { } - double cost() const { return _bitCount * _lookupCount; } + KeyMeta(const KeyMeta & rhs) noexcept + : _lookupCount(rhs.lookupCount()), + _bitCount(rhs._bitCount), + _chunkId(rhs._chunkId), + _chunkIndex(rhs._chunkIndex) + {} + KeyMeta & operator = (const KeyMeta & rhs) { + _lookupCount.store(rhs.lookupCount(), std::memory_order_release); + _bitCount = rhs._bitCount; + _chunkId = rhs._chunkId; + _chunkIndex = rhs._chunkIndex; + return *this; + } + double cost() const { return _bitCount * lookupCount(); } bool isCached() const { return _chunkId >= 0; } size_t bitCount() const { return _bitCount; } size_t chunkIndex() const { return _chunkIndex; } size_t chunkId() const { return _chunkId; } - size_t lookupCount() const { return _lookupCount; } - KeyMeta & incBits() { _bitCount++; return *this; } - KeyMeta & decBits() { _bitCount--; return *this; } + size_t lookupCount() const { return _lookupCount.load(std::memory_order_relaxed); } KeyMeta & lookup() { _lookupCount++; return *this; } KeyMeta & bitCount(uint32_t v) { _bitCount = v; return *this; } KeyMeta & chunkId(uint32_t v) { _chunkId = v; return *this; } KeyMeta & chunkIndex(uint32_t v) { _chunkIndex = v; return *this; } KeyMeta & unCache() { _chunkId = -1; return *this; } private: - size_t _lookupCount; + std::atomic<size_t> _lookupCount; uint32_t _bitCount; int32_t _chunkId; uint32_t _chunkIndex; diff --git a/searchlib/src/vespa/searchlib/features/fieldmatch/computer.cpp b/searchlib/src/vespa/searchlib/features/fieldmatch/computer.cpp index eab72818e53..7f97e3a4ca2 100644 --- a/searchlib/src/vespa/searchlib/features/fieldmatch/computer.cpp +++ b/searchlib/src/vespa/searchlib/features/fieldmatch/computer.cpp @@ -113,7 +113,7 @@ Computer::reset(uint32_t docId) void Computer::handleError(uint32_t fieldPos, uint32_t docId) const { - static int errcnt; + static std::atomic<int> errcnt(0); if (errcnt < 1000) { errcnt++; const FieldInfo * finfo = _splitter.get_query_env().getIndexEnvironment().getField(getFieldId()); diff --git a/searchlib/src/vespa/searchlib/features/onnx_feature.cpp b/searchlib/src/vespa/searchlib/features/onnx_feature.cpp index cdeb0515659..a330a4ff325 100644 --- a/searchlib/src/vespa/searchlib/features/onnx_feature.cpp +++ b/searchlib/src/vespa/searchlib/features/onnx_feature.cpp @@ -132,8 +132,7 @@ OnnxBlueprint::setup(const IIndexEnvironment &env, return fail("model setup failed: %s", ex.what()); } Onnx::WirePlanner planner; - for (size_t i = 0; i < _model->inputs().size(); ++i) { - const auto &model_input = _model->inputs()[i]; + for (const auto & model_input : _model->inputs()) { auto input_feature = model_cfg->input_feature(model_input.name); if (!input_feature.has_value()) { input_feature = fmt("rankingExpression(\"%s\")", normalize_name(model_input.name, "input").c_str()); @@ -151,8 +150,7 @@ OnnxBlueprint::setup(const IIndexEnvironment &env, } } planner.prepare_output_types(*_model); - for (size_t i = 0; i < _model->outputs().size(); ++i) { - const auto &model_output = _model->outputs()[i]; + for (const auto & model_output : _model->outputs()) { auto output_name = model_cfg->output_name(model_output.name); if (!output_name.has_value()) { output_name = normalize_name(model_output.name, "output"); diff --git a/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.cpp b/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.cpp index af7e19ee20d..4f9e88b323e 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.cpp @@ -20,9 +20,10 @@ BundledFieldsContext::add_field(uint32_t field_id) } void -BundledFieldsContext::add_uri_field(uint32_t uri_field_id) +BundledFieldsContext::add_uri_field(uint32_t uri_field_id, uint32_t uri_all_field_id) { _uri_fields.emplace_back(uri_field_id); + _uri_all_field_ids.emplace_back(uri_all_field_id); } } diff --git a/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.h b/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.h index fb1a68d7273..c058c14832d 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.h +++ b/searchlib/src/vespa/searchlib/memoryindex/bundled_fields_context.h @@ -16,16 +16,18 @@ class BundledFieldsContext vespalib::ISequencedTaskExecutor::ExecutorId _id; std::vector<uint32_t> _fields; std::vector<uint32_t> _uri_fields; + std::vector<uint32_t> _uri_all_field_ids; protected: BundledFieldsContext(vespalib::ISequencedTaskExecutor::ExecutorId id); ~BundledFieldsContext(); public: void add_field(uint32_t field_id); - void add_uri_field(uint32_t uri_field_id); + void add_uri_field(uint32_t uri_field_id, uint32_t uri_all_field_id); void set_id(vespalib::ISequencedTaskExecutor::ExecutorId id) { _id = id; } vespalib::ISequencedTaskExecutor::ExecutorId get_id() const noexcept { return _id; } const std::vector<uint32_t>& get_fields() const noexcept { return _fields; } const std::vector<uint32_t>& get_uri_fields() const noexcept { return _uri_fields; } + const std::vector<uint32_t>& get_uri_all_field_ids() const noexcept { return _uri_all_field_ids; } }; } diff --git a/searchlib/src/vespa/searchlib/memoryindex/document_inverter_context.cpp b/searchlib/src/vespa/searchlib/memoryindex/document_inverter_context.cpp index 8183cb005fe..93a12c24257 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/document_inverter_context.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/document_inverter_context.cpp @@ -15,20 +15,20 @@ template <typename Context> void make_contexts(const index::Schema& schema, const SchemaIndexFields& schema_index_fields, ISequencedTaskExecutor& executor, std::vector<Context>& contexts) { using ExecutorId = ISequencedTaskExecutor::ExecutorId; - using IdMapping = std::vector<std::tuple<ExecutorId, bool, uint32_t>>; + using IdMapping = std::vector<std::tuple<ExecutorId, bool, uint32_t, uint32_t>>; IdMapping map; for (uint32_t field_id : schema_index_fields._textFields) { // TODO: Add bias when sharing sequenced task executor between document types auto& name = schema.getIndexField(field_id).getName(); auto id = executor.getExecutorIdFromName(name); - map.emplace_back(id, false, field_id); + map.emplace_back(id, false, field_id, 0); } uint32_t uri_field_id = 0; for (auto& uri_field : schema_index_fields._uriFields) { // TODO: Add bias when sharing sequenced task executor between document types auto& name = schema.getIndexField(uri_field._all).getName(); auto id = executor.getExecutorIdFromName(name); - map.emplace_back(id, true, uri_field_id); + map.emplace_back(id, true, uri_field_id, uri_field._all); ++uri_field_id; } std::sort(map.begin(), map.end()); @@ -39,7 +39,7 @@ void make_contexts(const index::Schema& schema, const SchemaIndexFields& schema_ prev_id = std::get<0>(entry); } if (std::get<1>(entry)) { - contexts.back().add_uri_field(std::get<2>(entry)); + contexts.back().add_uri_field(std::get<2>(entry), std::get<3>(entry)); } else { contexts.back().add_field(std::get<2>(entry)); } diff --git a/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp index 643bcbb325e..86421711e32 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp +++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.cpp @@ -7,13 +7,14 @@ #include "field_index_collection.h" #include <vespa/document/fieldvalue/arrayfieldvalue.h> #include <vespa/document/fieldvalue/document.h> -#include <vespa/vespalib/util/isequencedtaskexecutor.h> #include <vespa/searchlib/index/field_length_calculator.h> #include <vespa/searchlib/index/schemautil.h> #include <vespa/searchlib/queryeval/create_blueprint_visitor_helper.h> #include <vespa/searchlib/queryeval/emptysearch.h> #include <vespa/searchlib/queryeval/leaf_blueprints.h> #include <vespa/vespalib/btree/btreenodeallocator.hpp> +#include <vespa/vespalib/data/slime/cursor.h> +#include <vespa/vespalib/util/isequencedtaskexecutor.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.memoryindex.memory_index"); @@ -47,6 +48,7 @@ using queryeval::FieldSpec; using queryeval::IRequestContext; using queryeval::Searchable; using vespalib::ISequencedTaskExecutor; +using vespalib::slime::Cursor; } @@ -251,4 +253,42 @@ MemoryIndex::get_field_length_info(const vespalib::string& field_name) const return FieldLengthInfo(); } +namespace { + +void +fields_to_slime(const std::vector<uint32_t>& field_ids, const Schema& schema, Cursor& array) +{ + for (uint32_t field_id : field_ids) { + assert(field_id < schema.getIndexFields().size()); + const auto& field = schema.getIndexField(field_id); + array.addString(field.getName()); + } +} + +void +write_context_to_slime(const BundledFieldsContext& ctx, const Schema& schema, Cursor& object) +{ + object.setLong("executor_id", ctx.get_id().getId()); + auto& fields = object.setArray("fields"); + fields_to_slime(ctx.get_fields(), schema, fields); + fields_to_slime(ctx.get_uri_all_field_ids(), schema, fields); +} + +} + +void +MemoryIndex::insert_write_context_state(Cursor& object) const +{ + auto& invert = object.setArray("invert"); + for (const auto& ctx : _inverter_context->get_invert_contexts()) { + auto& ctx_obj = invert.addObject(); + write_context_to_slime(ctx, _schema, ctx_obj); + } + auto& push = object.setArray("push"); + for (const auto& ctx : _inverter_context->get_push_contexts()) { + auto& ctx_obj = push.addObject(); + write_context_to_slime(ctx, _schema, ctx_obj); + } +} + } diff --git a/searchlib/src/vespa/searchlib/memoryindex/memory_index.h b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h index af76ed172ba..320c6fba277 100644 --- a/searchlib/src/vespa/searchlib/memoryindex/memory_index.h +++ b/searchlib/src/vespa/searchlib/memoryindex/memory_index.h @@ -17,7 +17,7 @@ namespace search::index { } namespace vespalib { class ISequencedTaskExecutor; } - +namespace vespalib::slime { struct Cursor; } namespace document { class Document; } namespace search::memoryindex { @@ -175,6 +175,8 @@ public: uint64_t getStaticMemoryFootprint() const { return _staticMemoryFootprint; } index::FieldLengthInfo get_field_length_info(const vespalib::string& field_name) const; + + void insert_write_context_state(vespalib::slime::Cursor& object) const; }; } diff --git a/storage/src/tests/distributor/pendingmessagetrackertest.cpp b/storage/src/tests/distributor/pendingmessagetrackertest.cpp index 8277281206d..35dc072b953 100644 --- a/storage/src/tests/distributor/pendingmessagetrackertest.cpp +++ b/storage/src/tests/distributor/pendingmessagetrackertest.cpp @@ -1,19 +1,24 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <tests/common/dummystoragelink.h> #include <vespa/document/base/testdocman.h> -#include <vespa/document/test/make_document_bucket.h> +#include <vespa/document/bucket/fixed_bucket_spaces.h> #include <vespa/document/test/make_bucket_space.h> +#include <vespa/document/test/make_document_bucket.h> #include <vespa/storage/distributor/pendingmessagetracker.h> #include <vespa/storage/frameworkimpl/component/storagecomponentregisterimpl.h> #include <vespa/storageapi/message/bucket.h> #include <vespa/storageapi/message/persistence.h> #include <vespa/storageframework/defaultimplementation/clock/fakeclock.h> -#include <tests/common/dummystoragelink.h> #include <vespa/vdslib/state/clusterstate.h> #include <vespa/vespalib/util/lambdatask.h> -#include <vespa/vespalib/gtest/gtest.h> #include <gmock/gmock.h> +#include <gtest/gtest.h> +#include <functional> +#include <vector> +using document::Bucket; +using document::BucketId; using document::test::makeDocumentBucket; using document::test::makeBucketSpace; using namespace ::testing; @@ -23,8 +28,7 @@ namespace storage::distributor { using namespace std::chrono_literals; struct PendingMessageTrackerTest : Test { - void insertMessages(PendingMessageTracker& tracker); - + std::vector<std::shared_ptr<api::StorageCommand>> insertMessages(PendingMessageTracker& tracker); }; namespace { @@ -33,11 +37,7 @@ class RequestBuilder { uint16_t _toNode; std::chrono::milliseconds _atTime; public: - RequestBuilder() - : _toNode(0), - _atTime() - { - } + RequestBuilder() noexcept : _toNode(0), _atTime() {} RequestBuilder& atTime(std::chrono::milliseconds t) { _atTime = t; @@ -59,14 +59,12 @@ makeStorageAddress(uint16_t node) { return {&_storage, lib::NodeType::STORAGE, node}; } -class Fixture -{ - StorageComponentRegisterImpl _compReg; +class Fixture { + StorageComponentRegisterImpl _compReg; framework::defaultimplementation::FakeClock _clock; - std::unique_ptr<PendingMessageTracker> _tracker; - document::TestDocMan _testDocMan; + std::unique_ptr<PendingMessageTracker> _tracker; + document::TestDocMan _testDocMan; public: - Fixture(); ~Fixture(); @@ -110,21 +108,26 @@ public: PendingMessageTracker& tracker() { return *_tracker; } auto& clock() { return _clock; } + std::vector<uint64_t> enumerate_msg_ids(const std::function<bool(const document::Bucket&)>& bucket_predicate) const { + std::vector<uint64_t> enumerated_ids; + auto insert_enumerated_ids = [&](uint64_t msg_id) { enumerated_ids.emplace_back(msg_id); }; + + _tracker->enumerate_matching_pending_bucket_ops(bucket_predicate, insert_enumerated_ids); + return enumerated_ids; + } + private: - std::string createDummyIdString(const document::BucketId& bucket) const { + static std::string createDummyIdString(const document::BucketId& bucket) { std::ostringstream id; id << "id:foo:testdoctype1:n=" << bucket.getId() << ":foo"; return id.str(); } - document::Document::SP createDummyDocumentForBucket(const document::BucketId& bucket) const - { + document::Document::SP createDummyDocumentForBucket(const document::BucketId& bucket) const { return _testDocMan.createDocument("foobar", createDummyIdString(bucket)); } - std::shared_ptr<api::RemoveCommand> createRemoveToNode( - uint16_t node) const - { + std::shared_ptr<api::RemoveCommand> createRemoveToNode(uint16_t node) const { document::BucketId bucket(16, 1234); auto cmd = std::make_shared<api::RemoveCommand>( makeDocumentBucket(bucket), @@ -195,9 +198,10 @@ TEST_F(PendingMessageTrackerTest, simple) { } } -void +std::vector<std::shared_ptr<api::StorageCommand>> PendingMessageTrackerTest::insertMessages(PendingMessageTracker& tracker) { + std::vector<std::shared_ptr<api::StorageCommand>> inserted; for (uint32_t i = 0; i < 4; i++) { std::ostringstream ost; ost << "id:footype:testdoc:n=1234:" << i; @@ -206,15 +210,19 @@ PendingMessageTrackerTest::insertMessages(PendingMessageTracker& tracker) document::DocumentId(ost.str()), 1000 + i); remove->setAddress(makeStorageAddress(i % 2)); tracker.insert(remove); + inserted.emplace_back(std::move(remove)); } for (uint32_t i = 0; i < 4; i++) { std::ostringstream ost; ost << "id:footype:testdoc:n=4567:" << i; - auto remove = std::make_shared<api::RemoveCommand>(makeDocumentBucket(document::BucketId(16, 4567)), document::DocumentId(ost.str()), 2000 + i); + auto remove = std::make_shared<api::RemoveCommand>(makeDocumentBucket(document::BucketId(16, 4567)), + document::DocumentId(ost.str()), 2000 + i); remove->setAddress(makeStorageAddress(i % 2)); tracker.insert(remove); + inserted.emplace_back(std::move(remove)); } + return inserted; } TEST_F(PendingMessageTrackerTest, start_page) { @@ -291,15 +299,13 @@ TEST_F(PendingMessageTrackerTest, multiple_messages) { namespace { -class TestChecker : public PendingMessageTracker::Checker -{ +class TestChecker : public PendingMessageTracker::Checker { public: uint8_t pri; - TestChecker() : pri(UINT8_MAX) {} + TestChecker() noexcept : pri(UINT8_MAX) {} - bool check(uint32_t msgType, uint16_t node, uint8_t p) override { - (void) node; + bool check(uint32_t msgType, [[maybe_unused]] uint16_t node, uint8_t p) override { if (msgType == api::MessageType::REMOVE_ID) { pri = p; return false; @@ -309,7 +315,6 @@ public: } }; - } TEST_F(PendingMessageTrackerTest, get_pending_message_types) { @@ -370,12 +375,10 @@ TEST_F(PendingMessageTrackerTest, has_pending_message) { namespace { -class OperationEnumerator : public PendingMessageTracker::Checker -{ +class OperationEnumerator : public PendingMessageTracker::Checker { std::ostringstream ss; public: - bool check(uint32_t msgType, uint16_t node, uint8_t p) override { - (void) p; + bool check(uint32_t msgType, uint16_t node, [[maybe_unused]] uint8_t p) override { ss << api::MessageType::get(static_cast<api::MessageType::Id>(msgType)).getName() << " -> " << node << "\n"; @@ -574,4 +577,58 @@ TEST_F(PendingMessageTrackerTest, request_bucket_info_with_bucket_tracked_with_s } } +namespace { + +auto bucket_id_eq_fn(BucketId matched_id) { + return [matched_id](auto& bucket) noexcept { + return bucket.getBucketId() == matched_id; + }; +} + +auto bucket_eq_fn(Bucket matched_bucket) { + return [matched_bucket](auto& bucket) noexcept { + return bucket == matched_bucket; + }; +} + +} + +TEST_F(PendingMessageTrackerTest, can_enumerate_all_message_ids_for_ops_to_matching_buckets) { + Fixture f; + BucketId bucket_id_a(16, 1234); + BucketId bucket_id_b(16, 4567); + // This inserts 4 ops for bucket {16, 1234} (bucket 'a') and 4 ops for {16, 4567} (bucket 'b') + auto inserted_ops = insertMessages(f.tracker()); + std::vector<uint64_t> bucket_a_msgs_ids, bucket_b_msgs_ids; + for (auto& op : inserted_ops) { + if (op->getBucketId() == bucket_id_a) { + bucket_a_msgs_ids.emplace_back(op->getMsgId()); + } else { + bucket_b_msgs_ids.emplace_back(op->getMsgId()); + } + } + + // Match all for bucket 'a' + EXPECT_THAT(f.enumerate_msg_ids(bucket_id_eq_fn(bucket_id_a)), UnorderedElementsAreArray(bucket_a_msgs_ids)); + // Match all for bucket 'b' + EXPECT_THAT(f.enumerate_msg_ids(bucket_id_eq_fn(bucket_id_b)), UnorderedElementsAreArray(bucket_b_msgs_ids)); + // Match no buckets + EXPECT_THAT(f.enumerate_msg_ids(bucket_id_eq_fn(BucketId(16, 7890))), IsEmpty()); + + const auto default_space = document::FixedBucketSpaces::default_space(); + const auto global_space = document::FixedBucketSpaces::global_space(); + + // Message to global bucket space (the former messages were all in the default space) + auto global_msg = std::make_shared<api::RequestBucketInfoCommand>(global_space, std::vector<BucketId>({bucket_id_a})); + global_msg->setAddress(makeStorageAddress(3)); + f.tracker().insert(global_msg); + + // Default space has the expected existing entries + EXPECT_THAT(f.enumerate_msg_ids(bucket_eq_fn(Bucket(default_space, bucket_id_a))), + UnorderedElementsAreArray(bucket_a_msgs_ids)); + // Global space has only 1 entry + EXPECT_THAT(f.enumerate_msg_ids(bucket_eq_fn(Bucket(global_space, bucket_id_a))), + ElementsAre(global_msg->getMsgId())); +} + } diff --git a/storage/src/tests/distributor/statecheckerstest.cpp b/storage/src/tests/distributor/statecheckerstest.cpp index 13c982f5a77..c249395ca50 100644 --- a/storage/src/tests/distributor/statecheckerstest.cpp +++ b/storage/src/tests/distributor/statecheckerstest.cpp @@ -93,7 +93,7 @@ struct StateCheckersTest : Test, DistributorStripeTestUtil { // Run checking only on this bucketid, but include all buckets // owned by it or owners of it, so we can detect inconsistent split. if (entry.getBucketId() == c.getBucketId()) { - c.entry = entry; + c.set_entry(entry); StateChecker::Result result(checker.check(c)); IdealStateOperation::UP op(result.createOperation()); diff --git a/storage/src/tests/distributor/top_level_bucket_db_updater_test.cpp b/storage/src/tests/distributor/top_level_bucket_db_updater_test.cpp index 074b1492a6e..d3af4cd564a 100644 --- a/storage/src/tests/distributor/top_level_bucket_db_updater_test.cpp +++ b/storage/src/tests/distributor/top_level_bucket_db_updater_test.cpp @@ -20,7 +20,6 @@ #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/text/stringtokenizer.h> #include <sstream> -#include <iomanip> using namespace storage::api; using namespace storage::lib; @@ -911,16 +910,11 @@ TEST_F(TopLevelBucketDBUpdaterTest, bit_change) { for (int i=0; cnt < 2; i++) { auto distribution = _component->getDistribution(); std::vector<uint16_t> distributors; - if (distribution->getIdealDistributorNode( - lib::ClusterState("bits:14 storage:1 distributor:2"), - document::BucketId(16, i)) - == 0) + if (distribution->getIdealDistributorNode(lib::ClusterState("bits:14 storage:1 distributor:2"), + document::BucketId(16, i)) == 0) { - vec.push_back(api::RequestBucketInfoReply::Entry( - document::BucketId(16, i), - api::BucketInfo(10,1,1))); - - bucketlist.push_back(document::BucketId(16, i)); + vec.emplace_back(document::BucketId(16, i), api::BucketInfo(10,1,1)); + bucketlist.emplace_back(16, i); cnt++; } } @@ -953,14 +947,10 @@ TEST_F(TopLevelBucketDBUpdaterTest, bit_change) { api::RequestBucketInfoReply::EntryVector &vec = sreply->getBucketInfo(); for (uint32_t i = 0; i < 3; ++i) { - vec.push_back(api::RequestBucketInfoReply::Entry( - document::BucketId(16, i), - api::BucketInfo(10,1,1))); + vec.emplace_back(document::BucketId(16, i), api::BucketInfo(10,1,1)); } - vec.push_back(api::RequestBucketInfoReply::Entry( - document::BucketId(16, 4), - api::BucketInfo(10,1,1))); + vec.emplace_back(document::BucketId(16, 4), api::BucketInfo(10,1,1)); } bucket_db_updater().onRequestBucketInfoReply(sreply); @@ -1047,9 +1037,7 @@ TEST_F(TopLevelBucketDBUpdaterTest, recheck_node) { EXPECT_EQ(bucket.getBucketId(), rbi.getBuckets()[0]); auto reply = std::make_shared<api::RequestBucketInfoReply>(rbi); - reply->getBucketInfo().push_back( - api::RequestBucketInfoReply::Entry(document::BucketId(16, 3), - api::BucketInfo(20, 10, 12, 50, 60, true, true))); + reply->getBucketInfo().emplace_back(document::BucketId(16, 3), api::BucketInfo(20, 10, 12, 50, 60, true, true)); stripe_bucket_db_updater.onRequestBucketInfoReply(reply); lib::ClusterState state("distributor:1 storage:3"); @@ -1110,8 +1098,8 @@ TEST_F(TopLevelBucketDBUpdaterTest, notify_bucket_change) { ASSERT_EQ(size_t(2), _sender.commands().size()); std::vector<api::BucketInfo> infos; - infos.push_back(api::BucketInfo(4567, 200, 2000, 400, 4000, true, true)); - infos.push_back(api::BucketInfo(8999, 300, 3000, 500, 5000, false, false)); + infos.emplace_back(4567, 200, 2000, 400, 4000, true, true); + infos.emplace_back(8999, 300, 3000, 500, 5000, false, false); for (int i = 0; i < 2; ++i) { auto& rbi = dynamic_cast<RequestBucketInfoCommand&>(*_sender.command(i)); @@ -1120,7 +1108,7 @@ TEST_F(TopLevelBucketDBUpdaterTest, notify_bucket_change) { EXPECT_EQ(bucket_id, rbi.getBuckets()[0]); auto reply = std::make_shared<api::RequestBucketInfoReply>(rbi); - reply->getBucketInfo().push_back(api::RequestBucketInfoReply::Entry(bucket_id, infos[i])); + reply->getBucketInfo().emplace_back(bucket_id, infos[i]); stripe_of_bucket(bucket_id).bucket_db_updater().onRequestBucketInfoReply(reply); } @@ -1166,10 +1154,7 @@ TEST_F(TopLevelBucketDBUpdaterTest, notify_bucket_change_from_node_down) { EXPECT_EQ(bucket_id, rbi.getBuckets()[0]); auto reply = std::make_shared<api::RequestBucketInfoReply>(rbi); - reply->getBucketInfo().push_back( - api::RequestBucketInfoReply::Entry( - bucket_id, - api::BucketInfo(8999, 300, 3000, 500, 5000, false, false))); + reply->getBucketInfo().emplace_back(bucket_id, api::BucketInfo(8999, 300, 3000, 500, 5000, false, false)); stripe_of_bucket(bucket_id).bucket_db_updater().onRequestBucketInfoReply(reply); // No change @@ -1233,9 +1218,9 @@ TEST_F(TopLevelBucketDBUpdaterTest, merge_reply) { add_nodes_to_stripe_bucket_db(bucket_id, "0=1234,1=1234,2=1234"); std::vector<api::MergeBucketCommand::Node> nodes; - nodes.push_back(api::MergeBucketCommand::Node(0)); - nodes.push_back(api::MergeBucketCommand::Node(1)); - nodes.push_back(api::MergeBucketCommand::Node(2)); + nodes.emplace_back(0); + nodes.emplace_back(1); + nodes.emplace_back(2); api::MergeBucketCommand cmd(makeDocumentBucket(bucket_id), nodes, 0); auto reply = std::make_shared<api::MergeBucketReply>(cmd); @@ -1253,9 +1238,7 @@ TEST_F(TopLevelBucketDBUpdaterTest, merge_reply) { EXPECT_EQ(bucket_id, req->getBuckets()[0]); auto reqreply = std::make_shared<api::RequestBucketInfoReply>(*req); - reqreply->getBucketInfo().push_back( - api::RequestBucketInfoReply::Entry(bucket_id, - api::BucketInfo(10 * (i + 1), 100 * (i +1), 1000 * (i+1)))); + reqreply->getBucketInfo().emplace_back(bucket_id, api::BucketInfo(10 * (i + 1), 100 * (i +1), 1000 * (i+1))); stripe_of_bucket(bucket_id).bucket_db_updater().onRequestBucketInfoReply(reqreply); } @@ -1275,7 +1258,7 @@ TEST_F(TopLevelBucketDBUpdaterTest, merge_reply_node_down) { add_nodes_to_stripe_bucket_db(bucket_id, "0=1234,1=1234,2=1234"); for (uint32_t i = 0; i < 3; ++i) { - nodes.push_back(api::MergeBucketCommand::Node(i)); + nodes.emplace_back(i); } api::MergeBucketCommand cmd(makeDocumentBucket(bucket_id), nodes, 0); @@ -1296,10 +1279,7 @@ TEST_F(TopLevelBucketDBUpdaterTest, merge_reply_node_down) { EXPECT_EQ(bucket_id, req->getBuckets()[0]); auto reqreply = std::make_shared<api::RequestBucketInfoReply>(*req); - reqreply->getBucketInfo().push_back( - api::RequestBucketInfoReply::Entry( - bucket_id, - api::BucketInfo(10 * (i + 1), 100 * (i +1), 1000 * (i+1)))); + reqreply->getBucketInfo().emplace_back(bucket_id, api::BucketInfo(10 * (i + 1), 100 * (i +1), 1000 * (i+1))); stripe_of_bucket(bucket_id).bucket_db_updater().onRequestBucketInfoReply(reqreply); } @@ -1338,10 +1318,7 @@ TEST_F(TopLevelBucketDBUpdaterTest, merge_reply_node_down_after_request_sent) { EXPECT_EQ(bucket_id, req->getBuckets()[0]); auto reqreply = std::make_shared<api::RequestBucketInfoReply>(*req); - reqreply->getBucketInfo().push_back( - api::RequestBucketInfoReply::Entry( - bucket_id, - api::BucketInfo(10 * (i + 1), 100 * (i +1), 1000 * (i+1)))); + reqreply->getBucketInfo().emplace_back(bucket_id, api::BucketInfo(10 * (i + 1), 100 * (i +1), 1000 * (i+1))); stripe_of_bucket(bucket_id).bucket_db_updater().onRequestBucketInfoReply(reqreply); } @@ -1360,7 +1337,7 @@ TEST_F(TopLevelBucketDBUpdaterTest, flush) { std::vector<api::MergeBucketCommand::Node> nodes; for (uint32_t i = 0; i < 3; ++i) { - nodes.push_back(api::MergeBucketCommand::Node(i)); + nodes.emplace_back(i); } api::MergeBucketCommand cmd(makeDocumentBucket(bucket_id), nodes, 0); @@ -1450,13 +1427,10 @@ TEST_F(TopLevelBucketDBUpdaterTest, pending_cluster_state_send_messages) { get_sent_nodes("distributor:4 storage:3", "distributor:4 .2.s:d storage:4")); - EXPECT_EQ("", - get_sent_nodes("distributor:4 storage:3", - "distributor:4 .0.s:d storage:4")); - - EXPECT_EQ("", - get_sent_nodes("distributor:3 storage:3", - "distributor:4 storage:3")); + EXPECT_TRUE(get_sent_nodes("distributor:4 storage:3", + "distributor:4 .0.s:d storage:4").empty()); + EXPECT_TRUE(get_sent_nodes("distributor:3 storage:3", + "distributor:4 storage:3").empty()); EXPECT_EQ(get_node_list({2}), get_sent_nodes("distributor:3 storage:3 .2.s:i", @@ -1470,29 +1444,21 @@ TEST_F(TopLevelBucketDBUpdaterTest, pending_cluster_state_send_messages) { get_sent_nodes("distributor:3 storage:4 .1.s:d .2.s:i", "distributor:3 storage:5")); - EXPECT_EQ("", - get_sent_nodes("distributor:1 storage:3", - "cluster:d")); - - EXPECT_EQ("", - get_sent_nodes("distributor:1 storage:3", - "distributor:1 storage:3")); - - EXPECT_EQ("", - get_sent_nodes("distributor:1 storage:3", - "cluster:d distributor:1 storage:6")); - - EXPECT_EQ("", - get_sent_nodes("distributor:3 storage:3", - "distributor:3 .2.s:m storage:3")); + EXPECT_TRUE(get_sent_nodes("distributor:1 storage:3", + "cluster:d").empty()); + EXPECT_TRUE(get_sent_nodes("distributor:1 storage:3", + "distributor:1 storage:3").empty()); + EXPECT_TRUE(get_sent_nodes("distributor:1 storage:3", + "cluster:d distributor:1 storage:6").empty()); + EXPECT_TRUE(get_sent_nodes("distributor:3 storage:3", + "distributor:3 .2.s:m storage:3").empty()); EXPECT_EQ(get_node_list({0, 1, 2}), get_sent_nodes("distributor:3 .2.s:m storage:3", "distributor:3 .2.s:d storage:3")); - EXPECT_EQ("", - get_sent_nodes("distributor:3 .2.s:m storage:3", - "distributor:3 storage:3")); + EXPECT_TRUE(get_sent_nodes("distributor:3 .2.s:m storage:3", + "distributor:3 storage:3").empty()); EXPECT_EQ(get_node_list({0, 1, 2}), get_sent_nodes_distribution_changed("distributor:3 storage:3")); @@ -1501,25 +1467,22 @@ TEST_F(TopLevelBucketDBUpdaterTest, pending_cluster_state_send_messages) { get_sent_nodes("distributor:10 storage:2", "distributor:10 .1.s:d storage:2")); - EXPECT_EQ("", - get_sent_nodes("distributor:2 storage:2", - "distributor:3 .2.s:i storage:2")); + EXPECT_TRUE(get_sent_nodes("distributor:2 storage:2", + "distributor:3 .2.s:i storage:2").empty()); EXPECT_EQ(get_node_list({0, 1, 2}), get_sent_nodes("distributor:3 storage:3", - "distributor:3 .2.s:s storage:3")); + "distributor:3 .2.s:s storage:3")); - EXPECT_EQ("", - get_sent_nodes("distributor:3 .2.s:s storage:3", - "distributor:3 .2.s:d storage:3")); + EXPECT_TRUE(get_sent_nodes("distributor:3 .2.s:s storage:3", + "distributor:3 .2.s:d storage:3").empty()); EXPECT_EQ(get_node_list({1}), get_sent_nodes("distributor:3 storage:3 .1.s:m", - "distributor:3 storage:3")); + "distributor:3 storage:3")); - EXPECT_EQ("", - get_sent_nodes("distributor:3 storage:3", - "distributor:3 storage:3 .1.s:m")); + EXPECT_TRUE(get_sent_nodes("distributor:3 storage:3", + "distributor:3 storage:3 .1.s:m").empty()); }; TEST_F(TopLevelBucketDBUpdaterTest, pending_cluster_state_receive) { @@ -1546,10 +1509,7 @@ TEST_F(TopLevelBucketDBUpdaterTest, pending_cluster_state_receive) { auto rep = std::make_shared<RequestBucketInfoReply>(*req); - rep->getBucketInfo().push_back( - RequestBucketInfoReply::Entry( - document::BucketId(16, i), - api::BucketInfo(i, i, i, i, i))); + rep->getBucketInfo().emplace_back(document::BucketId(16, i),api::BucketInfo(i, i, i, i, i)); ASSERT_TRUE(state->onRequestBucketInfoReply(rep)); ASSERT_EQ((i == (sender.commands().size() - 1)), state->done()); @@ -1573,9 +1533,8 @@ TEST_F(TopLevelBucketDBUpdaterTest, pending_cluster_state_with_group_down) { "distributor:6 .2.s:d .3.s:d storage:6")); // But don't fetch if not the entire group is down. - EXPECT_EQ("", - get_sent_nodes("distributor:6 storage:6", - "distributor:6 .2.s:d storage:6")); + EXPECT_EQ("", get_sent_nodes("distributor:6 storage:6", + "distributor:6 .2.s:d storage:6")); } TEST_F(TopLevelBucketDBUpdaterTest, pending_cluster_state_with_group_down_and_no_handover) { @@ -1612,23 +1571,16 @@ parse_input_data(const std::string& data, if (include_bucket_info) { vespalib::StringTokenizer tok4(tok3[j], "/"); - pending_transition.addNodeInfo( - document::BucketId(16, atoi(tok4[0].data())), - BucketCopy( - timestamp, - node, - api::BucketInfo( - atoi(tok4[1].data()), - atoi(tok4[2].data()), - atoi(tok4[3].data()), - atoi(tok4[2].data()), - atoi(tok4[3].data())))); + pending_transition.addNodeInfo(document::BucketId(16, atoi(tok4[0].data())), + BucketCopy(timestamp, node, + api::BucketInfo(atoi(tok4[1].data()), + atoi(tok4[2].data()), + atoi(tok4[3].data()), + atoi(tok4[2].data()), + atoi(tok4[3].data())))); } else { - pending_transition.addNodeInfo( - document::BucketId(16, atoi(tok3[j].data())), - BucketCopy(timestamp, - node, - api::BucketInfo(3, 3, 3, 3, 3))); + pending_transition.addNodeInfo(document::BucketId(16, atoi(tok3[j].data())), + BucketCopy(timestamp, node, api::BucketInfo(3, 3, 3, 3, 3))); } } } diff --git a/storage/src/vespa/storage/bucketdb/bucketcopy.h b/storage/src/vespa/storage/bucketdb/bucketcopy.h index ca629a6cd8e..5518e9ebe62 100644 --- a/storage/src/vespa/storage/bucketdb/bucketcopy.h +++ b/storage/src/vespa/storage/bucketdb/bucketcopy.h @@ -23,8 +23,7 @@ public: _info(info), _flags(0), _node(nodeIdx) - { - } + { } bool trusted() const noexcept { return _flags & TRUSTED; } diff --git a/storage/src/vespa/storage/bucketdb/bucketinfo.h b/storage/src/vespa/storage/bucketdb/bucketinfo.h index 9c024c31fd3..fe8fcd340c2 100644 --- a/storage/src/vespa/storage/bucketdb/bucketinfo.h +++ b/storage/src/vespa/storage/bucketdb/bucketinfo.h @@ -86,6 +86,7 @@ public: * Returns the bucket copy struct for the given node, null if nonexisting */ const BucketCopy* getNode(uint16_t node) const noexcept; + uint16_t internal_entry_index(uint16_t node) const noexcept; /** * Returns the number of nodes this entry has. diff --git a/storage/src/vespa/storage/bucketdb/bucketinfo.hpp b/storage/src/vespa/storage/bucketdb/bucketinfo.hpp index a8a1069d587..087c8b378b0 100644 --- a/storage/src/vespa/storage/bucketdb/bucketinfo.hpp +++ b/storage/src/vespa/storage/bucketdb/bucketinfo.hpp @@ -144,7 +144,18 @@ BucketInfoBase<NodeSeq>::getNode(uint16_t node) const noexcept { return &n; } } - return 0; + return nullptr; +} + +template <typename NodeSeq> +uint16_t +BucketInfoBase<NodeSeq>::internal_entry_index(uint16_t node) const noexcept { + for (uint16_t i = 0; i < _nodes.size(); i++) { + if (_nodes[i].getNode() == node) { + return i; + } + } + return 0xffff; // Not found signal } template <typename NodeSeq> @@ -221,7 +232,7 @@ BucketInfoBase<NodeSeq>::operator==(const BucketInfoBase<NodeSeq>& other) const return false; } - if (!(_nodes[i] == other._nodes[i])) { + if (_nodes[i] != other._nodes[i]) { return false; } } diff --git a/storage/src/vespa/storage/distributor/activecopy.cpp b/storage/src/vespa/storage/distributor/activecopy.cpp index 4c35d42a0e7..42f97c95b95 100644 --- a/storage/src/vespa/storage/distributor/activecopy.cpp +++ b/storage/src/vespa/storage/distributor/activecopy.cpp @@ -4,7 +4,6 @@ #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vespalib/stllike/asciistream.h> #include <algorithm> -#include <cassert> #include <ostream> namespace std { @@ -99,9 +98,8 @@ buildNodeList(const BucketDatabase::Entry& e,vespalib::ConstArrayRef<uint16_t> n SmallActiveCopyList result; result.reserve(nodeIndexes.size()); for (uint16_t nodeIndex : nodeIndexes) { - const BucketCopy *copy = e->getNode(nodeIndex); - assert(copy); - result.emplace_back(nodeIndex, *copy, idealState.lookup(nodeIndex)); + uint16_t entryIndex = e->internal_entry_index(nodeIndex); + result.emplace_back(nodeIndex, e->getNodeRef(entryIndex), idealState.lookup(nodeIndex), entryIndex); } return result; } @@ -132,7 +130,7 @@ ActiveCopy::calculate(const Node2Index & idealState, const lib::Distribution& di { IndexList validNodesWithCopy = buildValidNodeIndexList(e); if (validNodesWithCopy.empty()) { - return ActiveList(); + return {}; } std::vector<IndexList> groups; if (distribution.activePerGroup()) { @@ -154,15 +152,15 @@ ActiveCopy::calculate(const Node2Index & idealState, const lib::Distribution& di (inhibited_groups < max_activation_inhibited_out_of_sync_groups) && maybe_majority_info.valid()) { - const auto* candidate = e->getNode(best->_nodeIndex); - if (!candidate->getBucketInfo().equalDocumentInfo(maybe_majority_info) && !candidate->active()) { + const auto & candidate = e->getNodeRef(best->entryIndex()); + if (!candidate.getBucketInfo().equalDocumentInfo(maybe_majority_info) && !candidate.active()) { ++inhibited_groups; continue; // Do _not_ add candidate as activation target since it's out of sync with the majority } } result.emplace_back(*best); } - return ActiveList(std::move(result)); + return {std::move(result)}; } void @@ -170,8 +168,8 @@ ActiveList::print(std::ostream& out, bool verbose, const std::string& indent) co { out << "["; if (verbose) { - for (size_t i=0; i<_v.size(); ++i) { - out << "\n" << indent << " " << _v[i].nodeIndex() << " " << _v[i].getReason(); + for (const auto & copy : _v) { + out << "\n" << indent << " " << copy.nodeIndex() << " " << copy.getReason(); } if (!_v.empty()) { out << "\n" << indent; diff --git a/storage/src/vespa/storage/distributor/activecopy.h b/storage/src/vespa/storage/distributor/activecopy.h index 91dfb3f0bd0..2085b9632eb 100644 --- a/storage/src/vespa/storage/distributor/activecopy.h +++ b/storage/src/vespa/storage/distributor/activecopy.h @@ -19,13 +19,15 @@ public: : _nodeIndex(Index::invalid()), _ideal(Index::invalid()), _doc_count(0), + _entryIndex(Index::invalid()), _ready(false), _active(false) { } - ActiveCopy(uint16_t node, const BucketCopy & copy, uint16_t ideal) noexcept + ActiveCopy(uint16_t node, const BucketCopy & copy, uint16_t ideal, uint16_t entryIndex_in) noexcept : _nodeIndex(node), _ideal(ideal), _doc_count(copy.getDocumentCount()), + _entryIndex(entryIndex_in), _ready(copy.ready()), _active(copy.active()) { } @@ -36,12 +38,14 @@ public: static ActiveList calculate(const Node2Index & idealState, const lib::Distribution&, const BucketDatabase::Entry&, uint32_t max_activation_inhibited_out_of_sync_groups); uint16_t nodeIndex() const noexcept { return _nodeIndex; } + Index entryIndex() const noexcept { return Index(_entryIndex); } private: friend ActiveStateOrder; bool valid_ideal() const noexcept { return _ideal < Index::invalid(); } uint16_t _nodeIndex; uint16_t _ideal; uint32_t _doc_count; + uint16_t _entryIndex; // Index in BucketCopyList bool _ready; bool _active; }; diff --git a/storage/src/vespa/storage/distributor/idealstatemanager.cpp b/storage/src/vespa/storage/distributor/idealstatemanager.cpp index bc928ca3d41..59dbecc4397 100644 --- a/storage/src/vespa/storage/distributor/idealstatemanager.cpp +++ b/storage/src/vespa/storage/distributor/idealstatemanager.cpp @@ -49,31 +49,6 @@ IdealStateManager::print(std::ostream& out, bool verbose, const std::string& ind out << "IdealStateManager"; } -void -IdealStateManager::fillParentAndChildBuckets(StateChecker::Context& c) -{ - c.db.getAll(c.getBucketId(), c.entries); - if (c.entries.empty()) { - LOG(spam, "Did not find bucket %s in bucket database", c.bucket.toString().c_str()); - } -} -void -IdealStateManager::fillSiblingBucket(StateChecker::Context& c) -{ - c.siblingEntry = c.db.get(c.siblingBucket); -} - -BucketDatabase::Entry* -IdealStateManager::getEntryForPrimaryBucket(StateChecker::Context& c) -{ - for (auto & e : c.entries) { - if (e.getBucketId() == c.getBucketId() && ! e->getNodes().empty()) { - return &e; - } - } - return nullptr; -} - namespace { /* @@ -90,7 +65,7 @@ canOverwriteResult(const StateChecker::Result& existing, const StateChecker::Res } StateChecker::Result -IdealStateManager::runStateCheckers(StateChecker::Context& c) const +IdealStateManager::runStateCheckers(const StateChecker::Context& c) const { auto highestPri = StateChecker::Result::noMaintenanceNeeded(); // We go through _all_ active state checkers so that statistics can be @@ -114,13 +89,13 @@ IdealStateManager::verify_only_live_nodes_in_context(const StateChecker::Context if (_has_logged_phantom_replica_warning) { return; } - for (const auto& n : c.entry->getRawNodes()) { + for (const auto& n : c.entry()->getRawNodes()) { const uint16_t index = n.getNode(); const auto& state = c.systemState.getNodeState(lib::Node(lib::NodeType::STORAGE, index)); // Only nodes in Up, Initializing or Retired should ever be present in the DB. if (!state.getState().oneOf("uir")) { LOG(error, "%s in bucket DB is on node %u, which is in unavailable state %s. Current cluster state is '%s'", - c.entry.getBucketId().toString().c_str(), index, state.getState().toString().c_str(), + c.entry().getBucketId().toString().c_str(), index, state.getState().toString().c_str(), c.systemState.toString().c_str()); ASSERT_ONCE_OR_LOG(false, "Bucket DB contains replicas on unavailable node", 10000); _has_logged_phantom_replica_warning = true; @@ -133,16 +108,16 @@ IdealStateManager::generateHighestPriority(const document::Bucket& bucket, NodeM { auto& distributorBucketSpace = _op_ctx.bucket_space_repo().get(bucket.getBucketSpace()); StateChecker::Context c(node_context(), operation_context(), distributorBucketSpace, statsTracker, bucket); - fillParentAndChildBuckets(c); - fillSiblingBucket(c); + c.fillParentAndChildBuckets(); + c.fillSiblingBucket(); - BucketDatabase::Entry* e(getEntryForPrimaryBucket(c)); + const BucketDatabase::Entry* e(c.getEntryForPrimaryBucket()); if (!e) { return StateChecker::Result::noMaintenanceNeeded(); } LOG(spam, "Checking bucket %s", e->toString().c_str()); - c.entry = *e; + c.set_entry(*e); verify_only_live_nodes_in_context(c); return runStateCheckers(c); } @@ -162,23 +137,21 @@ IdealStateOperation::SP IdealStateManager::generateInterceptingSplit(BucketSpace bucketSpace, const BucketDatabase::Entry& e, api::StorageMessage::Priority pri) { + if ( ! e.valid()) return {}; + NodeMaintenanceStatsTracker statsTracker; document::Bucket bucket(bucketSpace, e.getBucketId()); auto& distributorBucketSpace = _op_ctx.bucket_space_repo().get(bucket.getBucketSpace()); StateChecker::Context c(node_context(), operation_context(), distributorBucketSpace, statsTracker, bucket); - if (e.valid()) { - c.entry = e; - - IdealStateOperation::UP operation(_splitBucketStateChecker->check(c).createOperation()); - if (operation.get()) { - operation->setPriority(pri); - operation->setIdealStateManager(this); - } + c.set_entry(e); - return operation; + IdealStateOperation::UP operation(_splitBucketStateChecker->check(c).createOperation()); + if (operation.get()) { + operation->setPriority(pri); + operation->setIdealStateManager(this); } - return {}; + return operation; } MaintenanceOperation::SP @@ -197,12 +170,12 @@ IdealStateManager::generateAll(const document::Bucket &bucket, NodeMaintenanceSt { auto& distributorBucketSpace = _op_ctx.bucket_space_repo().get(bucket.getBucketSpace()); StateChecker::Context c(node_context(), operation_context(), distributorBucketSpace, statsTracker, bucket); - fillParentAndChildBuckets(c); - fillSiblingBucket(c); - BucketDatabase::Entry* e(getEntryForPrimaryBucket(c)); + c.fillParentAndChildBuckets(); + c.fillSiblingBucket(); + const BucketDatabase::Entry* e(c.getEntryForPrimaryBucket()); std::vector<MaintenanceOperation::SP> operations; if (e) { - c.entry = *e; + c.set_entry(*e); } else { return operations; } diff --git a/storage/src/vespa/storage/distributor/idealstatemanager.h b/storage/src/vespa/storage/distributor/idealstatemanager.h index 39a662e4a81..949e7339fd4 100644 --- a/storage/src/vespa/storage/distributor/idealstatemanager.h +++ b/storage/src/vespa/storage/distributor/idealstatemanager.h @@ -41,15 +41,15 @@ public: static void print(std::ostream& out, bool verbose, const std::string& indent); // MaintenancePriorityGenerator interface - MaintenancePriorityAndType prioritize( - const document::Bucket& bucket, - NodeMaintenanceStatsTracker& statsTracker) const override; + MaintenancePriorityAndType prioritize(const document::Bucket& bucket, + NodeMaintenanceStatsTracker& statsTracker) const override; // MaintenanceOperationGenerator MaintenanceOperation::SP generate(const document::Bucket& bucket) const override; // MaintenanceOperationGenerator - std::vector<MaintenanceOperation::SP> generateAll(const document::Bucket& bucket, NodeMaintenanceStatsTracker& statsTracker) const override; + std::vector<MaintenanceOperation::SP> generateAll(const document::Bucket& bucket, + NodeMaintenanceStatsTracker& statsTracker) const override; /** * If the given bucket is too large, generate a split operation for it, @@ -72,12 +72,8 @@ public: private: void verify_only_live_nodes_in_context(const StateChecker::Context& c) const; - static void fillParentAndChildBuckets(StateChecker::Context& c); - static void fillSiblingBucket(StateChecker::Context& c); StateChecker::Result generateHighestPriority(const document::Bucket& bucket, NodeMaintenanceStatsTracker& statsTracker) const; - StateChecker::Result runStateCheckers(StateChecker::Context& c) const; - - static BucketDatabase::Entry* getEntryForPrimaryBucket(StateChecker::Context& c); + StateChecker::Result runStateCheckers(const StateChecker::Context& c) const; IdealStateMetricSet& _metrics; std::vector<StateChecker::SP> _stateCheckers; diff --git a/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp b/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp index e7832fd19e5..c7f858de608 100644 --- a/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/external/putoperation.cpp @@ -70,7 +70,7 @@ PutOperation::insertDatabaseEntryAndScheduleCreateBucket(const OperationTargetLi _op_ctx.distributor_config().max_activation_inhibited_out_of_sync_groups()); LOG(debug, "Active copies for bucket %s: %s", entry.getBucketId().toString().c_str(), active.toString().c_str()); for (uint32_t i=0; i<active.size(); ++i) { - BucketCopy copy(*entry->getNode(active[i].nodeIndex())); + BucketCopy copy(entry->getNodeRef(active[i].entryIndex())); copy.setActive(true); entry->updateNode(copy); } diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.cpp index 7bec6bbe53a..41767f0e3af 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/removebucketoperation.cpp @@ -20,8 +20,7 @@ RemoveBucketOperation::onStartInternal(DistributorStripeMessageSender& sender) BucketDatabase::Entry entry = _bucketSpace->getBucketDatabase().get(getBucketId()); - for (uint32_t i = 0; i < getNodes().size(); ++i) { - uint16_t node = getNodes()[i]; + for (uint16_t node : getNodes()) { const BucketCopy* copy(entry->getNode(node)); if (!copy) { LOG(debug, "Node %u was removed between scheduling remove operation and starting it; not sending DeleteBucket to it", node); @@ -31,7 +30,7 @@ RemoveBucketOperation::onStartInternal(DistributorStripeMessageSender& sender) auto msg = std::make_shared<api::DeleteBucketCommand>(getBucket()); setCommandMeta(*msg); msg->setBucketInfo(copy->getBucketInfo()); - msgs.push_back(std::make_pair(node, msg)); + msgs.emplace_back(node, msg); } _ok = true; diff --git a/storage/src/vespa/storage/distributor/operations/idealstate/setbucketstateoperation.cpp b/storage/src/vespa/storage/distributor/operations/idealstate/setbucketstateoperation.cpp index 9547bee6583..531f7f64b68 100644 --- a/storage/src/vespa/storage/distributor/operations/idealstate/setbucketstateoperation.cpp +++ b/storage/src/vespa/storage/distributor/operations/idealstate/setbucketstateoperation.cpp @@ -22,10 +22,7 @@ SetBucketStateOperation::~SetBucketStateOperation() = default; void SetBucketStateOperation::enqueueSetBucketStateCommand(uint16_t node, bool active) { - auto msg = std::make_shared<api::SetBucketStateCommand>(getBucket(), - active - ? api::SetBucketStateCommand::ACTIVE - : api::SetBucketStateCommand::INACTIVE); + auto msg = std::make_shared<api::SetBucketStateCommand>(getBucket(), api::SetBucketStateCommand::toState(active)); LOG(debug, "Enqueuing %s for %s to node %u", active ? "Activate" : "Deactivate", getBucketId().toString().c_str(), node); setCommandMeta(*msg); _tracker.queueCommand(std::move(msg), node); @@ -34,8 +31,8 @@ SetBucketStateOperation::enqueueSetBucketStateCommand(uint16_t node, bool active bool SetBucketStateOperation::shouldBeActive(uint16_t node) const { - for (uint32_t i=0, n=_wantedActiveNodes.size(); i<n; ++i) { - if (_wantedActiveNodes[i] == node) { + for (uint16_t wantedActiveNode : _wantedActiveNodes) { + if (wantedActiveNode == node) { return true; } } @@ -44,8 +41,8 @@ SetBucketStateOperation::shouldBeActive(uint16_t node) const void SetBucketStateOperation::activateNode(DistributorStripeMessageSender& sender) { - for (uint32_t i=0; i<_wantedActiveNodes.size(); ++i) { - enqueueSetBucketStateCommand(_wantedActiveNodes[i], true); + for (uint16_t wantedActiveNode : _wantedActiveNodes) { + enqueueSetBucketStateCommand(wantedActiveNode, true); } _tracker.flushQueue(sender); _ok = true; @@ -54,10 +51,9 @@ SetBucketStateOperation::activateNode(DistributorStripeMessageSender& sender) { void SetBucketStateOperation::deactivateNodes(DistributorStripeMessageSender& sender) { - const std::vector<uint16_t>& nodes(getNodes()); - for (size_t i = 0; i < nodes.size(); ++i) { - if (!shouldBeActive(nodes[i])) { - enqueueSetBucketStateCommand(nodes[i], false); + for (uint16_t node : getNodes()) { + if (!shouldBeActive(node)) { + enqueueSetBucketStateCommand(node, false); } } _tracker.flushQueue(sender); @@ -80,12 +76,10 @@ SetBucketStateOperation::onReceive(DistributorStripeMessageSender& sender, bool deactivate = false; if (reply->getResult().success()) { - BucketDatabase::Entry entry = - _bucketSpace->getBucketDatabase().get(rep.getBucketId()); + BucketDatabase::Entry entry = _bucketSpace->getBucketDatabase().get(rep.getBucketId()); if (entry.valid()) { const BucketCopy* copy = entry->getNode(node); - if (copy) { api::BucketInfo bInfo = copy->getBucketInfo(); @@ -96,23 +90,18 @@ SetBucketStateOperation::onReceive(DistributorStripeMessageSender& sender, bInfo.setActive(false); } - entry->updateNode( - BucketCopy(_manager->operation_context().generate_unique_timestamp(), - node, - bInfo).setTrusted(copy->trusted())); + entry->updateNode(BucketCopy(_manager->operation_context().generate_unique_timestamp(), node, bInfo) + .setTrusted(copy->trusted())); _bucketSpace->getBucketDatabase().update(entry); } } else { LOG(debug, "%s did not exist when receiving %s", - rep.getBucketId().toString().c_str(), - rep.toString(true).c_str()); + rep.getBucketId().toString().c_str(), rep.toString(true).c_str()); } } else { LOG(debug, "Failed setting state for %s on node %u: %s", - rep.getBucketId().toString().c_str(), - node, - reply->getResult().toString().c_str()); + rep.getBucketId().toString().c_str(), node, reply->getResult().toString().c_str()); _ok = false; } if (deactivate) { diff --git a/storage/src/vespa/storage/distributor/pendingmessagetracker.cpp b/storage/src/vespa/storage/distributor/pendingmessagetracker.cpp index 7b3cdacf702..c32b3b83c05 100644 --- a/storage/src/vespa/storage/distributor/pendingmessagetracker.cpp +++ b/storage/src/vespa/storage/distributor/pendingmessagetracker.cpp @@ -40,7 +40,8 @@ PendingMessageTracker::MessageEntry::toHtml() const { vespalib::asciistream ss; ss << "<li><i>Node " << nodeIdx << "</i>: " << "<b>" << vespalib::to_string(timeStamp) << "</b> " - << api::MessageType::get(api::MessageType::Id(msgType)).getName() << "(" << bucket.getBucketId() << ", priority=" << priority << ")</li>\n"; + << api::MessageType::get(api::MessageType::Id(msgType)).getName() + << "(" << bucket.getBucketId() << ", priority=" << priority << ")</li>\n"; return ss.str(); } @@ -83,7 +84,7 @@ std::vector<uint64_t> PendingMessageTracker::clearMessagesForNode(uint16_t node) { std::lock_guard guard(_lock); - MessagesByNodeAndBucket& idx(boost::multi_index::get<1>(_messages)); + auto& idx = boost::multi_index::get<IndexByNodeAndBucket>(_messages); auto range = pairAsRange(idx.equal_range(boost::make_tuple(node))); std::vector<uint64_t> erasedIds; @@ -97,6 +98,27 @@ PendingMessageTracker::clearMessagesForNode(uint16_t node) } void +PendingMessageTracker::enumerate_matching_pending_bucket_ops( + const std::function<bool(const document::Bucket&)>& bucket_predicate, + const std::function<void(uint64_t)>& msg_id_callback) const +{ + std::lock_guard guard(_lock); + const auto& idx = boost::multi_index::get<IndexByBucketAndType>(_messages); + auto iter = idx.begin(); + const auto last = idx.end(); + while (iter != last) { + const auto check_bucket = iter->bucket; + const bool match = bucket_predicate(check_bucket); + do { + if (match) { + msg_id_callback(iter->msgId); + } + ++iter; + } while ((iter != last) && (iter->bucket == check_bucket)); + } +} + +void PendingMessageTracker::insert(const std::shared_ptr<api::StorageMessage>& msg) { if (msg->getAddress()) { @@ -126,8 +148,8 @@ PendingMessageTracker::reply(const api::StorageReply& r) uint64_t msgId = r.getMsgId(); std::unique_lock guard(_lock); - MessagesByMsgId& msgs = boost::multi_index::get<0>(_messages); - MessagesByMsgId::iterator iter = msgs.find(msgId); + auto& msgs = boost::multi_index::get<IndexByMessageId>(_messages); + auto iter = msgs.find(msgId); if (iter != msgs.end()) { bucket = iter->bucket; _nodeInfo.decPending(r.getAddress()->getIndex()); @@ -184,7 +206,7 @@ bool range_is_empty_or_only_has_read_ops(const Range& range) noexcept { bool PendingMessageTracker::bucket_has_no_pending_write_ops(const document::Bucket& bucket) const noexcept { - auto& bucket_idx = boost::multi_index::get<2>(_messages); + auto& bucket_idx = boost::multi_index::get<IndexByBucketAndType>(_messages); auto pending_tasks_for_bucket = bucket_idx.equal_range(bucket); return range_is_empty_or_only_has_read_ops(pending_tasks_for_bucket); } @@ -243,30 +265,30 @@ runCheckerOnRange(PendingMessageTracker::Checker& checker, const Range& range) } void -PendingMessageTracker::checkPendingMessages(uint16_t node, const document::Bucket &bucket, Checker& checker) const +PendingMessageTracker::checkPendingMessages(uint16_t node, const document::Bucket& bucket, Checker& checker) const { std::lock_guard guard(_lock); - const MessagesByNodeAndBucket& msgs(boost::multi_index::get<1>(_messages)); + const auto& msgs = boost::multi_index::get<IndexByNodeAndBucket>(_messages); auto range = pairAsRange(msgs.equal_range(boost::make_tuple(node, bucket))); runCheckerOnRange(checker, range); } void -PendingMessageTracker::checkPendingMessages(const document::Bucket &bucket, Checker& checker) const +PendingMessageTracker::checkPendingMessages(const document::Bucket& bucket, Checker& checker) const { std::lock_guard guard(_lock); - const MessagesByBucketAndType& msgs(boost::multi_index::get<2>(_messages)); + const auto& msgs = boost::multi_index::get<IndexByBucketAndType>(_messages); auto range = pairAsRange(msgs.equal_range(boost::make_tuple(bucket))); runCheckerOnRange(checker, range); } bool -PendingMessageTracker::hasPendingMessage(uint16_t node, const document::Bucket &bucket, uint32_t messageType) const +PendingMessageTracker::hasPendingMessage(uint16_t node, const document::Bucket& bucket, uint32_t messageType) const { std::lock_guard guard(_lock); - const MessagesByNodeAndBucket& msgs(boost::multi_index::get<1>(_messages)); + const auto& msgs = boost::multi_index::get<IndexByNodeAndBucket>(_messages); auto range = msgs.equal_range(boost::make_tuple(node, bucket, messageType)); return (range.first != range.second); @@ -283,7 +305,7 @@ void PendingMessageTracker::getStatusPerBucket(std::ostream& out) const { std::lock_guard guard(_lock); - const MessagesByNodeAndBucket& msgs = boost::multi_index::get<1>(_messages); + const auto& msgs = boost::multi_index::get<IndexByNodeAndBucket>(_messages); using BucketMap = std::map<document::Bucket, std::vector<vespalib::string>>; BucketMap perBucketMsgs; for (const auto& msg : msgs) { @@ -312,9 +334,9 @@ void PendingMessageTracker::getStatusPerNode(std::ostream& out) const { std::lock_guard guard(_lock); - const MessagesByNodeAndBucket& msgs = boost::multi_index::get<1>(_messages); + const auto& msgs = boost::multi_index::get<IndexByNodeAndBucket>(_messages); int lastNode = -1; - for (const auto & node : msgs) { + for (const auto& node : msgs) { if (node.nodeIdx != lastNode) { if (lastNode != -1) { out << "</ul>\n"; diff --git a/storage/src/vespa/storage/distributor/pendingmessagetracker.h b/storage/src/vespa/storage/distributor/pendingmessagetracker.h index 4b5655d3f3c..736f2918401 100644 --- a/storage/src/vespa/storage/distributor/pendingmessagetracker.h +++ b/storage/src/vespa/storage/distributor/pendingmessagetracker.h @@ -2,22 +2,23 @@ #pragma once #include "nodeinfo.h" -#include <vespa/storageframework/generic/status/htmlstatusreporter.h> -#include <vespa/storageframework/generic/component/componentregister.h> -#include <vespa/storageframework/generic/component/component.h> #include <vespa/storageapi/message/bucket.h> +#include <vespa/storageframework/generic/component/component.h> +#include <vespa/storageframework/generic/component/componentregister.h> +#include <vespa/storageframework/generic/status/htmlstatusreporter.h> #include <vespa/vespalib/stllike/hash_set.h> -#include <boost/multi_index_container.hpp> +#include <boost/multi_index/composite_key.hpp> #include <boost/multi_index/identity.hpp> -#include <boost/multi_index/member.hpp> #include <boost/multi_index/mem_fun.hpp> +#include <boost/multi_index/member.hpp> #include <boost/multi_index/ordered_index.hpp> #include <boost/multi_index/sequenced_index.hpp> -#include <boost/multi_index/composite_key.hpp> -#include <set> -#include <unordered_map> +#include <boost/multi_index_container.hpp> #include <chrono> +#include <functional> #include <mutex> +#include <set> +#include <unordered_map> namespace storage::distributor { @@ -84,20 +85,20 @@ public: * passing it to the given type checker. * Breaks when the checker returns false. */ - void checkPendingMessages(uint16_t node, const document::Bucket &bucket, Checker& checker) const; + void checkPendingMessages(uint16_t node, const document::Bucket& bucket, Checker& checker) const; /** * Goes through each pending message (across all nodes) for the given bucket * and invokes the given checker with the node, message type and priority. * Breaks when the checker returns false. */ - void checkPendingMessages(const document::Bucket &bucket, Checker& checker) const; + void checkPendingMessages(const document::Bucket& bucket, Checker& checker) const; /** * Utility function for checking if there's a message of type * messageType pending to bucket bid on the given node. */ - bool hasPendingMessage(uint16_t node, const document::Bucket &bucket, uint32_t messageType) const; + bool hasPendingMessage(uint16_t node, const document::Bucket& bucket, uint32_t messageType) const; /** * Returns a vector containing the number of pending messages to each storage node. @@ -119,6 +120,17 @@ public: void run_once_no_pending_for_bucket(const document::Bucket& bucket, std::unique_ptr<DeferredTask> task); void abort_deferred_tasks(); + + /** + * For each distinct bucket with at least one pending message towards it: + * + * Iff `bucket_predicate(bucket) == true`, `msg_id_callback` is invoked once for _each_ + * message towards `bucket`, with the message ID as the argument. + * + * Note: `bucket_predicate` is only invoked once per distinct bucket. + */ + void enumerate_matching_pending_bucket_ops(const std::function<bool(const document::Bucket&)>& bucket_predicate, + const std::function<void(uint64_t)>& msg_id_callback) const; private: struct MessageEntry { TimePoint timeStamp; @@ -169,9 +181,11 @@ private: > >; - using MessagesByMsgId = Messages::nth_index<0>::type; - using MessagesByNodeAndBucket = Messages::nth_index<1>::type; - using MessagesByBucketAndType = Messages::nth_index<2>::type; + // Must match Messages::nth_index<N> + static constexpr uint32_t IndexByMessageId = 0; + static constexpr uint32_t IndexByNodeAndBucket = 1; + static constexpr uint32_t IndexByBucketAndType = 2; + using DeferredBucketTaskMap = std::unordered_multimap< document::Bucket, std::unique_ptr<DeferredTask>, diff --git a/storage/src/vespa/storage/distributor/statechecker.cpp b/storage/src/vespa/storage/distributor/statechecker.cpp index 7b30be53c13..70434aea987 100644 --- a/storage/src/vespa/storage/distributor/statechecker.cpp +++ b/storage/src/vespa/storage/distributor/statechecker.cpp @@ -4,7 +4,6 @@ #include "distributor_stripe_component.h" #include <vespa/vdslib/distribution/distribution.h> #include <vespa/vdslib/state/clusterstate.h> -#include <vespa/vespalib/stllike/hash_set_insert.hpp> #include <sstream> #include <vespa/log/log.h> @@ -78,11 +77,39 @@ StateChecker::Context::Context(const DistributorNodeContext& node_ctx_in, op_ctx(op_ctx_in), db(distributorBucketSpace.getBucketDatabase()), stats(statsTracker), - merges_inhibited_in_bucket_space(distributorBucketSpace.merges_inhibited()) + merges_inhibited_in_bucket_space(distributorBucketSpace.merges_inhibited()), + _entry() { } StateChecker::Context::~Context() = default; +void +StateChecker::Context::fillParentAndChildBuckets() +{ + db.getAll(getBucketId(), entries); + if (entries.empty()) { + LOG(spam, "Did not find bucket %s in bucket database", bucket.toString().c_str()); + } +} + +void +StateChecker::Context::fillSiblingBucket() +{ + siblingEntry = db.get(siblingBucket); +} + +const BucketDatabase::Entry* +StateChecker::Context::getEntryForPrimaryBucket() const +{ + for (auto & e : entries) { + if (e.getBucketId() == getBucketId() && ! e->getNodes().empty()) { + return &e; + } + } + return nullptr; +} + + std::string StateChecker::Context::toString() const { diff --git a/storage/src/vespa/storage/distributor/statechecker.h b/storage/src/vespa/storage/distributor/statechecker.h index d120b5e62d7..3635e025a21 100644 --- a/storage/src/vespa/storage/distributor/statechecker.h +++ b/storage/src/vespa/storage/distributor/statechecker.h @@ -44,8 +44,9 @@ public: * Context object used when generating operations and metrics for a * bucket. */ - struct Context + class Context { + public: Context(const DistributorNodeContext& node_ctx_in, const DistributorStripeOperationContext& op_ctx_in, const DistributorBucketSpace &distributorBucketSpace, @@ -59,7 +60,6 @@ public: // Per bucket document::Bucket bucket; document::BucketId siblingBucket; - BucketDatabase::Entry entry; BucketDatabase::Entry siblingEntry; std::vector<BucketDatabase::Entry> entries; @@ -86,6 +86,14 @@ public: document::BucketSpace getBucketSpace() const noexcept { return bucket.getBucketSpace(); } std::string toString() const; + void fillParentAndChildBuckets(); + void fillSiblingBucket(); + const BucketDatabase::Entry* getEntryForPrimaryBucket() const; + const BucketDatabase::Entry & entry() const noexcept { return _entry; } + + void set_entry(const BucketDatabase::Entry & e) { _entry = e; } + private: + BucketDatabase::Entry _entry; }; class ResultImpl @@ -130,7 +138,7 @@ public: * * @return Returns an operation to perform for the given bucket. */ - virtual Result check(Context& c) const = 0; + virtual Result check(const Context &c) const = 0; /** * Returns the name of this state checker. diff --git a/storage/src/vespa/storage/distributor/statecheckers.cpp b/storage/src/vespa/storage/distributor/statecheckers.cpp index 478faa38232..20320116e79 100644 --- a/storage/src/vespa/storage/distributor/statecheckers.cpp +++ b/storage/src/vespa/storage/distributor/statecheckers.cpp @@ -23,10 +23,10 @@ using document::BucketSpace; namespace storage::distributor { bool -SplitBucketStateChecker::validForSplit(Context& c) +SplitBucketStateChecker::validForSplit(const Context& c) { // Can't split if we have no nodes. - if (c.entry->getNodeCount() == 0) { + if (c.entry()->getNodeCount() == 0) { LOG(spam, "Can't split bucket %s, since it has no copies", c.bucket.toString().c_str()); return false; } @@ -40,9 +40,9 @@ SplitBucketStateChecker::validForSplit(Context& c) } double -SplitBucketStateChecker::getBucketSizeRelativeToMax(Context& c) +SplitBucketStateChecker::getBucketSizeRelativeToMax(const Context& c) { - auto highest = c.entry.getBucketInfo().getHighest(); + auto highest = c.entry().getBucketInfo().getHighest(); if (highest._documentCount < 2) { return 0; @@ -73,9 +73,9 @@ SplitBucketStateChecker::getBucketSizeRelativeToMax(Context& c) } StateChecker::Result -SplitBucketStateChecker::generateMinimumBucketSplitOperation(Context& c) +SplitBucketStateChecker::generateMinimumBucketSplitOperation(const Context& c) { - auto so = std::make_unique<SplitOperation>(c.node_ctx, BucketAndNodes(c.getBucket(), c.entry->getNodes()), + auto so = std::make_unique<SplitOperation>(c.node_ctx, BucketAndNodes(c.getBucket(), c.entry()->getNodes()), c.distributorConfig.getMinimalBucketSplit(), 0, 0); so->setPriority(c.distributorConfig.getMaintenancePriorities().splitDistributionBits); @@ -84,14 +84,14 @@ SplitBucketStateChecker::generateMinimumBucketSplitOperation(Context& c) } StateChecker::Result -SplitBucketStateChecker::generateMaxSizeExceededSplitOperation(Context& c) +SplitBucketStateChecker::generateMaxSizeExceededSplitOperation(const Context& c) { - auto so = std::make_unique<SplitOperation>(c.node_ctx, BucketAndNodes(c.getBucket(), c.entry->getNodes()), 58, + auto so = std::make_unique<SplitOperation>(c.node_ctx, BucketAndNodes(c.getBucket(), c.entry()->getNodes()), 58, c.distributorConfig.getSplitCount(), c.distributorConfig.getSplitSize()); so->setPriority(c.distributorConfig.getMaintenancePriorities().splitLargeBucket); - auto highest = c.entry.getBucketInfo().getHighest(); + auto highest = c.entry().getBucketInfo().getHighest(); vespalib::asciistream ost; ost << "[Splitting bucket because its maximum size (" << highest._totalDocumentSize << " b, " @@ -108,7 +108,7 @@ SplitBucketStateChecker::generateMaxSizeExceededSplitOperation(Context& c) } StateChecker::Result -SplitBucketStateChecker::check(Context& c) const { +SplitBucketStateChecker::check(const Context &c) const { if (!validForSplit(c)) { return StateChecker::Result::noMaintenanceNeeded(); } @@ -134,6 +134,22 @@ JoinBucketsStateChecker::isFirstSibling(const document::BucketId& bucketId) namespace { using ConstNodesRef = IdealServiceLayerNodesBundle::ConstNodesRef; +using Node2Index = IdealServiceLayerNodesBundle::Node2Index; + +bool +equalNodeSet(const Node2Index & node2Index, ConstNodesRef idealState, const BucketDatabase::Entry& dbEntry) +{ + if (idealState.size() != dbEntry->getNodeCount()) { + return false; + } + for (uint16_t i = 0; i < dbEntry->getNodeCount(); i++) { + const BucketCopy & info = dbEntry->getNodeRef(i); + if ( ! node2Index.lookup(info.getNode()).valid() ) { + return false; + } + } + return true; +} bool equalNodeSet(ConstNodesRef idealState, const BucketDatabase::Entry& dbEntry) @@ -154,7 +170,7 @@ equalNodeSet(ConstNodesRef idealState, const BucketDatabase::Entry& dbEntry) bool bucketAndSiblingReplicaLocationsEqualIdealState(const StateChecker::Context& context) { - if (!equalNodeSet(context.idealState(), context.entry)) { + if (!equalNodeSet(context.idealStateBundle.nonretired_or_maintenance_to_index(), context.idealState(), context.entry())) { return false; } std::vector<uint16_t> siblingIdealState = context.distribution.getIdealStorageNodes(context.systemState, context.siblingBucket); @@ -191,7 +207,7 @@ isInconsistentlySplit(const StateChecker::Context& c) bool contextBucketHasTooManyReplicas(const StateChecker::Context& c) { - return (c.entry->getNodeCount() > c.distribution.getRedundancy()); + return (c.entry()->getNodeCount() > c.distribution.getRedundancy()); } bool @@ -218,7 +234,7 @@ bucketHasMultipleChildren(const document::BucketId& bucket, const StateChecker:: bool JoinBucketsStateChecker::siblingsAreInSync(const Context& context) { - const auto& entry(context.entry); + const auto& entry(context.entry()); const auto& siblingEntry(context.siblingEntry); if (entry->getNodeCount() != siblingEntry->getNodeCount()) { @@ -275,7 +291,7 @@ JoinBucketsStateChecker::singleBucketJoinIsEnabled(const Context& c) bool JoinBucketsStateChecker::shouldJoin(const Context& c) { - if (c.entry->getNodeCount() == 0) { + if (c.entry()->getNodeCount() == 0) { LOG(spam, "Not joining bucket %s because it has no nodes", c.bucket.toString().c_str()); return false; } @@ -296,7 +312,7 @@ JoinBucketsStateChecker::shouldJoin(const Context& c) return false; } - if (c.entry->hasRecentlyCreatedEmptyCopy()) { + if (c.entry()->hasRecentlyCreatedEmptyCopy()) { return false; } @@ -334,7 +350,7 @@ JoinBucketsStateChecker::shouldJoin(const Context& c) uint64_t JoinBucketsStateChecker::getTotalUsedFileSize(const Context& c) { - return (c.entry.getBucketInfo().getHighestUsedFileSize() + return (c.entry().getBucketInfo().getHighestUsedFileSize() + c.getSiblingEntry().getBucketInfo().getHighestUsedFileSize()); } @@ -345,7 +361,7 @@ JoinBucketsStateChecker::getTotalUsedFileSize(const Context& c) uint64_t JoinBucketsStateChecker::getTotalMetaCount(const Context& c) { - return (c.entry.getBucketInfo().getHighestMetaCount() + return (c.entry().getBucketInfo().getHighestMetaCount() + c.getSiblingEntry().getBucketInfo().getHighestMetaCount()); } @@ -392,7 +408,7 @@ JoinBucketsStateChecker::computeJoinBucket(const Context& c) } StateChecker::Result -JoinBucketsStateChecker::check(Context& c) const +JoinBucketsStateChecker::check(const Context &c) const { // At this point in time, bucket is consistently split as the state checker // would otherwise be pre-empted by the inconsistent state checker. @@ -410,7 +426,7 @@ JoinBucketsStateChecker::check(Context& c) const sourceBuckets.push_back(c.getBucketId()); } sourceBuckets.push_back(c.getBucketId()); - auto op = std::make_unique<JoinOperation>(c.node_ctx, BucketAndNodes(joinedBucket, c.entry->getNodes()), sourceBuckets); + auto op = std::make_unique<JoinOperation>(c.node_ctx, BucketAndNodes(joinedBucket, c.entry()->getNodes()), sourceBuckets); op->setPriority(c.distributorConfig.getMaintenancePriorities().joinBuckets); vespalib::asciistream ost; ost << "[Joining buckets " << sourceBuckets[1].toString() << " and " << sourceBuckets[0].toString() @@ -471,7 +487,7 @@ SplitInconsistentStateChecker::getReason(const document::BucketId& bucketId, con } StateChecker::Result -SplitInconsistentStateChecker::check(Context& c) const +SplitInconsistentStateChecker::check(const Context &c) const { if (!isInconsistentlySplit(c)) { return Result::noMaintenanceNeeded(); @@ -481,7 +497,7 @@ SplitInconsistentStateChecker::check(Context& c) const return Result::noMaintenanceNeeded(); } - auto op = std::make_unique<SplitOperation>(c.node_ctx, BucketAndNodes(c.getBucket(), c.entry->getNodes()), + auto op = std::make_unique<SplitOperation>(c.node_ctx, BucketAndNodes(c.getBucket(), c.entry()->getNodes()), getHighestUsedBits(c.entries), 0, 0); op->setPriority(c.distributorConfig.getMaintenancePriorities().splitInconsistentBucket); @@ -641,7 +657,7 @@ MergeNodes::markMoveToIdealLocation(uint16_t node, uint8_t msgPriority) { void MergeNodes::markOutOfSync(const StateChecker::Context& c, uint8_t msgPriority) { - _reason << "[Synchronizing buckets with different checksums " << c.entry->toString() << "]"; + _reason << "[Synchronizing buckets with different checksums " << c.entry()->toString() << "]"; addProblem(OUT_OF_SYNC); updatePriority(msgPriority); } @@ -665,11 +681,11 @@ addStatisticsForNonIdealNodes(const StateChecker::Context& c, bool missingReplic { // Common case is that ideal state == actual state with no missing replicas. // If so, do nothing. - if (!missingReplica && (c.idealState().size() == c.entry->getNodeCount())) { + if (!missingReplica && (c.idealState().size() == c.entry()->getNodeCount())) { return; } - for (uint32_t j = 0; j < c.entry->getNodeCount(); ++j) { - const uint16_t node(c.entry->getNodeRef(j).getNode()); + for (uint32_t j = 0; j < c.entry()->getNodeCount(); ++j) { + const uint16_t node(c.entry()->getNodeRef(j).getNode()); if (!presentInIdealState(c, node)) { c.stats.incMovingOut(node, c.getBucketSpace()); } else if (missingReplica) { @@ -680,21 +696,21 @@ addStatisticsForNonIdealNodes(const StateChecker::Context& c, bool missingReplic } } -MergeNodes checkForNodesMissingFromIdealState(StateChecker::Context& c) __attribute__((noinline)); -MergeNodes checkIfBucketsAreOutOfSyncAndNeedMerging(StateChecker::Context& c) __attribute__((noinline)); +MergeNodes checkForNodesMissingFromIdealState(const StateChecker::Context& c) __attribute__((noinline)); +MergeNodes checkIfBucketsAreOutOfSyncAndNeedMerging(const StateChecker::Context& c) __attribute__((noinline)); MergeNodes -checkForNodesMissingFromIdealState(StateChecker::Context& c) +checkForNodesMissingFromIdealState(const StateChecker::Context& c) { MergeNodes ret; // Check if we need to add copies to get to ideal state. - if (!c.entry->emptyAndConsistent()) { + if (!c.entry()->emptyAndConsistent()) { bool hasMissingReplica = false; for (uint16_t node : c.idealState()) { bool found = false; - for (uint32_t j = 0; j < c.entry->getNodeCount(); j++) { - if (c.entry->getNodeRef(j).getNode() == node) { + for (uint32_t j = 0; j < c.entry()->getNodeCount(); j++) { + if (c.entry()->getNodeRef(j).getNode() == node) { found = true; break; } @@ -702,7 +718,7 @@ checkForNodesMissingFromIdealState(StateChecker::Context& c) if (!found) { const auto & mp = c.distributorConfig.getMaintenancePriorities(); - if (c.idealState().size() > c.entry->getNodeCount()) { + if (c.idealState().size() > c.entry()->getNodeCount()) { ret.markMissingReplica(node, mp.mergeTooFewCopies); } else { ret.markMoveToIdealLocation(node,mp.mergeMoveToIdealNode); @@ -717,20 +733,20 @@ checkForNodesMissingFromIdealState(StateChecker::Context& c) } void -addStatisticsForOutOfSyncCopies(StateChecker::Context& c) +addStatisticsForOutOfSyncCopies(const StateChecker::Context& c) { - const uint32_t n = c.entry->getNodeCount(); + const uint32_t n = c.entry()->getNodeCount(); for (uint32_t i = 0; i < n; ++i) { - const BucketCopy& cp(c.entry->getNodeRef(i)); + const BucketCopy& cp(c.entry()->getNodeRef(i)); c.stats.incSyncing(cp.getNode(), c.getBucketSpace()); } } MergeNodes -checkIfBucketsAreOutOfSyncAndNeedMerging(StateChecker::Context& c) +checkIfBucketsAreOutOfSyncAndNeedMerging(const StateChecker::Context& c) { MergeNodes ret; - if (!consistentApartFromEmptyBucketsInNonIdealLocationAndInvalidEntries(c.idealState(),c.entry.getBucketInfo())) { + if (!consistentApartFromEmptyBucketsInNonIdealLocationAndInvalidEntries(c.idealState(),c.entry().getBucketInfo())) { auto pri(c.distributorConfig.getMaintenancePriorities().mergeOutOfSyncCopies); ret.markOutOfSync(c, pri); addStatisticsForOutOfSyncCopies(c); @@ -741,9 +757,9 @@ checkIfBucketsAreOutOfSyncAndNeedMerging(StateChecker::Context& c) bool allCopiesAreInvalid(const StateChecker::Context& c) { - const uint32_t n = c.entry->getNodeCount(); + const uint32_t n = c.entry()->getNodeCount(); for (uint32_t i = 0; i < n; ++i) { - const BucketCopy& cp(c.entry->getNodeRef(i)); + const BucketCopy& cp(c.entry()->getNodeRef(i)); if (cp.valid()) { return false; } @@ -762,7 +778,7 @@ merging_effectively_disabled_for_state_checker(const StateChecker::Context& c) n } StateChecker::Result -SynchronizeAndMoveStateChecker::check(Context& c) const +SynchronizeAndMoveStateChecker::check(const Context &c) const { if (merging_effectively_disabled_for_state_checker(c)) { return Result::noMaintenanceNeeded(); @@ -780,9 +796,9 @@ SynchronizeAndMoveStateChecker::check(Context& c) const return Result::noMaintenanceNeeded(); } - assert(c.entry->getNodeCount() > 0); + assert(c.entry()->getNodeCount() > 0); - MergeNodes result(c.entry); + MergeNodes result(c.entry()); result += checkForNodesMissingFromIdealState(c); result += checkIfBucketsAreOutOfSyncAndNeedMerging(c); @@ -807,7 +823,7 @@ SynchronizeAndMoveStateChecker::check(Context& c) const return Result::createStoredResult(std::move(op), schedPri); } else { LOG(spam, "Bucket %s: No need for merge, as bucket is in consistent state (or inconsistent buckets are empty) %s", - c.bucket.toString().c_str(), c.entry->toString().c_str()); + c.bucket.toString().c_str(), c.entry()->toString().c_str()); return Result::noMaintenanceNeeded(); } } @@ -815,8 +831,8 @@ SynchronizeAndMoveStateChecker::check(Context& c) const bool DeleteExtraCopiesStateChecker::bucketHasNoData(const Context& c) { - return (c.entry->getHighestMetaCount() == 0 - && !c.entry->hasRecentlyCreatedEmptyCopy()); + return (c.entry()->getHighestMetaCount() == 0 + && !c.entry()->hasRecentlyCreatedEmptyCopy()); } bool @@ -845,10 +861,10 @@ DeleteExtraCopiesStateChecker::addToRemoveSet( uint32_t DeleteExtraCopiesStateChecker::numberOfIdealCopiesPresent(const Context& c) { - const uint32_t cnt = c.entry->getNodeCount(); + const uint32_t cnt = c.entry()->getNodeCount(); uint32_t idealCopies = 0; for (uint32_t i = 0; i < cnt; ++i) { - const BucketCopy& cp(c.entry->getNodeRef(i)); + const BucketCopy& cp(c.entry()->getNodeRef(i)); if (copyIsInIdealState(cp, c)) { ++idealCopies; } @@ -865,19 +881,19 @@ DeleteExtraCopiesStateChecker::numberOfIdealCopiesPresent(const Context& c) */ void DeleteExtraCopiesStateChecker::removeRedundantEmptyOrConsistentCopies( - Context& c, + const Context& c, std::vector<uint16_t>& removedCopies, vespalib::asciistream& reasons) { assert(removedCopies.empty()); - const bool copiesAreConsistent = c.entry->validAndConsistent(); - const uint32_t cnt = c.entry->getNodeCount(); + const bool copiesAreConsistent = c.entry()->validAndConsistent(); + const uint32_t cnt = c.entry()->getNodeCount(); // Always keep all ideal copies uint32_t keptIdealCopies = numberOfIdealCopiesPresent(c); uint32_t keptNonIdealCopies = 0; for (uint32_t i = 0; i < cnt; ++i) { - const BucketCopy& cp(c.entry->getNodeRef(i)); + const BucketCopy& cp(c.entry()->getNodeRef(i)); if (copyIsInIdealState(cp, c)) { continue; } @@ -898,9 +914,9 @@ DeleteExtraCopiesStateChecker::removeRedundantEmptyOrConsistentCopies( } StateChecker::Result -DeleteExtraCopiesStateChecker::check(Context& c) const +DeleteExtraCopiesStateChecker::check(const Context &c) const { - if (c.entry->hasInvalidCopy()) { + if (c.entry()->hasInvalidCopy()) { // Don't delete anything here. return Result::noMaintenanceNeeded(); } @@ -914,14 +930,14 @@ DeleteExtraCopiesStateChecker::check(Context& c) const std::vector<uint16_t> removedCopies; if (bucketHasNoData(c)) { - reasons << "[Removing all copies since bucket is empty:" << c.entry->toString() << "]"; + reasons << "[Removing all copies since bucket is empty:" << c.entry()->toString() << "]"; - for (uint32_t j = 0, cnt = c.entry->getNodeCount(); j < cnt; ++j) { - removedCopies.push_back(c.entry->getNodeRef(j).getNode()); + for (uint32_t j = 0, cnt = c.entry()->getNodeCount(); j < cnt; ++j) { + removedCopies.push_back(c.entry()->getNodeRef(j).getNode()); } - } else if (c.entry->getNodeCount() <= c.distribution.getRedundancy()) { + } else if (c.entry()->getNodeCount() <= c.distribution.getRedundancy()) { return Result::noMaintenanceNeeded(); - } else if (c.entry->hasRecentlyCreatedEmptyCopy()) { + } else if (c.entry()->hasRecentlyCreatedEmptyCopy()) { return Result::noMaintenanceNeeded(); } else { removeRedundantEmptyOrConsistentCopies(c, removedCopies, reasons); @@ -938,27 +954,34 @@ DeleteExtraCopiesStateChecker::check(Context& c) const return Result::noMaintenanceNeeded(); } +namespace { + bool -BucketStateStateChecker::shouldSkipActivationDueToMaintenance(const ActiveList& activeNodes, const Context& c) -{ +shouldSkipActivationDueToMaintenanceOrGatherOperationNodes(const ActiveList &activeNodes, + const StateChecker::Context &c, + std::vector<uint16_t> & operationNodes) { for (uint32_t i = 0; i < activeNodes.size(); ++i) { - const auto node_index = activeNodes[i].nodeIndex(); - const BucketCopy* cp(c.entry->getNode(node_index)); - if (!cp || cp->active()) { - continue; - } - if (!cp->ready()) { + const ActiveCopy & active = activeNodes[i]; + if ( ! active.entryIndex().valid()) continue; + const BucketCopy & cp(c.entry()->getNodeRef(active.entryIndex())); + if (cp.active()) continue; + + const auto node_index = active.nodeIndex(); + if (!cp.ready()) { if (!c.op_ctx.node_supported_features_repo().node_supported_features(node_index).no_implicit_indexing_of_active_buckets) { // If copy is not ready, we don't want to activate it if a node // is set in maintenance. Doing so would imply that we want proton // to start background indexing. - return containsMaintenanceNode(c.idealState(), c); + if (containsMaintenanceNode(c.idealState(), c)) return true; } // else: activation does not imply indexing, so we can safely do it at any time. } + operationNodes.push_back(node_index); } return false; } +} + /** * The copy we want to set active is, in prioritized order: * 1. The first ideal state copy that is trusted and ready @@ -970,7 +993,7 @@ BucketStateStateChecker::shouldSkipActivationDueToMaintenance(const ActiveList& * 7. Any valid copy if no copies are active */ StateChecker::Result -BucketStateStateChecker::check(Context& c) const +BucketStateStateChecker::check(const Context &c) const { if (c.distributorConfig.isBucketActivationDisabled()) { return Result::noMaintenanceNeeded(); @@ -981,29 +1004,28 @@ BucketStateStateChecker::check(Context& c) const } ActiveList activeNodes = ActiveCopy::calculate(c.idealStateBundle.nonretired_or_maintenance_to_index(), - c.distribution, c.entry, + c.distribution, c.entry(), c.distributorConfig.max_activation_inhibited_out_of_sync_groups()); if (activeNodes.empty()) { return Result::noMaintenanceNeeded(); } - if (shouldSkipActivationDueToMaintenance(activeNodes, c)) { + std::vector<uint16_t> operationNodes; + if (shouldSkipActivationDueToMaintenanceOrGatherOperationNodes(activeNodes, c, operationNodes)) { return Result::noMaintenanceNeeded(); } - vespalib::asciistream reason; - std::vector<uint16_t> operationNodes; - for (uint32_t i=0; i<activeNodes.size(); ++i) { - const BucketCopy* cp = c.entry->getNode(activeNodes[i].nodeIndex()); - if (cp == nullptr || cp->active()) { - continue; + for (uint16_t nodeIndex : operationNodes) { // Most of the time empty + for (uint32_t i = 0; i < activeNodes.size(); ++i) { + const ActiveCopy &active = activeNodes[i]; + if (nodeIndex == active.nodeIndex()) { + reason << "[Setting node " << active.nodeIndex() << " as active: " << active.getReason() << "]"; + } } - operationNodes.push_back(activeNodes[i].nodeIndex()); - reason << "[Setting node " << activeNodes[i].nodeIndex() << " as active: " << activeNodes[i].getReason() << "]"; } // Deactivate all copies that are currently marked as active. - for (uint32_t i = 0; i < c.entry->getNodeCount(); ++i) { - const BucketCopy& cp = c.entry->getNodeRef(i); + for (uint32_t i = 0; i < c.entry()->getNodeCount(); ++i) { + const BucketCopy& cp = c.entry()->getNodeRef(i); if (!cp.active()) { continue; } @@ -1050,32 +1072,32 @@ GarbageCollectionStateChecker::garbage_collection_disabled(const Context& c) noe bool GarbageCollectionStateChecker::needs_garbage_collection(const Context& c, vespalib::duration time_since_epoch) { - if (c.entry->getNodeCount() == 0) { + if (c.entry()->getNodeCount() == 0) { return false; } if (containsMaintenanceNode(c.idealState(), c)) { return false; } - std::chrono::seconds lastRunAt(c.entry->getLastGarbageCollectionTime()); + std::chrono::seconds lastRunAt(c.entry()->getLastGarbageCollectionTime()); return c.gcTimeCalculator.shouldGc(c.getBucketId(), time_since_epoch, lastRunAt); } StateChecker::Result -GarbageCollectionStateChecker::check(Context& c) const +GarbageCollectionStateChecker::check(const Context &c) const { if (garbage_collection_disabled(c)) { return Result::noMaintenanceNeeded(); } const vespalib::duration now(c.node_ctx.clock().getSystemTime().time_since_epoch()); - const std::chrono::seconds last_run_at(c.entry->getLastGarbageCollectionTime()); + const std::chrono::seconds last_run_at(c.entry()->getLastGarbageCollectionTime()); c.stats.update_observed_time_since_last_gc(now - last_run_at); if (needs_garbage_collection(c, now)) { - auto op = std::make_unique<GarbageCollectionOperation>(c.node_ctx, BucketAndNodes(c.getBucket(), c.entry->getNodes())); + auto op = std::make_unique<GarbageCollectionOperation>(c.node_ctx, BucketAndNodes(c.getBucket(), c.entry()->getNodes())); vespalib::asciistream reason; reason << "[Needs garbage collection: Last check at " - << c.entry->getLastGarbageCollectionTime() + << c.entry()->getLastGarbageCollectionTime() << ", current time " << vespalib::count_s(now) << ", configured interval " diff --git a/storage/src/vespa/storage/distributor/statecheckers.h b/storage/src/vespa/storage/distributor/statecheckers.h index b6cd5ba0534..e77e19e8c9d 100644 --- a/storage/src/vespa/storage/distributor/statecheckers.h +++ b/storage/src/vespa/storage/distributor/statecheckers.h @@ -8,19 +8,19 @@ namespace storage::distributor { class SynchronizeAndMoveStateChecker : public StateChecker { public: - Result check(Context& c) const override; + Result check(const Context &c) const override; const char* getName() const noexcept override { return "SynchronizeAndMove"; } }; class DeleteExtraCopiesStateChecker : public StateChecker { public: - Result check(Context& c) const override; + Result check(const Context &c) const override; const char* getName() const noexcept override { return "DeleteExtraCopies"; } private: static bool bucketHasNoData(const Context& c); - static void removeRedundantEmptyOrConsistentCopies(Context& c, std::vector<uint16_t>& removedCopies, vespalib::asciistream& reasons); + static void removeRedundantEmptyOrConsistentCopies(const Context& c, std::vector<uint16_t>& removedCopies, vespalib::asciistream& reasons); static bool copyIsInIdealState(const BucketCopy& cp, const Context& c); static bool enoughCopiesKept(uint32_t keptIdealCopies, uint32_t keptNonIdealCopies, const Context& c); static uint32_t numberOfIdealCopiesPresent(const Context& c); @@ -32,7 +32,7 @@ private: class JoinBucketsStateChecker : public StateChecker { public: - Result check(Context& c) const override; + Result check(const Context &c) const override; const char* getName() const noexcept override { return "JoinBuckets"; } private: static uint64_t getTotalUsedFileSize(const Context& c); @@ -49,20 +49,20 @@ private: class SplitBucketStateChecker : public StateChecker { public: - Result check(Context& c) const override; + Result check(const Context &c) const override; const char* getName() const noexcept override { return "SplitBucket"; } private: - static Result generateMinimumBucketSplitOperation(Context& c); - static Result generateMaxSizeExceededSplitOperation(Context& c); + static Result generateMinimumBucketSplitOperation(const Context& c); + static Result generateMaxSizeExceededSplitOperation(const Context& c); - static bool validForSplit(Context& c); - static double getBucketSizeRelativeToMax(Context& c); + static bool validForSplit(const Context& c); + static double getBucketSizeRelativeToMax(const Context& c); }; class SplitInconsistentStateChecker : public StateChecker { public: - Result check(Context& c) const override; + Result check(const Context &c) const override; const char* getName() const noexcept override { return "SplitInconsistentBuckets"; } private: @@ -75,16 +75,15 @@ class ActiveList; class BucketStateStateChecker : public StateChecker { - static bool shouldSkipActivationDueToMaintenance(const ActiveList& activeList, const Context& c); public: - Result check(Context& c) const override; + Result check(const Context &c) const override; const char* getName() const noexcept override { return "SetBucketState"; } }; class GarbageCollectionStateChecker : public StateChecker { public: - Result check(Context& c) const override; + Result check(const Context &c) const override; const char* getName() const noexcept override { return "GarbageCollection"; } private: static bool garbage_collection_disabled(const Context& c) noexcept; diff --git a/storage/src/vespa/storageapi/message/bucket.h b/storage/src/vespa/storageapi/message/bucket.h index e02b8fcd672..80565334331 100644 --- a/storage/src/vespa/storageapi/message/bucket.h +++ b/storage/src/vespa/storageapi/message/bucket.h @@ -106,7 +106,8 @@ public: uint16_t index; bool sourceOnly; - Node(uint16_t index_, bool sourceOnly_ = false) noexcept + Node(uint16_t index_) noexcept : Node(index_, false) { } + Node(uint16_t index_, bool sourceOnly_) noexcept : index(index_), sourceOnly(sourceOnly_) {} bool operator==(const Node& n) const noexcept @@ -471,21 +472,15 @@ public: class SetBucketStateCommand : public MaintenanceCommand { public: - enum BUCKET_STATE - { - INACTIVE, - ACTIVE - }; -private: - BUCKET_STATE _state; -public: + enum BUCKET_STATE { INACTIVE, ACTIVE }; SetBucketStateCommand(const document::Bucket &bucket, BUCKET_STATE state); void print(std::ostream& out, bool verbose, const std::string& indent) const override; BUCKET_STATE getState() const { return _state; } - DECLARE_STORAGECOMMAND(SetBucketStateCommand, onSetBucketState) + static BUCKET_STATE toState(bool active) noexcept { return active ? ACTIVE : INACTIVE; } + DECLARE_STORAGECOMMAND(SetBucketStateCommand, onSetBucketState); private: - vespalib::string getSummary() const override; + BUCKET_STATE _state; }; /** diff --git a/storage/src/vespa/storageapi/messageapi/bucketcommand.cpp b/storage/src/vespa/storageapi/messageapi/bucketcommand.cpp index c75267f560d..f202ce9da7d 100644 --- a/storage/src/vespa/storageapi/messageapi/bucketcommand.cpp +++ b/storage/src/vespa/storageapi/messageapi/bucketcommand.cpp @@ -7,10 +7,9 @@ using document::Bucket; using document::BucketId; using document::BucketSpace; -namespace storage { -namespace api { +namespace storage::api { -BucketCommand::BucketCommand(const MessageType& type, const Bucket &bucket) +BucketCommand::BucketCommand(const MessageType& type, const Bucket &bucket) noexcept : StorageCommand(type), _bucket(bucket), _originalBucket() @@ -42,5 +41,4 @@ BucketCommand::print(std::ostream& out, } } -} // api -} // storage +} diff --git a/storage/src/vespa/storageapi/messageapi/bucketcommand.h b/storage/src/vespa/storageapi/messageapi/bucketcommand.h index 605653681b5..f1d5cea377c 100644 --- a/storage/src/vespa/storageapi/messageapi/bucketcommand.h +++ b/storage/src/vespa/storageapi/messageapi/bucketcommand.h @@ -18,7 +18,7 @@ class BucketCommand : public StorageCommand { document::BucketId _originalBucket; protected: - BucketCommand(const MessageType& type, const document::Bucket &bucket); + BucketCommand(const MessageType& type, const document::Bucket &bucket) noexcept; public: DECLARE_POINTER_TYPEDEFS(BucketCommand); diff --git a/storage/src/vespa/storageapi/messageapi/bucketinfocommand.h b/storage/src/vespa/storageapi/messageapi/bucketinfocommand.h index 0f6627328a9..0675e639096 100644 --- a/storage/src/vespa/storageapi/messageapi/bucketinfocommand.h +++ b/storage/src/vespa/storageapi/messageapi/bucketinfocommand.h @@ -18,7 +18,7 @@ namespace storage::api { class BucketInfoCommand : public BucketCommand { protected: - BucketInfoCommand(const MessageType& type, const document::Bucket &bucket) + BucketInfoCommand(const MessageType& type, const document::Bucket &bucket) noexcept : BucketCommand(type, bucket) {} public: diff --git a/storage/src/vespa/storageapi/messageapi/maintenancecommand.h b/storage/src/vespa/storageapi/messageapi/maintenancecommand.h index 6bb36d0d32f..4b4612123a5 100644 --- a/storage/src/vespa/storageapi/messageapi/maintenancecommand.h +++ b/storage/src/vespa/storageapi/messageapi/maintenancecommand.h @@ -3,8 +3,7 @@ #include "bucketinfocommand.h" -namespace storage { -namespace api { +namespace storage::api { class MaintenanceCommand : public BucketInfoCommand { @@ -13,10 +12,10 @@ public: : BucketInfoCommand(type, bucket) {} MaintenanceCommand(const MaintenanceCommand &) = default; - MaintenanceCommand(MaintenanceCommand &&) = default; + MaintenanceCommand(MaintenanceCommand &&) noexcept = default; MaintenanceCommand & operator = (const MaintenanceCommand &) = delete; - MaintenanceCommand & operator = (MaintenanceCommand &&) = delete; - ~MaintenanceCommand(); + MaintenanceCommand & operator = (MaintenanceCommand &&) noexcept = delete; + ~MaintenanceCommand() override; const vespalib::string& getReason() const { return _reason; }; void setReason(vespalib::stringref reason) { _reason = reason; }; @@ -25,4 +24,3 @@ protected: }; } -} diff --git a/storage/src/vespa/storageapi/messageapi/storagecommand.cpp b/storage/src/vespa/storageapi/messageapi/storagecommand.cpp index 249e08362d4..c0da654627c 100644 --- a/storage/src/vespa/storageapi/messageapi/storagecommand.cpp +++ b/storage/src/vespa/storageapi/messageapi/storagecommand.cpp @@ -17,7 +17,7 @@ StorageCommand::StorageCommand(const StorageCommand& other) { } -StorageCommand::StorageCommand(const MessageType& type, Priority p) +StorageCommand::StorageCommand(const MessageType& type, Priority p) noexcept : StorageMessage(type, generateMsgId(), 0), _timeout(MAX_TIMEOUT), _sourceIndex(0xFFFF) diff --git a/storage/src/vespa/storageapi/messageapi/storagecommand.h b/storage/src/vespa/storageapi/messageapi/storagecommand.h index 06817a65b05..e5810ec638a 100644 --- a/storage/src/vespa/storageapi/messageapi/storagecommand.h +++ b/storage/src/vespa/storageapi/messageapi/storagecommand.h @@ -20,7 +20,8 @@ class StorageCommand : public StorageMessage { protected: StorageCommand(const StorageCommand& other); - explicit StorageCommand(const MessageType& type, Priority p = NORMAL); + explicit StorageCommand(const MessageType& type) noexcept : StorageCommand(type, NORMAL) { } + explicit StorageCommand(const MessageType& type, Priority p) noexcept; public: DECLARE_POINTER_TYPEDEFS(StorageCommand); diff --git a/tenant-cd-api/pom.xml b/tenant-cd-api/pom.xml index 33bafb2909a..bc17de1eb72 100644 --- a/tenant-cd-api/pom.xml +++ b/tenant-cd-api/pom.xml @@ -24,7 +24,7 @@ This version must match the string in all ExportPackage annotations in this module. It must also be in sync junit version specified in 'hosted-tenant-base'. --> - <hosted-tenant-base-junit-version>5.8.1</hosted-tenant-base-junit-version> + <hosted-tenant-base-junit-version>${junit.vespa.tenant.version}</hosted-tenant-base-junit-version> </properties> @@ -69,7 +69,7 @@ <dependency> <groupId>org.junit.platform</groupId> <artifactId>junit-platform-commons</artifactId> - <version>1.8.1</version> + <version>${junit.platform.vespa.tenant.version}</version> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/testutil/pom.xml b/testutil/pom.xml index 7f57b6defee..a26459c0b8a 100644 --- a/testutil/pom.xml +++ b/testutil/pom.xml @@ -24,7 +24,7 @@ <groupId>com.google.inject</groupId> <artifactId>guice</artifactId> <scope>provided</scope> - <classifier>no_aop</classifier> + </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index 50ab762bd9b..1c6a2957317 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -20,9 +20,6 @@ com.fasterxml.jackson.core:jackson-databind:2.15.2 com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.15.2 com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.15.2 com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.2 -com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.15.2 -com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.15.2 -com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.15.2 com.github.luben:zstd-jni:1.5.5-5 com.github.spotbugs:spotbugs-annotations:3.1.9 com.google.auth:google-auth-library-credentials:1.19.0 @@ -36,7 +33,7 @@ com.google.guava:guava:32.1.2-jre com.google.http-client:google-http-client:1.43.3 com.google.http-client:google-http-client-apache-v2:1.43.3 com.google.http-client:google-http-client-gson:1.42.3 -com.google.inject:guice:4.2.3:no_aop +com.google.inject:guice:6.0.0 com.google.j2objc:j2objc-annotations:2.8 com.google.protobuf:protobuf-java:3.24.2 com.ibm.icu:icu4j:73.2 @@ -81,9 +78,7 @@ io.prometheus:simpleclient_common:0.16.0 io.prometheus:simpleclient_tracer_common:0.16.0 io.prometheus:simpleclient_tracer_otel:0.16.0 io.prometheus:simpleclient_tracer_otel_agent:0.16.0 -jakarta.annotation:jakarta.annotation-api:1.3.5 -jakarta.validation:jakarta.validation-api:2.0.2 -jakarta.ws.rs:jakarta.ws.rs-api:2.1.6 +jakarta.inject:jakarta.inject-api:2.0.1 javax.activation:javax.activation-api:1.2.0 javax.annotation:javax.annotation-api:1.2 javax.inject:javax.inject:1 @@ -95,7 +90,7 @@ junit:junit:4.13.2 net.java.dev.jna:jna:5.13.0 net.openhft:zero-allocation-hashing:0.16 org.antlr:antlr-runtime:3.5.3 -org.antlr:antlr4-runtime:4.13.0 +org.antlr:antlr4-runtime:4.13.1 org.apache.aries.spifly:org.apache.aries.spifly.dynamic.bundle:1.3.6 org.apache.commons:commons-compress:1.23.0 org.apache.commons:commons-csv:1.10.0 @@ -106,7 +101,7 @@ org.apache.curator:curator-client:5.5.0 org.apache.curator:curator-framework:5.5.0 org.apache.curator:curator-recipes:5.5.0 org.apache.felix:org.apache.felix.framework:7.0.5 -org.apache.felix:org.apache.felix.log:1.0.1 +org.apache.felix:org.apache.felix.log:1.3.0 org.apache.httpcomponents:httpclient:4.5.14 org.apache.httpcomponents:httpcore:4.4.16 org.apache.httpcomponents:httpmime:4.5.14 @@ -171,34 +166,23 @@ org.eclipse.jetty.toolchain:jetty-jakarta-servlet-api:5.0.2 org.eclipse.sisu:org.eclipse.sisu.inject:0.3.5 org.eclipse.sisu:org.eclipse.sisu.plexus:0.3.5 org.fusesource.jansi:jansi:1.18 -org.glassfish.hk2:osgi-resource-locator:1.0.3 -org.glassfish.hk2.external:jakarta.inject:2.6.1 org.glassfish.jaxb:jaxb-core:4.0.3 org.glassfish.jaxb:jaxb-runtime:4.0.3 org.glassfish.jaxb:txw2:4.0.3 -org.glassfish.jersey.core:jersey-client:2.40 -org.glassfish.jersey.core:jersey-common:2.40 -org.glassfish.jersey.core:jersey-server:2.40 -org.glassfish.jersey.ext:jersey-entity-filtering:2.40 -org.glassfish.jersey.ext:jersey-proxy-client:2.40 -org.glassfish.jersey.media:jersey-media-json-jackson:2.40 -org.glassfish.jersey.media:jersey-media-multipart:2.40 org.hamcrest:hamcrest:2.2 org.hamcrest:hamcrest-core:2.2 org.hdrhistogram:HdrHistogram:2.1.12 org.iq80.snappy:snappy:0.4 org.json:json:20230618 +org.junit.jupiter:junit-jupiter-api:5.10.0 org.junit.jupiter:junit-jupiter-api:5.8.1 org.junit.jupiter:junit-jupiter-engine:5.8.1 org.junit.platform:junit-platform-commons:1.8.1 org.junit.platform:junit-platform-engine:1.8.1 org.junit.platform:junit-platform-launcher:1.8.1 -org.jvnet.mimepull:mimepull:1.10.0 org.kohsuke:libpam4j:1.11 org.lz4:lz4-java:1.8.0 org.opentest4j:opentest4j:1.3.0 -org.osgi:org.osgi.compendium:4.1.0 -org.osgi:org.osgi.core:4.1.0 org.ow2.asm:asm:9.5 org.ow2.asm:asm-analysis:9.5 org.ow2.asm:asm-commons:9.5 @@ -214,20 +198,22 @@ org.tukaani:xz:1.9 org.xerial.snappy:snappy-java:1.1.10.3 software.amazon.ion:ion-java:1.0.2 xerces:xercesImpl:2.12.2 -xml-apis:xml-apis:1.4.01 #[test-only] # Contains dependencies that are used exclusively in 'test' scope -com.google.inject:guice:4.2.3 com.google.jimfs:jimfs:1.3.0 net.bytebuddy:byte-buddy:1.14.7 net.bytebuddy:byte-buddy-agent:1.14.7 org.apache.curator:curator-test:5.5.0 org.assertj:assertj-core:3.24.2 -org.junit.jupiter:junit-jupiter:5.8.1 -org.junit.jupiter:junit-jupiter-params:5.8.1 +org.junit.jupiter:junit-jupiter:5.10.0 +org.junit.jupiter:junit-jupiter-engine:5.10.0 +org.junit.jupiter:junit-jupiter-params:5.10.0 +org.junit.platform:junit-platform-commons:1.10.0 +org.junit.platform:junit-platform-engine:1.10.0 +org.junit.vintage:junit-vintage-engine:5.10.0 org.junit.vintage:junit-vintage-engine:5.8.1 org.mockito:mockito-core:5.5.0 org.mockito:mockito-junit-jupiter:5.5.0 org.objenesis:objenesis:3.3 -org.wiremock:wiremock-standalone:3.0.1 +org.wiremock:wiremock-standalone:3.0.2 diff --git a/vespa-maven-plugin/pom.xml b/vespa-maven-plugin/pom.xml index d4f2e8649af..02bf660f6fc 100644 --- a/vespa-maven-plugin/pom.xml +++ b/vespa-maven-plugin/pom.xml @@ -93,6 +93,7 @@ <dependency> <groupId>org.junit.jupiter</groupId> <artifactId>junit-jupiter-engine</artifactId> + <version>${junit.vespa.tenant.version}</version> <scope>test</scope> </dependency> </dependencies> diff --git a/vespa-osgi-testrunner/pom.xml b/vespa-osgi-testrunner/pom.xml index 09ee87b6294..6bc974408a0 100644 --- a/vespa-osgi-testrunner/pom.xml +++ b/vespa-osgi-testrunner/pom.xml @@ -33,6 +33,12 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> + <dependency> + <groupId>org.junit.jupiter</groupId> + <artifactId>junit-jupiter-api</artifactId> + <version>${junit.vespa.tenant.version}</version> + <scope>provided</scope> + </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> @@ -43,7 +49,7 @@ <dependency> <groupId>org.junit.jupiter</groupId> <artifactId>junit-jupiter-engine</artifactId> - <version>5.8.1</version> + <version>${junit.vespa.tenant.version}</version> <exclusions> <exclusion> <groupId>org.junit.jupiter</groupId> @@ -66,7 +72,7 @@ <dependency> <groupId>org.junit.platform</groupId> <artifactId>junit-platform-launcher</artifactId> - <version>1.8.1</version> + <version>${junit.platform.vespa.tenant.version}</version> <exclusions> <exclusion> <groupId>org.junit.jupiter</groupId> diff --git a/vespalib/src/vespa/vespalib/fuzzy/sparse_state.h b/vespalib/src/vespa/vespalib/fuzzy/sparse_state.h index 54398c0b0d9..d20cfc07a9a 100644 --- a/vespalib/src/vespa/vespalib/fuzzy/sparse_state.h +++ b/vespalib/src/vespa/vespalib/fuzzy/sparse_state.h @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/util/sanitizers.h> +#include <vespa/config.h> #include <algorithm> #include <array> #include <cassert> diff --git a/vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp b/vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp index 7538e2acb50..f00ad523602 100644 --- a/vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp +++ b/vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp @@ -151,6 +151,16 @@ SequencedTaskExecutor::getStats() return accumulatedStats; } +std::vector<ExecutorStats> +SequencedTaskExecutor::get_raw_stats() +{ + std::vector<ExecutorStats> result; + for (auto& executor : _executors) { + result.push_back(executor->getStats()); + } + return result; +} + ISequencedTaskExecutor::ExecutorId SequencedTaskExecutor::getExecutorId(uint64_t componentId) const { auto id = getExecutorIdPerfect(componentId); diff --git a/vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h b/vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h index a4b1b82aacf..7e38e4e9512 100644 --- a/vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h +++ b/vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h @@ -30,6 +30,14 @@ public: ExecutorStats getStats() override; void wakeup() override; + /** + * Returns the ExecutorStats of each underlying executor. + * + * Calling this function resets the statistics of each underlying executor, + * similar to calling getStats(). + */ + std::vector<ExecutorStats> get_raw_stats(); + static std::unique_ptr<ISequencedTaskExecutor> create(Runnable::init_fun_t func, uint32_t threads); static std::unique_ptr<ISequencedTaskExecutor> |