diff options
| author | Morten Tokle <mortent@yahooinc.com> | 2023-05-05 12:40:43 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-05-05 12:40:43 +0200 |
| commit | 4ee7d2ab29c502e2145fe9eba9747f9720341be4 (patch) | |
| tree | 69ed743a50b6142a7120f31657c7a35f315be687 | |
| parent | 61563ece462c8dc0a6a38f275e7af7907172cddd (diff) | |
| parent | fba3ea069bad515c7fb8ffc9da26aa9e437ca3b0 (diff) | |
Merge branch 'master' into mortent/athenz-1.11.28
147 files changed, 1736 insertions, 933 deletions
diff --git a/client/go/go.mod b/client/go/go.mod index 94f69c8286a..c70ee5b75c8 100644 --- a/client/go/go.mod +++ b/client/go/go.mod @@ -3,32 +3,33 @@ module github.com/vespa-engine/vespa/client/go go 1.18 require ( - github.com/briandowns/spinner v1.16.0 - github.com/fatih/color v1.10.0 - github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 - github.com/mattn/go-colorable v0.1.8 - github.com/mattn/go-isatty v0.0.13 - github.com/pkg/browser v0.0.0-20210706143420-7d21f8c997e2 - github.com/spf13/cobra v1.4.0 + github.com/alessio/shellescape v1.4.1 + github.com/briandowns/spinner v1.23.0 + github.com/fatih/color v1.15.0 + github.com/goccy/go-json v0.10.2 + github.com/klauspost/compress v1.16.5 + github.com/mattn/go-colorable v0.1.13 + github.com/mattn/go-isatty v0.0.18 + github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 + github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.7.0 - github.com/zalando/go-keyring v0.1.1 + github.com/stretchr/testify v1.8.2 + github.com/zalando/go-keyring v0.2.2 golang.org/x/net v0.9.0 golang.org/x/sys v0.7.0 - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b + gopkg.in/yaml.v3 v3.0.1 ) require ( - github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect - github.com/danieljoos/wincred v1.1.0 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/danieljoos/wincred v1.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/godbus/dbus/v5 v5.0.4 // indirect - github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/godbus/dbus/v5 v5.1.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/stretchr/objx v0.1.1 // indirect + golang.org/x/term v0.7.0 // indirect golang.org/x/text v0.9.0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/client/go/go.sum b/client/go/go.sum index ac662c9fd43..9b79c215864 100644 --- a/client/go/go.sum +++ b/client/go/go.sum @@ -1,68 +1,70 @@ -github.com/briandowns/spinner v1.16.0 h1:DFmp6hEaIx2QXXuqSJmtfSBSAjRmpGiKG6ip2Wm/yOs= -github.com/briandowns/spinner v1.16.0/go.mod h1:QOuQk7x+EaDASo80FEXwlwiA+j/PPIcX3FScO+3/ZPQ= -github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU= -github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/danieljoos/wincred v1.1.0 h1:3RNcEpBg4IhIChZdFRSdlQt1QjCp1sMAPIrOnm7Yf8g= -github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7hqDjlFjiygg= +github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= +github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= +github.com/briandowns/spinner v1.23.0 h1:alDF2guRWqa/FOZZYWjlMIx2L6H0wyewPxo/CH4Pt2A= +github.com/briandowns/spinner v1.23.0/go.mod h1:rPG4gmXeN3wQV/TsAY4w8lPdIM6RX3yqeBQJSrbXjuE= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/danieljoos/wincred v1.1.2 h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0= +github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= -github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= -github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= -github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA= -github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/pkg/browser v0.0.0-20210706143420-7d21f8c997e2 h1:acNfDZXmm28D2Yg/c3ALnZStzNaZMSagpbr96vY6Zjc= -github.com/pkg/browser v0.0.0-20210706143420-7d21f8c997e2/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98= +github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= -github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/zalando/go-keyring v0.1.1 h1:w2V9lcx/Uj4l+dzAf1m9s+DJ1O8ROkEHnynonHjTcYE= -github.com/zalando/go-keyring v0.1.1/go.mod h1:OIC+OZ28XbmwFxU/Rp9V7eKzZjamBJwRzC8UFJH9+L8= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/zalando/go-keyring v0.2.2 h1:f0xmpYiSrHtSNAVgwip93Cg8tuF45HJM6rHq/A5RI/4= +github.com/zalando/go-keyring v0.2.2/go.mod h1:sI3evg9Wvpw3+n4SqplGSJUMwtDeROfD4nsFz4z9PG0= golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/client/go/internal/cli/auth/zts/zts.go b/client/go/internal/cli/auth/zts/zts.go index 2c66ff13e8b..0f73ea5912d 100644 --- a/client/go/internal/cli/auth/zts/zts.go +++ b/client/go/internal/cli/auth/zts/zts.go @@ -3,23 +3,39 @@ package zts import ( "encoding/json" "fmt" + "io" "net/http" "net/url" "strings" + "sync" "time" "github.com/vespa-engine/vespa/client/go/internal/util" ) -const DefaultURL = "https://zts.athenz.ouroath.com:4443" +const ( + DefaultURL = "https://zts.athenz.ouroath.com:4443" + expirySlack = 5 * time.Minute +) // Client is a client for Athenz ZTS, an authentication token service. type Client struct { client util.HTTPClient tokenURL *url.URL domain string + now func() time.Time + token Token + mu sync.Mutex +} + +// Token is an access token retrieved from ZTS. +type Token struct { + Value string + ExpiresAt time.Time } +func (t *Token) isExpired(now time.Time) bool { return t.ExpiresAt.Sub(now) < expirySlack } + // NewClient creates a new client for an Athenz ZTS service located at serviceURL. func NewClient(client util.HTTPClient, domain, serviceURL string) (*Client, error) { tokenURL, err := url.Parse(serviceURL) @@ -27,44 +43,60 @@ func NewClient(client util.HTTPClient, domain, serviceURL string) (*Client, erro return nil, err } tokenURL.Path = "/zts/v1/oauth2/token" - return &Client{tokenURL: tokenURL, client: client, domain: domain}, nil + return &Client{tokenURL: tokenURL, client: client, domain: domain, now: time.Now}, nil } func (c *Client) Authenticate(request *http.Request) error { - accessToken, err := c.AccessToken() - if err != nil { - return err + now := c.now() + if c.token.isExpired(now) { + c.mu.Lock() + if c.token.isExpired(now) { + accessToken, err := c.AccessToken() + if err != nil { + c.mu.Unlock() + return err + } + c.token = accessToken + } + c.mu.Unlock() } if request.Header == nil { request.Header = make(http.Header) } - request.Header.Add("Authorization", "Bearer "+accessToken) + request.Header.Add("Authorization", "Bearer "+c.token.Value) return nil } // AccessToken returns an access token within the domain configured in client c. -func (c *Client) AccessToken() (string, error) { - // TODO(mpolden): This should cache and re-use tokens until expiry +func (c *Client) AccessToken() (Token, error) { data := fmt.Sprintf("grant_type=client_credentials&scope=%s:domain", c.domain) req, err := http.NewRequest("POST", c.tokenURL.String(), strings.NewReader(data)) if err != nil { - return "", err + return Token{}, err } + now := c.now() response, err := c.client.Do(req, 10*time.Second) if err != nil { - return "", err + return Token{}, err } defer response.Body.Close() if response.StatusCode != http.StatusOK { - return "", fmt.Errorf("zts: got status %d from %s", response.StatusCode, c.tokenURL.String()) + return Token{}, fmt.Errorf("zts: got status %d from %s", response.StatusCode, c.tokenURL.String()) } var ztsResponse struct { AccessToken string `json:"access_token"` + ExpirySecs int `json:"expires_in"` + } + b, err := io.ReadAll(response.Body) + if err != nil { + return Token{}, err } - dec := json.NewDecoder(response.Body) - if err := dec.Decode(&ztsResponse); err != nil { - return "", err + if err := json.Unmarshal(b, &ztsResponse); err != nil { + return Token{}, err } - return ztsResponse.AccessToken, nil + return Token{ + Value: ztsResponse.AccessToken, + ExpiresAt: now.Add(time.Duration(ztsResponse.ExpirySecs) * time.Second), + }, nil } diff --git a/client/go/internal/cli/auth/zts/zts_test.go b/client/go/internal/cli/auth/zts/zts_test.go index 1c75a94ee03..15c60ed46d7 100644 --- a/client/go/internal/cli/auth/zts/zts_test.go +++ b/client/go/internal/cli/auth/zts/zts_test.go @@ -2,28 +2,57 @@ package zts import ( "testing" + "time" "github.com/vespa-engine/vespa/client/go/internal/mock" ) +type manualClock struct{ t time.Time } + +func (c *manualClock) now() time.Time { return c.t } +func (c *manualClock) advance(d time.Duration) { c.t = c.t.Add(d) } + func TestAccessToken(t *testing.T) { httpClient := mock.HTTPClient{} client, err := NewClient(&httpClient, "vespa.vespa", "http://example.com") if err != nil { t.Fatal(err) } + clock := &manualClock{t: time.Now()} + client.now = clock.now httpClient.NextResponseString(400, `{"message": "bad request"}`) _, err = client.AccessToken() if err == nil { t.Fatal("want error for non-ok response status") } - httpClient.NextResponseString(200, `{"access_token": "foo bar"}`) + httpClient.NextResponseString(200, `{"access_token": "foo", "expires_in": 3600}`) token, err := client.AccessToken() if err != nil { t.Fatal(err) } - want := "foo bar" - if token != want { - t.Errorf("got %q, want %q", token, want) + + // Token is cached + expiresAt := clock.now().Add(time.Hour) + assertToken(t, Token{Value: "foo", ExpiresAt: expiresAt}, token) + clock.advance(54 * time.Minute) + assertToken(t, Token{Value: "foo", ExpiresAt: expiresAt}, token) + + // Token is renewed when nearing expiry + clock.advance(time.Minute + time.Second) + httpClient.NextResponseString(200, `{"access_token": "bar", "expires_in": 1800}`) + token, err = client.AccessToken() + if err != nil { + t.Fatal(err) + } + expiresAt = clock.now().Add(30 * time.Minute) + assertToken(t, Token{Value: "bar", ExpiresAt: expiresAt}, token) +} + +func assertToken(t *testing.T, want, got Token) { + if want.Value != got.Value { + t.Errorf("got Value=%q, want %q", got.Value, want.Value) + } + if want.ExpiresAt != got.ExpiresAt { + t.Errorf("got ExpiresAt=%s, want %s", got.ExpiresAt, want.ExpiresAt) } } diff --git a/client/go/internal/cli/cmd/feed.go b/client/go/internal/cli/cmd/feed.go index 5b168ef79a2..2ea3ee1c4ed 100644 --- a/client/go/internal/cli/cmd/feed.go +++ b/client/go/internal/cli/cmd/feed.go @@ -93,15 +93,30 @@ $ cat docs.jsonl | vespa feed -`, return cmd } -func createServiceClients(service *vespa.Service, n int) []util.HTTPClient { - clients := make([]util.HTTPClient, 0, n) +func createServices(n int, timeout time.Duration, cli *CLI) ([]util.HTTPClient, string, error) { + if n < 1 { + return nil, "", fmt.Errorf("need at least one client") + } + target, err := cli.target(targetOptions{}) + if err != nil { + return nil, "", err + } + services := make([]util.HTTPClient, 0, n) + baseURL := "" for i := 0; i < n; i++ { - client := service.Client().Clone() + service, err := cli.service(target, vespa.DocumentService, 0, cli.config.cluster()) + if err != nil { + return nil, "", err + } + baseURL = service.BaseURL + // Create a separate HTTP client for each service + client := cli.httpClientFactory(timeout) // Feeding should always use HTTP/2 util.ForceHTTP2(client, service.TLSOptions.KeyPair, service.TLSOptions.CACertificate, service.TLSOptions.TrustAll) - clients = append(clients, client) + service.SetClient(client) + services = append(services, service) } - return clients + return services, baseURL, nil } func summaryTicker(secs int, cli *CLI, start time.Time, statsFunc func() document.Stats) *time.Ticker { @@ -130,21 +145,21 @@ func (opts feedOptions) compressionMode() (document.Compression, error) { } func feed(files []string, options feedOptions, cli *CLI) error { - service, err := documentService(cli) + timeout := time.Duration(options.timeoutSecs) * time.Second + clients, baseURL, err := createServices(options.connections, timeout, cli) if err != nil { return err } - clients := createServiceClients(service, options.connections) compression, err := options.compressionMode() if err != nil { return err } client, err := document.NewClient(document.ClientOptions{ Compression: compression, - Timeout: time.Duration(options.timeoutSecs) * time.Second, + Timeout: timeout, Route: options.route, TraceLevel: options.traceLevel, - BaseURL: service.BaseURL, + BaseURL: baseURL, NowFunc: cli.now, }, clients) if err != nil { diff --git a/client/go/internal/cli/cmd/feed_test.go b/client/go/internal/cli/cmd/feed_test.go index 467d55a0a6e..097d4ae5fa3 100644 --- a/client/go/internal/cli/cmd/feed_test.go +++ b/client/go/internal/cli/cmd/feed_test.go @@ -24,10 +24,9 @@ func (c *manualClock) now() time.Time { } func TestFeed(t *testing.T) { - httpClient := &mock.HTTPClient{} clock := &manualClock{tick: time.Second} cli, stdout, stderr := newTestCLI(t) - cli.httpClient = httpClient + httpClient := cli.httpClient.(*mock.HTTPClient) cli.now = clock.now td := t.TempDir() diff --git a/client/go/internal/cli/cmd/query_test.go b/client/go/internal/cli/cmd/query_test.go index 94b5a485b9d..1caf6d33e70 100644 --- a/client/go/internal/cli/cmd/query_test.go +++ b/client/go/internal/cli/cmd/query_test.go @@ -27,7 +27,7 @@ func TestQueryVerbose(t *testing.T) { cli.httpClient = client assert.Nil(t, cli.Run("query", "-v", "select from sources * where title contains 'foo'")) - assert.Equal(t, "curl http://127.0.0.1:8080/search/\\?timeout=10s\\&yql=select+from+sources+%2A+where+title+contains+%27foo%27\n", stderr.String()) + assert.Equal(t, "curl 'http://127.0.0.1:8080/search/?timeout=10s&yql=select+from+sources+%2A+where+title+contains+%27foo%27'\n", stderr.String()) assert.Equal(t, "{\n \"query\": \"result\"\n}\n", stdout.String()) } diff --git a/client/go/internal/cli/cmd/root.go b/client/go/internal/cli/cmd/root.go index 1b37ff00269..c4012024426 100644 --- a/client/go/internal/cli/cmd/root.go +++ b/client/go/internal/cli/cmd/root.go @@ -47,13 +47,14 @@ type CLI struct { config *Config version version.Version - httpClient util.HTTPClient - auth0Factory auth0Factory - ztsFactory ztsFactory - exec executor - isTerminal func() bool - spinner func(w io.Writer, message string, fn func() error) error - now func() time.Time + httpClient util.HTTPClient + httpClientFactory func(timeout time.Duration) util.HTTPClient + auth0Factory auth0Factory + ztsFactory ztsFactory + exec executor + isTerminal func() bool + spinner func(w io.Writer, message string, fn func() error) error + now func() time.Time } // ErrCLI is an error returned to the user. It wraps an exit status, a regular error and optional hints for resolving @@ -122,17 +123,19 @@ For detailed description of flags and configuration, see 'vespa help config'. if err != nil { return nil, err } + httpClientFactory := util.CreateClient cli := CLI{ Environment: env, Stdin: os.Stdin, Stdout: stdout, Stderr: stderr, - version: version, - cmd: cmd, - httpClient: util.CreateClient(time.Second * 10), - exec: &execSubprocess{}, - now: time.Now, + version: version, + cmd: cmd, + httpClient: httpClientFactory(time.Second * 10), + httpClientFactory: httpClientFactory, + exec: &execSubprocess{}, + now: time.Now, auth0Factory: func(httpClient util.HTTPClient, options auth0.Options) (vespa.Authenticator, error) { return auth0.NewClient(httpClient, options) }, diff --git a/client/go/internal/cli/cmd/testutil_test.go b/client/go/internal/cli/cmd/testutil_test.go index 492e40d8855..61d6c15c5a0 100644 --- a/client/go/internal/cli/cmd/testutil_test.go +++ b/client/go/internal/cli/cmd/testutil_test.go @@ -6,6 +6,7 @@ import ( "net/http" "path/filepath" "testing" + "time" "github.com/vespa-engine/vespa/client/go/internal/cli/auth/auth0" "github.com/vespa-engine/vespa/client/go/internal/mock" @@ -28,6 +29,7 @@ func newTestCLI(t *testing.T, envVars ...string) (*CLI, *bytes.Buffer, *bytes.Bu t.Fatal(err) } httpClient := &mock.HTTPClient{} + cli.httpClientFactory = func(timeout time.Duration) util.HTTPClient { return httpClient } cli.httpClient = httpClient cli.exec = &mock.Exec{} cli.auth0Factory = func(httpClient util.HTTPClient, options auth0.Options) (vespa.Authenticator, error) { diff --git a/client/go/internal/curl/curl.go b/client/go/internal/curl/curl.go index b70e0f824a3..daa60e6ff14 100644 --- a/client/go/internal/curl/curl.go +++ b/client/go/internal/curl/curl.go @@ -7,7 +7,7 @@ import ( "os/exec" "runtime" - "github.com/kballard/go-shellquote" + "github.com/alessio/shellescape" "github.com/vespa-engine/vespa/client/go/internal/util" ) @@ -77,7 +77,7 @@ func (c *Command) Args() []string { func (c *Command) String() string { args := []string{c.Path} args = append(args, c.Args()...) - return shellquote.Join(args...) + return shellescape.QuoteCommand(args) } func (c *Command) Header(key, value string) { diff --git a/client/go/internal/curl/curl_test.go b/client/go/internal/curl/curl_test.go index 448e1e5199f..c8bcfb7f56b 100644 --- a/client/go/internal/curl/curl_test.go +++ b/client/go/internal/curl/curl_test.go @@ -5,13 +5,12 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestPost(t *testing.T) { c, err := Post("https://example.com") - if err != nil { - t.Fatal(err) - } + require.Nil(t, err) c.PrivateKey = "key.pem" c.Certificate = "cert.pem" c.WithBodyFile("file.json") @@ -22,22 +21,18 @@ func TestPost(t *testing.T) { func TestGet(t *testing.T) { c, err := Get("https://example.com") - if err != nil { - t.Fatal(err) - } + require.Nil(t, err) c.PrivateKey = "key.pem" c.Certificate = "cert.pem" c.Param("yql", "select * from sources * where title contains 'foo';") c.Param("hits", "5") - assert.Equal(t, `curl --key key.pem --cert cert.pem https://example.com\?hits=5\&yql=select+%2A+from+sources+%2A+where+title+contains+%27foo%27%3B`, c.String()) + assert.Equal(t, `curl --key key.pem --cert cert.pem 'https://example.com?hits=5&yql=select+%2A+from+sources+%2A+where+title+contains+%27foo%27%3B'`, c.String()) } func TestRawArgs(t *testing.T) { c, err := RawArgs("https://example.com/search", "-v", "-m", "10", "-H", "foo: bar") - if err != nil { - t.Fatal(err) - } + assert.Nil(t, err) c.PrivateKey = "key.pem" c.Certificate = "cert.pem" diff --git a/client/go/internal/util/http.go b/client/go/internal/util/http.go index 8a67b24dffb..546098a204d 100644 --- a/client/go/internal/util/http.go +++ b/client/go/internal/util/http.go @@ -16,7 +16,6 @@ import ( type HTTPClient interface { Do(request *http.Request, timeout time.Duration) (response *http.Response, error error) - Clone() HTTPClient } type defaultHTTPClient struct { @@ -34,8 +33,6 @@ func (c *defaultHTTPClient) Do(request *http.Request, timeout time.Duration) (re return c.client.Do(request) } -func (c *defaultHTTPClient) Clone() HTTPClient { return CreateClient(c.client.Timeout) } - func ConfigureTLS(client HTTPClient, certificates []tls.Certificate, caCertificate []byte, trustAll bool) { c, ok := client.(*defaultHTTPClient) if !ok { @@ -81,8 +78,9 @@ func ForceHTTP2(client HTTPClient, certificates []tls.Certificate, caCertificate // https://github.com/golang/go/issues/16582 // https://github.com/golang/go/issues/22091 c.client.Transport = &http2.Transport{ - AllowHTTP: true, - DialTLSContext: dialFunc, + DisableCompression: true, + AllowHTTP: true, + DialTLSContext: dialFunc, } ConfigureTLS(client, certificates, caCertificate, trustAll) } diff --git a/client/go/internal/vespa/document/document.go b/client/go/internal/vespa/document/document.go index 214d1dc4797..8bc897d49b3 100644 --- a/client/go/internal/vespa/document/document.go +++ b/client/go/internal/vespa/document/document.go @@ -2,11 +2,12 @@ package document import ( "bufio" - "encoding/json" "fmt" "io" "strconv" "strings" + + "github.com/goccy/go-json" ) var asciiSpace = [256]uint8{'\t': 1, '\n': 1, '\v': 1, '\f': 1, '\r': 1, ' ': 1} @@ -158,22 +159,25 @@ func (d *Decoder) guessMode() error { if err := d.buf.UnreadByte(); err != nil { return err } - if d.array { - // prepare for decoding objects inside array - if _, err := d.dec.Token(); err != nil { - return err - } + if err := d.readArrayToken(true); err != nil { + return err } } return nil } -func (d *Decoder) readCloseToken() error { +func (d *Decoder) readArrayToken(open bool) error { if !d.array { return nil } - _, err := d.dec.Token() - return err + t, err := d.dec.Token() + if err != nil { + return err + } + if (open && t == json.Delim('[')) || (!open && t == json.Delim(']')) { + return nil + } + return fmt.Errorf("invalid array token: %q", t) } func (d *Decoder) Decode() (Document, error) { @@ -189,11 +193,10 @@ func (d *Decoder) decode() (Document, error) { return Document{}, err } if !d.dec.More() { - err := io.EOF - if tokenErr := d.readCloseToken(); tokenErr != nil { - err = tokenErr + if err := d.readArrayToken(false); err != nil { + return Document{}, err } - return Document{}, err + return Document{}, io.EOF } doc := jsonDocument{} if err := d.dec.Decode(&doc); err != nil { @@ -203,7 +206,7 @@ func (d *Decoder) decode() (Document, error) { } func NewDecoder(r io.Reader) *Decoder { - buf := bufio.NewReader(r) + buf := bufio.NewReaderSize(r, 1<<26) return &Decoder{ buf: buf, dec: json.NewDecoder(buf), diff --git a/client/go/internal/vespa/document/document_test.go b/client/go/internal/vespa/document/document_test.go index 111b9e37acc..bdf18586753 100644 --- a/client/go/internal/vespa/document/document_test.go +++ b/client/go/internal/vespa/document/document_test.go @@ -1,6 +1,7 @@ package document import ( + "fmt" "io" "reflect" "strings" @@ -173,8 +174,28 @@ func TestDocumentDecoder(t *testing.T) { t.Errorf("unexpected error: %s", err) } _, err = r.Decode() - wantErr := "invalid json at byte offset 60: invalid character '\\n' in string literal" + wantErr := "invalid json at byte offset 122: json: string of object unexpected end of JSON input" if err.Error() != wantErr { t.Errorf("want error %q, got %q", wantErr, err.Error()) } } + +func benchmarkDocumentDecoder(b *testing.B, size int) { + b.Helper() + input := fmt.Sprintf(`{"put": "id:ns:type::doc1", "fields": {"foo": "%s"}}`, strings.Repeat("s", size)) + r := strings.NewReader(input) + dec := NewDecoder(r) + b.ResetTimer() // ignore setup time + + for n := 0; n < b.N; n++ { + _, err := dec.Decode() + if err != nil { + b.Fatal(err) + } + r.Seek(0, 0) + } +} + +func BenchmarkDocumentDecoderSmall(b *testing.B) { benchmarkDocumentDecoder(b, 10) } + +func BenchmarkDocumentDecoderLarge(b *testing.B) { benchmarkDocumentDecoder(b, 10000) } diff --git a/client/go/internal/vespa/document/http.go b/client/go/internal/vespa/document/http.go index d42615d1e71..3581a791dbe 100644 --- a/client/go/internal/vespa/document/http.go +++ b/client/go/internal/vespa/document/http.go @@ -2,7 +2,6 @@ package document import ( "bytes" - "compress/gzip" "encoding/json" "fmt" "io" @@ -15,6 +14,8 @@ import ( "sync/atomic" "time" + "github.com/klauspost/compress/gzip" + "github.com/vespa-engine/vespa/client/go/internal/util" ) diff --git a/client/go/internal/vespa/document/http_test.go b/client/go/internal/vespa/document/http_test.go index 7e858f96020..489460b3ed7 100644 --- a/client/go/internal/vespa/document/http_test.go +++ b/client/go/internal/vespa/document/http_test.go @@ -275,6 +275,7 @@ func TestClientMethodAndURL(t *testing.T) { } func benchmarkClientSend(b *testing.B, compression Compression, document Document) { + b.Helper() httpClient := mock.HTTPClient{} client, _ := NewClient(ClientOptions{ Compression: compression, @@ -293,6 +294,7 @@ func BenchmarkClientSend(b *testing.B) { } func BenchmarkClientSendCompressed(b *testing.B) { - doc := Document{Create: true, Id: mustParseId("id:ns:type::doc1"), Operation: OperationUpdate, Body: []byte(`{"fields":{"foo": "my document"}}`)} + body := fmt.Sprintf(`{"fields":{"foo": "%s"}}`, strings.Repeat("my document", 100)) + doc := Document{Create: true, Id: mustParseId("id:ns:type::doc1"), Operation: OperationUpdate, Body: []byte(body)} benchmarkClientSend(b, CompressionGzip, doc) } diff --git a/client/go/internal/vespa/target.go b/client/go/internal/vespa/target.go index 9f3fd7f5c65..6dd64dd1275 100644 --- a/client/go/internal/vespa/target.go +++ b/client/go/internal/vespa/target.go @@ -110,7 +110,8 @@ func (s *Service) Do(request *http.Request, timeout time.Duration) (*http.Respon return s.httpClient.Do(request, timeout) } -func (s *Service) Client() util.HTTPClient { return s.httpClient } +// SetClient sets the HTTP client that this service should use. +func (s *Service) SetClient(client util.HTTPClient) { s.httpClient = client } // Wait polls the health check of this service until it succeeds or timeout passes. func (s *Service) Wait(timeout time.Duration) (int, error) { diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index 5ae43a21b41..a8b836d5846 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -21,7 +21,8 @@ <!-- MUST BE KEPT IN SYNC WITH parent/pom.xml --> <athenz.version>1.11.28</athenz.version> - <bouncycastle.version>1.72</bouncycastle.version> + + <bouncycastle.version>1.73</bouncycastle.version> <commons-codec.version>1.15</commons-codec.version> <felix.version>7.0.1</felix.version> <httpclient5.version>5.2.1</httpclient5.version> @@ -39,8 +40,8 @@ <aopalliance.version>1.0</aopalliance.version> <guava.version>27.1-jre</guava.version> <guice.version>4.2.3</guice.version> - <jackson2.version>2.14.2</jackson2.version> - <jackson-databind.version>2.14.2</jackson-databind.version> + <jackson2.version>2.15.0</jackson2.version> + <jackson-databind.version>2.15.0</jackson-databind.version> <javax.inject.version>1</javax.inject.version> <javax.servlet-api.version>3.1.0</javax.servlet-api.version> <javax.ws.rs-api.version>2.0.1</javax.ws.rs-api.version> diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java index e61b1e9119a..31b5d607058 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java @@ -7,9 +7,6 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.container.Container; -import com.yahoo.vespa.model.container.component.AccessLogComponent; -import com.yahoo.vespa.model.container.component.AccessLogComponent.AccessLogType; -import java.util.Optional; /** * Container that should be running on same host as the logserver. Sets up a handler for getting logs from logserver. @@ -19,11 +16,8 @@ public class LogserverContainer extends Container { public LogserverContainer(TreeConfigProducer<?> parent, DeployState deployState) { super(parent, "" + 0, 0, deployState); - if (deployState.isHosted() && deployState.getProperties().applicationId().instance().isTester()) useDynamicPorts(); - LogserverContainerCluster cluster = (LogserverContainerCluster) parent; - addComponent(new AccessLogComponent(cluster, AccessLogType.jsonAccessLog, - deployState.featureFlags().logFileCompressionAlgorithm("zstd"), - Optional.of(cluster.getName()), true)); + if (deployState.isHosted() && deployState.getProperties().applicationId().instance().isTester()) + useDynamicPorts(); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java index fef7d534c30..33915d48bd4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainerCluster.java @@ -6,6 +6,7 @@ import com.yahoo.config.model.producer.TreeConfigProducer; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.container.ContainerCluster; +import com.yahoo.vespa.model.container.component.AccessLogComponent; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SystemBindingPattern; @@ -22,6 +23,12 @@ public class LogserverContainerCluster extends ContainerCluster<LogserverContain addDefaultHandlersWithVip(); addLogHandler(); setJvmGCOptions(deployState.getProperties().jvmGCOptions(Optional.of(ClusterSpec.Type.admin))); + if (isHostedVespa()) + addComponent(new AccessLogComponent(this, + AccessLogComponent.AccessLogType.jsonAccessLog, + deployState.featureFlags().logFileCompressionAlgorithm("zstd"), + Optional.of(getName()), + isHostedVespa())); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index 660f411a2f1..a08058326d3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -16,15 +16,13 @@ import com.yahoo.vespa.config.content.FleetcontrollerConfig; import com.yahoo.vespa.config.content.reindexing.ReindexingConfig; import com.yahoo.vespa.model.application.validation.RestartConfigs; import com.yahoo.vespa.model.container.Container; -import com.yahoo.vespa.model.container.component.AccessLogComponent; +import com.yahoo.vespa.model.container.PlatformBundles; import com.yahoo.vespa.model.container.component.Component; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SimpleComponent; import com.yahoo.vespa.model.container.component.SystemBindingPattern; import com.yahoo.vespa.model.container.xml.ContainerModelBuilder; -import com.yahoo.vespa.model.container.PlatformBundles; -import java.util.Optional; import java.util.Set; import java.util.TreeSet; @@ -64,11 +62,6 @@ public class ClusterControllerContainer extends Container implements "com.yahoo.vespa.clustercontroller.apps.clustercontroller.StateRestApiV2Handler", "/cluster/v2/*", CLUSTERCONTROLLER_BUNDLE); - addComponent(new AccessLogComponent(containerCluster().orElse(null), - AccessLogComponent.AccessLogType.jsonAccessLog, - deployState.featureFlags().logFileCompressionAlgorithm("zstd"), - Optional.of("controller"), - deployState.isHosted())); // TODO: Why are bundles added here instead of in the cluster? addFileBundle("clustercontroller-apps"); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java index c65abfb0189..7b465e3eb48 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainerCluster.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.PlatformBundles; +import com.yahoo.vespa.model.container.component.AccessLogComponent; import java.nio.file.Path; import java.util.Collections; @@ -32,6 +33,12 @@ public class ClusterControllerContainerCluster extends ContainerCluster<ClusterC addDefaultHandlersWithVip(); this.reindexingContext = createReindexingContext(deployState); setJvmGCOptions(deployState.getProperties().jvmGCOptions(Optional.of(ClusterSpec.Type.admin))); + if (isHostedVespa()) + addComponent(new AccessLogComponent(this, + AccessLogComponent.AccessLogType.jsonAccessLog, + deployState.featureFlags().logFileCompressionAlgorithm("zstd"), + Optional.of("controller"), + isHostedVespa())); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index d69ddd1c5fd..8ec9b4e246b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -1,5 +1,4 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - package com.yahoo.vespa.model.admin.metricsproxy; import ai.vespa.metricsproxy.http.metrics.MetricsV2Handler; @@ -22,7 +21,6 @@ import com.yahoo.search.config.QrStartConfig; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.PortAllocBridge; import com.yahoo.vespa.model.container.Container; -import com.yahoo.vespa.model.container.component.AccessLogComponent; import java.time.Duration; import java.util.LinkedHashMap; @@ -65,10 +63,6 @@ public class MetricsProxyContainer extends Container implements setProp("clustertype", "admin"); setProp("index", String.valueOf(index)); addNodeSpecificComponents(); - addComponent(new AccessLogComponent(containerCluster().orElse(null), AccessLogComponent.AccessLogType.jsonAccessLog, - "zstd", - Optional.of("metrics-proxy"), - deployState.isHosted())); } private void addNodeSpecificComponents() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java index ada647b535d..78374cd4030 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerCluster.java @@ -32,6 +32,7 @@ import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.container.ContainerCluster; import com.yahoo.vespa.model.container.PlatformBundles; +import com.yahoo.vespa.model.container.component.AccessLogComponent; import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.container.component.SystemBindingPattern; @@ -99,6 +100,12 @@ public class MetricsProxyContainerCluster extends ContainerCluster<MetricsProxyC addPlatformBundle(METRICS_PROXY_BUNDLE_FILE); addClusterComponents(); + if (isHostedVespa()) + addComponent(new AccessLogComponent(this, + AccessLogComponent.AccessLogType.jsonAccessLog, + "zstd", + Optional.of("metrics-proxy"), + isHostedVespa())); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java index 0be3c825614..34c565871db 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java @@ -29,6 +29,7 @@ public class ContainerModelEvaluation implements public final static String EVALUATION_BUNDLE_NAME = "model-evaluation"; public final static String INTEGRATION_BUNDLE_NAME = "model-integration"; public final static String ONNXRUNTIME_BUNDLE_NAME = "container-onnxruntime.jar"; + public final static String ONNX_RUNTIME_CLASS = "ai.vespa.modelintegration.evaluator.OnnxRuntime"; private final static String EVALUATOR_NAME = ModelsEvaluator.class.getName(); private final static String REST_HANDLER_NAME = "ai.vespa.models.handler.ModelsEvaluationHandler"; @@ -44,7 +45,6 @@ public class ContainerModelEvaluation implements public ContainerModelEvaluation(ApplicationContainerCluster cluster, RankProfileList rankProfileList) { this.rankProfileList = Objects.requireNonNull(rankProfileList, "rankProfileList cannot be null"); cluster.addSimpleComponent(EVALUATOR_NAME, null, EVALUATION_BUNDLE_NAME); - cluster.addSimpleComponent("ai.vespa.modelintegration.evaluator.OnnxRuntime", null, INTEGRATION_BUNDLE_NAME); cluster.addComponent(ContainerModelEvaluation.getHandler()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java index 1aa4333a18c..49361b887c4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ConfigServerContainerModelBuilder.java @@ -62,7 +62,7 @@ public class ConfigServerContainerModelBuilder extends ContainerModelBuilder { } @Override - protected void addModelEvaluationBundles(ApplicationContainerCluster cluster) { + protected void addModelEvaluationRuntime(ApplicationContainerCluster cluster) { // Model evaluation bundles are pre-installed in the standalone container. } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 0270e91b238..ef5b29ccfb8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -221,8 +221,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { addAccessLogs(deployState, cluster, spec); addNodes(cluster, spec, context); + addModelEvaluationRuntime(cluster); addModelEvaluation(spec, cluster, context); // NOTE: Must be done after addNodes - addModelEvaluationBundles(cluster); addServerProviders(deployState, spec, cluster); @@ -705,13 +705,16 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return (child != null) ? Integer.parseInt(child.getTextContent()) : defaultValue; } - protected void addModelEvaluationBundles(ApplicationContainerCluster cluster) { + protected void addModelEvaluationRuntime(ApplicationContainerCluster cluster) { /* These bundles are added to all application container clusters, even if they haven't * declared 'model-evaluation' in services.xml, because there are many public API packages * in the model-evaluation bundle that could be used by customer code. */ cluster.addPlatformBundle(ContainerModelEvaluation.MODEL_EVALUATION_BUNDLE_FILE); cluster.addPlatformBundle(ContainerModelEvaluation.MODEL_INTEGRATION_BUNDLE_FILE); cluster.addPlatformBundle(ContainerModelEvaluation.ONNXRUNTIME_BUNDLE_FILE); + /* The ONNX runtime is always available for injection to any component */ + cluster.addSimpleComponent( + ContainerModelEvaluation.ONNX_RUNTIME_CLASS, null, ContainerModelEvaluation.INTEGRATION_BUNDLE_NAME); } private void addProcessing(DeployState deployState, Element spec, ApplicationContainerCluster cluster, ConfigModelContext context) { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/EndpointsChecker.java b/config-provisioning/src/main/java/com/yahoo/config/provision/EndpointsChecker.java index d9ced0177e5..c33a575e9b7 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/EndpointsChecker.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/EndpointsChecker.java @@ -19,29 +19,37 @@ import java.util.Optional; */ public interface EndpointsChecker { - record Endpoint(ClusterSpec.Id clusterName, + record Endpoint(ApplicationId applicationId, + ClusterSpec.Id clusterName, HttpURL url, Optional<InetAddress> ipAddress, Optional<DomainName> canonicalName, - boolean isPublic) { } + boolean isPublic, + CloudAccount account) { } /** Status sorted by increasing readiness. */ enum Status { endpointsUnavailable, containersUnhealthy, available } - record Availability(Status status, String message) { } + record Availability(Status status, String message) { + public static final Availability ready = new Availability(Status.available, "Endpoints are ready."); + } interface HostNameResolver { Optional<InetAddress> resolve(DomainName hostName); } interface CNameResolver { Optional<DomainName> resolve(DomainName hostName); } - interface ContainerHealthChecker { boolean healthy(Endpoint endpoint); } + interface HealthChecker { Availability healthy(Endpoint endpoint); } + + interface HealthCheckerProvider { + default HealthChecker getHealthChecker() { return __ -> Availability.ready; } + } - static EndpointsChecker of(ContainerHealthChecker containerHealthChecker) { - return zoneEndpoints -> endpointsAvailable(zoneEndpoints, EndpointsChecker::resolveHostName, EndpointsChecker::resolveCname, containerHealthChecker); + static EndpointsChecker of(HealthChecker healthChecker) { + return zoneEndpoints -> endpointsAvailable(zoneEndpoints, EndpointsChecker::resolveHostName, EndpointsChecker::resolveCname, healthChecker); } - static EndpointsChecker mock(HostNameResolver hostNameResolver, CNameResolver cNameResolver, ContainerHealthChecker containerHealthChecker) { - return zoneEndpoints -> endpointsAvailable(zoneEndpoints, hostNameResolver, cNameResolver, containerHealthChecker); + static EndpointsChecker mock(HostNameResolver hostNameResolver, CNameResolver cNameResolver, HealthChecker healthChecker) { + return zoneEndpoints -> endpointsAvailable(zoneEndpoints, hostNameResolver, cNameResolver, healthChecker); } Availability endpointsAvailable(List<Endpoint> zoneEndpoints); @@ -49,7 +57,7 @@ public interface EndpointsChecker { private static Availability endpointsAvailable(List<Endpoint> zoneEndpoints, HostNameResolver hostNameResolver, CNameResolver cNameResolver, - ContainerHealthChecker containerHealthChecker) { + HealthChecker healthChecker) { if (zoneEndpoints.isEmpty()) return new Availability(Status.endpointsUnavailable, "Endpoints not yet ready."); @@ -89,11 +97,13 @@ public interface EndpointsChecker { } } - for (Endpoint endpoint : zoneEndpoints) - if ( ! containerHealthChecker.healthy(endpoint)) - return new Availability(Status.containersUnhealthy, "Failed to get enough healthy responses from " + endpoint.url()); - - return new Availability(Status.available, "Endpoints are ready"); + Availability availability = Availability.ready; + for (Endpoint endpoint : zoneEndpoints) { + Availability candidate = healthChecker.healthy(endpoint); + if (candidate.status.compareTo(availability.status) < 0) + availability = candidate; + } + return availability; } /** Returns the IP address of the given host name, if any. */ diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java index 8004d4dc951..9ca10091129 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java @@ -117,7 +117,7 @@ public class NodeResources { } public boolean isDefault() { return this == getDefault(); } - public static Architecture getDefault() { return x86_64; } + public static Architecture getDefault() { return any; } } @@ -498,7 +498,7 @@ public class NodeResources { if (cpu == 0) cpu = 0.5; if (cpu == 2 && mem == 8 ) cpu = 1.5; if (cpu == 2 && mem == 12 ) cpu = 2.3; - return new NodeResources(cpu, mem, dsk, 0.3, DiskSpeed.getDefault(), StorageType.getDefault(), Architecture.x86_64); + return new NodeResources(cpu, mem, dsk, 0.3, DiskSpeed.getDefault(), StorageType.getDefault(), Architecture.any); } private static double validate(double value, String valueName) { diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java index 9351254034e..ae052c03556 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/NodeResourcesTest.java @@ -20,11 +20,11 @@ public class NodeResourcesTest { @Test void testToString() { - assertEquals("[vcpu: 1.0, memory: 10.0 Gb, disk 100.0 Gb, architecture: x86_64]", + assertEquals("[vcpu: 1.0, memory: 10.0 Gb, disk 100.0 Gb, architecture: any]", new NodeResources(1., 10., 100., 0).toString()); - assertEquals("[vcpu: 0.3, memory: 3.3 Gb, disk 33.3 Gb, bandwidth: 0.3 Gbps, architecture: x86_64]", + assertEquals("[vcpu: 0.3, memory: 3.3 Gb, disk 33.3 Gb, bandwidth: 0.3 Gbps, architecture: any]", new NodeResources(1 / 3., 10 / 3., 100 / 3., 0.3).toString()); - assertEquals("[vcpu: 0.7, memory: 9.0 Gb, disk 66.7 Gb, bandwidth: 0.7 Gbps, architecture: x86_64]", + assertEquals("[vcpu: 0.7, memory: 9.0 Gb, disk 66.7 Gb, bandwidth: 0.7 Gbps, architecture: any]", new NodeResources(2 / 3., 8.97, 200 / 3., 0.67).toString()); } diff --git a/configdefinitions/src/vespa/configserver.def b/configdefinitions/src/vespa/configserver.def index 5b8653d1a19..0f18298b434 100644 --- a/configdefinitions/src/vespa/configserver.def +++ b/configdefinitions/src/vespa/configserver.def @@ -15,7 +15,7 @@ zookeeper.barrierTimeout long default=360 zookeeperLocalhostAffinity bool default=true sessionLifetime long default=3600 # in seconds # How long to wait for all ZooKeeper servers to reach barrier after quorum has reached barrier. In seconds -barrierWaitForAllTimeout long default=1 +barrierWaitForAllTimeout long default=5 # Directories configModelPluginDir[] string diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 955b1bc8f4f..81de2e06b6c 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -21,7 +21,10 @@ import com.yahoo.config.provision.ApplicationTransaction; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.EndpointsChecker; import com.yahoo.config.provision.EndpointsChecker.Availability; +import com.yahoo.config.provision.EndpointsChecker.HealthCheckerProvider; +import com.yahoo.config.provision.EndpointsChecker.HealthChecker; import com.yahoo.config.provision.EndpointsChecker.Endpoint; +import com.yahoo.config.provision.EndpointsChecker.Status; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.InfraDeployer; @@ -172,6 +175,8 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye ConfigserverConfig configserverConfig, Orchestrator orchestrator, TesterClient testerClient, + Zone zone, + HealthCheckerProvider healthCheckers, Metric metric, SecretStore secretStore, FlagSource flagSource) { @@ -180,7 +185,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye infraDeployerProvider.getInfraDeployer(), configConvergenceChecker, httpProxy, - createEndpointsChecker(configserverConfig), + createEndpointsChecker(configserverConfig, zone, healthCheckers.getHealthChecker()), configserverConfig, orchestrator, new LogRetriever(), @@ -1222,28 +1227,36 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } - private static EndpointsChecker createEndpointsChecker(ConfigserverConfig config) { + private static EndpointsChecker createEndpointsChecker(ConfigserverConfig config, Zone zone, HealthChecker healthChecker) { CloseableHttpClient client = (SystemName.from(config.system()).isPublic() ? DefaultHttpClientBuilder.create(() -> null, "hosted-vespa-convergence-health-checker") : VespaHttpClientBuilder.custom().apacheBuilder().setUserAgent("hosted-vespa-convergence-health-checker")) .setDefaultHeaders(List.of(new BasicHeader(HttpHeaders.CONNECTION, "close"))) .build(); return EndpointsChecker.of(endpoint -> { + Availability health = healthChecker.healthy(endpoint); + if ( health.status() != Status.available // Unhealthy targets is the root cause, so return those details. + || endpoint.isPublic() // Controller checks /status.html on its own. + || endpoint.account().isEnclave(zone)) // Private endpoints in enclave are not reachable by us. + return health; + int remainingFailures = 3; - int remainingSuccesses = 100; + int remainingSuccesses = 10; while (remainingSuccesses > 0 && remainingFailures > 0) { try { if (client.execute(new HttpGet(endpoint.url().withPath(parse("/status.html")).asURI()), response -> response.getCode() == 200)) remainingSuccesses--; - else remainingFailures--; + else + throw new IOException("got non-200 status code"); } catch (Exception e) { log.log(Level.FINE, e, () -> "Failed to check " + endpoint + "status.html: " + e.getMessage()); - remainingFailures--; + if (--remainingFailures == 0) + return new Availability(Status.containersUnhealthy, "Failed to get enough healthy responses from " + endpoint.url()); } } - return remainingSuccesses == 0; + return Availability.ready; }); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/HealthCheckerProviderProvider.java b/configserver/src/main/java/com/yahoo/vespa/config/server/HealthCheckerProviderProvider.java new file mode 100644 index 00000000000..2d54f256a05 --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/HealthCheckerProviderProvider.java @@ -0,0 +1,19 @@ +package com.yahoo.vespa.config.server; + +import com.yahoo.config.provision.EndpointsChecker.HealthCheckerProvider; +import com.yahoo.container.di.componentgraph.Provider; + +/** + * Default stub for container health checker, overridden by node-repository when that is present. + * + * @author jonmv + */ +public class HealthCheckerProviderProvider implements Provider<HealthCheckerProvider> { + + @Override + public HealthCheckerProvider get() { return new HealthCheckerProvider() { }; } + + @Override + public void deconstruct() { } + +} diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java index 132d3d913e7..040f230a40e 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/ConfigConvergenceChecker.java @@ -86,19 +86,21 @@ public class ConfigConvergenceChecker extends AbstractComponent { /** Fetches the active config generation for all services in the given application. */ public Map<ServiceInfo, Long> getServiceConfigGenerations(Application application, Duration timeoutPerService) { - return getServiceConfigGenerations(application, timeoutPerService, true); + return getServiceConfigGenerations(application, timeoutPerService, new HostsToCheck(Set.of())); } /** * Fetches the active config generation for all services in the given application. Will not check services - * which defer config changes until restart if checkAll is false. + * which defer config changes until restart if checkAll is false. hostsToCheck are names to check, or empty to check all. */ - private Map<ServiceInfo, Long> getServiceConfigGenerations(Application application, Duration timeoutPerService, boolean checkAll) { + private Map<ServiceInfo, Long> getServiceConfigGenerations(Application application, + Duration timeoutPerService, + HostsToCheck hostsToCheck) { List<ServiceInfo> servicesToCheck = new ArrayList<>(); application.getModel().getHosts() .forEach(host -> host.getServices().stream() .filter(service -> serviceTypesToCheck.contains(service.getServiceType())) - .filter(serviceInfo -> shouldCheckService(checkAll, application, serviceInfo)) + .filter(serviceInfo -> shouldCheckService(hostsToCheck, application, serviceInfo)) .forEach(service -> getStatePort(service).ifPresent(port -> servicesToCheck.add(service)))); log.log(Level.FINE, "Services to check for config convergence: " + servicesToCheck); @@ -107,20 +109,20 @@ public class ConfigConvergenceChecker extends AbstractComponent { /** Checks all services in given application. Returns the minimum current generation of all services */ public ServiceListResponse checkConvergenceForAllServices(Application application, Duration timeoutPerService) { - return checkConvergence(application, timeoutPerService, true); + return checkConvergence(application, timeoutPerService, new HostsToCheck(Set.of())); } /** * Checks services except those which defer config changes until restart in the given application. * Returns the minimum current generation of those services. */ - public ServiceListResponse checkConvergenceUnlessDeferringChangesUntilRestart(Application application) { + public ServiceListResponse checkConvergenceUnlessDeferringChangesUntilRestart(Application application, Set<String> hostnames) { Duration timeoutPerService = Duration.ofSeconds(10); - return checkConvergence(application, timeoutPerService, false); + return checkConvergence(application, timeoutPerService, new HostsToCheck(hostnames)); } - private ServiceListResponse checkConvergence(Application application, Duration timeoutPerService, boolean checkAll) { - Map<ServiceInfo, Long> currentGenerations = getServiceConfigGenerations(application, timeoutPerService, checkAll); + private ServiceListResponse checkConvergence(Application application, Duration timeoutPerService, HostsToCheck hostsToCheck) { + Map<ServiceInfo, Long> currentGenerations = getServiceConfigGenerations(application, timeoutPerService, hostsToCheck); long currentGeneration = currentGenerations.values().stream().mapToLong(Long::longValue).min().orElse(-1); return new ServiceListResponse(currentGenerations, application.getApplicationGeneration(), currentGeneration); } @@ -142,8 +144,9 @@ public class ConfigConvergenceChecker extends AbstractComponent { } } - private boolean shouldCheckService(boolean checkServicesWithDeferChangesUntilRestart, Application application, ServiceInfo serviceInfo) { - if (checkServicesWithDeferChangesUntilRestart) return true; + private boolean shouldCheckService(HostsToCheck hostsToCheck, Application application, ServiceInfo serviceInfo) { + if (hostsToCheck.checkAll()) return true; + if ( ! hostsToCheck.check(serviceInfo.getHostName())) return false; if (isNotContainer(serviceInfo)) return true; return serviceIsInClusterWhichShouldBeChecked(application, serviceInfo); } @@ -307,6 +310,14 @@ public class ConfigConvergenceChecker extends AbstractComponent { .build(); } + private record HostsToCheck(Set<String> hostnames) { + + public boolean checkAll() { return hostnames.isEmpty(); } + + public boolean check(String hostname) { return checkAll() || hostnames.contains(hostname); } + + } + public static class ServiceResponse { public enum Status { ok, notFound, hostNotFound, error } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java index 66a5bc5a023..cab3e89c606 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/Deployment.java @@ -18,8 +18,6 @@ import com.yahoo.vespa.config.server.ApplicationRepository; import com.yahoo.vespa.config.server.ApplicationRepository.ActionTimer; import com.yahoo.vespa.config.server.ApplicationRepository.Activation; import com.yahoo.vespa.config.server.TimeoutBudget; -import com.yahoo.vespa.config.server.application.Application; -import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker; import com.yahoo.vespa.config.server.application.ConfigNotConvergedException; import com.yahoo.vespa.config.server.configchange.ConfigChangeActions; import com.yahoo.vespa.config.server.configchange.ReindexActions; @@ -175,9 +173,9 @@ public class Deployment implements com.yahoo.config.provision.Deployment { RestartActions restartActions = configChangeActions.getRestartActions().useForInternalRestart(internalRedeploy); if (restartActions.isEmpty()) return; - waitForConfigToConverge(applicationId); - Set<String> hostnames = restartActions.hostnames(); + waitForConfigToConverge(applicationId, hostnames); + provisioner.get().restart(applicationId, HostFilter.from(hostnames)); deployLogger.log(Level.INFO, String.format("Scheduled service restart of %d nodes: %s", hostnames.size(), hostnames.stream().sorted().collect(Collectors.joining(", ")))); @@ -186,30 +184,46 @@ public class Deployment implements com.yahoo.config.provision.Deployment { this.configChangeActions = configChangeActions.withRestartActions(new RestartActions()); } - private void waitForConfigToConverge(ApplicationId applicationId) { + private void waitForConfigToConverge(ApplicationId applicationId, Set<String> hostnames) { deployLogger.log(Level.INFO, "Wait for all services to use new config generation before restarting"); - while (true) { - try { - params.get().getTimeoutBudget().assertNotTimedOut( - () -> "Timeout exceeded while waiting for config convergence for " + applicationId); - } catch (UncheckedTimeoutException e) { - throw new ConfigNotConvergedException(e); - } + var convergenceChecker = applicationRepository.configConvergenceChecker(); + var app = applicationRepository.getActiveApplication(applicationId); - ConfigConvergenceChecker convergenceChecker = applicationRepository.configConvergenceChecker(); - Application app = applicationRepository.getActiveApplication(applicationId); - ServiceListResponse response = convergenceChecker.checkConvergenceUnlessDeferringChangesUntilRestart(app); + ServiceListResponse response = null; + while (timeLeft(applicationId, response)) { + response = convergenceChecker.checkConvergenceUnlessDeferringChangesUntilRestart(app, hostnames); if (response.converged) { deployLogger.log(Level.INFO, "Services converged on new config generation " + response.currentGeneration); return; } else { - deployLogger.log(Level.INFO, "Services did not converge on new config generation " + - response.wantedGeneration + ", current generation: " + response.currentGeneration + ", will retry"); + deployLogger.log(Level.INFO, "Services that did not converge on new config generation " + + response.wantedGeneration + ": " + + servicesNotConvergedFormatted(response) + ". Will retry"); try { Thread.sleep(5_000); } catch (InterruptedException e) { /* ignore */ } } } } + private boolean timeLeft(ApplicationId applicationId, ServiceListResponse response) { + try { + params.get().getTimeoutBudget().assertNotTimedOut( + () -> "Timeout exceeded while waiting for config convergence for " + applicationId + + ", wanted generation " + response.wantedGeneration + ", these services had another generation: " + + servicesNotConvergedFormatted(response)); + } catch (UncheckedTimeoutException e) { + throw new ConfigNotConvergedException(e); + } + return true; + } + + private String servicesNotConvergedFormatted(ServiceListResponse response) { + return response.services().stream() + .filter(service -> service.currentGeneration != response.wantedGeneration) + .map(service -> service.serviceInfo.getHostName() + ":" + service.serviceInfo.getServiceName() + + " on generation " + service.currentGeneration) + .collect(Collectors.joining(", ")); + } + private void storeReindexing(ApplicationId applicationId, long requiredSession) { applicationRepository.modifyReindexing(applicationId, reindexing -> { if (configChangeActions != null) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java index 62a1704b350..9a6e4632071 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandler.java @@ -10,6 +10,7 @@ import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.model.api.Model; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.EndpointsChecker.Availability; import com.yahoo.config.provision.EndpointsChecker.Endpoint; @@ -113,7 +114,7 @@ public class ApplicationHandler extends HttpHandler { public HttpResponse handlePOST(HttpRequest request) { Path path = new Path(request.getUri()); - if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/verify-endpoints")) return verifyEndpoints(request); + if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/verify-endpoints")) return verifyEndpoints(applicationId(path), request); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindex")) return triggerReindexing(applicationId(path), request); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/reindexing")) return enableReindexing(applicationId(path)); if (path.matches("/application/v2/tenant/{tenant}/application/{application}/environment/{ignore}/region/{ignore}/instance/{instance}/restart")) return restart(applicationId(path), request); @@ -332,17 +333,19 @@ public class ApplicationHandler extends HttpHandler { return new MessageResponse("Success"); } - private HttpResponse verifyEndpoints(HttpRequest request) { + private HttpResponse verifyEndpoints(ApplicationId applicationId, HttpRequest request) { byte[] data = uncheck(() -> request.getData().readAllBytes()); List<Endpoint> endpoints = new ArrayList<>(); SlimeUtils.jsonToSlime(data).get() .field("endpoints") .traverse((ArrayTraverser) (__, endpointObject) -> { - endpoints.add(new Endpoint(ClusterSpec.Id.from(endpointObject.field("clusterName").asString()), + endpoints.add(new Endpoint(applicationId, + ClusterSpec.Id.from(endpointObject.field("clusterName").asString()), HttpURL.from(URI.create(endpointObject.field("url").asString())), SlimeUtils.optionalString(endpointObject.field("ipAddress")).map(uncheck(InetAddress::getByName)), SlimeUtils.optionalString(endpointObject.field("canonicalName")).map(DomainName::of), - endpointObject.field("public").asBool())); + endpointObject.field("public").asBool(), + CloudAccount.from(endpointObject.field("account").asString()))); }); if (endpoints.isEmpty()) throw new IllegalArgumentException("No endpoints in request " + request); diff --git a/configserver/src/main/resources/configserver-app/services.xml b/configserver/src/main/resources/configserver-app/services.xml index bba7d9627dd..b6904467893 100644 --- a/configserver/src/main/resources/configserver-app/services.xml +++ b/configserver/src/main/resources/configserver-app/services.xml @@ -26,6 +26,7 @@ <component id="com.yahoo.vespa.config.server.tenant.TenantRepository" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.host.HostRegistry" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.ApplicationRepository" bundle="configserver" /> + <component id="com.yahoo.vespa.config.server.HealthCheckerProviderProvider" bundle="configserver" /> <component id="com.yahoo.vespa.config.server.version.VersionState" bundle="configserver" /> <component id="com.yahoo.config.provision.Zone" bundle="config-provisioning" /> <component id="com.yahoo.vespa.config.server.application.ConfigConvergenceChecker" bundle="configserver" /> diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockConfigConvergenceChecker.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockConfigConvergenceChecker.java index b4892caa05f..ccbdcde2c2e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/MockConfigConvergenceChecker.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockConfigConvergenceChecker.java @@ -5,14 +5,25 @@ import com.yahoo.vespa.config.server.application.Application; import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker; import java.time.Duration; +import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.Set; public class MockConfigConvergenceChecker extends ConfigConvergenceChecker { private final long wantedGeneration; + private final List<ServiceInfo> servicesThatFailFirstIteration; + + private int iteration = 0; public MockConfigConvergenceChecker(long wantedGeneration) { + this(wantedGeneration, List.of()); + } + + public MockConfigConvergenceChecker(long wantedGeneration, List<ServiceInfo> servicesThatFailFirstIteration) { this.wantedGeneration = wantedGeneration; + this.servicesThatFailFirstIteration = servicesThatFailFirstIteration; } @Override @@ -31,8 +42,16 @@ public class MockConfigConvergenceChecker extends ConfigConvergenceChecker { } @Override - public ServiceListResponse checkConvergenceUnlessDeferringChangesUntilRestart(Application application) { - return new ServiceListResponse(Map.of(), wantedGeneration, wantedGeneration); + public ServiceListResponse checkConvergenceUnlessDeferringChangesUntilRestart(Application application, Set<String> hostnames) { + iteration++; + if (servicesThatFailFirstIteration.isEmpty() || iteration > 1) + return new ServiceListResponse(Map.of(), wantedGeneration, wantedGeneration); + + Map<ServiceInfo, Long> services = new HashMap<>(); + for (var service : servicesThatFailFirstIteration) { + services.put(service, wantedGeneration - 1); + } + return new ServiceListResponse(services, wantedGeneration, wantedGeneration - 1); } } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java index 842416615e2..c1da4e0502f 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/deploy/HostedDeployTest.java @@ -22,9 +22,11 @@ import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; +import com.yahoo.slime.SlimeUtils; import com.yahoo.test.ManualClock; import com.yahoo.vespa.config.server.MockConfigConvergenceChecker; import com.yahoo.vespa.config.server.application.ApplicationReindexing; +import com.yahoo.vespa.config.server.application.ConfigConvergenceChecker; import com.yahoo.vespa.config.server.http.InternalServerException; import com.yahoo.vespa.config.server.http.InvalidApplicationException; import com.yahoo.vespa.config.server.http.UnknownVespaVersionException; @@ -41,8 +43,8 @@ import java.nio.file.Files; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -51,6 +53,7 @@ import java.util.stream.IntStream; import static com.yahoo.vespa.config.server.deploy.DeployTester.CountingModelFactory; import static com.yahoo.vespa.config.server.deploy.DeployTester.createFailingModelFactory; import static com.yahoo.vespa.config.server.deploy.DeployTester.createHostedModelFactory; +import static com.yahoo.yolean.Exceptions.findCause; import static com.yahoo.yolean.Exceptions.uncheck; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -445,8 +448,7 @@ public class HostedDeployTest { @Test public void testThatConfigChangeActionsAreCollectedFromAllModels() { List<Host> hosts = createHosts(9, "6.1.0", "6.2.0"); - List<ServiceInfo> services = List.of( - new ServiceInfo("serviceName", "serviceType", null, new HashMap<>(), "configId", "hostName")); + List<ServiceInfo> services = createServices(1); List<ModelFactory> modelFactories = List.of( new ConfigChangeActionsModelFactory(Version.fromString("6.1.0"), @@ -461,10 +463,41 @@ public class HostedDeployTest { } @Test + public void testConfigConvergenceBeforeRestart() { + List<Host> hosts = createHosts(9, "6.1.0", "6.2.0"); + List<ServiceInfo> services = createServices(1); + List<ServiceInfo> twoServices = createServices(2); + + List<ModelFactory> modelFactories = List.of( + new ConfigChangeActionsModelFactory(Version.fromString("6.2.0"), + new VespaRestartAction(ClusterSpec.Id.from("test"), "change", services))); + + DeployTester tester = createTester(hosts, + modelFactories, + prodZone, + Clock.systemUTC(), + new MockConfigConvergenceChecker(2L, services)); + var result = tester.deployApp("src/test/apps/hosted/", "6.2.0"); + DeployHandlerLogger deployLogger = result.deployLogger(); + + assertLogContainsMessage(deployLogger, "Scheduled service restart of 1 nodes: hostName0"); + assertLogContainsMessage(deployLogger, "Wait for all services to use new config generation before restarting"); + // Should only check convergence on 1 of the nodes + assertLogContainsMessage(deployLogger, "Services that did not converge on new config generation 2: hostName0:serviceName0 on generation 1. Will retry"); + assertLogContainsMessage(deployLogger, "Services converged on new config generation 2"); + } + + private void assertLogContainsMessage(DeployHandlerLogger log, String message) { + assertEquals(1, SlimeUtils.entriesStream(log.slime().get().field("log")) + .map(entry -> entry.field("message").asString()) + .filter(m -> m.contains(message)) + .count()); + } + + @Test public void testThatAllowedConfigChangeActionsAreActedUpon() { List<Host> hosts = createHosts(9, "6.1.0"); - List<ServiceInfo> services = List.of( - new ServiceInfo("serviceName", "serviceType", null, Map.of("clustername", "cluster"), "configId", "hostName")); + List<ServiceInfo> services = createServices(1); ManualClock clock = new ManualClock(Instant.EPOCH); List<ModelFactory> modelFactories = List.of( @@ -486,7 +519,7 @@ public class HostedDeployTest { assertEquals(9, tester.getAllocatedHostsOf(tester.applicationId()).getHosts().size()); assertTrue(prepareResult.configChangeActions().getRestartActions().isEmpty()); // Handled by deployment. assertEquals(Optional.of(ApplicationReindexing.empty() - .withPending("cluster", "music", prepareResult.sessionId())), + .withPending("cluster0", "music", prepareResult.sessionId())), tester.tenant().getApplicationRepo().database().readReindexingStatus(tester.applicationId())); } @@ -539,18 +572,39 @@ public class HostedDeployTest { return createTester(hosts, modelFactories, zone, Clock.systemUTC()); } - private DeployTester createTester(List<Host> hosts, List<ModelFactory> modelFactories, - Zone prodZone, Clock clock) { + private DeployTester createTester(List<Host> hosts, List<ModelFactory> modelFactories, Zone zone, Clock clock) { + return createTester(hosts, modelFactories, zone, clock, new MockConfigConvergenceChecker(2)); + } + + private DeployTester createTester(List<Host> hosts, + List<ModelFactory> modelFactories, + Zone zone, + Clock clock, + ConfigConvergenceChecker configConvergenceChecker) { return new DeployTester.Builder(temporaryFolder) .modelFactories(modelFactories) .clock(clock) - .zone(prodZone) + .zone(zone) .hostProvisioner(new InMemoryProvisioner(new Hosts(hosts), true, false)) - .configConvergenceChecker(new MockConfigConvergenceChecker(2)) - .hostedConfigserverConfig(prodZone) + .configConvergenceChecker(configConvergenceChecker) + .hostedConfigserverConfig(zone) .build(); } + private static List<ServiceInfo> createServices(int serviceCount) { + List<ServiceInfo> services = new ArrayList<>(); + + IntStream.range(0, serviceCount) + .forEach(i -> services.add(new ServiceInfo("serviceName" + i, + i == 0 ? "searchnode" : "container", + null, + Map.of("clustername", "cluster" + i), + "configId" + i, + "hostName" + i))); + + return services; + } + private static class ConfigChangeActionsModelFactory extends TestModelFactory { private final List<ConfigChangeAction> actions; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java index e8c4d819c31..306ba6da6f9 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/ApplicationHandlerTest.java @@ -10,6 +10,7 @@ import com.yahoo.config.model.api.PortInfo; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.EndpointsChecker; import com.yahoo.config.provision.EndpointsChecker.Availability; @@ -511,11 +512,13 @@ public class ApplicationHandlerTest { @Test public void testVerifyEndpoints() { - expectedEndpoints = List.of(new Endpoint(ClusterSpec.Id.from("bluster"), + expectedEndpoints = List.of(new Endpoint(ApplicationId.defaultId(), + ClusterSpec.Id.from("bluster"), HttpURL.from(URI.create("https://bluster.tld:1234")), Optional.of(uncheck(() -> InetAddress.getByName("4.3.2.1"))), Optional.of(DomainName.of("fluster.tld")), - false)); + false, + CloudAccount.empty)); availability = new Availability(EndpointsChecker.Status.available, "Endpoints are ready"); ApplicationHandler handler = createApplicationHandler(); HttpRequest request = createTestRequest(toUrlPath(applicationId, Zone.defaultZone(), true) + "/verify-endpoints", diff --git a/container-core/src/test/java/com/yahoo/container/di/ContainerTest.java b/container-core/src/test/java/com/yahoo/container/di/ContainerTest.java index 741f381c44c..9ea279ecb2b 100644 --- a/container-core/src/test/java/com/yahoo/container/di/ContainerTest.java +++ b/container-core/src/test/java/com/yahoo/container/di/ContainerTest.java @@ -297,7 +297,7 @@ public class ContainerTest extends ContainerTestBase { container.reloadConfig(3); dirConfigSource.awaitConfigChecked(10_000); - assertNotNull(newGraph.get(1, TimeUnit.SECONDS)); + assertNotNull(newGraph.get(30, TimeUnit.SECONDS)); container.shutdownConfigRetriever(); container.shutdown(newGraph.get()); diff --git a/container-dependency-versions/pom.xml b/container-dependency-versions/pom.xml index a95e0061992..a511e14de0a 100644 --- a/container-dependency-versions/pom.xml +++ b/container-dependency-versions/pom.xml @@ -243,8 +243,8 @@ <error-prone-annotations.version>2.18.0</error-prone-annotations.version> <guava.version>27.1-jre</guava.version> <guice.version>4.2.3</guice.version> - <jackson2.version>2.14.2</jackson2.version> - <jackson-databind.version>2.14.2</jackson-databind.version> + <jackson2.version>2.15.0</jackson2.version> + <jackson-databind.version>2.15.0</jackson-databind.version> <javax.inject.version>1</javax.inject.version> <javax.servlet-api.version>3.1.0</javax.servlet-api.version> <javax.ws.rs-api.version>2.0.1</javax.ws.rs-api.version> diff --git a/container-search/src/main/java/com/yahoo/prelude/hitfield/RawBase64.java b/container-search/src/main/java/com/yahoo/prelude/hitfield/RawBase64.java index 485e2c9a8c3..6660f1bf712 100644 --- a/container-search/src/main/java/com/yahoo/prelude/hitfield/RawBase64.java +++ b/container-search/src/main/java/com/yahoo/prelude/hitfield/RawBase64.java @@ -10,7 +10,7 @@ import java.util.Objects; * @author baldersheim */ public class RawBase64 implements Comparable<RawBase64> { - private final static Base64.Encoder encoder = Base64.getEncoder().withoutPadding(); + private final static Base64.Encoder encoder = Base64.getEncoder(); private final byte[] content; public RawBase64(byte[] content) { Objects.requireNonNull(content); diff --git a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java index 524001748c5..f8df6bbea03 100644 --- a/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java +++ b/container-search/src/main/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcher.java @@ -20,6 +20,7 @@ import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.result.Coverage; import com.yahoo.search.result.ErrorMessage; +import com.yahoo.search.result.FeatureData; import com.yahoo.search.result.Relevance; import com.yahoo.search.searchchain.Execution; import com.yahoo.searchlib.aggregation.Grouping; @@ -305,6 +306,9 @@ public class VdsStreamingSearcher extends VespaBackEndSearcher { if (hit instanceof SearchResult.HitWithSortBlob sortedHit) { fastHit.setSortData(sortedHit.getSortBlob(), query.getRanking().getSorting()); } + if (hit.getMatchFeatures().isPresent()) { + fastHit.setField("matchfeatures", new FeatureData(hit.getMatchFeatures().get())); + } fastHit.setFillable(); return fastHit; diff --git a/container-search/src/test/java/com/yahoo/search/grouping/result/GroupIdTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/result/GroupIdTestCase.java index 77ed858b14b..0443f507848 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/result/GroupIdTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/result/GroupIdTestCase.java @@ -48,8 +48,8 @@ public class GroupIdTestCase { assertEquals("group:long:69", new LongId(69L).toString()); assertEquals("group:long_bucket:6:9", new LongBucketId(6L, 9L).toString()); assertEquals("group:null", new NullId().toString()); - assertEquals("group:raw:Bgk", new RawId(new byte[]{6, 9}).toString()); - assertEquals("group:raw_bucket:Bgk:CQY", new RawBucketId(new byte[]{6, 9}, new byte[]{9, 6}).toString()); + assertEquals("group:raw:Bgk=", new RawId(new byte[]{6, 9}).toString()); + assertEquals("group:raw_bucket:Bgk=:CQY=", new RawBucketId(new byte[]{6, 9}, new byte[]{9, 6}).toString()); assertTrue(new RootId(0).toString().startsWith("group:root:")); assertEquals("group:string:69", new StringId("69").toString()); assertEquals("group:string_bucket:6:9", new StringBucketId("6", "9").toString()); diff --git a/container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultBuilderTestCase.java b/container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultBuilderTestCase.java index b0b48bb8731..4d52f37c751 100644 --- a/container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultBuilderTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/grouping/vespa/ResultBuilderTestCase.java @@ -60,12 +60,12 @@ public class ResultBuilderTestCase { assertGroupId("group:6.9", new FloatResultNode(6.9)); assertGroupId("group:69", new IntegerResultNode(69)); assertGroupId("group:null", new NullResultNode()); - assertGroupId("group:Bgk", new RawResultNode(new byte[]{6, 9})); + assertGroupId("group:Bgk=", new RawResultNode(new byte[]{6, 9})); assertGroupId("group:a", new StringResultNode("a")); assertGroupId("group:6.9:9.6", new FloatBucketResultNode(6.9, 9.6)); assertGroupId("group:6:9", new IntegerBucketResultNode(6, 9)); assertGroupId("group:a:b", new StringBucketResultNode("a", "b")); - assertGroupId("group:Bgk:CQY", new RawBucketResultNode(new RawResultNode(new byte[]{6, 9}), + assertGroupId("group:Bgk=:CQY=", new RawBucketResultNode(new RawResultNode(new byte[]{6, 9}), new RawResultNode(new byte[]{9, 6}))); } @@ -92,7 +92,7 @@ public class ResultBuilderTestCase { assertResult("69", new MinAggregationResult(new IntegerResultNode(69))); assertResult("69.3", new MinAggregationResult(new FloatResultNode(69.3))); assertResult("69.6", new MinAggregationResult(new StringResultNode("69.6"))); - assertResult("Bgk", new MinAggregationResult(new RawResultNode(new byte[]{6,9}))); + assertResult("Bgk=", new MinAggregationResult(new RawResultNode(new byte[]{6,9}))); } @Test diff --git a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java index 0ecf1db01e5..578ccec7f40 100644 --- a/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java +++ b/container-search/src/test/java/com/yahoo/vespa/streamingvisitors/VdsStreamingSearcherTestCase.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.streamingvisitors; +import com.yahoo.data.access.Inspectable; +import com.yahoo.data.access.helpers.MatchFeatureData; import com.yahoo.document.select.parser.TokenMgrException; import com.yahoo.messagebus.Trace; import com.yahoo.messagebus.routing.Route; @@ -26,6 +28,7 @@ import com.yahoo.vespa.streamingvisitors.tracing.TraceExporter; import org.junit.jupiter.api.Test; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -93,8 +96,12 @@ public class VdsStreamingSearcherTestCase { addResults(USERDOC_ID_PREFIX, 1, false); } else if (queryString.compareTo("onegroupinghit") == 0) { groupings.add(new Grouping()); + } else if (queryString.compareTo("match_features") == 0) { + addResults(USERDOC_ID_PREFIX, 1, false); + var matchFeatures = new MatchFeatureData(Arrays.asList("my_feature")).addHit(); + matchFeatures.set(0, 7.0); + hits.get(0).setMatchFeatures(matchFeatures); } - } private void addResults(String idPrefix, int hitCount, boolean emptyDocsum) { @@ -212,6 +219,16 @@ public class VdsStreamingSearcherTestCase { checkSearch(searcher, queryString, hitCount, null); } + private static void checkMatchFeatures(VdsStreamingSearcher searcher) { + String queryString = "/?streaming.selection=true&query=match_features"; + Result result = executeQuery(searcher, new Query(queryString)); + assertNull(result.hits().getError()); + assertEquals(result.hits().size(), 1); + Hit hit = result.hits().get(0); + var mf = hit.getField("matchfeatures"); + assertEquals(7.0, ((Inspectable) mf).inspect().field("my_feature").asDouble()); + } + @Test void testBasics() { MockVisitorFactory factory = new MockVisitorFactory(); @@ -249,6 +266,8 @@ public class VdsStreamingSearcherTestCase { checkSearch(searcher, "/?streaming.groupname=group1&query=twogrouphitsandoneuserhit", 2, GROUPDOC_ID_PREFIX); checkGrouping(searcher, "/?streaming.selection=true&query=onegroupinghit", 1); + + checkMatchFeatures(searcher); } @Test diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java index e29e8086c80..c148afb9190 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java @@ -26,7 +26,7 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.Teste public class MockTesterCloud implements TesterCloud { private final NameService nameService; - private final EndpointsChecker endpointsChecker = EndpointsChecker.mock(this::resolveHostName, this::resolveCname, __ -> true); + private final EndpointsChecker endpointsChecker = EndpointsChecker.mock(this::resolveHostName, this::resolveCname, __ -> Availability.ready); private List<LogEntry> log = new ArrayList<>(); private Status status = NOT_STARTED; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index c8ec38ec73b..fb437c3258b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -40,6 +40,7 @@ import com.yahoo.yolean.concurrent.Sleeper; import java.time.Clock; import java.time.Instant; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -50,6 +51,9 @@ import java.util.function.Predicate; import java.util.logging.Logger; import java.util.stream.Collectors; +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toSet; + /** * API to the controller. This contains the object model of everything the controller cares about, mainly tenants and * applications. The object model is persisted to curator. @@ -179,13 +183,23 @@ public class Controller extends AbstractComponent { log.info("Changing system version from " + printableVersion(currentStatus.systemVersion()) + " to " + printableVersion(newStatus.systemVersion())); } + Set<Version> obsoleteVersions = currentStatus.versions().stream().map(VespaVersion::versionNumber).collect(toSet()); + for (VespaVersion version : newStatus.versions()) { + obsoleteVersions.remove(version.versionNumber()); + VespaVersion current = currentStatus.version(version.versionNumber()); + if (current == null) + log.info("New version " + version.versionNumber().toFullString() + " added"); + else if ( ! current.confidence().equals(version.confidence())) + log.info("Confidence for version " + version.versionNumber().toFullString() + + " changed from " + current.confidence() + " to " + version.confidence()); + } + for (Version version : obsoleteVersions) + log.info("Version " + version.toFullString() + " is obsolete, and will be forgotten"); + curator.writeVersionStatus(newStatus); - // Removes confidence overrides for versions that no longer exist in the system - removeConfidenceOverride(version -> newStatus.versions().stream() - .noneMatch(vespaVersion -> vespaVersion.versionNumber() - .equals(version))); + removeConfidenceOverride(obsoleteVersions::contains); } - + /** Returns the latest known version status. Calling this is free but the status may be slightly out of date. */ public VersionStatus readVersionStatus() { return curator.readVersionStatus(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 7aacd93813c..71ab1c4d7da 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -356,7 +356,7 @@ public class InternalStepRunner implements StepRunner { } if (summary.converged()) { controller.jobController().locked(id, lockedRun -> lockedRun.withSummary(null)); - Availability availability = endpointsAvailable(id.application(), id.type().zone(), logger); + Availability availability = endpointsAvailable(id.application(), id.type().zone(), deployment.get(), logger); if (availability.status() == Status.available) { if (controller.routing().policies().processDnsChallenges(new DeploymentId(id.application(), id.type().zone()))) { logger.log("Installation succeeded!"); @@ -496,24 +496,26 @@ public class InternalStepRunner implements StepRunner { } } - private Availability endpointsAvailable(ApplicationId id, ZoneId zone, DualLogger logger) { - DeploymentId deployment = new DeploymentId(id, zone); - Map<ZoneId, List<Endpoint>> endpoints = controller.routing().readTestRunnerEndpointsOf(Set.of(deployment)); + private Availability endpointsAvailable(ApplicationId id, ZoneId zone, Deployment deployment, DualLogger logger) { + DeploymentId deploymentId = new DeploymentId(id, zone); + Map<ZoneId, List<Endpoint>> endpoints = controller.routing().readTestRunnerEndpointsOf(Set.of(deploymentId)); logEndpoints(endpoints, logger); - DeploymentRoutingContext context = controller.routing().of(deployment); + DeploymentRoutingContext context = controller.routing().of(deploymentId); boolean resolveEndpoints = context.routingMethod() == RoutingMethod.exclusive; return controller.serviceRegistry().testerCloud().verifyEndpoints( - deployment, + deploymentId, endpoints.getOrDefault(zone, List.of()) .stream() .map(endpoint -> { ClusterSpec.Id cluster = ClusterSpec.Id.from(endpoint.name()); RoutingPolicy policy = context.routingPolicy(cluster).get(); - return new EndpointsChecker.Endpoint(cluster, + return new EndpointsChecker.Endpoint(id, + cluster, HttpURL.from(endpoint.url()), policy.ipAddress().filter(__ -> resolveEndpoints).map(uncheck(InetAddress::getByName)), policy.canonicalName().filter(__ -> resolveEndpoints), - policy.isPublic()); + policy.isPublic(), + deployment.cloudAccount()); }).toList()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 49186be2089..2a6d62672f3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -52,8 +52,6 @@ import com.yahoo.slime.SlimeUtils; import com.yahoo.text.Text; import com.yahoo.vespa.athenz.api.OAuthCredentials; import com.yahoo.vespa.athenz.client.zms.ZmsClientException; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; @@ -167,7 +165,8 @@ import java.util.stream.Stream; import static com.yahoo.jdisc.Response.Status.BAD_REQUEST; import static com.yahoo.jdisc.Response.Status.CONFLICT; -import static com.yahoo.vespa.flags.FetchVector.Dimension.APPLICATION_ID; +import static com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource.APPLICATION_TEST_ZIP; +import static com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource.APPLICATION_ZIP; import static com.yahoo.yolean.Exceptions.uncheck; import static java.util.Comparator.comparingInt; import static java.util.Map.Entry.comparingByKey; @@ -189,7 +188,6 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private final Controller controller; private final AccessControlRequests accessControlRequests; private final TestConfigSerializer testConfigSerializer; - private final BooleanFlag failDeploymentOnMissingCertificateFile; @Inject public ApplicationApiHandler(ThreadedHttpRequestHandler.Context parentCtx, @@ -199,7 +197,6 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { this.controller = controller; this.accessControlRequests = accessControlRequests; this.testConfigSerializer = new TestConfigSerializer(controller.system()); - this.failDeploymentOnMissingCertificateFile = Flags.FAIL_DEPLOYMENT_ON_MISSING_CERTIFICATE_FILE.bindTo(controller.flagSource()); } @Override @@ -2435,7 +2432,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if ( ! dataParts.containsKey("applicationZip")) throw new IllegalArgumentException("Missing required form part 'applicationZip'"); - ApplicationPackage applicationPackage = new ApplicationPackage(dataParts.get(EnvironmentResource.APPLICATION_ZIP)); + ApplicationPackage applicationPackage = new ApplicationPackage(dataParts.get(APPLICATION_ZIP)); controller.applications().verifyApplicationIdentityConfiguration(id.tenant(), Optional.of(id.instance()), Optional.of(type.zone()), @@ -3071,14 +3068,8 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { }); ApplicationPackage applicationPackage = - new ApplicationPackage(dataParts.get(EnvironmentResource.APPLICATION_ZIP), - true, - controller.system().isPublic() && - failDeploymentOnMissingCertificateFile - .with(APPLICATION_ID, ApplicationId.from(tenant, application, "default").serializedForm()) - .value()); - - byte[] testPackage = dataParts.getOrDefault(EnvironmentResource.APPLICATION_TEST_ZIP, new byte[0]); + new ApplicationPackage(dataParts.get(APPLICATION_ZIP), true, controller.system().isPublic()); + byte[] testPackage = dataParts.getOrDefault(APPLICATION_TEST_ZIP, new byte[0]); Submission submission = new Submission(applicationPackage, testPackage, sourceUrl, sourceRevision, authorEmail, description, risk); TenantName tenantName = TenantName.from(tenant); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index de777475da2..2fc5ac10e6b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -27,6 +27,7 @@ import com.yahoo.vespa.hosted.controller.deployment.ConvergenceSummary; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.DelayCause; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.Readiness; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus.StepType; import com.yahoo.vespa.hosted.controller.deployment.JobController; import com.yahoo.vespa.hosted.controller.deployment.JobStatus; import com.yahoo.vespa.hosted.controller.deployment.Run; @@ -272,37 +273,19 @@ class JobControllerApiHandlerHelper { // TODO: recursively search dependents for what is the relevant partial change when this is a delay step ... Readiness readiness = stepStatus.job().map(jobsToRun::get).map(job -> job.get(0).readiness()) .orElse(stepStatus.readiness(change)); + Instant now = controller.clock().instant(); if (readiness.ok()) { stepObject.setLong("readyAt", readiness.at().toEpochMilli()); - if ( ! readiness.okAt(controller.clock().instant())) { - Instant until = readiness.at(); - stepObject.setLong("delayedUntil", readiness.at().toEpochMilli()); - switch (readiness.cause()) { - case paused -> stepObject.setLong("pausedUntil", until.toEpochMilli()); - case coolingDown -> stepObject.setLong("coolingDownUntil", until.toEpochMilli()); - case changeBlocked -> { - Readiness platformReadiness = stepStatus.readiness(Change.of(controller.systemVersion(versionStatus))); // Dummy version — just anything with a platform. - if (platformReadiness.cause() == DelayCause.changeBlocked) - stepObject.setLong("platformBlockedUntil", platformReadiness.at().toEpochMilli()); - Readiness applicationReadiness = stepStatus.readiness(Change.of(RevisionId.forProduction(1))); // Dummy version — just anything with an application. - if (applicationReadiness.cause() == DelayCause.changeBlocked) - stepObject.setLong("applicationBlockedUntil", applicationReadiness.at().toEpochMilli()); - } - } - } + if ( ! readiness.okAt(now)) stepObject.setLong("delayedUntil", readiness.at().toEpochMilli()); } - stepObject.setString("delayCause", - switch (readiness.cause()) { - case none -> null; - case invalidPackage -> "invalidPackage"; - case paused -> "paused"; - case coolingDown -> "coolingDown"; - case changeBlocked -> "changeBlocked"; - case blocked -> "blocked"; - case running -> "running"; - case notReady -> "notReady"; - case unverified -> "unverified"; - }); + if (readiness.cause() == DelayCause.coolingDown) stepObject.setLong("coolingDownUntil", readiness.at().toEpochMilli()); + if ( ! stepStatus.pausedUntil().okAt(now)) stepObject.setLong("pausedUntil", stepStatus.pausedUntil().at().toEpochMilli()); + Readiness platformReadiness = stepStatus.blockedUntil(Change.of(controller.systemVersion(versionStatus))); // Dummy version — just anything with a platform. + if ( ! platformReadiness.okAt(now)) + stepObject.setLong("platformBlockedUntil", platformReadiness.at().toEpochMilli()); + Readiness applicationReadiness = stepStatus.blockedUntil(Change.of(RevisionId.forProduction(1))); // Dummy version — just anything with an application. + if ( ! applicationReadiness.okAt(now)) + stepObject.setLong("applicationBlockedUntil", applicationReadiness.at().toEpochMilli()); if (stepStatus.type() == DeploymentStatus.StepType.delay) stepStatus.completedAt(change).ifPresent(completed -> stepObject.setLong("completedAt", completed.toEpochMilli())); @@ -362,7 +345,9 @@ class JobControllerApiHandlerHelper { } } - stepStatus.job().ifPresent(job -> { + boolean showDelayCause = true; + if (stepStatus.job().isPresent()) { + JobId job = stepStatus.job().get(); stepObject.setString("jobName", job.type().jobName()); URI baseUriForJob = baseUriForDeployments.resolve(baseUriForDeployments.getPath() + "/../instance/" + job.application().instance().value() + @@ -380,6 +365,7 @@ class JobControllerApiHandlerHelper { JobStatus jobStatus = status.jobs().get(job).get(); Cursor toRunArray = stepObject.setArray("toRun"); + showDelayCause = false; for (DeploymentStatus.Job versions : jobsToRun.getOrDefault(job, List.of())) { boolean running = jobStatus.lastTriggered() .map(run -> jobStatus.isRunning() @@ -389,12 +375,27 @@ class JobControllerApiHandlerHelper { if (running) continue; // Run will be contained in the "runs" array. + showDelayCause = true; Cursor runObject = toRunArray.addObject(); toSlime(runObject.setObject("versions"), versions.versions(), application); } toSlime(stepObject.setArray("runs"), jobStatus.runs().descendingMap().values(), application, 10, baseUriForJob); - }); + } + stepObject.setString("delayCause", + ! showDelayCause + ? (String) null + : switch (readiness.cause()) { + case none -> null; + case invalidPackage -> "invalidPackage"; + case paused -> "paused"; + case coolingDown -> "coolingDown"; + case changeBlocked -> "changeBlocked"; + case blocked -> "blocked"; + case running -> "running"; + case notReady -> "notReady"; + case unverified -> "unverified"; + }); } Cursor buildsArray = responseObject.setArray("builds"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json index 6793553faca..b103b579166 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview-2.json @@ -9,7 +9,6 @@ "declared": true, "instance": "default", "readyAt": 0, - "delayCause": null, "deploying": { "application": { "build": 3, @@ -148,7 +147,8 @@ } ] } - } + }, + "delayCause": null }, { "type": "test", @@ -156,7 +156,6 @@ "declared": false, "instance": "default", "readyAt": 0, - "delayCause": null, "jobName": "system-test", "url": "https://some.url:43/instance/default/job/system-test", "environment": "test", @@ -336,7 +335,8 @@ } ] } - ] + ], + "delayCause": null }, { "type": "test", @@ -346,7 +346,6 @@ "readyAt": 15153000, "delayedUntil": 15153000, "coolingDownUntil": 15153000, - "delayCause": "coolingDown", "jobName": "staging-test", "url": "https://some.url:43/instance/default/job/staging-test", "environment": "staging", @@ -769,7 +768,8 @@ } ] } - ] + ], + "delayCause": "coolingDown" }, { "type": "deployment", @@ -781,7 +781,6 @@ "instance": "default", "readyAt": 14403000, "delayedUntil": 14403000, - "delayCause": "running", "jobName": "production-us-central-1", "url": "https://some.url:43/instance/default/job/production-us-central-1", "environment": "prod", @@ -898,7 +897,8 @@ } ] } - ] + ], + "delayCause": null }, { "type": "test", @@ -907,7 +907,6 @@ ], "declared": true, "instance": "default", - "delayCause": "notReady", "jobName": "test-us-central-1", "url": "https://some.url:43/instance/default/job/test-us-central-1", "environment": "prod", @@ -1039,7 +1038,8 @@ } ] } - ] + ], + "delayCause": "notReady" }, { "type": "deployment", @@ -1048,7 +1048,6 @@ ], "declared": true, "instance": "default", - "delayCause": "notReady", "jobName": "production-us-west-1", "url": "https://some.url:43/instance/default/job/production-us-west-1", "environment": "prod", @@ -1148,7 +1147,8 @@ } ] } - ] + ], + "delayCause": "notReady" }, { "type": "deployment", @@ -1157,7 +1157,6 @@ ], "declared": true, "instance": "default", - "delayCause": "notReady", "jobName": "production-us-east-3", "url": "https://some.url:43/instance/default/job/production-us-east-3", "environment": "prod", @@ -1257,7 +1256,8 @@ } ] } - ] + ], + "delayCause": "notReady" } ], "builds": [ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json index 0b7c64c72a5..500f66a7cdb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-overview.json @@ -9,7 +9,6 @@ "declared": true, "instance": "instance1", "readyAt": 0, - "delayCause": null, "deploying": { "application": { "build": 4, @@ -60,7 +59,8 @@ ], "blockers": [ ] } - } + }, + "delayCause": null }, { "type": "test", @@ -69,7 +69,6 @@ "instance": "instance1", "readyAt": 0, "delayedUntil": 0, - "delayCause": "running", "jobName": "system-test", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test", "environment": "test", @@ -190,7 +189,8 @@ } ] } - ] + ], + "delayCause": null }, { "type": "test", @@ -199,7 +199,6 @@ "instance": "instance1", "readyAt": 0, "delayedUntil": 0, - "delayCause": "running", "jobName": "staging-test", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test", "environment": "staging", @@ -352,7 +351,8 @@ } ] } - ] + ], + "delayCause": null }, { "type": "deployment", @@ -361,7 +361,6 @@ ], "declared": true, "instance": "instance1", - "delayCause": "unverified", "jobName": "production-us-central-1", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-central-1", "environment": "prod", @@ -410,7 +409,8 @@ } ] } - ] + ], + "delayCause": "unverified" }, { "type": "deployment", @@ -419,7 +419,6 @@ ], "declared": true, "instance": "instance1", - "delayCause": "notReady", "jobName": "production-us-west-1", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-west-1", "environment": "prod", @@ -468,7 +467,8 @@ } ] } - ] + ], + "delayCause": "notReady" }, { "type": "deployment", @@ -477,7 +477,6 @@ ], "declared": true, "instance": "instance1", - "delayCause": "notReady", "jobName": "production-us-east-3", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-east-3", "environment": "prod", @@ -553,7 +552,8 @@ } ] } - ] + ], + "delayCause": "notReady" }, { "type": "instance", @@ -563,7 +563,6 @@ ], "declared": true, "instance": "instance2", - "delayCause": "notReady", "deploying": { "application": { "build": 4, @@ -614,7 +613,8 @@ ], "blockers": [ ] } - } + }, + "delayCause": "notReady" }, { "type": "deployment", @@ -623,7 +623,6 @@ ], "declared": true, "instance": "instance2", - "delayCause": "unverified", "jobName": "production-us-central-1", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance2/job/production-us-central-1", "environment": "prod", @@ -641,7 +640,8 @@ } } ], - "runs": [ ] + "runs": [ ], + "delayCause": "unverified" }, { "type": "deployment", @@ -650,7 +650,6 @@ ], "declared": true, "instance": "instance2", - "delayCause": "notReady", "jobName": "production-us-west-1", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance2/job/production-us-west-1", "environment": "prod", @@ -668,7 +667,8 @@ } } ], - "runs": [ ] + "runs": [ ], + "delayCause": "notReady" }, { "type": "deployment", @@ -677,7 +677,6 @@ ], "declared": true, "instance": "instance2", - "delayCause": "notReady", "jobName": "production-us-east-3", "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance2/job/production-us-east-3", "environment": "prod", @@ -695,7 +694,8 @@ } } ], - "runs": [ ] + "runs": [ ], + "delayCause": "notReady" } ], "builds": [ diff --git a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java index 795f8e93187..7a4f178b8ef 100644 --- a/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java +++ b/document/src/main/java/com/yahoo/document/json/JsonSerializationHelper.java @@ -47,7 +47,7 @@ import java.util.Map; */ public class JsonSerializationHelper { - private final static Base64.Encoder base64Encoder = Base64.getEncoder().withoutPadding(); // Important: _basic_ format + private final static Base64.Encoder base64Encoder = Base64.getEncoder(); // Important: _basic_ format static class JsonSerializationException extends RuntimeException { public JsonSerializationException(Exception base) { diff --git a/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java b/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java index d35693f785f..b2c92e6bb0b 100644 --- a/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java +++ b/document/src/main/java/com/yahoo/document/serialization/XmlSerializationHelper.java @@ -31,10 +31,9 @@ import java.util.Map; * @author <a href="mailto:humbe@yahoo-inc.com">Håkon Humberset</a> */ @Deprecated -@SuppressWarnings("removal") public class XmlSerializationHelper { - private final static Base64.Encoder base64Encoder = Base64.getEncoder().withoutPadding(); + private final static Base64.Encoder base64Encoder = Base64.getEncoder(); public static void printArrayXml(Array array, XmlStream xml) { List<FieldValue> lst = array.getValues(); diff --git a/document/src/test/java/com/yahoo/document/DocumentTestCase.java b/document/src/test/java/com/yahoo/document/DocumentTestCase.java index 4470865b636..33b77cb1878 100644 --- a/document/src/test/java/com/yahoo/document/DocumentTestCase.java +++ b/document/src/test/java/com/yahoo/document/DocumentTestCase.java @@ -52,7 +52,7 @@ public class DocumentTestCase extends DocumentTestCaseBase { " <mailid>emailfromalicetobob&someone</mailid>\n" + " <date>-2013512400</date>\n" + " <attachmentcount>2</attachmentcount>\n" + - " <rawfield binaryencoding=\"base64\">AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiYw</rawfield>\n"; + " <rawfield binaryencoding=\"base64\">AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiYw==</rawfield>\n"; private static final String SERTEST_DOC_AS_XML_WEIGHT1 = " <weightedfield>\n" + diff --git a/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java b/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java index af7469de31b..08a5c9a124c 100644 --- a/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java +++ b/document/src/test/java/com/yahoo/document/json/DocumentUpdateJsonSerializerTest.java @@ -504,7 +504,7 @@ public class DocumentUpdateJsonSerializerTest { " 'update': 'DOCUMENT_ID',", " 'fields': {", " 'raw_field': {", - " 'assign': 'RG9uJ3QgYmVsaWV2ZSBoaXMgbGllcw'", + " 'assign': 'RG9uJ3QgYmVsaWV2ZSBoaXMgbGllcw=='", " }", " }", "}" diff --git a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java index 4f15a2fe368..eab33afc3e4 100644 --- a/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java +++ b/document/src/test/java/com/yahoo/document/json/JsonWriterTestCase.java @@ -291,7 +291,7 @@ public class JsonWriterTestCase { String payload = new String( new JsonStringEncoder().quoteAsString( "c3RyaW5nIGxvbmcgZW5vdWdoIHRvIGVtaXQgbW9yZSB0aGFuIDc2IGJhc2U2NCBjaGFyYWN0ZXJzIGFuZC" + - "B3aGljaCBzaG91bGQgY2VydGFpbmx5IG5vdCBiZSBuZXdsaW5lLWRlbGltaXRlZCE")); + "B3aGljaCBzaG91bGQgY2VydGFpbmx5IG5vdCBiZSBuZXdsaW5lLWRlbGltaXRlZCE=")); String docId = "id:unittest:testraw::whee"; diff --git a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java index 22650fcdbf8..940217aa2b4 100644 --- a/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java +++ b/documentapi/src/test/java/com/yahoo/documentapi/messagebus/protocol/test/Messages60TestCase.java @@ -633,6 +633,8 @@ public class Messages60TestCase extends MessagesTestBase { @Override public void run() throws Exception { + test_result_with_match_features(); + Routable routable = deserialize("QueryResultMessage-1", DocumentProtocol.MESSAGE_QUERYRESULT, Language.CPP); assertTrue(routable instanceof QueryResultMessage); @@ -647,9 +649,11 @@ public class Messages60TestCase extends MessagesTestBase { com.yahoo.vdslib.SearchResult.Hit h = msg.getResult().getHit(0); assertEquals(89.0, h.getRank(), 1E-6); assertEquals("doc1", h.getDocId()); + assertFalse(h.getMatchFeatures().isPresent()); h = msg.getResult().getHit(1); assertEquals(109.0, h.getRank(), 1E-6); assertEquals("doc17", h.getDocId()); + assertFalse(h.getMatchFeatures().isPresent()); routable = deserialize("QueryResultMessage-3", DocumentProtocol.MESSAGE_QUERYRESULT, Language.CPP); assertTrue(routable instanceof QueryResultMessage); @@ -659,9 +663,11 @@ public class Messages60TestCase extends MessagesTestBase { h = msg.getResult().getHit(0); assertEquals(109.0, h.getRank(), 1E-6); assertEquals("doc17", h.getDocId()); + assertFalse(h.getMatchFeatures().isPresent()); h = msg.getResult().getHit(1); assertEquals(89.0, h.getRank(), 1E-6); assertEquals("doc1", h.getDocId()); + assertFalse(h.getMatchFeatures().isPresent()); routable = deserialize("QueryResultMessage-4", DocumentProtocol.MESSAGE_QUERYRESULT, Language.CPP); assertTrue(routable instanceof QueryResultMessage); @@ -673,32 +679,55 @@ public class Messages60TestCase extends MessagesTestBase { assertEquals(89.0, h.getRank(), 1E-6); assertEquals("doc1", h.getDocId()); byte[] b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); - assertEquals(9, b.length); - byte[] e = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '2' }; - for (int i = 0; i < b.length; i++) { - assertEquals(e[i], b[i]); - } + assertEqualsData(new byte[] { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '2' }, b); + h = msg.getResult().getHit(1); assertTrue(h instanceof SearchResult.HitWithSortBlob); assertEquals(109.0, h.getRank(), 1E-6); assertEquals("doc17", h.getDocId()); b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); - assertEquals(9, b.length); - byte[] d = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '1' }; - for (int i = 0; i < b.length; i++) { - assertEquals(d[i], b[i]); - } + assertEqualsData(new byte[] { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '1' }, b); + h = msg.getResult().getHit(2); assertTrue(h instanceof SearchResult.HitWithSortBlob); assertEquals(90.0, h.getRank(), 1E-6); assertEquals("doc18", h.getDocId()); b = ((SearchResult.HitWithSortBlob)h).getSortBlob(); - assertEquals(9, b.length); - byte[] c = { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '3' }; - for (int i = 0; i < b.length; i++) { - assertEquals(c[i], b[i]); + assertEqualsData(new byte[] { 's', 'o', 'r', 't', 'd', 'a', 't', 'a', '3' }, b); + } + + void assertEqualsData(byte[] exp, byte[] act) { + assertEquals(exp.length, act.length); + for (int i = 0; i < exp.length; ++i) { + assertEquals(exp[i], act[i]); } } + + void test_result_with_match_features() { + Routable routable = deserialize("QueryResultMessage-6", DocumentProtocol.MESSAGE_QUERYRESULT, Language.CPP); + assertTrue(routable instanceof QueryResultMessage); + + var msg = (QueryResultMessage)routable; + assertEquals(2, msg.getResult().getHitCount()); + + var h = msg.getResult().getHit(0); + assertTrue(h instanceof SearchResult.Hit); + assertEquals(7.0, h.getRank(), 1E-6); + assertEquals("doc2", h.getDocId()); + assertTrue(h.getMatchFeatures().isPresent()); + var mf = h.getMatchFeatures().get(); + assertEquals(12.0, mf.field("foo").asDouble(), 1E-6); + assertEqualsData(new byte[] { 'T', 'h', 'e', 'r', 'e' }, mf.field("bar").asData()); + + h = msg.getResult().getHit(1); + assertTrue(h instanceof SearchResult.Hit); + assertEquals(5.0, h.getRank(), 1E-6); + assertEquals("doc1", h.getDocId()); + assertTrue(h.getMatchFeatures().isPresent()); + mf = h.getMatchFeatures().get(); + assertEquals(1.0, mf.field("foo").asDouble(), 1E-6); + assertEqualsData(new byte[] { 'H', 'i' }, mf.field("bar").asData()); + } } public class testGetBucketListReply implements RunnableTest { diff --git a/documentapi/src/tests/messages/messages60test.cpp b/documentapi/src/tests/messages/messages60test.cpp index 89ab20373e7..58295ae2395 100644 --- a/documentapi/src/tests/messages/messages60test.cpp +++ b/documentapi/src/tests/messages/messages60test.cpp @@ -687,7 +687,7 @@ Messages60Test::testQueryResultMessage() sr3.set_match_features(FeatureValues(mf)); sr3.sort(); - EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 123u, serialize("QueryResultMessage-6", qrm3)); + EXPECT_EQUAL(MESSAGE_BASE_LENGTH + 125u, serialize("QueryResultMessage-6", qrm3)); routable = deserialize("QueryResultMessage-6", DocumentProtocol::MESSAGE_QUERYRESULT, LANG_CPP); if (!EXPECT_TRUE(routable)) { return false; @@ -709,6 +709,10 @@ Messages60Test::testQueryResultMessage() EXPECT_EQUAL(2u, mfv.size()); EXPECT_EQUAL(1.0, mfv[0].as_double()); EXPECT_EQUAL("Hi", mfv[1].as_data().make_string()); + const auto& mf_names = dr->get_match_features().names; + EXPECT_EQUAL(2u, mf_names.size()); + EXPECT_EQUAL("foo", mf_names[0]); + EXPECT_EQUAL("bar", mf_names[1]); return true; } diff --git a/documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-6.dat b/documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-6.dat Binary files differindex 229441aa9ba..efe7f8546a9 100644 --- a/documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-6.dat +++ b/documentapi/test/crosslanguagefiles/6.221-cpp-QueryResultMessage-6.dat diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 08a3285c9b4..4d1a3c7b19d 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -216,7 +216,7 @@ public class Flags { public static final UnboundDoubleFlag CONTAINER_SHUTDOWN_TIMEOUT = defineDoubleFlag( "container-shutdown-timeout", 50.0, - List.of("baldersheim"), "2021-09-25", "2023-05-01", + List.of("baldersheim"), "2021-09-25", "2023-12-31", "Timeout for shutdown of a jdisc container", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); @@ -224,13 +224,13 @@ public class Flags { // TODO: Move to a permanent flag public static final UnboundListFlag<String> ALLOWED_ATHENZ_PROXY_IDENTITIES = defineListFlag( "allowed-athenz-proxy-identities", List.of(), String.class, - List.of("bjorncs", "tokle"), "2021-02-10", "2023-05-01", + List.of("bjorncs", "tokle"), "2021-02-10", "2023-08-01", "Allowed Athenz proxy identities", "takes effect at redeployment"); public static final UnboundIntFlag MAX_ACTIVATION_INHIBITED_OUT_OF_SYNC_GROUPS = defineIntFlag( "max-activation-inhibited-out-of-sync-groups", 0, - List.of("vekterli"), "2021-02-19", "2023-05-01", + List.of("vekterli"), "2021-02-19", "2023-09-01", "Allows replicas in up to N content groups to not be activated " + "for query visibility if they are out of sync with a majority of other replicas", "Takes effect at redeployment", @@ -238,7 +238,7 @@ public class Flags { public static final UnboundDoubleFlag MIN_NODE_RATIO_PER_GROUP = defineDoubleFlag( "min-node-ratio-per-group", 0.0, - List.of("geirst", "vekterli"), "2021-07-16", "2023-05-01", + List.of("geirst", "vekterli"), "2021-07-16", "2023-09-01", "Minimum ratio of nodes that have to be available (i.e. not Down) in any hierarchic content cluster group for the group to be Up", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); @@ -302,7 +302,7 @@ public class Flags { public static final UnboundBooleanFlag ENABLE_PROXY_PROTOCOL_MIXED_MODE = defineFeatureFlag( "enable-proxy-protocol-mixed-mode", true, - List.of("tokle"), "2022-05-09", "2023-04-30", + List.of("tokle"), "2022-05-09", "2023-08-01", "Enable or disable proxy protocol mixed mode", "Takes effect on redeployment", APPLICATION_ID); @@ -331,7 +331,7 @@ public class Flags { public static final UnboundBooleanFlag RESTRICT_DATA_PLANE_BINDINGS = defineFeatureFlag( "restrict-data-plane-bindings", false, - List.of("mortent"), "2022-09-08", "2023-05-01", + List.of("mortent"), "2022-09-08", "2023-08-01", "Use restricted data plane bindings", "Takes effect at redeployment", APPLICATION_ID); @@ -352,7 +352,7 @@ public class Flags { public static final UnboundStringFlag CORE_ENCRYPTION_PUBLIC_KEY_ID = defineStringFlag( "core-encryption-public-key-id", "", - List.of("vekterli"), "2022-11-03", "2023-05-01", + List.of("vekterli"), "2022-11-03", "2023-10-01", "Specifies which public key to use for core dump encryption.", "Takes effect on the next tick.", ZONE_ID, NODE_TYPE, HOSTNAME); @@ -366,18 +366,11 @@ public class Flags { public static final UnboundBooleanFlag VESPA_ATHENZ_PROVIDER = defineFeatureFlag( "vespa-athenz-provider", false, - List.of("mortent"), "2023-02-22", "2023-05-01", + List.of("mortent"), "2023-02-22", "2023-08-01", "Enable athenz provider in public systems", "Takes effect on next config server container start", ZONE_ID); - public static final UnboundLongFlag ZOOKEEPER_BARRIER_WAIT_FOR_ALL_TIMEOUT = defineLongFlag( - "zookeeper-barrier-wait-for-all-timeout", 5, - List.of("hmusum"), "2023-03-28", "2023-05-28", - "Time to wait for all barrier members after getting response from quorum number of member", - "Takes effect on next config server container start", - ZONE_ID); - public static final UnboundBooleanFlag NODE_ADMIN_TENANT_SERVICE_REGISTRY = defineFeatureFlag( "node-admin-tenant-service-registry", false, List.of("olaa"), "2023-04-12", "2023-06-12", @@ -398,7 +391,7 @@ public class Flags { ZONE_ID, APPLICATION_ID); public static final UnboundBooleanFlag FAIL_DEPLOYMENT_ON_MISSING_CERTIFICATE_FILE = defineFeatureFlag( - "fail-on-missing-certificate-file", false, List.of("hmusum"), "2023-04-21", "2023-05-21", + "fail-on-missing-certificate-file", true, List.of("hmusum"), "2023-04-21", "2023-05-21", "Whether to fail in controller when a submitted application package has no certificate files", "Takes effect at redeployment", ZONE_ID); @@ -415,6 +408,12 @@ public class Flags { "Takes effect on application deployment", APPLICATION_ID); + public static final UnboundBooleanFlag USE_VESPA_ALMA_LINUX_X86_64_AMI = defineFeatureFlag( + "use-vespa-alma-linux-x86_64-ami", false, List.of("hmusum"), "2023-05-04", "2023-07-01", + "Whether to use VESPA-ALMALINUX-8-* AMI for x86_64 architecture", + "Takes effect when provisioning new AWS hosts", + APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/hosted-tenant-base/pom.xml b/hosted-tenant-base/pom.xml index bce3a7dc703..289854173c3 100644 --- a/hosted-tenant-base/pom.xml +++ b/hosted-tenant-base/pom.xml @@ -37,6 +37,7 @@ <maven-compiler-plugin.version>3.10.1</maven-compiler-plugin.version> <maven-surefire-plugin.version>2.22.2</maven-surefire-plugin.version> <maven-dependency-plugin.version>3.5.0</maven-dependency-plugin.version> + <jackson2.version>2.15.0</jackson2.version> <!-- NOTE: this must not be overriden by users, and must be in sync with junit version specified in 'tenant-cd-api' --> <vespa.junit.version>5.8.1</vespa.junit.version> <test.categories>!integration</test.categories> diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ScheduledQueue.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ScheduledQueue.java index 09483a7c7e5..457864929dc 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ScheduledQueue.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ScheduledQueue.java @@ -10,11 +10,10 @@ import java.util.Queue; */ class ScheduledQueue { - public static final int MILLIS_PER_SLOT = 100; - public static final int NUM_SLOTS = 512; - public static final int NUM_SLOTS_UNDILATED = 3; - public static final int SLOT_MASK = 511; // bitmask to modulo NUM_SLOTS - public static final int ITER_SHIFT = 9; // number of bits to shift off SLOT_MASK + public static final int MILLIS_PER_SLOT = 2; + public static final int NUM_SLOTS = 1024; + public static final int SLOT_MASK = NUM_SLOTS - 1; // bitmask to modulo NUM_SLOTS + public static final int ITER_SHIFT = Integer.numberOfTrailingZeros(NUM_SLOTS); // number of bits to shift off SLOT_MASK private final Entry[] slots = new Entry[NUM_SLOTS + 1]; private final int[] counts = new int[NUM_SLOTS + 1]; @@ -35,17 +34,16 @@ class ScheduledQueue { if (slots[NUM_SLOTS] == null && currentTimeMillis < nextTick) { return; } - int queueSize = queueSize() + out.size(); drainTo(NUM_SLOTS, 0, out); - for (int i = 0; currentTimeMillis >= nextTick && (queueSize > out.size()); i++, nextTick += MILLIS_PER_SLOT) { - if (i < NUM_SLOTS_UNDILATED) { - if (++currSlot >= NUM_SLOTS) { - currSlot = 0; - currIter++; - } - drainTo(currSlot, currIter, out); + while (currentTimeMillis >= nextTick) { + if (++currSlot >= NUM_SLOTS) { + currSlot = 0; + currIter++; } + drainTo(currSlot, currIter, out); + nextTick += MILLIS_PER_SLOT; } + } private void drainTo(int slot, int iter, Queue<Object> out) { diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/TimeoutManagerImpl.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/TimeoutManagerImpl.java index 7a3898b2946..4e651cb9013 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/TimeoutManagerImpl.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/TimeoutManagerImpl.java @@ -2,6 +2,7 @@ package com.yahoo.jdisc.core; import com.google.inject.Inject; +import com.yahoo.concurrent.SystemTimer; import com.yahoo.jdisc.Request; import com.yahoo.jdisc.ResourceReference; import com.yahoo.jdisc.Response; @@ -14,6 +15,7 @@ import com.yahoo.jdisc.handler.RequestHandler; import com.yahoo.jdisc.handler.ResponseHandler; import java.nio.ByteBuffer; +import java.time.Duration; import java.util.LinkedList; import java.util.Queue; import java.util.concurrent.ThreadFactory; @@ -60,7 +62,7 @@ public class TimeoutManagerImpl { return new ManagedRequestHandler(handler); } - synchronized int queueSize() { return scheduler.queueSize(); } + int queueSize() { return scheduler.queueSize(); } Timer timer() { return timer; @@ -91,7 +93,7 @@ public class TimeoutManagerImpl { private class ManagerTask implements Runnable { - boolean oneMoreCheck(int timeoutMS) { + boolean oneMoreCheck(long timeoutMS) { synchronized (done) { if (!done.get()) { try { @@ -106,7 +108,9 @@ public class TimeoutManagerImpl { @Override public void run() { - while (oneMoreCheck(ScheduledQueue.MILLIS_PER_SLOT)) { + Duration desiredTimeout = Duration.ofMillis(ScheduledQueue.MILLIS_PER_SLOT); + Duration actualTimeout = SystemTimer.adjustTimeoutByDetectedHz(desiredTimeout); + while (oneMoreCheck(actualTimeout.toMillis())) { checkTasks(timer.currentTimeMillis()); } } diff --git a/jdisc_core/src/test/java/com/yahoo/jdisc/core/ScheduledQueueTestCase.java b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ScheduledQueueTestCase.java index cd963caa8d2..b5eef973d5b 100644 --- a/jdisc_core/src/test/java/com/yahoo/jdisc/core/ScheduledQueueTestCase.java +++ b/jdisc_core/src/test/java/com/yahoo/jdisc/core/ScheduledQueueTestCase.java @@ -5,12 +5,10 @@ import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.LinkedList; -import java.util.List; import java.util.Queue; import static com.yahoo.jdisc.core.ScheduledQueue.MILLIS_PER_SLOT; import static com.yahoo.jdisc.core.ScheduledQueue.NUM_SLOTS; -import static com.yahoo.jdisc.core.ScheduledQueue.NUM_SLOTS_UNDILATED; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -95,7 +93,7 @@ public class ScheduledQueueTestCase { @Test void requireThatEntriesDoNotExpireMoreThanOnce() { ScheduledQueue queue = new ScheduledQueue(0); - Object foo = scheduleAt(queue, NUM_SLOTS * MILLIS_PER_SLOT + 50); + Object foo = scheduleAt(queue, NUM_SLOTS * MILLIS_PER_SLOT + MILLIS_PER_SLOT/2); long now = 0; for (int i = 0; i < NUM_SLOTS; ++i, now += MILLIS_PER_SLOT) { @@ -115,25 +113,6 @@ public class ScheduledQueueTestCase { assertDrainTo(queue, 0, foo); } - @Test - void requireThatDrainToPerformsTimeDilationWhenOverloaded() { - ScheduledQueue queue = new ScheduledQueue(0); - List<Object> payloads = new LinkedList<>(); - for (int i = 1; i <= NUM_SLOTS_UNDILATED + 1; ++i) { - payloads.add(scheduleAt(queue, i * MILLIS_PER_SLOT)); - } - - Queue<Object> expired = new LinkedList<>(); - long currentTimeMillis = payloads.size() * MILLIS_PER_SLOT; - queue.drainTo(currentTimeMillis, expired); - assertEquals(NUM_SLOTS_UNDILATED, expired.size()); - - expired = new LinkedList<>(); - currentTimeMillis += MILLIS_PER_SLOT; - queue.drainTo(currentTimeMillis, expired); - assertEquals(1, expired.size()); - } - private static Object scheduleAt(ScheduledQueue queue, long expireAtMillis) { Object obj = new Object(); queue.newEntry(obj).scheduleAt(expireAtMillis); diff --git a/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp b/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp index 849c8311bd0..7a9ef50ce20 100644 --- a/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp +++ b/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.cpp @@ -56,7 +56,15 @@ void CfHandler::doConfigure() { vespalib::string CfHandler::clientCertFile() const { static const vespalib::string certDir = "/var/lib/sia/certs/"; if (_lastConfig && !_lastConfig->role.empty()) { - return certDir + _lastConfig->role + ".pem"; + return certDir + _lastConfig->role + ".cert.pem"; + } + return ""; +} + +vespalib::string CfHandler::clientKeyFile() const { + static const vespalib::string certDir = "/var/lib/sia/keys/"; + if (_lastConfig && !_lastConfig->role.empty()) { + return certDir + _lastConfig->role + ".key.pem"; } return ""; } diff --git a/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.h b/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.h index ceaa0db1011..beca68b52ec 100644 --- a/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.h +++ b/logforwarder/src/apps/vespa-logforwarder-start/cf-handler.h @@ -19,6 +19,7 @@ public: CfHandler(); virtual ~CfHandler(); vespalib::string clientCertFile() const; + vespalib::string clientKeyFile() const; void start(const char *configId); void check(); diff --git a/logforwarder/src/apps/vespa-logforwarder-start/splunk-starter.cpp b/logforwarder/src/apps/vespa-logforwarder-start/splunk-starter.cpp index 9c09a426ea7..23c2565b0af 100644 --- a/logforwarder/src/apps/vespa-logforwarder-start/splunk-starter.cpp +++ b/logforwarder/src/apps/vespa-logforwarder-start/splunk-starter.cpp @@ -36,6 +36,17 @@ cfFilePath(const vespalib::string &parent, const vespalib::string &filename) { return path + "/" + filename; } +void appendFile(FILE *target, const vespalib::string &filename) { + FILE *fp = fopen(filename.c_str(), "r"); + if (fp != NULL) { + int c; + while (EOF != (c = fgetc(fp))) { + fputc(c, target); + } + fclose(fp); + } +} + } // namespace <unnamed> void SplunkStarter::gotConfig(const LogforwarderConfig& config) { @@ -82,13 +93,23 @@ void SplunkStarter::gotConfig(const LogforwarderConfig& config) { } } vespalib::string clientCert = clientCertFile(); - if (! clientCert.empty()) { + vespalib::string clientKey = clientKeyFile(); + if (!clientCert.empty() && !clientKey.empty()) { + vespalib::string certPath = cfFilePath(config.splunkHome, "clientcert.pem"); + tmpPath = certPath + ".new"; + fp = fopen(tmpPath.c_str(), "w"); + appendFile(fp, clientCert); + appendFile(fp, clientKey); + appendFile(fp, "/etc/ssl/certs/ca-bundle.crt"); + fclose(fp); + rename(tmpPath.c_str(), certPath.c_str()); + path = cfFilePath(config.splunkHome, "outputs.conf"); tmpPath = path + ".new"; fp = fopen(tmpPath.c_str(), "w"); if (fp != NULL) { fprintf(fp, "[tcpout]\n"); - fprintf(fp, "clientCert = %s\n", clientCert.c_str()); + fprintf(fp, "clientCert = %s\n", certPath.c_str()); fclose(fp); rename(tmpPath.c_str(), path.c_str()); } diff --git a/maven-plugins/allowed-maven-dependencies.txt b/maven-plugins/allowed-maven-dependencies.txt index 5272dd21a31..29c5fa69429 100644 --- a/maven-plugins/allowed-maven-dependencies.txt +++ b/maven-plugins/allowed-maven-dependencies.txt @@ -3,9 +3,9 @@ #[non-test] # Contains dependencies that are not used exclusively in 'test' scope aopalliance:aopalliance:1.0 -com.fasterxml.jackson.core:jackson-annotations:2.14.2 -com.fasterxml.jackson.core:jackson-core:2.14.2 -com.fasterxml.jackson.core:jackson-databind:2.14.2 +com.fasterxml.jackson.core:jackson-annotations:2.15.0 +com.fasterxml.jackson.core:jackson-core:2.15.0 +com.fasterxml.jackson.core:jackson-databind:2.15.0 com.google.errorprone:error_prone_annotations:2.18.0 com.google.guava:failureaccess:1.0.1 com.google.guava:guava:27.1-jre diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Cgroup.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Cgroup.java new file mode 100644 index 00000000000..e40e3c6c003 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Cgroup.java @@ -0,0 +1,163 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.cgroup; + +import com.yahoo.vespa.defaults.Defaults; +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; +import com.yahoo.vespa.hosted.node.admin.container.ContainerId; +import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; + +import java.nio.file.FileSystem; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Consumer; +import java.util.logging.Logger; + +/** + * Represents a cgroup in the control group v2 hierarchy, see + * <a href="https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html">Control Group v2</a>. + * + * @author hakonhall + */ +public class Cgroup { + private static final Logger logger = Logger.getLogger(Cgroup.class.getName()); + + private static Map<String, Consumer<UnixPath>> cgroupDirectoryCallbacks = new HashMap<>(); + + private final Path root; + private final Path relativePath; + + public static Cgroup root(FileSystem fileSystem) { + return new Cgroup(fileSystem.getPath("/sys/fs/cgroup"), fileSystem.getPath("")); + } + + private Cgroup(Path root, Path relativePath) { + this.root = root.normalize(); + this.relativePath = this.root.relativize(this.root.resolve(relativePath).normalize()); + if (this.relativePath.toString().equals("..") || this.relativePath.toString().startsWith("../")) { + throw new IllegalArgumentException("Invalid cgroup relative path: " + relativePath); + } + } + + /** Whether this cgroup actually exists in the kernel / on the file system. */ + public boolean exists() { return unixPath().resolve("cgroup.controllers").exists(); } + + /** Creates this cgroup if it does not already exist, and return this. */ + public Cgroup create() { + if (unixPath().createDirectory()) { + // cgroup automatically creates various files in a newly created cgroup directory. A unit test may simulate + // this by registering consumers before the test is run. + Consumer<UnixPath> callback = cgroupDirectoryCallbacks.get(relativePath.toString()); + if (callback != null) + callback.accept(unixPath()); + } + return this; + } + + /** Whether v2 cgroup is enabled on this host. */ + public boolean v2CgroupIsEnabled() { return resolveRoot().exists(); } + + /** + * Resolve the given path against the path of this cgroup, and return the resulting cgroup. + * If the given path is absolute, it is resolved against the root of the cgroup hierarchy. + */ + public Cgroup resolve(String path) { + Path effectivePath = fileSystem().getPath(path); + if (effectivePath.isAbsolute()) { + return new Cgroup(root, fileSystem().getPath("/").relativize(effectivePath)); + } else { + return new Cgroup(root, relativePath.resolve(path)); + } + } + + /** Returns the root cgroup, possibly this. */ + public Cgroup resolveRoot() { return isRoot() ? this : new Cgroup(root, fileSystem().getPath("")); } + + /** Returns the cgroup of a system service assuming this is the root, e.g. vespa-host-admin -> system.slice/vespa-host-admin.service. */ + public Cgroup resolveSystemService(String name) { return resolve("system.slice").resolve(serviceNameOf(name)); } + + /** Returns the root cgroup of the given Podman container. */ + public Cgroup resolveContainer(ContainerId containerId) { return resolve("/machine.slice/libpod-" + containerId + ".scope/container"); } + + /** Returns the root cgroup of the container, or otherwise the root cgroup. */ + public Cgroup resolveRoot(Optional<ContainerId> containerId) { return containerId.map(this::resolveContainer).orElseGet(this::resolveRoot); } + + /** Returns the absolute path to this cgroup. */ + public Path path() { return root.resolve(relativePath); } + + /** Returns the UnixPath of {@link #path()}. */ + public UnixPath unixPath() { return new UnixPath(path()); } + + public String read(String filename) { + return unixPath().resolve(filename).readUtf8File(); + } + + public Optional<String> readIfExists(String filename) { + return unixPath().resolve(filename).readUtf8FileIfExists().map(String::strip); + } + + public List<String> readLines(String filename) { + return unixPath().resolve(filename).readUtf8File().lines().toList(); + } + + public Optional<Integer> readIntIfExists(String filename) { + return unixPath().resolve(filename).readUtf8FileIfExists().map(String::strip).map(Integer::parseInt); + } + + public Size readSize(String filename) { return Size.from(read(filename).stripTrailing()); } + + public boolean convergeFileContent(TaskContext context, String filename, String content, boolean apply) { + UnixPath path = unixPath().resolve(filename); + String currentContent = path.readUtf8File(); + if (ensureSuffixNewline(currentContent).equals(ensureSuffixNewline(content))) return false; + + if (apply) { + context.recordSystemModification(logger, "Updating " + path + " from '" + currentContent.stripTrailing() + + "' to '" + content.stripTrailing() + "'"); + path.writeUtf8File(content); + } + return true; + } + + /** The kernel appears to append a newline if none exist, when writing to files in cgroupfs. */ + private static String ensureSuffixNewline(String content) { + return content.endsWith("\n") ? content : content + "\n"; + } + + /** Returns an instance representing core interface files (cgroup.* files). */ + public CgroupCore core() { return new CgroupCore(this); } + + /** Returns the CPU controller of this cgroup (cpu.* files). */ + public CpuController cpu() { return new CpuController(this); } + + /** Returns the memory controller of this cgroup (memory.* files). */ + public MemoryController memory() { return new MemoryController(this); } + + /** + * Wraps {@code command} to ensure it is executed in this cgroup. + * + * <p>WARNING: This method must be called only after vespa-cgexec has been installed.</p> + */ + public String[] wrapCommandForExecutionInCgroup(String... command) { + String[] fullCommand = new String[3 + command.length]; + fullCommand[0] = Defaults.getDefaults().vespaHome() + "/bin/vespa-cgexec"; + fullCommand[1] = "-g"; + fullCommand[2] = relativePath.toString(); + System.arraycopy(command, 0, fullCommand, 3, command.length); + return fullCommand; + } + + public static void unitTesting_atCgroupCreation(String relativePath, Consumer<UnixPath> callback) { + cgroupDirectoryCallbacks.put(relativePath, callback); + } + + private boolean isRoot() { return relativePath.toString().isEmpty(); } + + private static String serviceNameOf(String name) { + return name.indexOf('.') == -1 ? name + ".service" : name; + } + + private FileSystem fileSystem() { return root.getFileSystem(); } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupCore.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupCore.java new file mode 100644 index 00000000000..68f27549e1b --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupCore.java @@ -0,0 +1,34 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.cgroup; + +import java.util.List; + +/** + * Utility methods for accessing the cgroup core interface files, i.e. all cgroup.* files. + * + * @author hakonhall + */ +public class CgroupCore { + private final Cgroup cgroup; + + CgroupCore(Cgroup cgroup) { this.cgroup = cgroup; } + + public List<Integer> getPidsInCgroup() { + return cgroup.readLines("cgroup.procs") + .stream() + .map(Integer::parseInt) + .toList(); + } + + /** Whether the given PID is a member of this cgroup. */ + public boolean isMember(int pid) { + return getPidsInCgroup().contains(pid); + } + + /** Move the given PID to this cgroup, but return false if it was already a member. */ + public boolean addMember(int pid) { + if (isMember(pid)) return false; + cgroup.unixPath().resolve("cgroup.procs").writeUtf8File(Integer.toString(pid)); + return true; + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/CpuController.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/CpuController.java new file mode 100644 index 00000000000..c273a6237b4 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/CpuController.java @@ -0,0 +1,111 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.cgroup; + +import com.yahoo.collections.Pair; +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; + +import java.util.Arrays; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import static java.lang.Integer.parseInt; + +/** + * Represents a cgroup v2 CPU controller, i.e. all cpu.* files. + * + * @author hakonhall + */ +public class CpuController { + private final Cgroup cgroup; + + CpuController(Cgroup cgroup) { + this.cgroup = cgroup; + } + + /** + * The maximum bandwidth limit of the format "QUOTA PERIOD", which indicates that the cgroup may consume + * up to QUOTA in each PERIOD duration. A quota of "max" indicates no limit. + */ + public record Max(Size quota, int period) { + public String toFileContent() { return quota + " " + period + '\n'; } + } + + /** + * Returns the maximum CPU usage, or empty if cgroup is not found. + * + * @see Max + */ + public Optional<Max> readMax() { + return cgroup.readIfExists("cpu.max") + .map(content -> { + String[] parts = content.strip().split(" "); + return new Max(Size.from(parts[0]), parseInt(parts[1])); + }); + } + + /** + * Update CPU quota and period for the given container ID. Set quota to -1 value for unlimited. + * + * @see #readMax() + * @see Max + */ + public boolean updateMax(TaskContext context, int quota, int period) { + Max max = new Max(quota < 0 ? Size.max() : Size.from(quota), period); + return cgroup.convergeFileContent(context, "cpu.max", max.toFileContent(), true); + } + + /** @return The weight in the range [1, 10000], or empty if not found. */ + private Optional<Integer> readWeight() { + return cgroup.readIntIfExists("cpu.weight"); + } + + /** @return The number of shares allocated to this cgroup for purposes of CPU time scheduling, or empty if not found. */ + public Optional<Integer> readShares() { + return readWeight().map(CpuController::weightToShares); + } + + public boolean updateShares(TaskContext context, int shares) { + return cgroup.convergeFileContent(context, "cpu.weight", sharesToWeight(shares) + "\n", true); + } + + // Must be same as in crun: https://github.com/containers/crun/blob/72c6e60ade0e4716fe2d8353f0d97d72cc8d1510/src/libcrun/cgroup.c#L3061 + // TODO: Migrate to weights + public static int sharesToWeight(int shares) { return (int) (1 + ((shares - 2L) * 9999) / 262142); } + public static int weightToShares(int weight) { return (int) (2 + ((weight - 1L) * 262142) / 9999); } + + public enum StatField { + TOTAL_USAGE_USEC("usage_usec"), + USER_USAGE_USEC("user_usec"), + SYSTEM_USAGE_USEC("system_usec"), + TOTAL_PERIODS("nr_periods"), + THROTTLED_PERIODS("nr_throttled"), + THROTTLED_TIME_USEC("throttled_usec"); + + private final String name; + + StatField(String name) { + this.name = name; + } + + long parseValue(String value) { + return Long.parseLong(value); + } + + static Optional<StatField> fromField(String fieldName) { + return Arrays.stream(values()) + .filter(field -> fieldName.equals(field.name)) + .findFirst(); + } + } + + public Map<StatField, Long> readStats() { + return cgroup.readLines("cpu.stat") + .stream() + .map(line -> line.split("\\s+")) + .filter(parts -> parts.length == 2) + .flatMap(parts -> StatField.fromField(parts[0]).stream().map(field -> new Pair<>(field, field.parseValue(parts[1])))) + .collect(Collectors.toMap(Pair::getFirst, Pair::getSecond)); + } + +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/MemoryController.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/MemoryController.java new file mode 100644 index 00000000000..840cd025917 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/MemoryController.java @@ -0,0 +1,48 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.cgroup; + +import java.util.List; +import java.util.Optional; + +/** + * Represents a cgroup v2 memory controller, i.e. all memory.* files. + * + * @author hakonhall + */ +public class MemoryController { + private final Cgroup cgroup; + + MemoryController(Cgroup cgroup) { + this.cgroup = cgroup; + } + + /** @return Maximum amount of memory that can be used by the cgroup and its descendants. */ + public Size readMax() { + return cgroup.readSize("memory.max"); + } + + /** @return The total amount of memory currently being used by the cgroup and its descendants, in bytes. */ + public Size readCurrent() { + return cgroup.readSize("memory.current"); + } + + /** @return The total amount of memory currently being used by the cgroup and its descendants, in bytes. */ + public Optional<Size> readCurrentIfExists() { + return cgroup.readIfExists("memory.current").map(Size::from); + } + + /** @return Number of bytes used to cache filesystem data, including tmpfs and shared memory. */ + public Size readFileSystemCache() { + return Size.from(readField(cgroup.readLines("memory.stat"), "file")); + } + + private static String readField(List<String> lines, String fieldName) { + return lines.stream() + .map(line -> line.split("\\s+")) + .filter(fields -> fields.length == 2) + .filter(fields -> fieldName.equals(fields[0])) + .map(fields -> fields[1]) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("No such field: " + fieldName)); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Size.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Size.java new file mode 100644 index 00000000000..5e6ca7de8bd --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/Size.java @@ -0,0 +1,67 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.cgroup; + +import java.util.Objects; + +/** + * Represents a number of bytes or possibly "max". + * + * @author hakonhall + */ +public class Size { + private static final String MAX = "max"; + + private final boolean max; + private final long value; + + public static Size max() { + return new Size(true, 0); + } + + public static Size from(long value) { + return new Size(false, value); + } + + public static Size from(String value) { + return value.equals(MAX) ? new Size(true, 0) : new Size(false, Long.parseLong(value)); + } + + private Size(boolean max, long value) { + this.max = max; + this.value = value; + } + + public boolean isMax() { + return max; + } + + /** Returns the value, i.e. the number of "bytes" if applicable. Throws if this is max. */ + public long value() { + if (max) throw new IllegalStateException("Value is max"); + return value; + } + + public String toFileContent() { return toString() + '\n'; } + + @Override + public String toString() { return max ? MAX : Long.toString(value); } + + public boolean isGreaterThan(Size that) { + if (that.max) return false; + if (this.max) return true; + return this.value > that.value; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Size size = (Size) o; + return max == size.max && value == size.value; + } + + @Override + public int hashCode() { + return Objects.hash(max, value); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/package-info.java new file mode 100644 index 00000000000..c0a23166ccc --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/cgroup/package-info.java @@ -0,0 +1,9 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +/** + * @author hakonhall + */ +@ExportPackage +package com.yahoo.vespa.hosted.node.admin.cgroup; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2.java deleted file mode 100644 index 3cb34e066ff..00000000000 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2.java +++ /dev/null @@ -1,188 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; - -import com.yahoo.collections.Pair; -import com.yahoo.vespa.defaults.Defaults; -import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; - -import java.io.IOException; -import java.nio.file.FileSystem; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.OptionalInt; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * Read and write interface to the cgroup v2 of a Podman container. - * - * @see <a href="https://www.kernel.org/doc/html/latest/admin-guide/cgroup-v2.html">CGroups V2</a> - * @author freva - */ -public class CGroupV2 { - - private static final Logger logger = Logger.getLogger(CGroupV2.class.getName()); - private static final String MAX = "max"; - - private final Path rootCgroupPath; - - public CGroupV2(FileSystem fileSystem) { - this.rootCgroupPath = fileSystem.getPath("/sys/fs/cgroup"); - } - - /** - * Wraps {@code command} to ensure it is executed in the given cgroup. - * - * <p>WARNING: This method must be called only after vespa-cgexec has been installed.</p> - * - * @param cgroup The cgroup to execute the command in, e.g. /sys/fs/cgroup/system.slice/wireguard.scope. - * @param command The command to execute in the cgroup. - * @see #cgroupRootPath() - * @see #cgroupPath(ContainerId) - */ - public String[] wrapForExecutionIn(Path cgroup, String... command) { - String[] fullCommand = new String[3 + command.length]; - fullCommand[0] = Defaults.getDefaults().vespaHome() + "/bin/vespa-cgexec"; - fullCommand[1] = "-g"; - fullCommand[2] = cgroup.toString(); - System.arraycopy(command, 0, fullCommand, 3, command.length); - return fullCommand; - } - - /** - * Returns quota and period values used for CPU scheduling. This serves as hard cap on CPU usage by allowing - * the CGroupV2 to use up to {@code quota} each {@code period}. If uncapped, quota will be negative. - * - * @param containerId full container ID. - * @return CPU quota and period for the given container. Empty if CGroupV2 for this container is not found. - */ - public Optional<Pair<Integer, Integer>> cpuQuotaPeriod(ContainerId containerId) { - return cpuMaxPath(containerId).readUtf8FileIfExists() - .map(s -> { - String[] parts = s.strip().split(" "); - return new Pair<>(MAX.equals(parts[0]) ? -1 : Integer.parseInt(parts[0]), Integer.parseInt(parts[1])); - }); - } - - /** @return number of shares allocated to this CGroupV2 for purposes of CPU time scheduling, empty if CGroupV2 not found */ - public OptionalInt cpuShares(ContainerId containerId) { - return cpuWeightPath(containerId).readUtf8FileIfExists() - .map(s -> OptionalInt.of(weightToShares(Integer.parseInt(s.strip())))) - .orElseGet(OptionalInt::empty); - } - - /** Update CPU quota and period for the given container ID, set quota to -1 value for unlimited */ - public boolean updateCpuQuotaPeriod(NodeAgentContext context, ContainerId containerId, int cpuQuotaUs, int periodUs) { - String wanted = String.format("%s %d", cpuQuotaUs < 0 ? MAX : cpuQuotaUs, periodUs); - return writeCGroupsValue(context, cpuMaxPath(containerId), wanted); - } - - public boolean updateCpuShares(NodeAgentContext context, ContainerId containerId, int shares) { - return writeCGroupsValue(context, cpuWeightPath(containerId), Integer.toString(sharesToWeight(shares))); - } - - enum CpuStatField { - TOTAL_USAGE_USEC("usage_usec"), - USER_USAGE_USEC("user_usec"), - SYSTEM_USAGE_USEC("system_usec"), - TOTAL_PERIODS("nr_periods"), - THROTTLED_PERIODS("nr_throttled"), - THROTTLED_TIME_USEC("throttled_usec"); - - private final String name; - - CpuStatField(String name) { - this.name = name; - } - - long parseValue(String value) { - return Long.parseLong(value); - } - - static Optional<CpuStatField> fromField(String fieldName) { - return Arrays.stream(values()) - .filter(field -> fieldName.equals(field.name)) - .findFirst(); - } - } - - public Map<CpuStatField, Long> cpuStats(ContainerId containerId) throws IOException { - return Files.readAllLines(cgroupPath(containerId).resolve("cpu.stat")).stream() - .map(line -> line.split("\\s+")) - .filter(parts -> parts.length == 2) - .flatMap(parts -> CpuStatField.fromField(parts[0]).stream().map(field -> new Pair<>(field, field.parseValue(parts[1])))) - .collect(Collectors.toMap(Pair::getFirst, Pair::getSecond)); - } - - /** @return Maximum amount of memory that can be used by the cgroup and its descendants. */ - public long memoryLimitInBytes(ContainerId containerId) throws IOException { - String limit = Files.readString(cgroupPath(containerId).resolve("memory.max")).strip(); - return MAX.equals(limit) ? -1L : Long.parseLong(limit); - } - - /** @return The total amount of memory currently being used by the cgroup and its descendants. */ - public long memoryUsageInBytes(ContainerId containerId) throws IOException { - return parseLong(cgroupPath(containerId).resolve("memory.current")); - } - - /** @return Number of bytes used to cache filesystem data, including tmpfs and shared memory. */ - public long memoryCacheInBytes(ContainerId containerId) throws IOException { - return parseLong(cgroupPath(containerId).resolve("memory.stat"), "file"); - } - - /** Returns the cgroup v2 mount point path (/sys/fs/cgroup). */ - public Path cgroupRootPath() { - return rootCgroupPath; - } - - /** Returns the cgroup directory of the Podman container, and which appears as the root cgroup within the container. */ - public Path cgroupPath(ContainerId containerId) { - // crun path, runc path is without the 'container' directory - return rootCgroupPath.resolve("machine.slice/libpod-" + containerId + ".scope/container"); - } - - private UnixPath cpuMaxPath(ContainerId containerId) { - return new UnixPath(cgroupPath(containerId).resolve("cpu.max")); - } - - private UnixPath cpuWeightPath(ContainerId containerId) { - return new UnixPath(cgroupPath(containerId).resolve("cpu.weight")); - } - - private static boolean writeCGroupsValue(NodeAgentContext context, UnixPath unixPath, String value) { - String currentValue = unixPath.readUtf8File().strip(); - if (currentValue.equals(value)) return false; - - context.recordSystemModification(logger, "Updating " + unixPath + " from " + currentValue + " to " + value); - unixPath.writeUtf8File(value); - return true; - } - - // Must be same as in crun: https://github.com/containers/crun/blob/72c6e60ade0e4716fe2d8353f0d97d72cc8d1510/src/libcrun/cgroup.c#L3061 - static int sharesToWeight(int shares) { return (int) (1 + ((shares - 2L) * 9999) / 262142); } - static int weightToShares(int weight) { return (int) (2 + ((weight - 1L) * 262142) / 9999); } - - static long parseLong(Path path) throws IOException { - return Long.parseLong(Files.readString(path).trim()); - } - - static long parseLong(Path path, String fieldName) throws IOException { - return parseLong(Files.readAllLines(path), fieldName); - } - - static long parseLong(List<String> lines, String fieldName) { - for (String line : lines) { - String[] fields = line.split("\\s+"); - if (fields.length != 2) - throw new IllegalArgumentException("Expected line on the format 'key value', got: '" + line + "'"); - - if (fieldName.equals(fields[0])) return Long.parseLong(fields[1]); - } - throw new IllegalArgumentException("No such field: " + fieldName); - } -} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java index ce2a6bb22ac..264035b86a1 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperations.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.hosted.node.admin.container; import com.yahoo.config.provision.DockerImage; +import com.yahoo.jdisc.Timer; +import com.yahoo.vespa.hosted.node.admin.cgroup.Cgroup; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; import com.yahoo.vespa.hosted.node.admin.container.image.ContainerImageDownloader; import com.yahoo.vespa.hosted.node.admin.container.image.ContainerImagePruner; @@ -12,7 +14,6 @@ import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandLine; import com.yahoo.vespa.hosted.node.admin.task.util.process.CommandResult; import java.nio.file.FileSystem; -import java.time.Clock; import java.time.Duration; import java.util.List; import java.util.Objects; @@ -33,10 +34,10 @@ public class ContainerOperations { private final ContainerImagePruner imagePruner; private final ContainerStatsCollector containerStatsCollector; - public ContainerOperations(ContainerEngine containerEngine, CGroupV2 cgroup, FileSystem fileSystem) { + public ContainerOperations(ContainerEngine containerEngine, Cgroup cgroup, FileSystem fileSystem, Timer timer) { this.containerEngine = Objects.requireNonNull(containerEngine); this.imageDownloader = new ContainerImageDownloader(containerEngine); - this.imagePruner = new ContainerImagePruner(containerEngine, Clock.systemUTC()); + this.imagePruner = new ContainerImagePruner(containerEngine, timer); this.containerStatsCollector = new ContainerStatsCollector(containerEngine, cgroup, fileSystem); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollector.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollector.java index 870809123a9..8244666f9e0 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollector.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollector.java @@ -1,6 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.container; +import com.yahoo.vespa.hosted.node.admin.cgroup.Cgroup; +import com.yahoo.vespa.hosted.node.admin.cgroup.CpuController; +import com.yahoo.vespa.hosted.node.admin.cgroup.Size; +import com.yahoo.vespa.hosted.node.admin.cgroup.MemoryController; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser; @@ -27,18 +31,18 @@ import java.util.stream.Stream; class ContainerStatsCollector { private final ContainerEngine containerEngine; - private final CGroupV2 cgroup; private final FileSystem fileSystem; + private final Cgroup rootCgroup; private final int onlineCpus; - ContainerStatsCollector(ContainerEngine containerEngine, CGroupV2 cgroup, FileSystem fileSystem) { - this(containerEngine, cgroup, fileSystem, Runtime.getRuntime().availableProcessors()); + ContainerStatsCollector(ContainerEngine containerEngine, Cgroup rootCgroup, FileSystem fileSystem) { + this(containerEngine, rootCgroup, fileSystem, Runtime.getRuntime().availableProcessors()); } - ContainerStatsCollector(ContainerEngine containerEngine, CGroupV2 cgroup, FileSystem fileSystem, int onlineCpus) { + ContainerStatsCollector(ContainerEngine containerEngine, Cgroup rootCgroup, FileSystem fileSystem, int onlineCpus) { this.containerEngine = Objects.requireNonNull(containerEngine); - this.cgroup = Objects.requireNonNull(cgroup); this.fileSystem = Objects.requireNonNull(fileSystem); + this.rootCgroup = Objects.requireNonNull(rootCgroup); this.onlineCpus = onlineCpus; } @@ -52,6 +56,10 @@ class ContainerStatsCollector { return Optional.of(new ContainerStats(networkStats, memoryStats, cpuStats, gpuStats)); } catch (NoSuchFileException ignored) { return Optional.empty(); // Container disappeared while we collected stats + } catch (UncheckedIOException e) { + if (e.getCause() != null && e.getCause() instanceof NoSuchFileException) + return Optional.empty(); + throw e; } catch (IOException e) { throw new UncheckedIOException(e); } @@ -83,21 +91,22 @@ class ContainerStatsCollector { } private ContainerStats.CpuStats collectCpuStats(ContainerId containerId) throws IOException { - Map<CGroupV2.CpuStatField, Long> cpuStats = cgroup.cpuStats(containerId); + Map<CpuController.StatField, Long> cpuStats = rootCgroup.resolveContainer(containerId).cpu().readStats(); return new ContainerStats.CpuStats(onlineCpus, systemCpuUsage(), - cpuStats.get(CGroupV2.CpuStatField.TOTAL_USAGE_USEC), - cpuStats.get(CGroupV2.CpuStatField.SYSTEM_USAGE_USEC), - cpuStats.get(CGroupV2.CpuStatField.THROTTLED_TIME_USEC), - cpuStats.get(CGroupV2.CpuStatField.TOTAL_PERIODS), - cpuStats.get(CGroupV2.CpuStatField.THROTTLED_PERIODS)); + cpuStats.get(CpuController.StatField.TOTAL_USAGE_USEC), + cpuStats.get(CpuController.StatField.SYSTEM_USAGE_USEC), + cpuStats.get(CpuController.StatField.THROTTLED_TIME_USEC), + cpuStats.get(CpuController.StatField.TOTAL_PERIODS), + cpuStats.get(CpuController.StatField.THROTTLED_PERIODS)); } private ContainerStats.MemoryStats collectMemoryStats(ContainerId containerId) throws IOException { - long memoryLimitInBytes = cgroup.memoryLimitInBytes(containerId); - long memoryUsageInBytes = cgroup.memoryUsageInBytes(containerId); - long cachedInBytes = cgroup.memoryCacheInBytes(containerId); - return new ContainerStats.MemoryStats(cachedInBytes, memoryUsageInBytes, memoryLimitInBytes); + MemoryController memoryController = rootCgroup.resolveContainer(containerId).memory(); + Size max = memoryController.readMax(); + long memoryUsageInBytes = memoryController.readCurrent().value(); + long cachedInBytes = memoryController.readFileSystemCache().value(); + return new ContainerStats.MemoryStats(cachedInBytes, memoryUsageInBytes, max.isMax() ? -1 : max.value()); } private ContainerStats.NetworkStats collectNetworkStats(String iface, int containerPid) throws IOException { diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePruner.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePruner.java index 8fcd7893f64..372b8522b0c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePruner.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePruner.java @@ -2,11 +2,11 @@ package com.yahoo.vespa.hosted.node.admin.container.image; import com.yahoo.collections.Pair; +import com.yahoo.jdisc.Timer; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; import com.yahoo.vespa.hosted.node.admin.container.ContainerEngine; import com.yahoo.vespa.hosted.node.admin.container.PartialContainer; -import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; @@ -51,14 +51,14 @@ public class ContainerImagePruner { private static final Logger LOG = Logger.getLogger(ContainerImagePruner.class.getName()); - private final Clock clock; + private final Timer timer; private final ContainerEngine containerEngine; private final Map<String, Instant> lastTimeUsedByImageId = new ConcurrentHashMap<>(); - public ContainerImagePruner(ContainerEngine containerEngine, Clock clock) { + public ContainerImagePruner(ContainerEngine containerEngine, Timer timer) { this.containerEngine = Objects.requireNonNull(containerEngine); - this.clock = Objects.requireNonNull(clock); + this.timer = Objects.requireNonNull(timer); } /** @@ -112,7 +112,7 @@ public class ContainerImagePruner { } private Set<String> updateRecentlyUsedImageIds(List<Image> images, List<PartialContainer> containers, Duration minImageAgeToDelete) { - final Instant now = clock.instant(); + final Instant now = timer.currentTime(); // Add any already downloaded image to the list once images.forEach(image -> lastTimeUsedByImageId.putIfAbsent(image.id(), now)); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index f0182ae36e4..9a548fba431 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -5,6 +5,7 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.NodeType; +import com.yahoo.jdisc.Timer; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; import com.yahoo.vespa.hosted.node.admin.container.Container; import com.yahoo.vespa.hosted.node.admin.container.ContainerName; @@ -27,7 +28,6 @@ import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal; import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; -import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.time.ZoneOffset; @@ -55,7 +55,7 @@ public class StorageMaintainer { private final CoredumpHandler coredumpHandler; private final DiskCleanup diskCleanup; private final SyncClient syncClient; - private final Clock clock; + private final Timer timer; private final Path archiveContainerStoragePath; // We cache disk usage to avoid doing expensive disk operations so often @@ -65,12 +65,12 @@ public class StorageMaintainer { .build(); public StorageMaintainer(Terminal terminal, CoredumpHandler coredumpHandler, DiskCleanup diskCleanup, - SyncClient syncClient, Clock clock, Path archiveContainerStoragePath) { + SyncClient syncClient, Timer timer, Path archiveContainerStoragePath) { this.terminal = terminal; this.coredumpHandler = coredumpHandler; this.diskCleanup = diskCleanup; this.syncClient = syncClient; - this.clock = clock; + this.timer = timer; this.archiveContainerStoragePath = archiveContainerStoragePath; } @@ -138,7 +138,7 @@ public class StorageMaintainer { } private List<DiskCleanupRule> createCleanupRules(NodeAgentContext context) { - Instant start = clock.instant(); + Instant start = timer.currentTime(); double oneMonthSeconds = Duration.ofDays(30).getSeconds(); Function<Instant, Double> monthNormalizer = instant -> Duration.between(instant, start).getSeconds() / oneMonthSeconds; List<DiskCleanupRule> rules = new ArrayList<>(); @@ -176,7 +176,7 @@ public class StorageMaintainer { public void archiveNodeStorage(NodeAgentContext context) { ContainerPath logsDirInContainer = context.paths().underVespaHome("logs"); Path containerLogsInArchiveDir = archiveContainerStoragePath - .resolve(context.containerName().asString() + "_" + DATE_TIME_FORMATTER.format(clock.instant()) + logsDirInContainer.pathInContainer()); + .resolve(context.containerName().asString() + "_" + DATE_TIME_FORMATTER.format(timer.currentTime()) + logsDirInContainer.pathInContainer()); // Files.move() does not support moving non-empty directories across providers, move using host paths UnixPath containerLogsOnHost = new UnixPath(logsDirInContainer.pathOnHost()); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java index 8c293fb40c9..af47f00a21e 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.coredump; import com.yahoo.config.provision.DockerImage; +import com.yahoo.jdisc.Timer; import com.yahoo.security.KeyId; import com.yahoo.security.SecretSharedKey; import com.yahoo.vespa.flags.FetchVector; @@ -28,7 +29,6 @@ import java.io.OutputStream; import java.io.UncheckedIOException; import java.nio.file.Files; import java.nio.file.Path; -import java.time.Clock; import java.util.Comparator; import java.util.List; import java.util.Optional; @@ -65,7 +65,7 @@ public class CoredumpHandler { private final String crashPatchInContainer; private final Path doneCoredumpsPath; private final Metrics metrics; - private final Clock clock; + private final Timer timer; private final Supplier<String> coredumpIdSupplier; private final SecretSharedKeySupplier secretSharedKeySupplier; private final StringFlag coreEncryptionPublicKeyIdFlag; @@ -75,23 +75,23 @@ public class CoredumpHandler { * @param doneCoredumpsPath path on host where processed core dumps are stored */ public CoredumpHandler(CoreCollector coreCollector, Cores cores, - String crashPathInContainer, Path doneCoredumpsPath, Metrics metrics, + String crashPathInContainer, Path doneCoredumpsPath, Metrics metrics, Timer timer, SecretSharedKeySupplier secretSharedKeySupplier, FlagSource flagSource) { this(coreCollector, cores, crashPathInContainer, doneCoredumpsPath, - metrics, Clock.systemUTC(), () -> UUID.randomUUID().toString(), secretSharedKeySupplier, + metrics, timer, () -> UUID.randomUUID().toString(), secretSharedKeySupplier, flagSource); } CoredumpHandler(CoreCollector coreCollector, Cores cores, String crashPathInContainer, Path doneCoredumpsPath, Metrics metrics, - Clock clock, Supplier<String> coredumpIdSupplier, + Timer timer, Supplier<String> coredumpIdSupplier, SecretSharedKeySupplier secretSharedKeySupplier, FlagSource flagSource) { this.coreCollector = coreCollector; this.cores = cores; this.crashPatchInContainer = crashPathInContainer; this.doneCoredumpsPath = doneCoredumpsPath; this.metrics = metrics; - this.clock = clock; + this.timer = timer; this.coredumpIdSupplier = coredumpIdSupplier; this.secretSharedKeySupplier = secretSharedKeySupplier; this.coreEncryptionPublicKeyIdFlag = Flags.CORE_ENCRYPTION_PUBLIC_KEY_ID.bindTo(flagSource); @@ -276,7 +276,7 @@ public class CoredumpHandler { private boolean isReadyForProcessing(FileFinder.FileAttributes fileAttributes) { // Wait at least a minute until we start processing a core/heap dump to ensure that // kernel/JVM has finished writing it - return clock.instant().minusSeconds(60).isAfter(fileAttributes.lastModifiedTime()); + return timer.currentTime().minusSeconds(60).isAfter(fileAttributes.lastModifiedTime()); } void processAndReportSingleCoreDump(NodeAgentContext context, ContainerPath coreDumpDirectory, diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java index 57a6ceb68aa..6119c77242c 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/identity/AthenzCredentialsMaintainer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.identity; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.jdisc.Timer; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; import com.yahoo.security.Pkcs10Csr; @@ -45,7 +46,6 @@ import java.nio.file.Path; import java.security.KeyPair; import java.security.PrivateKey; import java.security.cert.X509Certificate; -import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.List; @@ -76,7 +76,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { private final URI ztsEndpoint; private final Path ztsTrustStorePath; - private final Clock clock; + private final Timer timer; private final String certificateDnsSuffix; private final ServiceIdentityProvider hostIdentityProvider; private final IdentityDocumentClient identityDocumentClient; @@ -92,7 +92,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { String certificateDnsSuffix, ServiceIdentityProvider hostIdentityProvider, FlagSource flagSource, - Clock clock) { + Timer timer) { this.ztsEndpoint = ztsEndpoint; this.ztsTrustStorePath = ztsTrustStorePath; this.certificateDnsSuffix = certificateDnsSuffix; @@ -101,7 +101,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { configServerInfo.getLoadBalancerEndpoint(), hostIdentityProvider, new AthenzIdentityVerifier(Set.of(configServerInfo.getConfigServerIdentity()))); - this.clock = clock; + this.timer = timer; this.tenantServiceIdentityFlag = Flags.NODE_ADMIN_TENANT_SERVICE_REGISTRY.bindTo(flagSource); this.useNewIdentityDocumentLayout = Flags.NEW_IDDOC_LAYOUT.bindTo(flagSource); } @@ -144,7 +144,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { } X509Certificate certificate = readCertificateFromFile(certificateFile); - Instant now = clock.instant(); + Instant now = timer.currentTime(); Instant expiry = certificate.getNotAfter().toInstant(); var doc = EntityBindingsMapper.readSignedIdentityDocumentFromFile(identityDocumentFile); if (refreshIdentityDocument(doc, context)) { @@ -192,11 +192,13 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { try { var roleCertificatePath = siaDirectory.resolve("certs") .resolve(String.format("%s.cert.pem", role)); + var roleKeyPath = siaDirectory.resolve("keys") + .resolve(String.format("%s.key.pem", role)); if (!Files.exists(roleCertificatePath)) { - writeRoleCertificate(context, privateKeyFile, certificateFile, roleCertificatePath, identity, identityDocument, role); + writeRoleCredentials(context, privateKeyFile, certificateFile, roleCertificatePath, roleKeyPath, identity, identityDocument, role); modified = true; } else if (shouldRefreshCertificate(context, roleCertificatePath)) { - writeRoleCertificate(context, privateKeyFile, certificateFile, roleCertificatePath, identity, identityDocument, role); + writeRoleCredentials(context, privateKeyFile, certificateFile, roleCertificatePath, roleKeyPath, identity, identityDocument, role); modified = true; } } catch (IOException e) { @@ -208,33 +210,38 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { private boolean shouldRefreshCertificate(NodeAgentContext context, ContainerPath certificatePath) throws IOException { var certificate = readCertificateFromFile(certificatePath); - var now = clock.instant(); + var now = timer.currentTime(); var shouldRefresh = now.isAfter(certificate.getNotAfter().toInstant()) || now.isBefore(certificate.getNotBefore().toInstant().plus(REFRESH_PERIOD)); return !shouldThrottleRefreshAttempts(context.containerName(), now) && shouldRefresh; } - private void writeRoleCertificate(NodeAgentContext context, + private void writeRoleCredentials(NodeAgentContext context, ContainerPath privateKeyFile, ContainerPath certificateFile, ContainerPath roleCertificatePath, + ContainerPath roleKeyPath, AthenzIdentity identity, IdentityDocument identityDocument, String role) throws IOException { HostnameVerifier ztsHostNameVerifier = (hostname, sslSession) -> true; + var keyPair = KeyUtils.generateKeypair(KeyAlgorithm.RSA); var athenzRole = AthenzRole.fromResourceNameString(role); - var privateKey = KeyUtils.fromPemEncodedPrivateKey(new String(Files.readAllBytes(privateKeyFile))); - var containerIdentitySslContext = new SslContextBuilder().withKeyStore(privateKeyFile, certificateFile) + var containerIdentitySslContext = new SslContextBuilder() + .withKeyStore(privateKeyFile, certificateFile) .withTrustStore(ztsTrustStorePath) .build(); - try (ZtsClient ztsClient = new DefaultZtsClient.Builder(ztsEndpoint(identityDocument)).withSslContext(containerIdentitySslContext).withHostnameVerifier(ztsHostNameVerifier).build()) { + try (ZtsClient ztsClient = new DefaultZtsClient.Builder(ztsEndpoint(identityDocument)) + .withSslContext(containerIdentitySslContext) + .withHostnameVerifier(ztsHostNameVerifier) + .build()) { var csrGenerator = new CsrGenerator(certificateDnsSuffix, identityDocument.providerService().getFullName()); var csr = csrGenerator.generateRoleCsr( - identity, athenzRole, identityDocument.providerUniqueId(), identityDocument.clusterType(), KeyUtils.toKeyPair(privateKey)); + identity, athenzRole, identityDocument.providerUniqueId(), identityDocument.clusterType(), keyPair); var roleCertificate = ztsClient.getRoleCertificate(athenzRole, csr); - writeFile(roleCertificatePath, X509CertificateUtils.toPem(roleCertificate)); + writePrivateKeyAndCertificate(roleKeyPath, keyPair.getPrivate(), roleCertificatePath, roleCertificate); context.log(logger, "Role certificate successfully retrieved written to file " + roleCertificatePath.pathInContainer()); } } @@ -256,7 +263,7 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { ContainerPath certificateFile = (ContainerPath) SiaUtils.getCertificateFile(containerSiaDirectory, context.identity()); try { X509Certificate certificate = readCertificateFromFile(certificateFile); - Instant now = clock.instant(); + Instant now = timer.currentTime(); Instant expiry = certificate.getNotAfter().toInstant(); return Duration.between(now, expiry); } catch (IOException e) { @@ -278,9 +285,10 @@ public class AthenzCredentialsMaintainer implements CredentialsMaintainer { var privateKeyFile = (ContainerPath) SiaUtils.getPrivateKeyFile(siaDirectory, athenzIdentity); var certificateFile = (ContainerPath) SiaUtils.getCertificateFile(siaDirectory, athenzIdentity); try { - return Files.deleteIfExists(identityDocumentFile) || - Files.deleteIfExists(privateKeyFile) || - Files.deleteIfExists(certificateFile); + var modified = Files.deleteIfExists(identityDocumentFile); + modified |= Files.deleteIfExists(privateKeyFile); + modified |= Files.deleteIfExists(certificateFile); + return modified; } catch (IOException e) { throw new UncheckedIOException(e); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java index ea393979bf6..716e2f92e74 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImpl.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudName; +import com.yahoo.jdisc.Timer; import com.yahoo.text.Lowercase; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository; @@ -19,7 +20,6 @@ import com.yahoo.yolean.concurrent.Sleeper; import java.io.UncheckedIOException; import java.net.URI; -import java.time.Clock; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; @@ -44,20 +44,20 @@ public class VespaServiceDumperImpl implements VespaServiceDumper { private final ContainerOperations container; private final SyncClient syncClient; private final NodeRepository nodeRepository; - private final Clock clock; + private final Timer timer; private final ArtifactProducers artifactProducers; - public VespaServiceDumperImpl(ContainerOperations container, SyncClient syncClient, NodeRepository nodeRepository) { - this(ArtifactProducers.createDefault(Sleeper.DEFAULT), container, syncClient, nodeRepository, Clock.systemUTC()); + public VespaServiceDumperImpl(ContainerOperations container, SyncClient syncClient, NodeRepository nodeRepository, Timer timer) { + this(ArtifactProducers.createDefault(Sleeper.DEFAULT), container, syncClient, nodeRepository, timer); } // For unit testing VespaServiceDumperImpl(ArtifactProducers producers, ContainerOperations container, SyncClient syncClient, - NodeRepository nodeRepository, Clock clock) { + NodeRepository nodeRepository, Timer timer) { this.container = container; this.syncClient = syncClient; this.nodeRepository = nodeRepository; - this.clock = clock; + this.timer = timer; this.artifactProducers = producers; } @@ -65,7 +65,7 @@ public class VespaServiceDumperImpl implements VespaServiceDumper { public void processServiceDumpRequest(NodeAgentContext context) { if (context.zone().getCloudName().equals(CloudName.GCP)) return; - Instant startedAt = clock.instant(); + Instant startedAt = timer.currentTime(); NodeSpec nodeSpec = context.node(); ServiceDumpReport request; try { @@ -119,7 +119,7 @@ public class VespaServiceDumperImpl implements VespaServiceDumper { producedArtifacts.addAll(producer.produceArtifacts(producerCtx)); } uploadArtifacts(context, destination, producedArtifacts); - storeReport(context, ServiceDumpReport.createSuccessReport(request, startedAt, clock.instant(), destination)); + storeReport(context, ServiceDumpReport.createSuccessReport(request, startedAt, timer.currentTime(), destination)); } catch (Exception e) { handleFailure(context, request, startedAt, e, e.getMessage()); } finally { @@ -157,13 +157,13 @@ public class VespaServiceDumperImpl implements VespaServiceDumper { private void handleFailure(NodeAgentContext context, ServiceDumpReport requestOrNull, Instant startedAt, Exception failure, String message) { context.log(log, Level.WARNING, failure.toString(), failure); - ServiceDumpReport report = ServiceDumpReport.createErrorReport(requestOrNull, startedAt, clock.instant(), message); + ServiceDumpReport report = ServiceDumpReport.createErrorReport(requestOrNull, startedAt, timer.currentTime(), message); storeReport(context, report); } private void handleFailure(NodeAgentContext context, ServiceDumpReport requestOrNull, Instant startedAt, String message) { context.log(log, Level.WARNING, message); - ServiceDumpReport report = ServiceDumpReport.createErrorReport(requestOrNull, startedAt, clock.instant(), message); + ServiceDumpReport report = ServiceDumpReport.createErrorReport(requestOrNull, startedAt, timer.currentTime(), message); storeReport(context, report); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java index f168523a1ef..cd7eee8ba50 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImpl.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.nodeadmin; +import com.yahoo.jdisc.Timer; import com.yahoo.vespa.hosted.node.admin.container.ContainerStats; import com.yahoo.vespa.hosted.node.admin.container.metrics.Counter; import com.yahoo.vespa.hosted.node.admin.container.metrics.Dimensions; @@ -13,7 +14,6 @@ import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentFactory; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentScheduler; import java.nio.file.FileSystem; -import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.HashSet; @@ -37,7 +37,7 @@ public class NodeAdminImpl implements NodeAdmin { private final NodeAgentWithSchedulerFactory nodeAgentWithSchedulerFactory; - private final Clock clock; + private final Timer timer; private final Duration freezeTimeout; private final Duration spread; private boolean previousWantFrozen; @@ -54,27 +54,27 @@ public class NodeAdminImpl implements NodeAdmin { private final Metrics metrics; private Dimensions previousMemoryOverheadDimensions = null; - public NodeAdminImpl(NodeAgentFactory nodeAgentFactory, Metrics metrics, Clock clock, FileSystem fileSystem) { - this(nodeAgentContext -> create(clock, nodeAgentFactory, nodeAgentContext), - metrics, clock, NODE_AGENT_FREEZE_TIMEOUT, NODE_AGENT_SPREAD, new ProcMeminfoReader(fileSystem)); + public NodeAdminImpl(NodeAgentFactory nodeAgentFactory, Metrics metrics, Timer timer, FileSystem fileSystem) { + this(nodeAgentContext -> create(timer, nodeAgentFactory, nodeAgentContext), + metrics, timer, NODE_AGENT_FREEZE_TIMEOUT, NODE_AGENT_SPREAD, new ProcMeminfoReader(fileSystem)); } public NodeAdminImpl(NodeAgentFactory nodeAgentFactory, Metrics metrics, - Clock clock, Duration freezeTimeout, Duration spread, ProcMeminfoReader procMeminfoReader) { - this(nodeAgentContext -> create(clock, nodeAgentFactory, nodeAgentContext), - metrics, clock, freezeTimeout, spread, procMeminfoReader); + Timer timer, Duration freezeTimeout, Duration spread, ProcMeminfoReader procMeminfoReader) { + this(nodeAgentContext -> create(timer, nodeAgentFactory, nodeAgentContext), + metrics, timer, freezeTimeout, spread, procMeminfoReader); } NodeAdminImpl(NodeAgentWithSchedulerFactory nodeAgentWithSchedulerFactory, - Metrics metrics, Clock clock, Duration freezeTimeout, Duration spread, + Metrics metrics, Timer timer, Duration freezeTimeout, Duration spread, ProcMeminfoReader procMeminfoReader) { this.nodeAgentWithSchedulerFactory = nodeAgentWithSchedulerFactory; - this.clock = clock; + this.timer = timer; this.freezeTimeout = freezeTimeout; this.spread = spread; this.previousWantFrozen = true; this.isFrozen = true; - this.startOfFreezeConvergence = clock.instant(); + this.startOfFreezeConvergence = timer.currentTime(); this.numberOfUnhandledExceptions = metrics.declareCounter("unhandled_exceptions", new Dimensions(Map.of("src", "node-agents"))); @@ -104,7 +104,7 @@ public class NodeAdminImpl implements NodeAdmin { }); Duration timeBetweenNodeAgents = spread.dividedBy(Math.max(nodeAgentContextsByHostname.size() - 1, 1)); - Instant nextAgentStart = clock.instant(); + Instant nextAgentStart = timer.currentTime(); // At this point, nodeAgentContextsByHostname and nodeAgentWithSchedulerByHostname should have the same keys for (Map.Entry<String, NodeAgentContext> entry : nodeAgentContextsByHostname.entrySet()) { nodeAgentWithSchedulerByHostname.get(entry.getKey()).scheduleTickWith(entry.getValue(), nextAgentStart); @@ -158,7 +158,7 @@ public class NodeAdminImpl implements NodeAdmin { public boolean setFrozen(boolean wantFrozen) { if (wantFrozen != previousWantFrozen) { if (wantFrozen) { - this.startOfFreezeConvergence = clock.instant(); + this.startOfFreezeConvergence = timer.currentTime(); } else { this.startOfFreezeConvergence = null; } @@ -188,7 +188,7 @@ public class NodeAdminImpl implements NodeAdmin { if (startOfFreezeConvergence == null) { return Duration.ZERO; } else { - return Duration.between(startOfFreezeConvergence, clock.instant()); + return Duration.between(startOfFreezeConvergence, timer.currentTime()); } } @@ -252,8 +252,8 @@ public class NodeAdminImpl implements NodeAdmin { NodeAgentWithScheduler create(NodeAgentContext context); } - private static NodeAgentWithScheduler create(Clock clock, NodeAgentFactory nodeAgentFactory, NodeAgentContext context) { - NodeAgentContextManager contextManager = new NodeAgentContextManager(clock, context); + private static NodeAgentWithScheduler create(Timer timer, NodeAgentFactory nodeAgentFactory, NodeAgentContext context) { + NodeAgentContextManager contextManager = new NodeAgentContextManager(timer, context); NodeAgent nodeAgent = nodeAgentFactory.create(contextManager, context); return new NodeAgentWithScheduler(nodeAgent, contextManager); } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java index 00fe7198667..54d59aac3c8 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManager.java @@ -1,7 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.nodeagent; -import java.time.Clock; +import com.yahoo.jdisc.Timer; + import java.time.Duration; import java.time.Instant; import java.util.Objects; @@ -14,7 +15,7 @@ import java.util.Objects; public class NodeAgentContextManager implements NodeAgentContextSupplier, NodeAgentScheduler { private final Object monitor = new Object(); - private final Clock clock; + private final Timer timer; private NodeAgentContext currentContext; private NodeAgentContext nextContext; @@ -24,8 +25,8 @@ public class NodeAgentContextManager implements NodeAgentContextSupplier, NodeAg private boolean interrupted = false; private boolean isWaitingForNextContext = false; - public NodeAgentContextManager(Clock clock, NodeAgentContext context) { - this.clock = clock; + public NodeAgentContextManager(Timer timer, NodeAgentContext context) { + this.timer = timer; this.currentContext = context; } @@ -48,8 +49,8 @@ public class NodeAgentContextManager implements NodeAgentContextSupplier, NodeAg boolean successful; long remainder; - long end = clock.instant().plus(timeout).toEpochMilli(); - while (!(successful = isFrozen == frozen) && (remainder = end - clock.millis()) > 0) { + long end = timer.currentTime().plus(timeout).toEpochMilli(); + while (!(successful = isFrozen == frozen) && (remainder = end - timer.currentTimeMillis()) > 0) { try { monitor.wait(remainder); // Wait with timeout until the supplier is has reached wanted frozen state } catch (InterruptedException ignored) { } diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java index 025a04a15d6..64efeb85e63 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImpl.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneApi; +import com.yahoo.jdisc.Timer; import com.yahoo.vespa.flags.DoubleFlag; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; @@ -32,7 +33,6 @@ import com.yahoo.vespa.hosted.node.admin.maintenance.servicedump.VespaServiceDum import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; -import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; @@ -70,7 +70,7 @@ public class NodeAgentImpl implements NodeAgent { private final List<CredentialsMaintainer> credentialsMaintainers; private final Optional<AclMaintainer> aclMaintainer; private final Optional<HealthChecker> healthChecker; - private final Clock clock; + private final Timer timer; private final Duration warmUpDuration; private final DoubleFlag containerCpuCap; private final VespaServiceDumper serviceDumper; @@ -109,10 +109,10 @@ public class NodeAgentImpl implements NodeAgent { Orchestrator orchestrator, ContainerOperations containerOperations, RegistryCredentialsProvider registryCredentialsProvider, StorageMaintainer storageMaintainer, FlagSource flagSource, List<CredentialsMaintainer> credentialsMaintainers, - Optional<AclMaintainer> aclMaintainer, Optional<HealthChecker> healthChecker, Clock clock, + Optional<AclMaintainer> aclMaintainer, Optional<HealthChecker> healthChecker, Timer timer, VespaServiceDumper serviceDumper, List<ContainerWireguardTask> wireguardTasks) { this(contextSupplier, nodeRepository, orchestrator, containerOperations, registryCredentialsProvider, - storageMaintainer, flagSource, credentialsMaintainers, aclMaintainer, healthChecker, clock, + storageMaintainer, flagSource, credentialsMaintainers, aclMaintainer, healthChecker, timer, DEFAULT_WARM_UP_DURATION, serviceDumper, wireguardTasks); } @@ -120,7 +120,7 @@ public class NodeAgentImpl implements NodeAgent { Orchestrator orchestrator, ContainerOperations containerOperations, RegistryCredentialsProvider registryCredentialsProvider, StorageMaintainer storageMaintainer, FlagSource flagSource, List<CredentialsMaintainer> credentialsMaintainers, - Optional<AclMaintainer> aclMaintainer, Optional<HealthChecker> healthChecker, Clock clock, + Optional<AclMaintainer> aclMaintainer, Optional<HealthChecker> healthChecker, Timer timer, Duration warmUpDuration, VespaServiceDumper serviceDumper, List<ContainerWireguardTask> wireguardTasks) { this.contextSupplier = contextSupplier; @@ -132,7 +132,7 @@ public class NodeAgentImpl implements NodeAgent { this.credentialsMaintainers = credentialsMaintainers; this.aclMaintainer = aclMaintainer; this.healthChecker = healthChecker; - this.clock = clock; + this.timer = timer; this.warmUpDuration = warmUpDuration; this.containerCpuCap = PermanentFlags.CONTAINER_CPU_CAP.bindTo(flagSource); this.serviceDumper = serviceDumper; @@ -232,7 +232,7 @@ public class NodeAgentImpl implements NodeAgent { Optional<DropDocumentsReport> report = context.node().reports().getReport(DropDocumentsReport.reportId(), DropDocumentsReport.class); if (report.isPresent() && report.get().startedAt() == null && report.get().readiedAt() != null) { - newNodeAttributes.withReport(DropDocumentsReport.reportId(), report.get().withStartedAt(clock.millis()).toJsonNode()); + newNodeAttributes.withReport(DropDocumentsReport.reportId(), report.get().withStartedAt(timer.currentTimeMillis()).toJsonNode()); changed = true; } @@ -400,7 +400,7 @@ public class NodeAgentImpl implements NodeAgent { ContainerResources wantedContainerResources = getContainerResources(context); if (healthChecker.isPresent() && firstSuccessfulHealthCheckInstant - .map(clock.instant().minus(warmUpDuration(context))::isBefore) + .map(timer.currentTime().minus(warmUpDuration(context))::isBefore) .orElse(true)) return existingContainer; @@ -450,7 +450,7 @@ public class NodeAgentImpl implements NodeAgent { container.ifPresent(c -> removeContainer(context, c, List.of("Dropping documents"), true)); FileFinder.from(context.paths().underVespaHome("var/db/vespa/search")).deleteRecursively(context); nodeRepository.updateNodeAttributes(context.node().hostname(), - new NodeAttributes().withReport(DropDocumentsReport.reportId(), report.get().withDroppedAt(clock.millis()).toJsonNode())); + new NodeAttributes().withReport(DropDocumentsReport.reportId(), report.get().withDroppedAt(timer.currentTimeMillis()).toJsonNode())); } throw ConvergenceException.ofTransient("Documents already dropped, waiting for signal to start the container"); @@ -529,9 +529,9 @@ public class NodeAgentImpl implements NodeAgent { if (healthChecker.isPresent()) { healthChecker.get().verifyHealth(context); if (firstSuccessfulHealthCheckInstant.isEmpty()) - firstSuccessfulHealthCheckInstant = Optional.of(clock.instant()); + firstSuccessfulHealthCheckInstant = Optional.of(timer.currentTime()); - Duration timeLeft = Duration.between(clock.instant(), firstSuccessfulHealthCheckInstant.get().plus(warmUpDuration(context))); + Duration timeLeft = Duration.between(timer.currentTime(), firstSuccessfulHealthCheckInstant.get().plus(warmUpDuration(context))); if (!container.get().resources().equalsCpu(getContainerResources(context))) throw ConvergenceException.ofTransient("Refusing to resume until warm up period ends (" + (timeLeft.isNegative() ? "next tick" : "in " + timeLeft) + ")"); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java index fbef3def446..94c2df1a8b8 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPath.java @@ -23,6 +23,7 @@ import java.nio.file.attribute.PosixFilePermissions; import java.nio.file.attribute.UserPrincipal; import java.nio.file.attribute.UserPrincipalLookupService; import java.time.Instant; +import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.Stream; @@ -99,6 +100,22 @@ public class UnixPath { } } + public List<String> readLines() { + return uncheck(() -> Files.readAllLines(path)); + } + + /** Create an empty file and return true, or false if the file already exists (the file may not be regular). */ + public boolean create() { + try { + Files.createFile(path); + return true; + } catch (FileAlreadyExistsException ignored) { + return false; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + public UnixPath writeUtf8File(String content, OpenOption... options) { return writeBytes(content.getBytes(StandardCharsets.UTF_8), options); } @@ -209,15 +226,16 @@ public class UnixPath { return this; } - /** Create directory with given permissions, unless it already exists, and return this. */ - public UnixPath createDirectory(String... permissions) { + /** Create directory with given permissions and return true, or false if it already exists. */ + public boolean createDirectory(String... permissions) { try { Files.createDirectory(path, permissionsAsFileAttributes(permissions)); } catch (FileAlreadyExistsException ignore) { + return false; } catch (IOException e) { throw new UncheckedIOException(e); } - return this; + return true; } public UnixPath createDirectories(String... permissions) { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2Test.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupTest.java index 789f31f75c6..27580082020 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/CGroupV2Test.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/cgroup/CgroupTest.java @@ -1,7 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.node.admin.container; -import com.yahoo.collections.Pair; +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.cgroup; + +import com.yahoo.vespa.hosted.node.admin.container.ContainerId; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; @@ -12,16 +14,15 @@ import java.io.IOException; import java.nio.file.FileSystem; import java.util.Map; import java.util.Optional; -import java.util.OptionalInt; - -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.SYSTEM_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.THROTTLED_PERIODS; -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.THROTTLED_TIME_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.TOTAL_PERIODS; -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.TOTAL_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.USER_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.sharesToWeight; -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.weightToShares; + +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.SYSTEM_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.THROTTLED_PERIODS; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.THROTTLED_TIME_USEC; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.TOTAL_PERIODS; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.TOTAL_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.USER_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.sharesToWeight; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.weightToShares; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -29,47 +30,48 @@ import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author freva */ -public class CGroupV2Test { +public class CgroupTest { private static final ContainerId containerId = new ContainerId("4aec78cc"); private final FileSystem fileSystem = TestFileSystem.create(); - private final CGroupV2 cgroup = new CGroupV2(fileSystem); + private final Cgroup containerCgroup = Cgroup.root(fileSystem).resolveContainer(containerId); + private final CpuController containerCpu = containerCgroup.cpu(); private final NodeAgentContext context = NodeAgentContextImpl.builder("node123.yahoo.com").fileSystem(fileSystem).build(); private final UnixPath cgroupRoot = new UnixPath(fileSystem.getPath("/sys/fs/cgroup/machine.slice/libpod-4aec78cc.scope/container")).createDirectories(); @Test public void updates_cpu_quota_and_period() { - assertEquals(Optional.empty(), cgroup.cpuQuotaPeriod(containerId)); + assertEquals(Optional.empty(), containerCgroup.cpu().readMax()); cgroupRoot.resolve("cpu.max").writeUtf8File("max 100000\n"); - assertEquals(Optional.of(new Pair<>(-1, 100000)), cgroup.cpuQuotaPeriod(containerId)); + assertEquals(Optional.of(new CpuController.Max(Size.max(), 100000)), containerCpu.readMax()); cgroupRoot.resolve("cpu.max").writeUtf8File("456 123456\n"); - assertEquals(Optional.of(new Pair<>(456, 123456)), cgroup.cpuQuotaPeriod(containerId)); + assertEquals(Optional.of(new CpuController.Max(Size.from(456), 123456)), containerCpu.readMax()); - assertFalse(cgroup.updateCpuQuotaPeriod(context, containerId, 456, 123456)); + containerCgroup.cpu().updateMax(context, 456, 123456); - assertTrue(cgroup.updateCpuQuotaPeriod(context, containerId, 654, 123456)); - assertEquals(Optional.of(new Pair<>(654, 123456)), cgroup.cpuQuotaPeriod(containerId)); - assertEquals("654 123456", cgroupRoot.resolve("cpu.max").readUtf8File()); + assertTrue(containerCgroup.cpu().updateMax(context, 654, 123456)); + assertEquals(Optional.of(new CpuController.Max(Size.from(654), 123456)), containerCpu.readMax()); + assertEquals("654 123456\n", cgroupRoot.resolve("cpu.max").readUtf8File()); - assertTrue(cgroup.updateCpuQuotaPeriod(context, containerId, -1, 123456)); - assertEquals(Optional.of(new Pair<>(-1, 123456)), cgroup.cpuQuotaPeriod(containerId)); - assertEquals("max 123456", cgroupRoot.resolve("cpu.max").readUtf8File()); + assertTrue(containerCgroup.cpu().updateMax(context, -1, 123456)); + assertEquals(Optional.of(new CpuController.Max(Size.max(), 123456)), containerCpu.readMax()); + assertEquals("max 123456\n", cgroupRoot.resolve("cpu.max").readUtf8File()); } @Test public void updates_cpu_shares() { - assertEquals(OptionalInt.empty(), cgroup.cpuShares(containerId)); + assertEquals(Optional.empty(), containerCgroup.cpu().readShares()); cgroupRoot.resolve("cpu.weight").writeUtf8File("1\n"); - assertEquals(OptionalInt.of(2), cgroup.cpuShares(containerId)); + assertEquals(Optional.of(2), containerCgroup.cpu().readShares()); - assertFalse(cgroup.updateCpuShares(context, containerId, 2)); + assertFalse(containerCgroup.cpu().updateShares(context, 2)); - assertTrue(cgroup.updateCpuShares(context, containerId, 12345)); - assertEquals(OptionalInt.of(12323), cgroup.cpuShares(containerId)); + assertTrue(containerCgroup.cpu().updateShares(context, 12345)); + assertEquals(Optional.of(12323), containerCgroup.cpu().readShares()); } @Test @@ -82,16 +84,16 @@ public class CGroupV2Test { "throttled_usec 14256\n"); assertEquals(Map.of(TOTAL_USAGE_USEC, 17794243L, USER_USAGE_USEC, 16099205L, SYSTEM_USAGE_USEC, 1695038L, - TOTAL_PERIODS, 12465L, THROTTLED_PERIODS, 25L, THROTTLED_TIME_USEC, 14256L), cgroup.cpuStats(containerId)); + TOTAL_PERIODS, 12465L, THROTTLED_PERIODS, 25L, THROTTLED_TIME_USEC, 14256L), containerCgroup.cpu().readStats()); } @Test public void reads_memory_metrics() throws IOException { cgroupRoot.resolve("memory.current").writeUtf8File("2525093888\n"); - assertEquals(2525093888L, cgroup.memoryUsageInBytes(containerId)); + assertEquals(2525093888L, containerCgroup.memory().readCurrent().value()); cgroupRoot.resolve("memory.max").writeUtf8File("4322885632\n"); - assertEquals(4322885632L, cgroup.memoryLimitInBytes(containerId)); + assertEquals(4322885632L, containerCgroup.memory().readMax().value()); cgroupRoot.resolve("memory.stat").writeUtf8File("anon 3481600\n" + "file 69206016\n" + @@ -102,7 +104,7 @@ public class CGroupV2Test { "shmem 8380416\n" + "file_mapped 1081344\n" + "file_dirty 135168\n"); - assertEquals(69206016L, cgroup.memoryCacheInBytes(containerId)); + assertEquals(69206016L, containerCgroup.memory().readFileSystemCache().value()); } @Test diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java index 701dd33cf55..567a23ed09d 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerOperationsTest.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.hosted.node.admin.container; import com.yahoo.config.provision.DockerImage; +import com.yahoo.jdisc.test.TestTimer; +import com.yahoo.vespa.hosted.node.admin.cgroup.Cgroup; import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.jupiter.api.Test; @@ -11,9 +13,11 @@ import java.time.Instant; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.stream.Collectors; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; /** * @author mpolden @@ -23,7 +27,8 @@ public class ContainerOperationsTest { private final TestTaskContext context = new TestTaskContext(); private final ContainerEngineMock containerEngine = new ContainerEngineMock(); private final FileSystem fileSystem = TestFileSystem.create(); - private final ContainerOperations containerOperations = new ContainerOperations(containerEngine, new CGroupV2(fileSystem), fileSystem); + private final TestTimer timer = new TestTimer(); + private final ContainerOperations containerOperations = new ContainerOperations(containerEngine, mock(Cgroup.class), fileSystem, timer); @Test void no_managed_containers_running() { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java index 72c5d016a47..d4598c8923f 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/ContainerStatsCollectorTest.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.container; +import com.yahoo.vespa.hosted.node.admin.cgroup.Cgroup; +import com.yahoo.vespa.hosted.node.admin.cgroup.Size; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; @@ -8,6 +10,7 @@ import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; import com.yahoo.vespa.hosted.node.admin.task.util.process.TestTerminal; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.jupiter.api.Test; +import org.mockito.Answers; import java.io.IOException; import java.nio.file.FileSystem; @@ -17,12 +20,12 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.SYSTEM_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.THROTTLED_PERIODS; -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.THROTTLED_TIME_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.TOTAL_PERIODS; -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.TOTAL_USAGE_USEC; -import static com.yahoo.vespa.hosted.node.admin.container.CGroupV2.CpuStatField.USER_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.SYSTEM_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.THROTTLED_PERIODS; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.THROTTLED_TIME_USEC; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.TOTAL_PERIODS; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.TOTAL_USAGE_USEC; +import static com.yahoo.vespa.hosted.node.admin.cgroup.CpuController.StatField.USER_USAGE_USEC; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.eq; @@ -37,11 +40,10 @@ public class ContainerStatsCollectorTest { private final TestTerminal testTerminal = new TestTerminal(); private final ContainerEngineMock containerEngine = new ContainerEngineMock(testTerminal); private final FileSystem fileSystem = TestFileSystem.create(); - private final CGroupV2 cgroup = mock(CGroupV2.class); + private final Cgroup cgroup = mock(Cgroup.class, Answers.RETURNS_DEEP_STUBS); private final NodeAgentContext context = NodeAgentContextImpl.builder(NodeSpec.Builder.testSpec("c1").build()) .fileSystem(TestFileSystem.create()) .build(); - @Test void collect() throws Exception { ContainerStatsCollector collector = new ContainerStatsCollector(containerEngine, cgroup, fileSystem, 24); @@ -92,17 +94,17 @@ public class ContainerStatsCollectorTest { " eth0: 22280813 118083 3 4 0 0 0 0 19859383 115415 5 6 0 0 0 0\n"); } - private void mockMemoryStats(ContainerId containerId) throws IOException { - when(cgroup.memoryUsageInBytes(eq(containerId))).thenReturn(1228017664L); - when(cgroup.memoryLimitInBytes(eq(containerId))).thenReturn(2147483648L); - when(cgroup.memoryCacheInBytes(eq(containerId))).thenReturn(470790144L); + private void mockMemoryStats(ContainerId containerId) { + when(cgroup.resolveContainer(eq(containerId)).memory().readCurrent()).thenReturn(Size.from(1228017664L)); + when(cgroup.resolveContainer(eq(containerId)).memory().readMax()).thenReturn(Size.from(2147483648L)); + when(cgroup.resolveContainer(eq(containerId)).memory().readFileSystemCache()).thenReturn(Size.from(470790144L)); } private void mockCpuStats(ContainerId containerId) throws IOException { UnixPath proc = new UnixPath(fileSystem.getPath("/proc")); proc.createDirectories(); - when(cgroup.cpuStats(eq(containerId))).thenReturn(Map.of( + when(cgroup.resolveContainer(eq(containerId)).cpu().readStats()).thenReturn(Map.of( TOTAL_USAGE_USEC, 691675615472L, SYSTEM_USAGE_USEC, 262190000000L, USER_USAGE_USEC, 40900L, TOTAL_PERIODS, 1L, THROTTLED_PERIODS, 2L, THROTTLED_TIME_USEC, 3L)); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java index 2ef6780dff6..79701f59994 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImagePrunerTest.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.container.image; import com.yahoo.config.provision.DockerImage; -import com.yahoo.test.ManualClock; +import com.yahoo.jdisc.test.TestTimer; import com.yahoo.vespa.hosted.node.admin.component.TaskContext; import com.yahoo.vespa.hosted.node.admin.component.TestTaskContext; import com.yahoo.vespa.hosted.node.admin.container.Container; @@ -130,8 +130,8 @@ public class ContainerImagePrunerTest { private final ContainerEngineMock containerEngine = new ContainerEngineMock(); private final TaskContext context = new TestTaskContext(); - private final ManualClock clock = new ManualClock(); - private final ContainerImagePruner pruner = new ContainerImagePruner(containerEngine, clock); + private final TestTimer timer = new TestTimer(); + private final ContainerImagePruner pruner = new ContainerImagePruner(containerEngine, timer); private final Map<String, Integer> removalCountByImageId = new HashMap<>(); private boolean initialized = false; @@ -165,7 +165,7 @@ public class ContainerImagePrunerTest { initialized = true; } - clock.advance(Duration.ofMinutes(minutesAfter)); + timer.advance(Duration.ofMinutes(minutesAfter)); pruner.removeUnusedImages(context, excludedRefs, Duration.ofHours(1).minusSeconds(1)); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java index 7676d0e1790..b0bcea01f79 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerTester.java @@ -4,10 +4,11 @@ package com.yahoo.vespa.hosted.node.admin.integration; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; +import com.yahoo.jdisc.test.TestTimer; import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.hosted.node.admin.cgroup.Cgroup; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator; -import com.yahoo.vespa.hosted.node.admin.container.CGroupV2; import com.yahoo.vespa.hosted.node.admin.container.ContainerEngineMock; import com.yahoo.vespa.hosted.node.admin.container.ContainerName; import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; @@ -30,9 +31,7 @@ import org.mockito.InOrder; import org.mockito.Mockito; import java.nio.file.FileSystem; -import java.time.Clock; import java.time.Duration; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.concurrent.Phaser; @@ -60,7 +59,8 @@ public class ContainerTester implements AutoCloseable { private final ContainerEngineMock containerEngine = new ContainerEngineMock(); private final FileSystem fileSystem = TestFileSystem.create(); - final ContainerOperations containerOperations = spy(new ContainerOperations(containerEngine, new CGroupV2(fileSystem), fileSystem)); + private final TestTimer timer = new TestTimer(); + final ContainerOperations containerOperations = spy(new ContainerOperations(containerEngine, mock(Cgroup.class), fileSystem, timer)); final NodeRepoMock nodeRepository = spy(new NodeRepoMock()); final Orchestrator orchestrator = mock(Orchestrator.class); final StorageMaintainer storageMaintainer = mock(StorageMaintainer.class); @@ -85,7 +85,6 @@ public class ContainerTester implements AutoCloseable { NodeSpec hostSpec = NodeSpec.Builder.testSpec(HOST_HOSTNAME.value()).type(NodeType.host).build(); nodeRepository.updateNodeSpec(hostSpec); - Clock clock = Clock.systemUTC(); Metrics metrics = new Metrics(); FileSystem fileSystem = TestFileSystem.create(); ProcMeminfoReader procMeminfoReader = mock(ProcMeminfoReader.class); @@ -94,7 +93,7 @@ public class ContainerTester implements AutoCloseable { NodeAgentFactory nodeAgentFactory = (contextSupplier, nodeContext) -> new NodeAgentImpl(contextSupplier, nodeRepository, orchestrator, containerOperations, () -> RegistryCredentials.none, storageMaintainer, flagSource, - Collections.emptyList(), Optional.empty(), Optional.empty(), clock, Duration.ofSeconds(-1), + List.of(), Optional.empty(), Optional.empty(), timer, Duration.ofSeconds(-1), VespaServiceDumper.DUMMY_INSTANCE, List.of()) { @Override public void converge(NodeAgentContext context) { super.converge(context); @@ -109,7 +108,7 @@ public class ContainerTester implements AutoCloseable { phaser.arriveAndDeregister(); } }; - nodeAdmin = new NodeAdminImpl(nodeAgentFactory, metrics, clock, Duration.ofMillis(10), Duration.ZERO, procMeminfoReader); + nodeAdmin = new NodeAdminImpl(nodeAgentFactory, metrics, timer, Duration.ofMillis(10), Duration.ZERO, procMeminfoReader); NodeAgentContextFactory nodeAgentContextFactory = (nodeSpec, acl) -> NodeAgentContextImpl.builder(nodeSpec).acl(acl).fileSystem(fileSystem).build(); nodeAdminStateUpdater = new NodeAdminStateUpdater(nodeAgentContextFactory, nodeRepository, orchestrator, diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java index daae19478ed..1b770788995 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainerTest.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.hosted.node.admin.maintenance; import com.yahoo.config.provision.NodeResources; -import com.yahoo.test.ManualClock; +import com.yahoo.jdisc.test.TestTimer; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.maintenance.coredump.CoredumpHandler; import com.yahoo.vespa.hosted.node.admin.maintenance.disk.DiskCleanup; @@ -44,9 +44,9 @@ public class StorageMaintainerTest { private final CoredumpHandler coredumpHandler = mock(CoredumpHandler.class); private final DiskCleanup diskCleanup = mock(DiskCleanup.class); private final SyncClient syncClient = mock(SyncClient.class); - private final ManualClock clock = new ManualClock(Instant.ofEpochSecond(1234567890)); + private final TestTimer timer = new TestTimer(Instant.ofEpochSecond(1234567890)); private final FileSystem fileSystem = TestFileSystem.create(); - private final StorageMaintainer storageMaintainer = new StorageMaintainer(terminal, coredumpHandler, diskCleanup, syncClient, clock, + private final StorageMaintainer storageMaintainer = new StorageMaintainer(terminal, coredumpHandler, diskCleanup, syncClient, timer, fileSystem.getPath("/data/vespa/storage/container-archive")); @Test @@ -87,7 +87,7 @@ public class StorageMaintainerTest { // Archive container-1 storageMaintainer.archiveNodeStorage(context1); - clock.advance(Duration.ofSeconds(3)); + timer.advance(Duration.ofSeconds(3)); storageMaintainer.archiveNodeStorage(context1); // container-1 should be gone from container-storage diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java index b0bd1b7b68f..061eef94c2b 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandlerTest.java @@ -2,10 +2,10 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.coredump; import com.yahoo.config.provision.DockerImage; +import com.yahoo.jdisc.test.TestTimer; import com.yahoo.security.KeyId; import com.yahoo.security.SealedSharedKey; import com.yahoo.security.SecretSharedKey; -import com.yahoo.test.ManualClock; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.node.admin.configserver.cores.CoreDumpMetadata; @@ -63,14 +63,14 @@ public class CoredumpHandlerTest { private final CoreCollector coreCollector = mock(CoreCollector.class); private final Cores cores = mock(Cores.class); private final Metrics metrics = new Metrics(); - private final ManualClock clock = new ManualClock(); + private final TestTimer timer = new TestTimer(); @SuppressWarnings("unchecked") private final Supplier<String> coredumpIdSupplier = mock(Supplier.class); private final SecretSharedKeySupplier secretSharedKeySupplier = mock(SecretSharedKeySupplier.class); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); private final CoredumpHandler coredumpHandler = new CoredumpHandler(coreCollector, cores, containerCrashPath.pathInContainer(), - doneCoredumpsPath, metrics, clock, coredumpIdSupplier, secretSharedKeySupplier, + doneCoredumpsPath, metrics, timer, coredumpIdSupplier, secretSharedKeySupplier, flagSource); @Test @@ -86,7 +86,7 @@ public class CoredumpHandlerTest { assertEquals(Optional.empty(), enqueuedPath); // bash.core.431 finished writing... and 2 more have since been written - clock.advance(Duration.ofMinutes(3)); + timer.advance(Duration.ofMinutes(3)); createFileAged(crashPath.resolve("vespa-proton.core.119"), Duration.ofMinutes(10)); createFileAged(crashPath.resolve("vespa-slobrok.core.673"), Duration.ofMinutes(5)); @@ -280,7 +280,7 @@ public class CoredumpHandlerTest { private Path createFileAged(Path path, Duration age) { return uncheck(() -> Files.setLastModifiedTime( Files.createFile(path), - FileTime.from(clock.instant().minus(age)))); + FileTime.from(timer.currentTime().minus(age)))); } private static byte[] bytesOf(String str) { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java index 554a319f08b..cbe42c90a20 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/servicedump/VespaServiceDumperImplTest.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.node.admin.maintenance.servicedump; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.yahoo.test.ManualClock; +import com.yahoo.jdisc.test.TestTimer; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeSpec; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeState; import com.yahoo.vespa.hosted.node.admin.container.ContainerOperations; @@ -26,7 +26,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant; import java.util.List; -import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.mockito.ArgumentMatchers.any; @@ -71,11 +70,11 @@ class VespaServiceDumperImplTest { .thenReturn(new CommandResult(null, 0, "")); SyncClient syncClient = createSyncClientMock(); NodeRepoMock nodeRepository = new NodeRepoMock(); - ManualClock clock = new ManualClock(Instant.ofEpochMilli(1600001000000L)); + TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L)); NodeSpec nodeSpec = createNodeSpecWithDumpRequest(nodeRepository, List.of("perf-report"), new ServiceDumpReport.DumpOptions(true, 45.0, null)); VespaServiceDumper reporter = new VespaServiceDumperImpl( - ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, clock); + ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer); NodeAgentContextImpl context = NodeAgentContextImpl.builder(nodeSpec) .fileSystem(fileSystem) .build(); @@ -112,12 +111,12 @@ class VespaServiceDumperImplTest { .thenReturn(new CommandResult(null, 0, "name=host-admin success")); SyncClient syncClient = createSyncClientMock(); NodeRepoMock nodeRepository = new NodeRepoMock(); - ManualClock clock = new ManualClock(Instant.ofEpochMilli(1600001000000L)); + TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L)); NodeSpec nodeSpec = createNodeSpecWithDumpRequest( nodeRepository, List.of("jvm-jfr"), new ServiceDumpReport.DumpOptions(null, null, null)); VespaServiceDumper reporter = new VespaServiceDumperImpl( - ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, clock); + ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer); NodeAgentContextImpl context = NodeAgentContextImpl.builder(nodeSpec) .fileSystem(fileSystem) .build(); @@ -156,11 +155,11 @@ class VespaServiceDumperImplTest { .thenReturn(new CommandResult(null, 0, "name=host-admin success")); SyncClient syncClient = createSyncClientMock(); NodeRepoMock nodeRepository = new NodeRepoMock(); - ManualClock clock = new ManualClock(Instant.ofEpochMilli(1600001000000L)); + TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L)); NodeSpec nodeSpec = createNodeSpecWithDumpRequest(nodeRepository, List.of("perf-report", "jvm-jfr"), new ServiceDumpReport.DumpOptions(true, 20.0, null)); VespaServiceDumper reporter = new VespaServiceDumperImpl( - ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, clock); + ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer); NodeAgentContextImpl context = NodeAgentContextImpl.builder(nodeSpec) .fileSystem(fileSystem) .build(); @@ -179,7 +178,7 @@ class VespaServiceDumperImplTest { ContainerOperations operations = mock(ContainerOperations.class); SyncClient syncClient = createSyncClientMock(); NodeRepoMock nodeRepository = new NodeRepoMock(); - ManualClock clock = new ManualClock(Instant.ofEpochMilli(1600001000000L)); + TestTimer timer = new TestTimer(Instant.ofEpochMilli(1600001000000L)); JsonNodeFactory fac = new ObjectMapper().getNodeFactory(); ObjectNode invalidRequest = new ObjectNode(fac) .set("dumpOptions", new ObjectNode(fac).put("duration", "invalidDurationDataType")); @@ -189,7 +188,7 @@ class VespaServiceDumperImplTest { .build(); nodeRepository.updateNodeSpec(spec); VespaServiceDumper reporter = new VespaServiceDumperImpl( - ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, clock); + ArtifactProducers.createDefault(Sleeper.NOOP), operations, syncClient, nodeRepository, timer); NodeAgentContextImpl context = NodeAgentContextImpl.builder(spec) .fileSystem(fileSystem) .build(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java index 96c18517bfe..7c58007a91a 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeadmin/NodeAdminImplTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.nodeadmin; -import com.yahoo.test.ManualClock; +import com.yahoo.jdisc.test.TestTimer; import com.yahoo.vespa.hosted.node.admin.container.metrics.Metrics; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextImpl; @@ -17,7 +17,9 @@ import java.util.Set; import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl.NodeAgentWithScheduler; import static com.yahoo.vespa.hosted.node.admin.nodeadmin.NodeAdminImpl.NodeAgentWithSchedulerFactory; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.eq; @@ -34,10 +36,10 @@ import static org.mockito.Mockito.when; public class NodeAdminImplTest { private final NodeAgentWithSchedulerFactory nodeAgentWithSchedulerFactory = mock(NodeAgentWithSchedulerFactory.class); - private final ManualClock clock = new ManualClock(); + private final TestTimer timer = new TestTimer(); private final ProcMeminfoReader procMeminfoReader = mock(ProcMeminfoReader.class); private final NodeAdminImpl nodeAdmin = new NodeAdminImpl(nodeAgentWithSchedulerFactory, - new Metrics(), clock, Duration.ZERO, Duration.ZERO, procMeminfoReader); + new Metrics(), timer, Duration.ZERO, Duration.ZERO, procMeminfoReader); @Test void nodeAgentsAreProperlyLifeCycleManaged() { @@ -129,19 +131,19 @@ public class NodeAdminImplTest { // Initially everything is frozen to force convergence assertTrue(nodeAdmin.isFrozen()); assertTrue(nodeAdmin.subsystemFreezeDuration().isZero()); - clock.advance(Duration.ofSeconds(1)); + timer.advance(Duration.ofSeconds(1)); assertEquals(Duration.ofSeconds(1), nodeAdmin.subsystemFreezeDuration()); // Unfreezing floors freeze duration assertTrue(nodeAdmin.setFrozen(false)); // Unfreeze everything assertTrue(nodeAdmin.subsystemFreezeDuration().isZero()); - clock.advance(Duration.ofSeconds(1)); + timer.advance(Duration.ofSeconds(1)); assertTrue(nodeAdmin.subsystemFreezeDuration().isZero()); // Advancing time now will make freeze duration proceed according to clock assertTrue(nodeAdmin.setFrozen(true)); assertTrue(nodeAdmin.subsystemFreezeDuration().isZero()); - clock.advance(Duration.ofSeconds(1)); + timer.advance(Duration.ofSeconds(1)); assertEquals(Duration.ofSeconds(1), nodeAdmin.subsystemFreezeDuration()); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java index 82c3f79f39f..a0a06f0fb1a 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentContextManagerTest.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.node.admin.nodeagent; +import com.yahoo.jdisc.core.SystemTimer; import com.yahoo.vespa.test.file.TestFileSystem; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; -import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.Optional; @@ -13,7 +13,10 @@ import java.util.concurrent.Callable; import java.util.concurrent.CountDownLatch; import static com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContextSupplier.ContextSupplierInterruptedException; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * @author freva @@ -22,15 +25,15 @@ public class NodeAgentContextManagerTest { private static final int TIMEOUT = 10_000; - private final Clock clock = Clock.systemUTC(); + private final SystemTimer timer = new SystemTimer(); private final NodeAgentContext initialContext = generateContext(); - private final NodeAgentContextManager manager = new NodeAgentContextManager(clock, initialContext); + private final NodeAgentContextManager manager = new NodeAgentContextManager(timer, initialContext); @Test @Timeout(TIMEOUT) void context_is_ignored_unless_scheduled_while_waiting() { NodeAgentContext context1 = generateContext(); - manager.scheduleTickWith(context1, clock.instant()); + manager.scheduleTickWith(context1, timer.currentTime()); assertSame(initialContext, manager.currentContext()); AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); @@ -38,7 +41,7 @@ public class NodeAgentContextManagerTest { assertFalse(async.isCompleted()); NodeAgentContext context2 = generateContext(); - manager.scheduleTickWith(context2, clock.instant()); + manager.scheduleTickWith(context2, timer.currentTime()); assertSame(context2, async.awaitResult().response.get()); assertSame(context2, manager.currentContext()); @@ -51,13 +54,13 @@ public class NodeAgentContextManagerTest { manager.waitUntilWaitingForNextContext(); NodeAgentContext context1 = generateContext(); - Instant returnAt = clock.instant().plusMillis(500); + Instant returnAt = timer.currentTime().plusMillis(500); manager.scheduleTickWith(context1, returnAt); assertSame(context1, async.awaitResult().response.get()); assertSame(context1, manager.currentContext()); // Is accurate to a millisecond - assertFalse(clock.instant().plusMillis(1).isBefore(returnAt)); + assertFalse(timer.currentTime().plusMillis(1).isBefore(returnAt)); } @Test @@ -68,7 +71,7 @@ public class NodeAgentContextManagerTest { assertFalse(async.isCompleted()); NodeAgentContext context1 = generateContext(); - manager.scheduleTickWith(context1, clock.instant()); + manager.scheduleTickWith(context1, timer.currentTime()); async.awaitResult(); assertEquals(Optional.of(context1), async.response); @@ -98,7 +101,7 @@ public class NodeAgentContextManagerTest { NodeAgentContext context1 = generateContext(); AsyncExecutor<NodeAgentContext> async = new AsyncExecutor<>(manager::nextContext); manager.waitUntilWaitingForNextContext(); - manager.scheduleTickWith(context1, clock.instant()); + manager.scheduleTickWith(context1, timer.currentTime()); assertSame(context1, async.awaitResult().response.get()); assertTrue(manager.setFrozen(false, Duration.ZERO)); @@ -108,9 +111,9 @@ public class NodeAgentContextManagerTest { @Timeout(TIMEOUT) void setFrozen_blocks_at_least_for_duration_of_timeout() { long wantedDurationMillis = 100; - long start = clock.millis(); + long start = timer.currentTimeMillis(); assertFalse(manager.setFrozen(false, Duration.ofMillis(wantedDurationMillis))); - long actualDurationMillis = clock.millis() - start; + long actualDurationMillis = timer.currentTimeMillis() - start; assertTrue(actualDurationMillis >= wantedDurationMillis); } diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index 2db5314dbf2..0913e1d040a 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -6,7 +6,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.test.ManualClock; +import com.yahoo.jdisc.test.TestTimer; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeAttributes; @@ -78,7 +78,7 @@ public class NodeAgentImplTest { private final HealthChecker healthChecker = mock(HealthChecker.class); private final CredentialsMaintainer credentialsMaintainer = mock(CredentialsMaintainer.class); private final InMemoryFlagSource flagSource = new InMemoryFlagSource(); - private final ManualClock clock = new ManualClock(Instant.now()); + private final TestTimer timer = new TestTimer(Instant.now()); private final FileSystem fileSystem = TestFileSystem.create(); @BeforeEach @@ -654,7 +654,7 @@ public class NodeAgentImplTest { } catch (ConvergenceException e) { assertEquals(healthCheckException, e); } - clock.advance(Duration.ofSeconds(30)); + timer.advance(Duration.ofSeconds(30)); } doNothing().when(healthChecker).verifyHealth(any()); @@ -669,7 +669,7 @@ public class NodeAgentImplTest { inOrder.verify(containerOperations, never()).updateContainer(any(), any(), any()); - clock.advance(Duration.ofSeconds(31)); + timer.advance(Duration.ofSeconds(31)); nodeAgent.doConverge(context); inOrder.verify(orchestrator, never()).suspend(any()); @@ -739,7 +739,7 @@ public class NodeAgentImplTest { inOrder.verify(containerOperations, never()).updateContainer(any(), any(), any()); - clock.advance(Duration.ofSeconds(31)); + timer.advance(Duration.ofSeconds(31)); nodeAgent.doConverge(context); inOrder.verify(orchestrator, times(1)).resume(eq(hostName)); } @@ -767,7 +767,7 @@ public class NodeAgentImplTest { nodeAgent.converge(context); verify(containerOperations).removeContainer(eq(context), any()); assertFalse(indexPath.exists()); - inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes().withReport(DropDocumentsReport.reportId(), new DropDocumentsReport(1L, clock.millis(), null, null).toJsonNode()))); + inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes().withReport(DropDocumentsReport.reportId(), new DropDocumentsReport(1L, timer.currentTimeMillis(), null, null).toJsonNode()))); inOrder.verifyNoMoreInteractions(); // After droppedAt and before readiedAt are set, we cannot proceed @@ -784,13 +784,13 @@ public class NodeAgentImplTest { inOrder.verifyNoMoreInteractions(); mockGetContainer(dockerImage, ContainerResources.from(0, 2, 16), true); - clock.advance(Duration.ofSeconds(31)); + timer.advance(Duration.ofSeconds(31)); nodeAgent.converge(context); verify(containerOperations, times(1)).startContainer(eq(context)); verify(containerOperations, never()).removeContainer(eq(context), any()); inOrder.verify(nodeRepository).updateNodeAttributes(eq(hostName), eq(new NodeAttributes() .withRebootGeneration(0) - .withReport(DropDocumentsReport.reportId(), new DropDocumentsReport(1L, 2L, 3L, clock.millis()).toJsonNode()))); + .withReport(DropDocumentsReport.reportId(), new DropDocumentsReport(1L, 2L, 3L, timer.currentTimeMillis()).toJsonNode()))); inOrder.verifyNoMoreInteractions(); } @@ -844,7 +844,7 @@ public class NodeAgentImplTest { return new NodeAgentImpl(contextSupplier, nodeRepository, orchestrator, containerOperations, () -> RegistryCredentials.none, storageMaintainer, flagSource, List.of(credentialsMaintainer), Optional.of(aclMaintainer), Optional.of(healthChecker), - clock, warmUpDuration, VespaServiceDumper.DUMMY_INSTANCE, List.of()); + timer, warmUpDuration, VespaServiceDumper.DUMMY_INSTANCE, List.of()); } private void mockGetContainer(DockerImage dockerImage, boolean isRunning) { @@ -860,7 +860,7 @@ public class NodeAgentImplTest { Optional.of(new Container( containerId, ContainerName.fromHostname(hostName), - clock.instant(), + timer.currentTime(), isRunning ? Container.State.running : Container.State.exited, "image-id-1", dockerImage, diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java index ad8c6ea3a35..bbe96272b4b 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/task/util/file/UnixPathTest.java @@ -10,8 +10,14 @@ import java.nio.charset.StandardCharsets; import java.nio.file.FileSystem; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; /** * @author hakonhall @@ -38,6 +44,15 @@ public class UnixPathTest { path.writeUtf8File(original); String fromFile = path.readUtf8File(); assertEquals(original, fromFile); + assertEquals(List.of("foo", "bar"), path.readLines()); + } + + @Test + void touch() { + UnixPath path = new UnixPath(fs.getPath("example.txt")); + assertTrue(path.create()); + assertEquals("", path.readUtf8File()); + assertFalse(path.create()); } @Test @@ -74,9 +89,10 @@ public class UnixPathTest { Path path = fs.getPath("dir"); UnixPath unixPath = new UnixPath(path); String permissions = "rwxr-xr--"; - unixPath.createDirectory(permissions); + assertTrue(unixPath.createDirectory(permissions)); assertTrue(unixPath.isDirectory()); assertEquals(permissions, unixPath.getPermissions()); + assertFalse(unixPath.createDirectory(permissions)); } @Test diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java index 313cf45e1ee..4f33e079d8f 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.lb; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.EndpointsChecker.HealthChecker; import com.yahoo.config.provision.NodeType; /** @@ -9,7 +10,7 @@ import com.yahoo.config.provision.NodeType; * * @author mpolden */ -public interface LoadBalancerService { +public interface LoadBalancerService extends HealthChecker { /** * Provisions load balancers from the given specification. Implementations are expected to be idempotent diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java index 751f3d46059..a79766a577d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java @@ -4,6 +4,8 @@ package com.yahoo.vespa.hosted.provision.lb; import ai.vespa.http.DomainName; import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.EndpointsChecker.Availability; +import com.yahoo.config.provision.EndpointsChecker.Endpoint; import com.yahoo.config.provision.NodeType; import java.util.Collections; @@ -85,4 +87,9 @@ public class LoadBalancerServiceMock implements LoadBalancerService { instances.remove(loadBalancer.id()); } + @Override + public Availability healthy(Endpoint endpoint) { + return Availability.ready; + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java index f9f26852b0d..e49d1b302cf 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.hosted.provision.lb; import ai.vespa.http.DomainName; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.EndpointsChecker.Availability; +import com.yahoo.config.provision.EndpointsChecker.Endpoint; import com.yahoo.config.provision.NodeType; import java.util.List; @@ -68,4 +70,9 @@ public class SharedLoadBalancerService implements LoadBalancerService { return nodeType == NodeType.tenant && clusterType.isContainer(); } + @Override + public Availability healthy(Endpoint endpoint) { + return Availability.ready; + } + } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java index 6e301b7724c..65039aaca77 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionServiceProvider.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; +import com.yahoo.config.provision.EndpointsChecker.HealthCheckerProvider; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerService; import java.util.Optional; @@ -10,11 +11,12 @@ import java.util.Optional; * * @author freva */ -public interface ProvisionServiceProvider { +public interface ProvisionServiceProvider extends HealthCheckerProvider { Optional<LoadBalancerService> getLoadBalancerService(); Optional<HostProvisioner> getHostProvisioner(); HostResourcesCalculator getHostResourcesCalculator(); + } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java index 40ca30d758e..d75f51680d7 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/AutoscalingMaintainerTest.java @@ -154,10 +154,10 @@ public class AutoscalingMaintainerTest { @Test public void test_toString() { - assertEquals("4 nodes with [vcpu: 1.0, memory: 2.0 Gb, disk 4.0 Gb, bandwidth: 1.0 Gbps, architecture: x86_64] (total: [vcpu: 4.0, memory: 8.0 Gb, disk 16.0 Gb, bandwidth: 4.0 Gbps, architecture: x86_64])", + assertEquals("4 nodes with [vcpu: 1.0, memory: 2.0 Gb, disk 4.0 Gb, bandwidth: 1.0 Gbps, architecture: any] (total: [vcpu: 4.0, memory: 8.0 Gb, disk 16.0 Gb, bandwidth: 4.0 Gbps, architecture: any])", AutoscalingMaintainer.toString(new ClusterResources(4, 1, new NodeResources(1, 2, 4, 1)))); - assertEquals("4 nodes (in 2 groups) with [vcpu: 1.0, memory: 2.0 Gb, disk 4.0 Gb, bandwidth: 1.0 Gbps, architecture: x86_64] (total: [vcpu: 4.0, memory: 8.0 Gb, disk 16.0 Gb, bandwidth: 4.0 Gbps, architecture: x86_64])", + assertEquals("4 nodes (in 2 groups) with [vcpu: 1.0, memory: 2.0 Gb, disk 4.0 Gb, bandwidth: 1.0 Gbps, architecture: any] (total: [vcpu: 4.0, memory: 8.0 Gb, disk 16.0 Gb, bandwidth: 4.0 Gbps, architecture: any])", AutoscalingMaintainer.toString(new ClusterResources(4, 2, new NodeResources(1, 2, 4, 1)))); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java index e6c183d02ce..f73d6f2ce01 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/ScalingSuggestionsMaintainerTest.java @@ -73,9 +73,9 @@ public class ScalingSuggestionsMaintainerTest { new TestMetric()); maintainer.maintain(); - assertEquals("8 nodes with [vcpu: 3.2, memory: 4.5 Gb, disk 10.0 Gb, bandwidth: 0.1 Gbps, architecture: x86_64]", + assertEquals("8 nodes with [vcpu: 3.2, memory: 4.5 Gb, disk 10.0 Gb, bandwidth: 0.1 Gbps, architecture: any]", suggestionOf(app1, cluster1, tester).resources().get().toString()); - assertEquals("8 nodes with [vcpu: 3.6, memory: 4.4 Gb, disk 11.8 Gb, bandwidth: 0.1 Gbps, architecture: x86_64]", + assertEquals("8 nodes with [vcpu: 3.6, memory: 4.4 Gb, disk 11.8 Gb, bandwidth: 0.1 Gbps, architecture: any]", suggestionOf(app2, cluster2, tester).resources().get().toString()); // Utilization goes way down @@ -83,14 +83,14 @@ public class ScalingSuggestionsMaintainerTest { addMeasurements(0.10f, 0.10f, 0.10f, 0, 500, app1, tester.nodeRepository()); maintainer.maintain(); assertEquals("Suggestion stays at the peak value observed", - "8 nodes with [vcpu: 3.2, memory: 4.5 Gb, disk 10.0 Gb, bandwidth: 0.1 Gbps, architecture: x86_64]", + "8 nodes with [vcpu: 3.2, memory: 4.5 Gb, disk 10.0 Gb, bandwidth: 0.1 Gbps, architecture: any]", suggestionOf(app1, cluster1, tester).resources().get().toString()); // Utilization is still way down and a week has passed tester.clock().advance(Duration.ofDays(7)); addMeasurements(0.10f, 0.10f, 0.10f, 0, 500, app1, tester.nodeRepository()); maintainer.maintain(); assertEquals("Peak suggestion has been outdated", - "3 nodes with [vcpu: 1.2, memory: 4.0 Gb, disk 10.0 Gb, bandwidth: 0.1 Gbps, architecture: x86_64]", + "3 nodes with [vcpu: 1.2, memory: 4.0 Gb, disk 10.0 Gb, bandwidth: 0.1 Gbps, architecture: any]", suggestionOf(app1, cluster1, tester).resources().get().toString()); assertTrue(shouldSuggest(app1, cluster1, tester)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java index 98c17eb4d5e..40b035968bd 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java @@ -326,7 +326,7 @@ public class OsVersionsTest { .account(CloudAccount.from("000000000000")) .build()); - provisionInfraApplication(hostCount, infraApplication, NodeType.host, NodeResources.StorageType.remote); + provisionInfraApplication(hostCount, infraApplication, NodeType.host, NodeResources.StorageType.remote, NodeResources.Architecture.x86_64); Supplier<NodeList> hostNodes = () -> tester.nodeRepository().nodes().list().nodeType(NodeType.host); // New target is set @@ -535,13 +535,20 @@ public class OsVersionsTest { return provisionInfraApplication(nodeCount, infraApplication, NodeType.host); } - private List<Node> provisionInfraApplication(int nodeCount, ApplicationId application, NodeType nodeType) { + private List<Node> provisionInfraApplication(int nodeCount, ApplicationId application, + NodeType nodeType) { return provisionInfraApplication(nodeCount, application, nodeType, NodeResources.StorageType.local); } - private List<Node> provisionInfraApplication(int nodeCount, ApplicationId application, NodeType nodeType, NodeResources.StorageType storageType) { + private List<Node> provisionInfraApplication(int nodeCount, ApplicationId application, + NodeType nodeType, NodeResources.StorageType storageType) { + return provisionInfraApplication(nodeCount, application, nodeType, storageType, NodeResources.Architecture.x86_64); + } + + private List<Node> provisionInfraApplication(int nodeCount, ApplicationId application, NodeType nodeType, + NodeResources.StorageType storageType, NodeResources.Architecture architecture) { var nodes = tester.makeReadyNodes(nodeCount, new NodeResources(48, 128, 2000, 10, - NodeResources.DiskSpeed.fast, storageType), + NodeResources.DiskSpeed.fast, storageType, architecture), nodeType, 10); tester.prepareAndActivateInfraApplication(application, nodeType); tester.clock().advance(Duration.ofDays(1).plusSeconds(1)); // Let grace period pass diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java index 12a8e4d9386..ec8a8f637c8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/HostCapacityTest.java @@ -98,11 +98,14 @@ public class HostCapacityTest { @Test public void unusedCapacityOf() { - assertEquals(new NodeResources(5, 40, 80, 2, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote), + assertEquals(new NodeResources(5, 40, 80, 2, + NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote, NodeResources.Architecture.x86_64), capacity.unusedCapacityOf(host1)); - assertEquals(new NodeResources(5, 60, 80, 4.5, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote), + assertEquals(new NodeResources(5, 60, 80, 4.5, + NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote, NodeResources.Architecture.x86_64), capacity.unusedCapacityOf(host3)); - assertEquals(new NodeResources(7, 100, 120, 5, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote), + assertEquals(new NodeResources(7, 100, 120, 5, + NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote, NodeResources.Architecture.x86_64), capacity.unusedCapacityOf(host4)); doAnswer(invocation -> { @@ -110,17 +113,19 @@ public class HostCapacityTest { return totalHostResources.subtract(new NodeResources(1, 2, 3, 0.5, NodeResources.DiskSpeed.any)); }).when(hostResourcesCalculator).advertisedResourcesOf(any()); - assertEquals(new NodeResources(4, 38, 77, 1.5, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote), + assertEquals(new NodeResources(4, 38, 77, 1.5, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote, NodeResources.Architecture.x86_64), capacity.unusedCapacityOf(host1)); - assertEquals(new NodeResources(4, 58, 77, 4, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote), + assertEquals(new NodeResources(4, 58, 77, 4, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote, NodeResources.Architecture.x86_64), capacity.unusedCapacityOf(host3)); } @Test public void availableCapacityOf() { - assertEquals(new NodeResources(5, 40, 80, 2, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote), + assertEquals(new NodeResources(5, 40, 80, 2, + NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote, NodeResources.Architecture.x86_64), capacity.availableCapacityOf(host1)); - assertEquals(new NodeResources(5, 60, 80, 4.5, NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote), + assertEquals(new NodeResources(5, 60, 80, 4.5, + NodeResources.DiskSpeed.fast, NodeResources.StorageType.remote, NodeResources.Architecture.x86_64), capacity.availableCapacityOf(host3)); assertEquals(NodeResources.zero(), capacity.availableCapacityOf(host4)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java index 0744d82c85b..f40c8037f41 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java @@ -440,7 +440,7 @@ public class VirtualNodeProvisioningTest { catch (Exception e) { assertEquals("No room for 3 nodes as 2 of 4 hosts are exclusive", "Could not satisfy request for 3 nodes with " + - "[vcpu: 2.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, architecture: x86_64] " + + "[vcpu: 2.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, architecture: any] " + "in tenant2.app2 container cluster 'my-container' 6.39: " + "Node allocation failure on group 0: " + "Not enough suitable nodes available due to host exclusivity constraints", @@ -467,7 +467,7 @@ public class VirtualNodeProvisioningTest { } catch (NodeAllocationException e) { assertEquals("Could not satisfy request for 2 nodes with " + - "[vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: remote, architecture: x86_64] " + + "[vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: remote, architecture: any] " + "in tenant.app1 content cluster 'my-content'" + " 6.42: Node allocation failure on group 0", e.getMessage()); @@ -549,8 +549,8 @@ public class VirtualNodeProvisioningTest { } catch (IllegalArgumentException e) { assertEquals("No allocation possible within limits: " + - "from 2 nodes with [vcpu: 1.0, memory: 5.0 Gb, disk 10.0 Gb, bandwidth: 1.0 Gbps, architecture: x86_64] " + - "to 4 nodes with [vcpu: 1.0, memory: 5.0 Gb, disk 10.0 Gb, bandwidth: 1.0 Gbps, architecture: x86_64]", + "from 2 nodes with [vcpu: 1.0, memory: 5.0 Gb, disk 10.0 Gb, bandwidth: 1.0 Gbps, architecture: any] " + + "to 4 nodes with [vcpu: 1.0, memory: 5.0 Gb, disk 10.0 Gb, bandwidth: 1.0 Gbps, architecture: any]", e.getMessage()); } } @@ -573,9 +573,9 @@ public class VirtualNodeProvisioningTest { } catch (IllegalArgumentException e) { assertEquals("No allocation possible within limits: " + - "from 2 nodes with [vcpu: 20.0, memory: 37.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, architecture: x86_64] " + - "to 4 nodes with [vcpu: 20.0, memory: 37.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, architecture: x86_64]. " + - "Nearest allowed node resources: [vcpu: 20.0, memory: 40.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: remote, architecture: x86_64]", + "from 2 nodes with [vcpu: 20.0, memory: 37.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, architecture: any] " + + "to 4 nodes with [vcpu: 20.0, memory: 37.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, architecture: any]. " + + "Nearest allowed node resources: [vcpu: 20.0, memory: 40.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: remote, architecture: any]", e.getMessage()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json index 4c18b614075..92e5425e84e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application1.json @@ -14,7 +14,7 @@ "bandwidthGbps" : 1.0, "diskSpeed" : "fast", "storageType" : "any", - "architecture":"x86_64" + "architecture":"any" } }, "max" : { @@ -27,7 +27,7 @@ "bandwidthGbps" : 1.0, "diskSpeed" : "fast", "storageType" : "any", - "architecture":"x86_64" + "architecture":"any" } }, "current" : { @@ -56,7 +56,7 @@ "bandwidthGbps": 1.0, "diskSpeed": "fast", "storageType": "any", - "architecture": "x86_64" + "architecture": "any" } }, "at" : 123, @@ -89,7 +89,7 @@ "bandwidthGbps": 1.0, "diskSpeed": "fast", "storageType": "any", - "architecture": "x86_64" + "architecture": "any" } }, "at" : 123, @@ -121,7 +121,7 @@ "bandwidthGbps": 0.0, "diskSpeed": "fast", "storageType": "any", - "architecture":"x86_64" + "architecture":"any" } }, "to": { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2.json index ea4d5e01ebc..cba56e1c51e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/application2.json @@ -14,7 +14,7 @@ "bandwidthGbps": 1.0, "diskSpeed": "fast", "storageType": "any", - "architecture":"x86_64" + "architecture":"any" } }, "max": { @@ -27,7 +27,7 @@ "bandwidthGbps": 1.0, "diskSpeed": "fast", "storageType": "any", - "architecture":"x86_64" + "architecture":"any" } }, "current": { @@ -75,7 +75,7 @@ "bandwidthGbps": 0.0, "diskSpeed": "fast", "storageType": "any", - "architecture":"x86_64" + "architecture":"any" } }, "to": { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-container1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-container1.json index a77e79170e7..8ef88eae97d 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-container1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/docker-container1.json @@ -25,7 +25,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0,"diskSpeed":"fast", "storageType":"any","architecture":"x86_64" }, + "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0,"diskSpeed":"fast", "storageType":"any","architecture":"any" }, "rebootGeneration": 0, "currentRebootGeneration": 0, "failCount": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node1.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node1.json index 0b14c5769f6..d90ed692f1c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node1.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node1.json @@ -24,7 +24,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any","architecture":"x86_64" }, + "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any","architecture":"any" }, "rebootGeneration": 0, "currentRebootGeneration": 0, "failCount": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node10.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node10.json index 25f7b2a9c2d..bec194ea325 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node10.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node10.json @@ -25,7 +25,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any","architecture":"x86_64" }, + "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any","architecture":"any" }, "rebootGeneration": 0, "currentRebootGeneration": 0, "vespaVersion": "5.104.142", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node11.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node11.json index 72131465f29..d7e07f02f3a 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node11.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node11.json @@ -5,9 +5,9 @@ "type": "tenant", "hostname": "host11.yahoo.com", "parentHostname": "parent.host.yahoo.com", - "flavor": "[vcpu: 1.0, memory: 1.0 Gb, disk 100.0 Gb, bandwidth: 0.3 Gbps, architecture: x86_64]", - "resources":{"vcpu":1.0,"memoryGb":1.0,"diskGb":100.0,"bandwidthGbps":0.3,"diskSpeed":"fast","storageType":"any","architecture":"x86_64"}, - "realResources":{"vcpu":1.0,"memoryGb":1.0,"diskGb":100.0,"bandwidthGbps":0.3,"diskSpeed":"fast","storageType":"any","architecture":"x86_64"}, + "flavor": "[vcpu: 1.0, memory: 1.0 Gb, disk 100.0 Gb, bandwidth: 0.3 Gbps, architecture: any]", + "resources":{"vcpu":1.0,"memoryGb":1.0,"diskGb":100.0,"bandwidthGbps":0.3,"diskSpeed":"fast","storageType":"any","architecture":"any"}, + "realResources":{"vcpu":1.0,"memoryGb":1.0,"diskGb":100.0,"bandwidthGbps":0.3,"diskSpeed":"fast","storageType":"any","architecture":"any"}, "environment": "DOCKER_CONTAINER", "rebootGeneration": 0, "currentRebootGeneration": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node13.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node13.json index f1fb2def612..73c34a7fa9e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node13.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node13.json @@ -24,7 +24,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":10.0, "memoryGb":48.0, "diskGb":500.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any","architecture":"x86_64" }, + "requestedResources": { "vcpu":10.0, "memoryGb":48.0, "diskGb":500.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any","architecture":"any" }, "rebootGeneration": 0, "currentRebootGeneration": 0, "failCount": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node14.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node14.json index c19ca13066d..abb0ba57e49 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node14.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node14.json @@ -24,7 +24,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":10.0, "memoryGb":48.0, "diskGb":500.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any","architecture":"x86_64" }, + "requestedResources": { "vcpu":10.0, "memoryGb":48.0, "diskGb":500.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any","architecture":"any" }, "rebootGeneration": 0, "currentRebootGeneration": 0, "failCount": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node2.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node2.json index 40e32f5113e..9cd675163f0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node2.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node2.json @@ -24,7 +24,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any","architecture":"x86_64" }, + "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any","architecture":"any" }, "rebootGeneration": 0, "currentRebootGeneration": 0, "failCount": 0, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-after-changes.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-after-changes.json index 6354fb6a894..03621c40f67 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-after-changes.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-after-changes.json @@ -26,7 +26,7 @@ "currentRestartGeneration": 1, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any","architecture":"x86_64" }, + "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any","architecture":"any" }, "orchestratorStatus": "ALLOWED_TO_BE_DOWN", "suspendedSinceMillis": 0, "rebootGeneration": 2, diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-wg.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-wg.json index 6510bdde0c8..a1883ba4b25 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-wg.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-wg.json @@ -25,7 +25,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any","architecture":"x86_64" }, + "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any","architecture":"any" }, "rebootGeneration": 0, "currentRebootGeneration": 0, "vespaVersion": "6.41.0", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-with-hostnames.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-with-hostnames.json index 6eae33eda15..50007fd6610 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-with-hostnames.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4-with-hostnames.json @@ -25,7 +25,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any","architecture":"x86_64" }, + "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any","architecture":"any" }, "rebootGeneration": 0, "currentRebootGeneration": 0, "vespaVersion": "6.41.0", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4.json index 961c03570e6..f206adf4366 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node4.json @@ -25,7 +25,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any","architecture":"x86_64" }, + "requestedResources": { "vcpu":1.0, "memoryGb":4.0, "diskGb":100.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any","architecture":"any" }, "rebootGeneration": 0, "currentRebootGeneration": 0, "vespaVersion": "6.41.0", diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6.json b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6.json index de286d1f55f..69316b1ca7f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6.json +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/responses/node6.json @@ -24,7 +24,7 @@ "currentRestartGeneration": 0, "wantedDockerImage": "docker-registry.domain.tld:8080/dist/vespa:6.42.0", "wantedVespaVersion": "6.42.0", - "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any","architecture":"x86_64" }, + "requestedResources": { "vcpu":2.0, "memoryGb":8.0, "diskGb":50.0, "bandwidthGbps":1.0, "diskSpeed":"fast", "storageType":"any","architecture":"any" }, "rebootGeneration": 0, "currentRebootGeneration": 0, "failCount": 0, diff --git a/parent/pom.xml b/parent/pom.xml index 2701acc218c..71e0c35eb6c 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -1125,7 +1125,7 @@ <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> <athenz.version>1.11.28</athenz.version> <!-- WARNING: sync cloud-tenant-base-dependencies-enforcer/pom.xml --> - <aws-sdk.version>1.12.331</aws-sdk.version> + <aws-sdk.version>1.12.460</aws-sdk.version> <!-- Athenz END --> <!-- WARNING: If you change curator version, you also need to update @@ -1134,7 +1134,7 @@ find zkfacade/src/main/java/org/apache/curator -name package-info.java | \ xargs perl -pi -e 's/major = [0-9]+, minor = [0-9]+, micro = [0-9]+/major = 5, minor = 3, micro = 0/g' --> - <bouncycastle.version>1.72</bouncycastle.version> + <bouncycastle.version>1.73</bouncycastle.version> <curator.version>5.4.0</curator.version> <commons-codec.version>1.15</commons-codec.version> <commons-io.version>2.11.0</commons-io.version> diff --git a/searchsummary/src/vespa/juniper/rpinterface.cpp b/searchsummary/src/vespa/juniper/rpinterface.cpp index afda82c1110..702cfb11f4d 100644 --- a/searchsummary/src/vespa/juniper/rpinterface.cpp +++ b/searchsummary/src/vespa/juniper/rpinterface.cpp @@ -37,7 +37,7 @@ void SetDebug(unsigned int mask) } -Juniper::Juniper(IJuniperProperties* props, Fast_WordFolder* wordfolder, int api_version) : +Juniper::Juniper(IJuniperProperties* props, const Fast_WordFolder* wordfolder, int api_version) : _props(props), _wordfolder(wordfolder), _modifier(new QueryModifier()) diff --git a/searchsummary/src/vespa/juniper/rpinterface.h b/searchsummary/src/vespa/juniper/rpinterface.h index 5bfee1e2691..b6612f2a5b7 100644 --- a/searchsummary/src/vespa/juniper/rpinterface.h +++ b/searchsummary/src/vespa/juniper/rpinterface.h @@ -89,7 +89,7 @@ public: * compatibility between versions. */ Juniper(IJuniperProperties* props, - Fast_WordFolder* wordfolder, int api_version = JUNIPER_RP_ABI_VERSION); + const Fast_WordFolder* wordfolder, int api_version = JUNIPER_RP_ABI_VERSION); /** Deinitialize the Juniper subsystem. Release all remaining resources * associated with Juniper - reverse the effect of the Init function. * Assumes that all Result objects have been released. @@ -137,7 +137,7 @@ public: private: IJuniperProperties * _props; - Fast_WordFolder * _wordfolder; + const Fast_WordFolder * _wordfolder; std::unique_ptr<QueryModifier> _modifier; }; diff --git a/storage/src/tests/persistence/testandsettest.cpp b/storage/src/tests/persistence/testandsettest.cpp index 1aa359de634..b17837d67c5 100644 --- a/storage/src/tests/persistence/testandsettest.cpp +++ b/storage/src/tests/persistence/testandsettest.cpp @@ -123,6 +123,28 @@ TEST_F(TestAndSetTest, conditional_put_executed_on_condition_match) { assertTestDocumentFoundAndMatchesContent(NEW_CONTENT); } +TEST_F(TestAndSetTest, conditional_put_not_executed_when_no_document_and_no_create) { + api::Timestamp putTimestamp = 200; + testDoc->setValue(testDoc->getField("content"), NEW_CONTENT); + auto putUp = std::make_shared<api::PutCommand>(BUCKET, testDoc, putTimestamp); + setTestCondition(*putUp); + + ASSERT_EQ(fetchResult(asyncHandler->handlePut(*putUp, createTracker(putUp, BUCKET))).getResult(), api::ReturnCode::Result::TEST_AND_SET_CONDITION_FAILED); + EXPECT_EQ("", dumpBucket(BUCKET_ID)); +} + +TEST_F(TestAndSetTest, conditional_put_executed_when_no_document_but_create_is_enabled) { + api::Timestamp putTimestamp = 200; + testDoc->setValue(testDoc->getField("content"), NEW_CONTENT); + auto putUp = std::make_shared<api::PutCommand>(BUCKET, testDoc, putTimestamp); + setTestCondition(*putUp); + putUp->set_create_if_non_existent(true); + + ASSERT_EQ(fetchResult(asyncHandler->handlePut(*putUp, createTracker(putUp, BUCKET))).getResult(), api::ReturnCode::Result::OK); + EXPECT_EQ(expectedDocEntryString(putTimestamp, testDocId), dumpBucket(BUCKET_ID)); + assertTestDocumentFoundAndMatchesContent(NEW_CONTENT); +} + TEST_F(TestAndSetTest, conditional_remove_not_executed_on_condition_mismatch) { // Put document with mismatching header api::Timestamp timestampOne = 0; diff --git a/storage/src/tests/storageapi/mbusprot/storageprotocoltest.cpp b/storage/src/tests/storageapi/mbusprot/storageprotocoltest.cpp index ec8d6855abc..72bb6cf9fb5 100644 --- a/storage/src/tests/storageapi/mbusprot/storageprotocoltest.cpp +++ b/storage/src/tests/storageapi/mbusprot/storageprotocoltest.cpp @@ -176,6 +176,7 @@ TEST_P(StorageProtocolTest, put) { EXPECT_EQ(*_testDoc, *cmd2->getDocument()); EXPECT_EQ(Timestamp(14), cmd2->getTimestamp()); EXPECT_EQ(Timestamp(13), cmd2->getUpdateTimestamp()); + EXPECT_EQ(false, cmd2->get_create_if_non_existent()); auto reply = std::make_shared<PutReply>(*cmd2); ASSERT_TRUE(reply->hasDocument()); @@ -803,6 +804,15 @@ TEST_P(StorageProtocolTest, put_command_with_condition) { EXPECT_EQ(cmd->getCondition().getSelection(), cmd2->getCondition().getSelection()); } +TEST_P(StorageProtocolTest, put_command_with_create_flag) { + auto cmd = std::make_shared<PutCommand>(_bucket, _testDoc, 14); + EXPECT_EQ(false, cmd->get_create_if_non_existent()); + cmd->set_create_if_non_existent(true); + EXPECT_EQ(true, cmd->get_create_if_non_existent()); + auto cmd2 = copyCommand(cmd); + EXPECT_EQ(cmd->get_create_if_non_existent(), cmd2->get_create_if_non_existent()); +} + TEST_P(StorageProtocolTest, update_command_with_condition) { auto update = std::make_shared<document::DocumentUpdate>( _docMan.getTypeRepo(), *_testDoc->getDataType(), _testDoc->getId()); @@ -877,7 +887,7 @@ TEST_P(StorageProtocolTest, track_memory_footprint_for_some_messages) { EXPECT_EQ(sizeof(BucketCommand), sizeof(StorageCommand) + 24); EXPECT_EQ(sizeof(BucketInfoCommand), sizeof(BucketCommand)); EXPECT_EQ(sizeof(TestAndSetCommand), sizeof(BucketInfoCommand) + sizeof(vespalib::string)); - EXPECT_EQ(sizeof(PutCommand), sizeof(TestAndSetCommand) + 32); + EXPECT_EQ(sizeof(PutCommand), sizeof(TestAndSetCommand) + 40); EXPECT_EQ(sizeof(UpdateCommand), sizeof(TestAndSetCommand) + 32); EXPECT_EQ(sizeof(RemoveCommand), sizeof(TestAndSetCommand) + 112); EXPECT_EQ(sizeof(GetCommand), sizeof(BucketInfoCommand) + sizeof(TestAndSetCondition) + 184); diff --git a/storage/src/vespa/storage/persistence/asynchandler.cpp b/storage/src/vespa/storage/persistence/asynchandler.cpp index 60c6d507416..42d44394f3a 100644 --- a/storage/src/vespa/storage/persistence/asynchandler.cpp +++ b/storage/src/vespa/storage/persistence/asynchandler.cpp @@ -159,7 +159,7 @@ AsyncHandler::handlePut(api::PutCommand& cmd, MessageTracker::UP trackerUP) cons tracker.setMetric(metrics); metrics.request_size.addValue(cmd.getApproxByteSize()); - if (tasConditionExists(cmd) && !tasConditionMatches(cmd, tracker, tracker.context())) { + if (tasConditionExists(cmd) && !tasConditionMatches(cmd, tracker, tracker.context(), cmd.get_create_if_non_existent())) { // Will also count condition parse failures etc as TaS failures, but // those results _will_ increase the error metrics as well. metrics.test_and_set_failed.inc(); diff --git a/storage/src/vespa/storageapi/mbusprot/protobuf/feed.proto b/storage/src/vespa/storageapi/mbusprot/protobuf/feed.proto index b02e1eab0aa..b115cca8263 100644 --- a/storage/src/vespa/storageapi/mbusprot/protobuf/feed.proto +++ b/storage/src/vespa/storageapi/mbusprot/protobuf/feed.proto @@ -17,6 +17,7 @@ message PutRequest { uint64 new_timestamp = 3; uint64 expected_old_timestamp = 4; // If zero; no expectation. TestAndSetCondition condition = 5; + bool create_if_non_existent = 6; } message PutResponse { diff --git a/storage/src/vespa/storageapi/mbusprot/protocolserialization7.cpp b/storage/src/vespa/storageapi/mbusprot/protocolserialization7.cpp index f2dcf07c01a..3f1ab1e5fe1 100644 --- a/storage/src/vespa/storageapi/mbusprot/protocolserialization7.cpp +++ b/storage/src/vespa/storageapi/mbusprot/protocolserialization7.cpp @@ -421,6 +421,7 @@ void ProtocolSerialization7::onEncode(GBBuf& buf, const api::PutCommand& msg) co if (msg.getDocument()) { set_document(*req.mutable_document(), *msg.getDocument()); } + req.set_create_if_non_existent(msg.get_create_if_non_existent()); }); } @@ -438,6 +439,7 @@ api::StorageCommand::UP ProtocolSerialization7::onDecodePutCommand(BBuf& buf) co if (req.has_condition()) { cmd->setCondition(get_tas_condition(req.condition())); } + cmd->set_create_if_non_existent(req.create_if_non_existent()); return cmd; }); } diff --git a/storage/src/vespa/storageapi/message/persistence.h b/storage/src/vespa/storageapi/message/persistence.h index d010c295ca7..e6f2023fa48 100644 --- a/storage/src/vespa/storageapi/message/persistence.h +++ b/storage/src/vespa/storageapi/message/persistence.h @@ -48,7 +48,7 @@ class PutCommand : public TestAndSetCommand { DocumentSP _doc; Timestamp _timestamp; Timestamp _updateTimestamp; - + bool _create_if_non_existent = false; public: PutCommand(const document::Bucket &bucket, const DocumentSP&, Timestamp); ~PutCommand() override; @@ -67,6 +67,8 @@ public: const document::DocumentId& getDocumentId() const override; Timestamp getTimestamp() const { return _timestamp; } const document::DocumentType * getDocumentType() const override; + void set_create_if_non_existent(bool value) noexcept { _create_if_non_existent = value; } + bool get_create_if_non_existent() const noexcept { return _create_if_non_existent; } vespalib::string getSummary() const override; void print(std::ostream& out, bool verbose, const std::string& indent) const override; diff --git a/streamingvisitors/src/vespa/searchvisitor/searchenvironment.cpp b/streamingvisitors/src/vespa/searchvisitor/searchenvironment.cpp index 4da9164021d..096f9ab4c50 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchenvironment.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchenvironment.cpp @@ -14,11 +14,11 @@ namespace streaming { __thread SearchEnvironment::EnvMap * SearchEnvironment::_localEnvMap = nullptr; -SearchEnvironment::Env::Env(const vespalib::string & muffens, const config::ConfigUri & configUri, Fast_NormalizeWordFolder & wf) : - _configId(configUri.getConfigId()), - _configurer(std::make_unique<config::SimpleConfigRetriever>(createKeySet(configUri.getConfigId()), configUri.getContext()), this), - _vsmAdapter(std::make_unique<VSMAdapter>(muffens, _configId, wf)), - _rankManager(std::make_unique<RankManager>(_vsmAdapter.get())) +SearchEnvironment::Env::Env(const config::ConfigUri& configUri, const Fast_NormalizeWordFolder& wf) + : _configId(configUri.getConfigId()), + _configurer(std::make_unique<config::SimpleConfigRetriever>(createKeySet(configUri.getConfigId()), configUri.getContext()), this), + _vsmAdapter(std::make_unique<VSMAdapter>(_configId, wf)), + _rankManager(std::make_unique<RankManager>(_vsmAdapter.get())) { _configurer.start(); @@ -79,7 +79,7 @@ SearchEnvironment::getEnv(const vespalib::string & searchCluster) EnvMap::iterator found = _envMap.find(searchCluster); if (found == _envMap.end()) { LOG(debug, "Init VSMAdapter with config id = '%s'", searchCluster.c_str()); - Env::SP env = std::make_shared<Env>("*", searchClusterUri, _wordFolder); + Env::SP env = std::make_shared<Env>(searchClusterUri, _wordFolder); _envMap[searchCluster] = std::move(env); found = _envMap.find(searchCluster); } diff --git a/streamingvisitors/src/vespa/searchvisitor/searchenvironment.h b/streamingvisitors/src/vespa/searchvisitor/searchenvironment.h index 844c99ad347..15a6edd667d 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchenvironment.h +++ b/streamingvisitors/src/vespa/searchvisitor/searchenvironment.h @@ -18,7 +18,7 @@ private: class Env : public config::SimpleConfigurable { public: using SP = std::shared_ptr<Env>; - Env(const vespalib::string & muffens, const config::ConfigUri & configUri, Fast_NormalizeWordFolder & wf); + Env(const config::ConfigUri& configUri, const Fast_NormalizeWordFolder& wf); ~Env() override; const vsm::VSMAdapter * getVSMAdapter() const { return _vsmAdapter.get(); } const RankManager * getRankManager() const { return _rankManager.get(); } diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index 105196a3249..7a987a39d0e 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -25,6 +25,7 @@ #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/text/stringtokenizer.h> #include <vespa/fnet/databuffer.h> +#include <optional> #include <vespa/log/log.h> LOG_SETUP(".visitor.instance.searchvisitor"); @@ -48,6 +49,20 @@ using vsm::FieldPath; using vsm::StorageDocument; using vsm::StringFieldIdTMap; +namespace { + +std::optional<vespalib::string> +extract_search_cluster(const vdslib::Parameters& params) +{ + Parameters::ValueRef searchClusterBlob; + if (params.lookup("searchcluster", searchClusterBlob)) { + return {{searchClusterBlob.data(), searchClusterBlob.size()}}; + } + return std::nullopt; +} + +} + class ForceWordfolderInit { public: @@ -354,11 +369,10 @@ void SearchVisitor::init(const Parameters & params) _summaryGenerator.set_location(valueRef); } - Parameters::ValueRef searchClusterBlob; - if (params.lookup("searchcluster", searchClusterBlob)) { - LOG(spam, "Received searchcluster blob of %zd bytes", searchClusterBlob.size()); - vespalib::string searchCluster(searchClusterBlob.data(), searchClusterBlob.size()); - _vsmAdapter = _env.getVSMAdapter(searchCluster); + auto search_cluster = extract_search_cluster(params); + if (search_cluster.has_value()) { + LOG(spam, "Received searchcluster blob of %zd bytes", search_cluster.value().size()); + _vsmAdapter = _env.getVSMAdapter(search_cluster.value()); if ( params.lookup("sort", valueRef) ) { search::uca::UcaConverterFactory ucaFactory; @@ -396,7 +410,7 @@ void SearchVisitor::init(const Parameters & params) setupAttributeVectorsForSorting(_sortSpec); - const RankManager * rm = _env.getRankManager(searchCluster); + const RankManager * rm = _env.getRankManager(search_cluster.value()); _rankController.setRankManagerSnapshot(rm->getSnapshot()); _rankController.setupRankProcessors(_query, location, wantedSummaryCount, _attrMan, _attributeFields); diff --git a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp index 8a3226ee52a..50ee380cac9 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp @@ -174,9 +174,8 @@ VSMConfigSnapshot::VSMConfigSnapshot(const vespalib::string & configId, const co { } VSMConfigSnapshot::~VSMConfigSnapshot() = default; -VSMAdapter::VSMAdapter(const vespalib::string & highlightindexes, const vespalib::string & configId, Fast_WordFolder & wordFolder) - : _highlightindexes(highlightindexes), - _configId(configId), +VSMAdapter::VSMAdapter(const vespalib::string & configId, const Fast_WordFolder& wordFolder) + : _configId(configId), _wordFolder(wordFolder), _fieldsCfg(), _docsumTools(), diff --git a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h index ba87ccfef05..2ca627c837b 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h +++ b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h @@ -103,16 +103,15 @@ public: class VSMAdapter { public: - VSMAdapter(const vespalib::string & highlightindexes, const vespalib::string & configId, Fast_WordFolder & wordFolder); + VSMAdapter(const vespalib::string& configId, const Fast_WordFolder& wordFolder); virtual ~VSMAdapter(); VsmfieldsHandle getFieldsConfig() const { return _fieldsCfg.get(); } DocsumToolsPtr getDocsumTools() const { return _docsumTools.get(); } void configure(const VSMConfigSnapshot & snapshot); private: - vespalib::string _highlightindexes; const vespalib::string _configId; - Fast_WordFolder & _wordFolder; + const Fast_WordFolder& _wordFolder; vespalib::PtrHolder<VsmfieldsConfig> _fieldsCfg; vespalib::PtrHolder<DocsumTools> _docsumTools; std::unique_ptr<JuniperProperties> _juniperProps; diff --git a/vdslib/src/main/java/com/yahoo/vdslib/SearchResult.java b/vdslib/src/main/java/com/yahoo/vdslib/SearchResult.java index e263c292bc8..b7c9b1b71b5 100644 --- a/vdslib/src/main/java/com/yahoo/vdslib/SearchResult.java +++ b/vdslib/src/main/java/com/yahoo/vdslib/SearchResult.java @@ -1,28 +1,40 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vdslib; +import com.yahoo.data.access.helpers.MatchFeatureData; import com.yahoo.vespa.objects.BufferSerializer; import com.yahoo.vespa.objects.Deserializer; import java.io.UnsupportedEncodingException; import java.nio.ByteOrder; +import java.util.ArrayList; import java.util.Map; +import java.util.Optional; import java.util.TreeMap; public class SearchResult { public static class Hit implements Comparable<Hit> { private String docId; - private double rank; + private double rank; + private MatchFeatureData.HitValue matchFeatures; public Hit(Hit h) { docId = h.docId; rank = h.rank; + matchFeatures = h.matchFeatures; } public Hit(String docId, double rank) { this.rank = rank; this.docId = docId; + this.matchFeatures = null; + } + final public String getDocId() { return docId; } + final public double getRank() { return rank; } + final public Optional<MatchFeatureData.HitValue> getMatchFeatures() { + return Optional.ofNullable(matchFeatures); } - final public String getDocId() { return docId; } - final public double getRank() { return rank; } final public void setRank(double rank) { this.rank = rank; } + final public void setMatchFeatures(MatchFeatureData.HitValue matchFeatures) { + this.matchFeatures = matchFeatures; + } public int compareTo(Hit h) { return (h.rank < rank) ? -1 : (h.rank > rank) ? 1 : 0; // Sort order: descending } @@ -49,12 +61,19 @@ public class SearchResult { private Hit[] hits; private TreeMap<Integer, byte []> aggregatorList; private TreeMap<Integer, byte []> groupingList; + private static int EXTENSION_FLAGS_PRESENT = -1; + private static int MATCH_FEATURES_PRESENT_MASK = 1; public SearchResult(Deserializer buf) { BufferSerializer bser = (BufferSerializer) buf; // TODO: dirty cast. must do this differently bser.order(ByteOrder.BIG_ENDIAN); this.totalHits = buf.getInt(null); int numHits = buf.getInt(null); + int extensionFlags = 0; + if (hasExtensionFlags(numHits)) { + extensionFlags = buf.getInt(null); + numHits = buf.getInt(null); + } hits = new Hit[numHits]; if (numHits != 0) { int docIdBufferLength = buf.getInt(null); @@ -101,7 +120,45 @@ public class SearchResult { groupingList.put(aggrId, buf.getBytes(null, aggrLength)); } + if (hasMatchFeatures(extensionFlags)) { + deserializeMatchFeatures(buf, numHits); + } } + + private void deserializeMatchFeatures(Deserializer buf, int numHits) { + var featureNames = new ArrayList<String>(); + int numFeatures = buf.getInt(null); + for (int i = 0; i < numFeatures; ++i) { + featureNames.add(buf.getString(null)); + } + var factory = new MatchFeatureData(featureNames); + for (int i = 0; i < numHits; ++i) { + var matchFeatures = factory.addHit(); + for (int j = 0; j < numFeatures; ++j) { + byte featureType = buf.getByte(null); + if (isDoubleFeature(featureType)) { + matchFeatures.set(j, buf.getDouble(null)); + } else { + int bufLength = buf.getInt(null); + matchFeatures.set(j, buf.getBytes(null, bufLength)); + } + } + hits[i].setMatchFeatures(matchFeatures); + } + } + + private static boolean hasExtensionFlags(int numHits) { + return numHits == EXTENSION_FLAGS_PRESENT; + } + + private static boolean hasMatchFeatures(int extensionFlags) { + return (extensionFlags & MATCH_FEATURES_PRESENT_MASK) != 0; + } + + private static boolean isDoubleFeature(byte featureType) { + return featureType == 0; + } + /** * Constructs a new message from a byte buffer. * diff --git a/vdslib/src/vespa/vdslib/container/searchresult.cpp b/vdslib/src/vespa/vdslib/container/searchresult.cpp index c8bc331d1a8..e348c9d9e13 100644 --- a/vdslib/src/vespa/vdslib/container/searchresult.cpp +++ b/vdslib/src/vespa/vdslib/container/searchresult.cpp @@ -11,21 +11,21 @@ namespace vdslib { namespace { // Magic value for hit count to enable extension flags -constexpr uint32_t enable_extension_flags_magic = 0xffffffffu; +constexpr uint32_t extension_flags_present = 0xffffffffu; // Extension flag values -constexpr uint32_t match_features_present = 1; +constexpr uint32_t match_features_present_mask = 1; // Selector values for feature value constexpr uint8_t feature_value_is_double = 0; constexpr uint8_t feature_value_is_data = 1; inline bool has_match_features(uint32_t extension_flags) { - return ((extension_flags & match_features_present) != 0); + return ((extension_flags & match_features_present_mask) != 0); } inline bool must_serialize_extension_flags(uint32_t extension_flags, uint32_t hit_count) { - return ((extension_flags != 0) || (hit_count == enable_extension_flags_magic)); + return ((extension_flags != 0) || (hit_count == extension_flags_present)); } } @@ -155,7 +155,7 @@ SearchResult::deserialize(document::ByteBuffer & buf) uint32_t numResults(0), bufSize(0); buf.getIntNetwork(tmp); numResults = tmp; uint32_t extension_flags = 0u; - if (numResults == enable_extension_flags_magic) { + if (numResults == extension_flags_present) { buf.getIntNetwork(tmp); extension_flags = tmp; buf.getIntNetwork(tmp); @@ -189,7 +189,7 @@ void SearchResult::serialize(vespalib::GrowableByteBuffer & buf) const uint32_t hitCount = std::min(_hits.size(), _wantedHits); uint32_t extension_flags = calc_extension_flags(hitCount); if (must_serialize_extension_flags(extension_flags, hitCount)) { - buf.putInt(enable_extension_flags_magic); + buf.putInt(extension_flags_present); buf.putInt(extension_flags); } buf.putInt(hitCount); @@ -241,7 +241,7 @@ SearchResult::calc_extension_flags(uint32_t hit_count) const noexcept { uint32_t extension_flags = 0u; if (!_match_features.names.empty() && hit_count != 0) { - extension_flags |= match_features_present; + extension_flags |= match_features_present_mask; } return extension_flags; } @@ -251,7 +251,7 @@ SearchResult::get_match_features_serialized_size(uint32_t hit_count) const noexc { uint32_t size = sizeof(uint32_t); for (auto& name : _match_features.names) { - size += sizeof(uint32_t) + name.size(); + size += sizeof(uint32_t) + name.size() + 1; } for (uint32_t i = 0; i < hit_count; ++i) { auto mfv = get_match_feature_values(i); @@ -271,7 +271,7 @@ SearchResult::serialize_match_features(vespalib::GrowableByteBuffer& buf, uint32 { buf.putInt(_match_features.names.size()); for (auto& name : _match_features.names) { - buf.putString(name); + buf.put_c_string(name); } for (uint32_t i = 0; i < hit_count; ++i) { auto mfv = get_match_feature_values(i); @@ -301,10 +301,11 @@ SearchResult::deserialize_match_features(document::ByteBuffer& buf) _match_features.names.resize(num_features); for (auto& name : _match_features.names) { buf.getIntNetwork(tmp); - name.resize(tmp); - if (tmp != 0) { - buf.getBytes(&name[0], tmp); + if (tmp > 1) { + name.resize(tmp - 1); + buf.getBytes(&name[0], tmp - 1); } + buf.getByte(selector); // Read and ignore the nul-termination. } uint32_t hit_count = _hits.size(); uint32_t num_values = num_features * hit_count; diff --git a/vespa-athenz/pom.xml b/vespa-athenz/pom.xml index 04172e1d49e..a6f83fb102f 100644 --- a/vespa-athenz/pom.xml +++ b/vespa-athenz/pom.xml @@ -196,6 +196,11 @@ </dependency> <dependency> <!-- needed by auth-core --> <groupId>io.jsonwebtoken</groupId> + <artifactId>jjwt-api</artifactId> + <scope>compile</scope> + </dependency> + <dependency> <!-- needed by auth-core --> + <groupId>io.jsonwebtoken</groupId> <artifactId>jjwt-jackson</artifactId> <scope>compile</scope> <exclusions> diff --git a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt index 8d9265bc083..a408486d2a5 100644 --- a/vespa-dependencies-enforcer/allowed-maven-dependencies.txt +++ b/vespa-dependencies-enforcer/allowed-maven-dependencies.txt @@ -9,20 +9,20 @@ backport-util-concurrent:backport-util-concurrent:3.1 ch.qos.logback:logback-classic:1.2.10 ch.qos.logback:logback-core:1.2.10 classworlds:classworlds:1.1-alpha-2 -com.amazonaws:aws-java-sdk-core:1.12.331 -com.amazonaws:aws-java-sdk-ssm:1.12.331 -com.amazonaws:aws-java-sdk-sts:1.12.331 -com.amazonaws:jmespath-java:1.12.331 +com.amazonaws:aws-java-sdk-core:1.12.460 +com.amazonaws:aws-java-sdk-ssm:1.12.460 +com.amazonaws:aws-java-sdk-sts:1.12.460 +com.amazonaws:jmespath-java:1.12.460 com.auth0:java-jwt:3.10.0 -com.fasterxml.jackson.core:jackson-annotations:2.14.2 -com.fasterxml.jackson.core:jackson-core:2.14.2 -com.fasterxml.jackson.core:jackson-databind:2.14.2 -com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.14.2 -com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.14.2 -com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.14.2 -com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.14.2 -com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.14.2 -com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.14.2 +com.fasterxml.jackson.core:jackson-annotations:2.15.0 +com.fasterxml.jackson.core:jackson-core:2.15.0 +com.fasterxml.jackson.core:jackson-databind:2.15.0 +com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.15.0 +com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.15.0 +com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.15.0 +com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:2.15.0 +com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:2.15.0 +com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.15.0 com.github.spotbugs:spotbugs-annotations:3.1.9 com.google.code.findbugs:jsr305:3.0.2 com.google.errorprone:error_prone_annotations:2.18.0 @@ -140,9 +140,9 @@ org.apache.zookeeper:zookeeper:3.8.1 org.apache.zookeeper:zookeeper-jute:3.8.0 org.apache.zookeeper:zookeeper-jute:3.8.1 org.apiguardian:apiguardian-api:1.1.2 -org.bouncycastle:bcpkix-jdk18on:1.72 -org.bouncycastle:bcprov-jdk18on:1.72 -org.bouncycastle:bcutil-jdk18on:1.72 +org.bouncycastle:bcpkix-jdk18on:1.73 +org.bouncycastle:bcprov-jdk18on:1.73 +org.bouncycastle:bcutil-jdk18on:1.73 org.codehaus.plexus:plexus-archiver:4.2.1 org.codehaus.plexus:plexus-cipher:2.0 org.codehaus.plexus:plexus-classworlds:2.6.0 diff --git a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpRequestStrategy.java b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpRequestStrategy.java index 4e2256f40b4..ce86ad59ffe 100644 --- a/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpRequestStrategy.java +++ b/vespa-feed-client/src/main/java/ai/vespa/feed/client/impl/HttpRequestStrategy.java @@ -9,7 +9,6 @@ import ai.vespa.feed.client.FeedException; import ai.vespa.feed.client.HttpResponse ; import ai.vespa.feed.client.OperationStats; -import javax.net.ssl.SSLException; import java.io.IOException; import java.nio.channels.CancelledKeyException; import java.util.Map; @@ -139,9 +138,9 @@ class HttpRequestStrategy implements RequestStrategy { */ private boolean retry(HttpRequest request, Throwable thrown, int attempt) { breaker.failure(thrown); - if ( (thrown instanceof IOException && ! (thrown instanceof SSLException)) // General IO problems, but not SSL, which is irrecoverable. - || (thrown instanceof CancellationException) // TLS session disconnect. - || (thrown instanceof CancelledKeyException)) { // Selection cancelled. + if ( (thrown instanceof IOException) // General IO problems. + || (thrown instanceof CancellationException) // TLS session disconnect. + || (thrown instanceof CancelledKeyException)) { // Selection cancelled. log.log(FINER, thrown, () -> "Failed attempt " + attempt + " at " + request); return retry(request, attempt); } diff --git a/vespaclient-java/src/main/java/com/yahoo/vespa/security/tool/CliUtils.java b/vespaclient-java/src/main/java/com/yahoo/vespa/security/tool/CliUtils.java index a60c3647b41..b09ae17cd77 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespa/security/tool/CliUtils.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespa/security/tool/CliUtils.java @@ -8,6 +8,7 @@ import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Paths; +import java.nio.file.attribute.PosixFilePermissions; /** * @author vekterli @@ -43,7 +44,10 @@ public class CliUtils { return stdOut; } else { // TODO fail if file already exists? - return Files.newOutputStream(Paths.get(pathOrDash)); + var privFilePerms = PosixFilePermissions.fromString("rw-------"); + var outPath = Paths.get(pathOrDash); + Files.createFile(outPath, PosixFilePermissions.asFileAttribute(privFilePerms)); + return Files.newOutputStream(outPath); } } diff --git a/vespaclient-java/src/test/java/com/yahoo/vespa/security/tool/CryptoToolsTest.java b/vespaclient-java/src/test/java/com/yahoo/vespa/security/tool/CryptoToolsTest.java index f55278342e1..05d7e8c9511 100644 --- a/vespaclient-java/src/test/java/com/yahoo/vespa/security/tool/CryptoToolsTest.java +++ b/vespaclient-java/src/test/java/com/yahoo/vespa/security/tool/CryptoToolsTest.java @@ -64,6 +64,12 @@ public class CryptoToolsTest { Files.writeString(keyPath, contents); } + private static void assertOnlyFileOwnerHasAccessRights(Path file) throws IOException { + var actualFilePerms = Files.getPosixFilePermissions(file); + var expectedPerms = PosixFilePermissions.fromString("rw-------"); + assertEquals(expectedPerms, actualFilePerms); + } + @Test void top_level_help_page_printed_if_help_option_given() throws IOException { verifyStdoutMatchesFile(List.of("--help"), "expected-help-output.txt"); @@ -180,9 +186,7 @@ public class CryptoToolsTest { "--private-out-file", absPathOf(privKeyFile), "--public-out-file", absPathOf(pubKeyFile))); assertEquals(0, procOut.exitCode()); - var privKeyPerms = Files.getPosixFilePermissions(privKeyFile); - var expectedPerms = PosixFilePermissions.fromString("rw-------"); - assertEquals(expectedPerms, privKeyPerms); + assertOnlyFileOwnerHasAccessRights(privKeyFile); } private static final String TEST_PRIV_KEY = "GFg54SaGNCmcSGufZCx68SKLGuAFrASoDeMk3t5AjU6L"; @@ -381,6 +385,7 @@ public class CryptoToolsTest { assertEquals("", procOut.stdErr()); assertEquals(greatSecret, Files.readString(decryptedPath)); + assertOnlyFileOwnerHasAccessRights(decryptedPath); } @Test diff --git a/vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp b/vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp index 424ad7ba470..fa0dc9bf99e 100644 --- a/vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp +++ b/vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp @@ -76,6 +76,14 @@ GrowableByteBuffer::putString(vespalib::stringref v) } void +GrowableByteBuffer::put_c_string(vespalib::stringref v) +{ + putInt(v.size() + 1); + putBytes(v.data(), v.size()); + putByte(0); +} + +void GrowableByteBuffer::putByte(uint8_t v) { putBytes(reinterpret_cast<const char*>(&v), sizeof(v)); diff --git a/vespalib/src/vespa/vespalib/util/growablebytebuffer.h b/vespalib/src/vespa/vespalib/util/growablebytebuffer.h index b0fb30606d4..61698868dba 100644 --- a/vespalib/src/vespa/vespalib/util/growablebytebuffer.h +++ b/vespalib/src/vespa/vespalib/util/growablebytebuffer.h @@ -68,11 +68,17 @@ public: void putDouble(double v); /** - Adds a string to the buffer. + Adds a string to the buffer (without nul-termination). */ void putString(vespalib::stringref v); /** + * Adds a string to the buffer (including nul-termination). + * This matches com.yahoo.vespa.objects.Deserializer.getString. + */ + void put_c_string(vespalib::stringref v); + + /** Adds a single byte to the buffer. */ void putByte(uint8_t v); |