diff options
77 files changed, 1047 insertions, 236 deletions
diff --git a/client/go/go.mod b/client/go/go.mod index f698d42be7b..b60efceb469 100644 --- a/client/go/go.mod +++ b/client/go/go.mod @@ -10,7 +10,7 @@ require ( github.com/go-json-experiment/json v0.0.0-20230216065249-540f01442424 github.com/klauspost/compress v1.17.1 github.com/mattn/go-colorable v0.1.13 - github.com/mattn/go-isatty v0.0.19 + github.com/mattn/go-isatty v0.0.20 github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 diff --git a/client/go/go.sum b/client/go/go.sum index bba03a1ffe8..509b06eb7eb 100644 --- a/client/go/go.sum +++ b/client/go/go.sum @@ -41,6 +41,8 @@ github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp9 github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/client/js/app/yarn.lock b/client/js/app/yarn.lock index a00131e7ba0..a3795b594b2 100644 --- a/client/js/app/yarn.lock +++ b/client/js/app/yarn.lock @@ -15,7 +15,7 @@ "@jridgewell/gen-mapping" "^0.3.0" "@jridgewell/trace-mapping" "^0.3.9" -"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.12.13", "@babel/code-frame@^7.22.10", "@babel/code-frame@^7.22.13", "@babel/code-frame@^7.22.5": +"@babel/code-frame@^7.0.0", "@babel/code-frame@^7.12.13", "@babel/code-frame@^7.22.13", "@babel/code-frame@^7.22.5": version "7.22.13" resolved "https://registry.yarnpkg.com/@babel/code-frame/-/code-frame-7.22.13.tgz#e3c1c099402598483b7a8c46a721d1038803755e" integrity sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w== @@ -91,7 +91,7 @@ json5 "^2.2.3" semver "^6.3.1" -"@babel/generator@^7.22.10", "@babel/generator@^7.22.15", "@babel/generator@^7.7.2": +"@babel/generator@^7.22.15", "@babel/generator@^7.7.2": version "7.22.15" resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.22.15.tgz#1564189c7ec94cb8f77b5e8a90c4d200d21b2339" integrity sha512-Zu9oWARBqeVOW0dZOjXc3JObrzuqothQ3y/n1kUtrjCoCPLkXUwMvOo/F/TCfoHMbWIFlWwpZtkZVb9ga4U2pA== @@ -111,6 +111,16 @@ "@jridgewell/trace-mapping" "^0.3.17" jsesc "^2.5.1" +"@babel/generator@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/generator/-/generator-7.23.0.tgz#df5c386e2218be505b34837acbcb874d7a983420" + integrity sha512-lN85QRR+5IbYrMWM6Y4pE/noaQtg4pNiqeNGX60eqOfo6gtEj6uw/JagelB8vVztSd7R6M5n1+PQkDbHbBRU4g== + dependencies: + "@babel/types" "^7.23.0" + "@jridgewell/gen-mapping" "^0.3.2" + "@jridgewell/trace-mapping" "^0.3.17" + jsesc "^2.5.1" + "@babel/helper-compilation-targets@^7.22.15": version "7.22.15" resolved "https://registry.yarnpkg.com/@babel/helper-compilation-targets/-/helper-compilation-targets-7.22.15.tgz#0698fc44551a26cf29f18d4662d5bf545a6cfc52" @@ -138,13 +148,13 @@ resolved "https://registry.yarnpkg.com/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz#96159db61d34a29dba454c959f5ae4a649ba9167" integrity sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA== -"@babel/helper-function-name@^7.22.5": - version "7.22.5" - resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.22.5.tgz#ede300828905bb15e582c037162f99d5183af1be" - integrity sha512-wtHSq6jMRE3uF2otvfuD3DIvVhOsSNshQl0Qrd7qC9oQJzHvOL4qQXlQn2916+CXGywIjpGuIkoyZRRxHPiNQQ== +"@babel/helper-function-name@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz#1f9a3cdbd5b2698a670c30d2735f9af95ed52759" + integrity sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw== dependencies: - "@babel/template" "^7.22.5" - "@babel/types" "^7.22.5" + "@babel/template" "^7.22.15" + "@babel/types" "^7.23.0" "@babel/helper-hoist-variables@^7.22.5": version "7.22.5" @@ -261,7 +271,7 @@ chalk "^2.4.2" js-tokens "^4.0.0" -"@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.20.7", "@babel/parser@^7.22.11", "@babel/parser@^7.22.15", "@babel/parser@^7.22.16": +"@babel/parser@^7.1.0", "@babel/parser@^7.14.7", "@babel/parser@^7.20.7", "@babel/parser@^7.22.15", "@babel/parser@^7.22.16": version "7.22.16" resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.22.16.tgz#180aead7f247305cce6551bea2720934e2fa2c95" integrity sha512-+gPfKv8UWeKKeJTUxe59+OobVcrYHETCsORl61EmSkmgymguYk/X5bp7GuUIXaFsc6y++v8ZxPsLSSuujqDphA== @@ -271,6 +281,11 @@ resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.22.13.tgz#23fb17892b2be7afef94f573031c2f4b42839a2b" integrity sha512-3l6+4YOvc9wx7VlCSw4yQfcBo01ECA8TicQfbnCPuCEpRQrf+gTUyGdxNw+pyTUyywp6JRD1w0YQs9TpBXYlkw== +"@babel/parser@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/parser/-/parser-7.23.0.tgz#da950e622420bf96ca0d0f2909cdddac3acd8719" + integrity sha512-vvPKKdMemU85V9WE/l5wZEmImpCtLqbnTvqDS2U1fJ96KrxoW7KrXhNsNCblQlg8Ck4b85yxdTyelsMUgFUXiw== + "@babel/plugin-syntax-async-generators@^7.8.4": version "7.8.4" resolved "https://registry.yarnpkg.com/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz#a983fb1aeb2ec3f6ed042a210f640e90e786fe0d" @@ -415,51 +430,19 @@ "@babel/parser" "^7.22.15" "@babel/types" "^7.22.15" -"@babel/traverse@^7.22.11", "@babel/traverse@^7.22.17": - version "7.22.17" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.22.17.tgz#b23c203ab3707e3be816043081b4a994fcacec44" - integrity sha512-xK4Uwm0JnAMvxYZxOVecss85WxTEIbTa7bnGyf/+EgCL5Zt3U7htUpEOWv9detPlamGKuRzCqw74xVglDWpPdg== +"@babel/traverse@^7.22.11", "@babel/traverse@^7.22.15", "@babel/traverse@^7.22.17", "@babel/traverse@^7.22.20", "@babel/traverse@^7.22.8": + version "7.23.2" + resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.23.2.tgz#329c7a06735e144a506bdb2cad0268b7f46f4ad8" + integrity sha512-azpe59SQ48qG6nu2CzcMLbxUudtN+dOM9kDbUqGq3HXUJRlo7i8fvPoxQUzYgLZ4cMVmuZgm8vvBpNeRhd6XSw== dependencies: "@babel/code-frame" "^7.22.13" - "@babel/generator" "^7.22.15" - "@babel/helper-environment-visitor" "^7.22.5" - "@babel/helper-function-name" "^7.22.5" - "@babel/helper-hoist-variables" "^7.22.5" - "@babel/helper-split-export-declaration" "^7.22.6" - "@babel/parser" "^7.22.16" - "@babel/types" "^7.22.17" - debug "^4.1.0" - globals "^11.1.0" - -"@babel/traverse@^7.22.15", "@babel/traverse@^7.22.20": - version "7.22.20" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.22.20.tgz#db572d9cb5c79e02d83e5618b82f6991c07584c9" - integrity sha512-eU260mPZbU7mZ0N+X10pxXhQFMGTeLb9eFS0mxehS8HZp9o1uSnFeWQuG1UPrlxgA7QoUzFhOnilHDp0AXCyHw== - dependencies: - "@babel/code-frame" "^7.22.13" - "@babel/generator" "^7.22.15" + "@babel/generator" "^7.23.0" "@babel/helper-environment-visitor" "^7.22.20" - "@babel/helper-function-name" "^7.22.5" - "@babel/helper-hoist-variables" "^7.22.5" - "@babel/helper-split-export-declaration" "^7.22.6" - "@babel/parser" "^7.22.16" - "@babel/types" "^7.22.19" - debug "^4.1.0" - globals "^11.1.0" - -"@babel/traverse@^7.22.8": - version "7.22.11" - resolved "https://registry.yarnpkg.com/@babel/traverse/-/traverse-7.22.11.tgz#71ebb3af7a05ff97280b83f05f8865ac94b2027c" - integrity sha512-mzAenteTfomcB7mfPtyi+4oe5BZ6MXxWcn4CX+h4IRJ+OOGXBrWU6jDQavkQI9Vuc5P+donFabBfFCcmWka9lQ== - dependencies: - "@babel/code-frame" "^7.22.10" - "@babel/generator" "^7.22.10" - "@babel/helper-environment-visitor" "^7.22.5" - "@babel/helper-function-name" "^7.22.5" + "@babel/helper-function-name" "^7.23.0" "@babel/helper-hoist-variables" "^7.22.5" "@babel/helper-split-export-declaration" "^7.22.6" - "@babel/parser" "^7.22.11" - "@babel/types" "^7.22.11" + "@babel/parser" "^7.23.0" + "@babel/types" "^7.23.0" debug "^4.1.0" globals "^11.1.0" @@ -481,6 +464,15 @@ "@babel/helper-validator-identifier" "^7.22.15" to-fast-properties "^2.0.0" +"@babel/types@^7.23.0": + version "7.23.0" + resolved "https://registry.yarnpkg.com/@babel/types/-/types-7.23.0.tgz#8c1f020c9df0e737e4e247c0619f58c68458aaeb" + integrity sha512-0oIyUfKoI3mSqMvsxBdclDwxXKXAUA8v/apZbc+iSyARYou1o8ZGDxbUYyLFoW2arqS2jDGqJuZvv1d/io1axg== + dependencies: + "@babel/helper-string-parser" "^7.22.5" + "@babel/helper-validator-identifier" "^7.22.20" + to-fast-properties "^2.0.0" + "@bcoe/v8-coverage@^0.2.3": version "0.2.3" resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" @@ -1245,10 +1237,10 @@ dependencies: "@babel/runtime" "^7.13.10" -"@remix-run/router@1.9.0": - version "1.9.0" - resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.9.0.tgz#9033238b41c4cbe1e961eccb3f79e2c588328cf6" - integrity sha512-bV63itrKBC0zdT27qYm6SDZHlkXwFL1xMBuhkn+X7l0+IIhNaH5wuuvZKp6eKhCD4KFhujhfhCT1YxXW6esUIA== +"@remix-run/router@1.10.0": + version "1.10.0" + resolved "https://registry.yarnpkg.com/@remix-run/router/-/router-1.10.0.tgz#e2170dc2049b06e65bbe883adad0e8ddf8291278" + integrity sha512-Lm+fYpMfZoEucJ7cMxgt4dYt8jLfbpwRCzAjm9UgSLOkmlqo9gupxt6YX3DY0Fk155NT9l17d/ydi+964uS9Lw== "@sinclair/typebox@^0.27.8": version "0.27.8" @@ -4676,19 +4668,19 @@ react-remove-scroll@^2.5.5: use-sidecar "^1.1.2" react-router-dom@^6: - version "6.16.0" - resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.16.0.tgz#86f24658da35eb66727e75ecbb1a029e33ee39d9" - integrity sha512-aTfBLv3mk/gaKLxgRDUPbPw+s4Y/O+ma3rEN1u8EgEpLpPe6gNjIsWt9rxushMHHMb7mSwxRGdGlGdvmFsyPIg== + version "6.17.0" + resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-6.17.0.tgz#ea73f89186546c1cf72b10fcb7356d874321b2ad" + integrity sha512-qWHkkbXQX+6li0COUUPKAUkxjNNqPJuiBd27dVwQGDNsuFBdMbrS6UZ0CLYc4CsbdLYTckn4oB4tGDuPZpPhaQ== dependencies: - "@remix-run/router" "1.9.0" - react-router "6.16.0" + "@remix-run/router" "1.10.0" + react-router "6.17.0" -react-router@6.16.0: - version "6.16.0" - resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.16.0.tgz#abbf3d5bdc9c108c9b822a18be10ee004096fb81" - integrity sha512-VT4Mmc4jj5YyjpOi5jOf0I+TYzGpvzERy4ckNSvSh2RArv8LLoCxlsZ2D+tc7zgjxcY34oTz2hZaeX5RVprKqA== +react-router@6.17.0: + version "6.17.0" + resolved "https://registry.yarnpkg.com/react-router/-/react-router-6.17.0.tgz#7b680c4cefbc425b57537eb9c73bedecbdc67c1e" + integrity sha512-YJR3OTJzi3zhqeJYADHANCGPUu9J+6fT5GLv82UWRGSxu6oJYCKVmxUcaBQuGm9udpWmPsvpme/CdHumqgsoaA== dependencies: - "@remix-run/router" "1.9.0" + "@remix-run/router" "1.10.0" react-style-singleton@^2.2.1: version "2.2.1" diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java index 36d6efdf59b..d262c7bc862 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationFile.java @@ -27,21 +27,21 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { } /** - * Check whether or not this file is a directory. + * Checks whether this file is a directory. * * @return true if it is, false if not. */ public abstract boolean isDirectory(); /** - * Test whether or not this file exists. + * Tests whether this file exists. * * @return true if it exists, false if not. */ public abstract boolean exists(); /** - * Create a {@link Reader} for the contents of this file. + * Creates a {@link Reader} for the contents of this file. * * @return A {@link Reader} that should be closed after use. * @throws FileNotFoundException if the file is not found. @@ -50,7 +50,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { /** - * Create an {@link InputStream} for the contents of this file. + * Creates an {@link InputStream} for the contents of this file. * * @return An {@link InputStream} that should be closed after use. * @throws FileNotFoundException if the file is not found. @@ -58,7 +58,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { public abstract InputStream createInputStream() throws FileNotFoundException; /** - * Create a directory at the path represented by this file. Parent directories will + * Creates a directory at the path represented by this file. Parent directories will * be automatically created. * * @return this @@ -67,7 +67,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { public abstract ApplicationFile createDirectory(); /** - * Write the contents from this reader to this file. Any existing content will be overwritten! + * Writes the contents from supplied reader to this file. Any existing content will be overwritten! * * @param input A reader pointing to the content that should be written. * @return this @@ -82,7 +82,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { public abstract ApplicationFile appendFile(String value); /** - * List the files under this directory. If this is file, an empty list is returned. + * Lists the files under this directory. If this is file, an empty list is returned. * Only immediate files/subdirectories are returned. * * @return a list of files in this directory. @@ -92,7 +92,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { } /** - * List the files under this directory. If this is file, an empty list is returned. + * Lists the files under this directory. If this is a file, an empty list is returned. * Only immediate files/subdirectories are returned. * * @param filter A filter functor for filtering path names @@ -101,7 +101,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { public abstract List<ApplicationFile> listFiles(PathFilter filter); /** - * List the files in this directory, optionally list files for subdirectories recursively as well. + * Lists the files in this directory, optionally lists files for subdirectories recursively as well. * * @param recurse Set to true if all files in the directory tree should be returned. * @return a list of files in this directory. @@ -121,7 +121,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { } /** - * Delete the file pointed to by this. If it is a non-empty directory, the operation will throw. + * Deletes the file pointed to by this. If this is a non-empty directory, the operation will throw. * * @return this. * @throws RuntimeException if the file is a directory and not empty. @@ -129,7 +129,7 @@ public abstract class ApplicationFile implements Comparable<ApplicationFile> { public abstract ApplicationFile delete(); /** - * Get the path that this file represents. + * Gets the path that this file represents. * * @return a Path */ diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java index ddb6b004070..94b456b3f5e 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java @@ -155,7 +155,8 @@ public class SummaryClass extends Derived { summaryField.getTransform() == SummaryTransform.GEOPOS || summaryField.getTransform() == SummaryTransform.POSITIONS || summaryField.getTransform() == SummaryTransform.MATCHED_ELEMENTS_FILTER || - summaryField.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER) + summaryField.getTransform() == SummaryTransform.MATCHED_ATTRIBUTE_ELEMENTS_FILTER || + summaryField.getTransform() == SummaryTransform.LINGUISTICS_TOKENS) { return summaryField.getSingleSource(); } else if (summaryField.getTransform().isDynamic()) { diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java index c1e6dd2aea3..54a4883fa00 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java @@ -92,6 +92,8 @@ public class SummaryClassField { return Type.FEATUREDATA; } else if (transform != null && transform.equals(SummaryTransform.SUMMARYFEATURES)) { return Type.FEATUREDATA; + } else if (transform != null && transform.equals(SummaryTransform.LINGUISTICS_TOKENS)) { + return Type.JSONSTRING; } else { return Type.LONGSTRING; } diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java index 7c6d62580cb..61f68defe40 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java @@ -217,6 +217,8 @@ public class ConvertParsedFields { transform = SummaryTransform.MATCHED_ELEMENTS_FILTER; } else if (parsed.getDynamic()) { transform = SummaryTransform.DYNAMICTEASER; + } else if (parsed.getLinguisticsTokens()) { + transform = SummaryTransform.LINGUISTICS_TOKENS; } if (parsed.getBolded()) { transform = transform.bold(); diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java index 1d5d73635e7..446981f1ba4 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java @@ -18,6 +18,7 @@ class ParsedSummaryField extends ParsedBlock { private boolean isMEO = false; private boolean isFull = false; private boolean isBold = false; + private boolean isLinguisticsTokens = false; private final List<String> sources = new ArrayList<>(); private final List<String> destinations = new ArrayList<>(); @@ -37,6 +38,7 @@ class ParsedSummaryField extends ParsedBlock { boolean getDynamic() { return isDyn; } boolean getFull() { return isFull; } boolean getMatchedElementsOnly() { return isMEO; } + boolean getLinguisticsTokens() { return isLinguisticsTokens; } void addDestination(String dst) { destinations.add(dst); } void addSource(String src) { sources.add(src); } @@ -44,6 +46,7 @@ class ParsedSummaryField extends ParsedBlock { void setDynamic() { this.isDyn = true; } void setFull() { this.isFull = true; } void setMatchedElementsOnly() { this.isMEO = true; } + void setLinguisticsTokens() { this.isLinguisticsTokens = true; } void setType(ParsedType value) { verifyThat(type == null, "Cannot change type from ", type, "to", value); this.type = value; diff --git a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java index 1d279242895..e54f8d3e881 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java @@ -78,7 +78,8 @@ public class IndexingOutputs extends Processor { return; } dynamicSummary.add(summaryName); - } else if (summaryTransform != SummaryTransform.ATTRIBUTE) { + } else if (summaryTransform != SummaryTransform.ATTRIBUTE && + summaryTransform != SummaryTransform.LINGUISTICS_TOKENS) { staticSummary.add(summaryName); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java index 575a3a748e6..c7c1606951e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryTransform.java @@ -23,7 +23,8 @@ public enum SummaryTransform { MATCHED_ELEMENTS_FILTER("matchedelementsfilter"), MATCHED_ATTRIBUTE_ELEMENTS_FILTER("matchedattributeelementsfilter"), COPY("copy"), - DOCUMENT_ID("documentid"); + DOCUMENT_ID("documentid"), + LINGUISTICS_TOKENS("linguistics-tokens"); private final String name; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 9821f3b9568..f434d056bfc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -166,7 +166,8 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat UserConfiguredFiles files = new UserConfiguredFiles(deployState.getFileRegistry(), deployState.getDeployLogger(), deployState.featureFlags(), - userConfiguredUrls); + userConfiguredUrls, + deployState.getApplicationPackage()); for (Component<?, ?> component : getAllComponents()) { files.register(component); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java index 47ae2f40414..b10da29ee04 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFiles.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.model.filedistribution; import com.yahoo.config.FileReference; import com.yahoo.config.ModelReference; +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ModelContext; @@ -37,14 +39,17 @@ public class UserConfiguredFiles implements Serializable { private final DeployLogger logger; private final UserConfiguredUrls userConfiguredUrls; private final String unknownConfigDefinition; + private final ApplicationPackage applicationPackage; public UserConfiguredFiles(FileRegistry fileRegistry, DeployLogger logger, ModelContext.FeatureFlags featureFlags, - UserConfiguredUrls userConfiguredUrls) { + UserConfiguredUrls userConfiguredUrls, + ApplicationPackage applicationPackage) { this.fileRegistry = fileRegistry; this.logger = logger; this.userConfiguredUrls = userConfiguredUrls; this.unknownConfigDefinition = featureFlags.unknownConfigDefinition(); + this.applicationPackage = applicationPackage; } /** @@ -155,8 +160,8 @@ public class UserConfiguredFiles implements Serializable { path = Path.fromString(builder.getValue()); } - File file = path.toFile(); - if (file.isDirectory() && (file.listFiles() == null || file.listFiles().length == 0)) + ApplicationFile file = applicationPackage.getFile(path); + if (file.isDirectory() && (file.listFiles() == null || file.listFiles().isEmpty())) throw new IllegalArgumentException("Directory '" + path.getRelative() + "' is empty"); FileReference reference = registeredFiles.get(path); diff --git a/config-model/src/main/javacc/SchemaParser.jj b/config-model/src/main/javacc/SchemaParser.jj index ae4c3b365d8..a5238afc86a 100644 --- a/config-model/src/main/javacc/SchemaParser.jj +++ b/config-model/src/main/javacc/SchemaParser.jj @@ -201,6 +201,7 @@ TOKEN : | < FULL: "full" > | < STATIC: "static" > | < DYNAMIC: "dynamic" > +| < LINGUISTICS_TOKENS: "linguistics-tokens" > | < MATCHED_ELEMENTS_ONLY: "matched-elements-only" > | < SSCONTEXTUAL: "contextual" > | < SSOVERRIDE: "override" > @@ -1128,6 +1129,7 @@ void summaryInFieldShort(ParsedField field) : <COLON> ( <DYNAMIC> { psf.setDynamic(); } | <MATCHED_ELEMENTS_ONLY> { psf.setMatchedElementsOnly(); } | (<FULL> | <STATIC>) { psf.setFull(); } + | <LINGUISTICS_TOKENS> { psf.setLinguisticsTokens(); } ) } @@ -1173,6 +1175,7 @@ void summaryTransform(ParsedSummaryField field) : { } ( <DYNAMIC> { field.setDynamic(); } | <MATCHED_ELEMENTS_ONLY> { field.setMatchedElementsOnly(); } | (<FULL> | <STATIC>) { field.setFull(); } + | <LINGUISTICS_TOKENS> { field.setLinguisticsTokens(); } ) } @@ -2712,6 +2715,7 @@ String identifier() : { } | <INLINE> | <INPUTS> | <INTEGER> + | <LINGUISTICS_TOKENS> | <LITERAL> | <LOCALE> | <LONG> diff --git a/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java index 1f18a5ed49b..4128baddcb7 100644 --- a/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java @@ -227,6 +227,19 @@ public class SummaryTestCase extends AbstractSchemaTestCase { } @Test + void linguistics_tokenizer_override() throws ParseException { + var schema = buildSchema("field foo type string { indexing: summary }", + joinLines("document-summary bar {", + " summary baz type string {", + " source: foo ", + " linguistics-tokens", + " }", + " from-disk", + "}")); + assertOverride(schema, "baz", SummaryTransform.LINGUISTICS_TOKENS.getName(), "foo", "bar"); + } + + @Test void documentid_summary_transform_requires_disk_access() { assertFalse(SummaryTransform.DOCUMENT_ID.isInMemory()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java index 523b0e74be1..92fb89a5c4c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/filedistribution/UserConfiguredFilesTest.java @@ -9,6 +9,7 @@ import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.producer.UserConfigRepo; +import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.MockRoot; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionKey; @@ -19,6 +20,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import java.io.File; import java.nio.ByteBuffer; import java.nio.file.Path; import java.util.HashMap; @@ -69,12 +71,23 @@ public class UserConfiguredFilesTest { public String toString() { return export().toString(); } } - private UserConfiguredFiles userConfiguredFiles() { return new UserConfiguredFiles(fileRegistry, new BaseDeployLogger(), new TestProperties(), - new ApplicationContainerCluster.UserConfiguredUrls()); + new ApplicationContainerCluster.UserConfiguredUrls(), + new MockApplicationPackage.Builder().build()); + } + + private UserConfiguredFiles userConfiguredFiles(File root, com.yahoo.path.Path path) { + return new UserConfiguredFiles(fileRegistry, + new BaseDeployLogger(), + new TestProperties(), + new ApplicationContainerCluster.UserConfiguredUrls(), + new MockApplicationPackage.Builder() + .withRoot(root) + .withFiles(Map.of(path, "")) + .build()); } @BeforeEach @@ -289,13 +302,14 @@ public class UserConfiguredFilesTest { } @Test - void require_that_using_empty_dir_gives_sane_error_message(@TempDir Path tempDir) { - String relativeTempDir = tempDir.toString().substring(tempDir.toString().lastIndexOf("target")); + void require_that_using_empty_dir_fails(@TempDir Path tempDir) { + String relativeTempDir = tempDir.toString().substring(tempDir.toString().lastIndexOf("target") + 7); try { def.addPathDef("pathVal"); builder.setField("pathVal", relativeTempDir); fileRegistry.pathToRef.put(relativeTempDir, new FileReference("bazshash")); - userConfiguredFiles().register(producer); + userConfiguredFiles(tempDir.toFile().getParentFile(), + com.yahoo.path.Path.fromString(tempDir.toFile().getAbsolutePath())).register(producer); fail("Should have thrown exception"); } catch (IllegalArgumentException e) { assertEquals("Invalid config in services.xml for 'mynamespace.myname': Directory '" + relativeTempDir + "' is empty", @@ -303,4 +317,18 @@ public class UserConfiguredFilesTest { } } + @Test + void require_that_using_non_existing_dir_fails() { + String relativeTempDir = "non-existing"; + try { + def.addPathDef("pathVal"); + builder.setField("pathVal", relativeTempDir); + userConfiguredFiles().register(producer); + fail("Should have thrown exception"); + } catch (IllegalArgumentException e) { + assertEquals("Invalid config in services.xml for 'mynamespace.myname': No such file or directory '" + relativeTempDir + "'", + e.getMessage()); + } + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MockPricingController.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MockPricingController.java index 6fe7017e3b7..ee0df3adbfb 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MockPricingController.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/MockPricingController.java @@ -17,20 +17,25 @@ import static java.math.BigDecimal.valueOf; public class MockPricingController implements PricingController { + private static final BigDecimal cpuCost = new BigDecimal("1.00"); + private static final BigDecimal memoryCost = new BigDecimal("0.10"); + private static final BigDecimal diskCost = new BigDecimal("0.005"); + @Override public Prices priceForApplications(List<ApplicationResources> applicationResources, PricingInfo pricingInfo, Plan plan) { ApplicationResources resources = applicationResources.get(0); - BigDecimal listPrice = resources.vcpu().multiply(valueOf(1000)) - .add(resources.memoryGb().multiply(valueOf(100))) - .add(resources.diskGb().multiply(valueOf(10))) - .add(resources.enclaveVcpu().multiply(valueOf(1000)) - .add(resources.enclaveMemoryGb().multiply(valueOf(100))) - .add(resources.enclaveDiskGb().multiply(valueOf(10)))); - - BigDecimal supportLevelCost = pricingInfo.supportLevel() == BASIC ? new BigDecimal("-160.00") : new BigDecimal("800.00"); + + BigDecimal listPrice = resources.vcpu().multiply(cpuCost) + .add(resources.memoryGb().multiply(memoryCost) + .add(resources.diskGb().multiply(diskCost)) + .add(resources.enclaveVcpu().multiply(cpuCost) + .add(resources.enclaveMemoryGb().multiply(memoryCost)) + .add(resources.enclaveDiskGb().multiply(diskCost)))); + + BigDecimal supportLevelCost = pricingInfo.supportLevel() == BASIC ? new BigDecimal("-1.00") : new BigDecimal("8.00"); BigDecimal listPriceWithSupport = listPrice.add(supportLevelCost); - BigDecimal enclaveDiscount = isEnclave(resources) ? new BigDecimal("-15.1234") : BigDecimal.ZERO; - BigDecimal volumeDiscount = new BigDecimal("-5.64315634"); + BigDecimal enclaveDiscount = isEnclave(resources) ? new BigDecimal("-0.15") : BigDecimal.ZERO; + BigDecimal volumeDiscount = new BigDecimal("-0.1"); BigDecimal appTotalAmount = listPrice.add(supportLevelCost).add(enclaveDiscount).add(volumeDiscount); List<PriceInformation> appPrices = applicationResources.stream() @@ -42,9 +47,12 @@ public class MockPricingController implements PricingController { .toList(); PriceInformation sum = PriceInformation.sum(appPrices); - var committedAmountDiscount = new BigDecimal("-1.23"); + var committedAmountDiscount = new BigDecimal("-0.2"); var totalAmount = sum.totalAmount().add(committedAmountDiscount); - var totalPrice = new PriceInformation(ZERO, ZERO, committedAmountDiscount, ZERO, totalAmount); + var enclave = ZERO; + if (resources.enclave() && totalAmount.compareTo(new BigDecimal("14.00")) < 0) + enclave = new BigDecimal("14.00").subtract(totalAmount); + var totalPrice = new PriceInformation(ZERO, ZERO, committedAmountDiscount, enclave, totalAmount); return new Prices(appPrices, totalPrice); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java index 95b2ba9f8f8..f5543002a26 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingController.java @@ -130,6 +130,7 @@ public interface BillingController { default void updateCache(List<TenantName> tenants) {} + /** Export a bill to a payment service. Returns the invoice ID in the external system. */ default String exportBill(Bill bill, String exportMethod, CloudTenant tenant) { return "NOT_IMPLEMENTED"; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingDatabaseClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingDatabaseClientMock.java index 300c1658c29..ee7679f54ca 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingDatabaseClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingDatabaseClientMock.java @@ -29,6 +29,7 @@ public class BillingDatabaseClientMock implements BillingDatabaseClient { private final Map<Bill.Id, Bill.StatusHistory> statuses = new HashMap<>(); private final Map<Bill.Id, ZonedDateTime> startTimes = new HashMap<>(); private final Map<Bill.Id, ZonedDateTime> endTimes = new HashMap<>(); + private final Map<Bill.Id, String> exportedInvoiceIds = new HashMap<>(); private final ZonedDateTime startTime = LocalDate.of(2020, 4, 1).atStartOfDay(ZoneId.of("UTC")); private final ZonedDateTime endTime = LocalDate.of(2020, 5, 1).atStartOfDay(ZoneId.of("UTC")); @@ -74,7 +75,8 @@ public class BillingDatabaseClientMock implements BillingDatabaseClient { var status = statuses.getOrDefault(billId, Bill.StatusHistory.open(clock)); var start = startTimes.getOrDefault(billId, startTime); var end = endTimes.getOrDefault(billId, endTime); - return invoice.map(tenant -> new Bill(billId, tenant, status, lines, start, end)); + var exportedId = exportedInvoiceId(billId); + return invoice.map(tenant -> new Bill(billId, tenant, status, lines, start, end, exportedId)); } @Override @@ -157,7 +159,7 @@ public class BillingDatabaseClientMock implements BillingDatabaseClient { var status = statuses.get(invoiceId); var start = startTimes.get(invoiceId); var end = endTimes.get(invoiceId); - return new Bill(invoiceId, tenant, status, items, start, end); + return new Bill(invoiceId, tenant, status, items, start, end, exportedInvoiceId(invoiceId)); }) .toList(); } @@ -171,7 +173,7 @@ public class BillingDatabaseClientMock implements BillingDatabaseClient { var status = statuses.get(invoiceId); var start = startTimes.get(invoiceId); var end = endTimes.get(invoiceId); - return new Bill(invoiceId, tenant, status, items, start, end); + return new Bill(invoiceId, tenant, status, items, start, end, exportedInvoiceId(invoiceId)); }) .toList(); } @@ -180,9 +182,14 @@ public class BillingDatabaseClientMock implements BillingDatabaseClient { public void maintain() {} @Override - public void setExportedInvoiceId(Bill.Id billId, String invoiceId) { } + public void setExportedInvoiceId(Bill.Id billId, String invoiceId) { + exportedInvoiceIds.put(billId, invoiceId); + } @Override public void setExportedInvoiceItemId(String lineItemId, String invoiceItemId) { } + private String exportedInvoiceId(Bill.Id billId) { + return exportedInvoiceIds.getOrDefault(billId, null); + } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingReporter.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingReporter.java index 7339555e578..0d6d840591c 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingReporter.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingReporter.java @@ -6,4 +6,7 @@ import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; public interface BillingReporter { BillingReference maintainTenant(CloudTenant tenant); + + InvoiceUpdate maintainInvoice(Bill bill); + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingReporterMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingReporterMock.java index 29c7fbbf410..9531745556f 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingReporterMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/BillingReporterMock.java @@ -18,4 +18,10 @@ public class BillingReporterMock implements BillingReporter { public BillingReference maintainTenant(CloudTenant tenant) { return new BillingReference(UUID.randomUUID().toString(), clock.instant()); } + + @Override + public InvoiceUpdate maintainInvoice(Bill bill) { + return new InvoiceUpdate(0,0,1); + } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/InvoiceUpdate.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/InvoiceUpdate.java new file mode 100644 index 00000000000..6ca3cf6ebb1 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/InvoiceUpdate.java @@ -0,0 +1,45 @@ +package com.yahoo.vespa.hosted.controller.api.integration.billing; + +/** + * Helper to track changes to an invoice. + * + * @author gjoranv + */ +public record InvoiceUpdate(int itemsAdded, int itemsRemoved, int itemsModified) { + public boolean isEmpty() { + return itemsAdded == 0 && itemsRemoved == 0 && itemsModified == 0; + } + + public static InvoiceUpdate empty() { + return new InvoiceUpdate(0, 0, 0); + } + + public static class Counter { + private int itemsAdded = 0; + private int itemsRemoved = 0; + private int itemsModified = 0; + + public void addedItem() { + itemsAdded++; + } + + public void removedItem() { + itemsRemoved++; + } + + public void modifiedItem() { + itemsModified++; + } + + public void add(InvoiceUpdate other) { + itemsAdded += other.itemsAdded; + itemsRemoved += other.itemsRemoved; + itemsModified += other.itemsModified; + } + + public InvoiceUpdate finish() { + return new InvoiceUpdate(itemsAdded, itemsRemoved, itemsModified); + } + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java index 8ef14dd60ba..52a41f8da56 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/billing/MockBillingController.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.billing; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import java.math.BigDecimal; import java.time.Clock; @@ -22,6 +23,7 @@ import java.util.stream.Stream; public class MockBillingController implements BillingController { private final Clock clock; + private final BillingDatabaseClient dbClient; PlanId defaultPlan = PlanId.from("trial"); List<TenantName> tenants = new ArrayList<>(); @@ -32,8 +34,9 @@ public class MockBillingController implements BillingController { Map<TenantName, List<Bill.LineItem>> unusedLineItems = new HashMap<>(); Map<TenantName, CollectionMethod> collectionMethod = new HashMap<>(); - public MockBillingController(Clock clock) { + public MockBillingController(Clock clock, BillingDatabaseClient dbClient) { this.clock = clock; + this.dbClient = dbClient; } @Override @@ -134,7 +137,7 @@ public class MockBillingController implements BillingController { "line-item-id", description, amount, - "some-plan", + "paid", agent, ZonedDateTime.now())); } @@ -203,6 +206,14 @@ public class MockBillingController implements BillingController { return count < limit; } + @Override + public String exportBill(Bill bill, String exportMethod, CloudTenant tenant) { + // Replace bill with a copy with exportedId set + var exportedId = "EXT-ID-123"; + dbClient.setExportedInvoiceId(bill.id(), exportedId); + return exportedId; + } + public void setTenants(List<TenantName> tenants) { this.tenants = tenants; } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/PendingMailVerification.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/PendingMailVerification.java index 057c8bad89b..9c4bbc88f1f 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/PendingMailVerification.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/tenant/PendingMailVerification.java @@ -76,6 +76,7 @@ public class PendingMailVerification { public enum MailType { TENANT_CONTACT, - NOTIFICATIONS + NOTIFICATIONS, + BILLING } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/MailVerifier.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/MailVerifier.java index 4d29abaa212..ecdfc5990c0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/MailVerifier.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/MailVerifier.java @@ -86,6 +86,7 @@ public class MailVerifier { case NOTIFICATIONS -> withTenantContacts(oldTenantInfo, pendingMailVerification); case TENANT_CONTACT -> oldTenantInfo.withContact(oldTenantInfo.contact() .withEmail(oldTenantInfo.contact().email().withVerification(true))); + case BILLING -> withVerifiedBillingMail(oldTenantInfo); }; tenantController.lockOrThrow(tenant.name(), LockedTenant.Cloud.class, lockedTenant -> { @@ -111,6 +112,13 @@ public class MailVerifier { return oldInfo.withContacts(new TenantContacts(newContacts)); } + private TenantInfo withVerifiedBillingMail(TenantInfo oldInfo) { + var verifiedMail = oldInfo.billingContact().contact().email().withVerification(true); + var billingContact = oldInfo.billingContact() + .withContact(oldInfo.billingContact().contact().withEmail(verifiedMail)); + return oldInfo.withBilling(billingContact); + } + private void writePendingVerification(PendingMailVerification pendingMailVerification) { try (var lock = curatorDb.lockPendingMailVerification(pendingMailVerification.getVerificationCode())) { curatorDb.writePendingMailVerification(pendingMailVerification); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java index 3aeba07630b..15396649d2f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainer.java @@ -5,8 +5,11 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.LockedTenant; +import com.yahoo.vespa.hosted.controller.api.integration.billing.Bill; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; +import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingDatabaseClient; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingReporter; +import com.yahoo.vespa.hosted.controller.api.integration.billing.InvoiceUpdate; import com.yahoo.vespa.hosted.controller.api.integration.billing.Plan; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistry; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; @@ -23,18 +26,25 @@ public class BillingReportMaintainer extends ControllerMaintainer { private final BillingReporter reporter; private final BillingController billing; + private final BillingDatabaseClient databaseClient; + private final PlanRegistry plans; public BillingReportMaintainer(Controller controller, Duration interval) { super(controller, interval, null, Set.of(SystemName.PublicCd)); - this.reporter = controller.serviceRegistry().billingReporter(); - this.billing = controller.serviceRegistry().billingController(); - this.plans = controller.serviceRegistry().planRegistry(); + reporter = controller.serviceRegistry().billingReporter(); + billing = controller.serviceRegistry().billingController(); + databaseClient = controller.serviceRegistry().billingDatabase(); + plans = controller.serviceRegistry().planRegistry(); } @Override protected double maintain() { maintainTenants(); + + var updates = maintainInvoices(); + log.fine("Updated invoices: " + updates); + return 0.0; } @@ -53,6 +63,19 @@ public class BillingReportMaintainer extends ControllerMaintainer { }); } + InvoiceUpdate maintainInvoices() { + var billsNeedingMaintenance = databaseClient.readBills().stream() + .filter(bill -> bill.getExportedId().isPresent()) + .filter(exported -> ! exported.status().equals("ISSUED")) // TODO: This status does not yet exist. + .toList(); + + var updates = new InvoiceUpdate.Counter(); + for (var bill : billsNeedingMaintenance) { + updates.add(reporter.maintainInvoice(bill)); + } + return updates.finish(); + } + private Map<TenantName, CloudTenant> cloudTenants() { return controller().tenants().asList() .stream() @@ -74,4 +97,5 @@ public class BillingReportMaintainer extends ControllerMaintainer { .flatMap(p -> billing.tenantsWithPlan(tenants, p.id()).stream()) .toList(); } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java index 6c50afe7fb2..9358a648b43 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirer.java @@ -3,24 +3,39 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.ListFlag; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; +import com.yahoo.vespa.hosted.controller.notification.Notification; +import com.yahoo.vespa.hosted.controller.notification.NotificationSource; +import com.yahoo.vespa.hosted.controller.persistence.TrialNotifications; import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Optional; import java.util.function.Predicate; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import static com.yahoo.vespa.hosted.controller.persistence.TrialNotifications.State.EXPIRED; +import static com.yahoo.vespa.hosted.controller.persistence.TrialNotifications.State.EXPIRES_IMMEDIATELY; +import static com.yahoo.vespa.hosted.controller.persistence.TrialNotifications.State.EXPIRES_SOON; +import static com.yahoo.vespa.hosted.controller.persistence.TrialNotifications.State.MID_CHECK_IN; +import static com.yahoo.vespa.hosted.controller.persistence.TrialNotifications.State.SIGNED_UP; +import static com.yahoo.vespa.hosted.controller.persistence.TrialNotifications.State.UNKNOWN; + /** * Expires unused tenants from Vespa Cloud. - * <p> - * TODO: Should support sending notifications some time before the various expiry events happen. * * @author ogronnesby */ @@ -30,17 +45,20 @@ public class CloudTrialExpirer extends ControllerMaintainer { private static final Duration nonePlanAfter = Duration.ofDays(14); private static final Duration tombstoneAfter = Duration.ofDays(91); private final ListFlag<String> extendedTrialTenants; + private final BooleanFlag cloudTrialNotificationEnabled; public CloudTrialExpirer(Controller controller, Duration interval) { super(controller, interval, null, SystemName.allOf(SystemName::isPublic)); this.extendedTrialTenants = PermanentFlags.EXTENDED_TRIAL_TENANTS.bindTo(controller().flagSource()); + this.cloudTrialNotificationEnabled = Flags.CLOUD_TRIAL_NOTIFICATIONS.bindTo(controller().flagSource()); } @Override protected double maintain() { var a = tombstoneNonePlanTenants(); var b = moveInactiveTenantsToNonePlan(); - return (a ? 0.0 : -0.5) + (b ? 0.0 : -0.5); + var c = notifyTenants(); + return (a ? 0.0 : -(1D/3)) + (b ? 0.0 : -(1D/3) + (c ? 0.0 : -(1D/3))); } private boolean moveInactiveTenantsToNonePlan() { @@ -76,6 +94,87 @@ public class CloudTrialExpirer extends ControllerMaintainer { return tombstoneTenants(idleOldPlanTenants); } + private boolean notifyTenants() { + try { + var currentStatus = controller().curator().readTrialNotifications() + .map(TrialNotifications::tenants).orElse(List.of()); + log.fine(() -> "Current: %s".formatted(currentStatus)); + var currentStatusByTenant = new HashMap<TenantName, TrialNotifications.Status>(); + currentStatus.forEach(status -> currentStatusByTenant.put(status.tenant(), status)); + var updatedStatus = new ArrayList<TrialNotifications.Status>(); + var now = controller().clock().instant(); + + for (var tenant : controller().tenants().asList()) { + + var status = currentStatusByTenant.get(tenant.name()); + var state = status == null ? UNKNOWN : status.state(); + var plan = controller().serviceRegistry().billingController().getPlan(tenant.name()).value(); + var ageInDays = Duration.between(tenant.createdAt(), now).toDays(); + + // TODO Replace stubs with proper email content stored in templates. + + var enabled = cloudTrialNotificationEnabled.with(FetchVector.Dimension.TENANT_ID, tenant.name().value()).value(); + if (!enabled) { + if (status != null) updatedStatus.add(status); + } else if (!List.of("none", "trial").contains(plan)) { + // Ignore tenants that are on a paid plan and skip from inclusion in updated data structure + } else if (status == null && "trial".equals(plan) && ageInDays <= 1) { + updatedStatus.add(updatedStatus(tenant, now, SIGNED_UP)); + queueNotification(tenant, "Welcome to Vespa Cloud", "Welcome to Vespa Cloud", + "Welcome to Vespa Cloud! We hope you will enjoy your trial. " + + "Please reach out to us if you have any questions or feedback."); + } else if ("none".equals(plan) && !List.of(EXPIRED).contains(state)) { + updatedStatus.add(updatedStatus(tenant, now, EXPIRED)); + queueNotification(tenant, "Your Vespa Cloud trial has expired", "Your Vespa Cloud trial has expired", + "Your Vespa Cloud trial has expired. " + + "Please reach out to us if you have any questions or feedback."); + } else if ("trial".equals(plan) && ageInDays >= 13 + && !List.of(EXPIRES_IMMEDIATELY, EXPIRED).contains(state)) { + updatedStatus.add(updatedStatus(tenant, now, EXPIRES_IMMEDIATELY)); + queueNotification(tenant, "Your Vespa Cloud trial expires tomorrow", "Your Vespa Cloud trial expires tomorrow", + "Your Vespa Cloud trial expires tomorrow. " + + "Please reach out to us if you have any questions or feedback."); + } else if ("trial".equals(plan) && ageInDays >= 12 + && !List.of(EXPIRES_SOON, EXPIRES_IMMEDIATELY, EXPIRED).contains(state)) { + updatedStatus.add(updatedStatus(tenant, now, EXPIRES_SOON)); + queueNotification(tenant, "Your Vespa Cloud trial expires in 2 days", "Your Vespa Cloud trial expires in 2 days", + "Your Vespa Cloud trial expires in 2 days. " + + "Please reach out to us if you have any questions or feedback."); + } else if ("trial".equals(plan) && ageInDays >= 7 + && !List.of(MID_CHECK_IN, EXPIRES_SOON, EXPIRES_IMMEDIATELY, EXPIRED).contains(state)) { + updatedStatus.add(updatedStatus(tenant, now, MID_CHECK_IN)); + queueNotification(tenant, "How is your Vespa Cloud trial going?", "How is your Vespa Cloud trial going?", + "How is your Vespa Cloud trial going? " + + "Please reach out to us if you have any questions or feedback."); + } else { + updatedStatus.add(status); + } + } + log.fine(() -> "Updated: %s".formatted(updatedStatus)); + controller().curator().writeTrialNotifications(new TrialNotifications(updatedStatus)); + return true; + } catch (Exception e) { + log.log(Level.WARNING, "Failed to process trial notifications", e); + return false; + } + } + + private void queueNotification(Tenant tenant, String consoleMsg, String emailSubject, String emailMsg) { + var mail = Optional.of(Notification.MailContent.fromTemplate("default-mail-content") + .subject(emailSubject) + .with("mailMessageTemplate", "cloud-trial-notification") + .with("cloudTrialMessage", emailMsg) + .build()); + var source = NotificationSource.from(tenant.name()); + // Remove previous notification to ensure new notification is sent by email + controller().notificationsDb().removeNotification(source, Notification.Type.account); + controller().notificationsDb().setNotification( + source, Notification.Type.account, Notification.Level.info, List.of(consoleMsg), mail); + } + + private static TrialNotifications.Status updatedStatus(Tenant t, Instant i, TrialNotifications.State s) { + return new TrialNotifications.Status(t.name(), s, i); + } private boolean tenantIsCloudTenant(Tenant tenant) { return tenant.type() == Tenant.Type.cloud; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java index 48e9d1f6786..40c24c6f339 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java @@ -68,8 +68,12 @@ public record Notification(Instant at, Notification.Type type, Notification.Leve /** * Application cluster is reindexing document(s) */ - reindex + reindex, + /** + * Account, e.g. expiration of trial plan + */ + account } public static class MailContent { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java index 287342f1290..e752e13eddd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/NotificationsDb.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing; +import com.yahoo.vespa.hosted.controller.notification.Notification.MailContent; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import java.time.Clock; @@ -63,18 +64,24 @@ public class NotificationsDb { setNotification(source, type, level, List.of(message)); } + public void setNotification(NotificationSource source, Type type, Level level, List<String> messages) { + setNotification(source, type, level, messages, Optional.empty()); + } + /** * Add a notification with given source and type. If a notification with same source and type - * already exists, it'll be replaced by this one instead + * already exists, it'll be replaced by this one instead. + * Email content is not persisted here. The email dispatcher is responsible for reliable delivery. */ - public void setNotification(NotificationSource source, Type type, Level level, List<String> messages) { + public void setNotification(NotificationSource source, Type type, Level level, List<String> messages, + Optional<MailContent> mailContent) { Optional<Notification> changed = Optional.empty(); try (Mutex lock = curatorDb.lockNotifications(source.tenant())) { var existingNotifications = curatorDb.readNotifications(source.tenant()); List<Notification> notifications = existingNotifications.stream() .filter(notification -> !source.equals(notification.source()) || type != notification.type()) .collect(Collectors.toCollection(ArrayList::new)); - var notification = new Notification(clock.instant(), type, level, source, messages); + var notification = new Notification(clock.instant(), type, level, source, messages, mailContent); if (!notificationExists(notification, existingNotifications, false)) { changed = Optional.of(notification); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java index c1e1f075552..6468a4c397b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notifier.java @@ -72,6 +72,7 @@ public class Notifier { registerTemplate(repo, "mail"); registerTemplate(repo, "default-mail-content"); registerTemplate(repo, "notification-message"); + registerTemplate(repo, "cloud-trial-notification"); return v; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index a2a4cf809b1..cef62438a53 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -111,6 +111,7 @@ public class CuratorDb { private static final Path mailVerificationRoot = root.append("mailVerification"); private static final Path dataPlaneTokenRoot = root.append("dataplaneTokens"); private static final Path certificatePoolRoot = root.append("certificatePool"); + private static final Path trialNotificationsRoot = root.append("trialNotifications"); private final NodeVersionSerializer nodeVersionSerializer = new NodeVersionSerializer(); private final VersionStatusSerializer versionStatusSerializer = new VersionStatusSerializer(nodeVersionSerializer); @@ -816,6 +817,16 @@ public class CuratorDb { return curator.getChildren(certificatePoolRoot).stream().flatMap(id -> readUnassignedCertificate(id).stream()).toList(); } + // -------------- Cloud trial notification -------------------------------- + + public void writeTrialNotifications(TrialNotifications tn) { + curator.set(trialNotificationsRoot, asJson(tn.toSlime())); + } + + public Optional<TrialNotifications> readTrialNotifications() { + return readSlime(trialNotificationsRoot).map(TrialNotifications::fromSlime); + } + // -------------- Paths --------------------------------------------------- private static Path upgradesPerMinutePath() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java index fa688436256..7915a833be6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializer.java @@ -103,6 +103,7 @@ public class NotificationsSerializer { case deployment -> "deployment"; case feedBlock -> "feedBlock"; case reindex -> "reindex"; + case account -> "account"; }; } @@ -114,6 +115,7 @@ public class NotificationsSerializer { case "deployment" -> Notification.Type.deployment; case "feedBlock" -> Notification.Type.feedBlock; case "reindex" -> Notification.Type.reindex; + case "account" -> Notification.Type.account; default -> throw new IllegalArgumentException("Unknown serialized notification type value '" + field.asString() + "'"); }; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java index 381a5eaaa26..4404063456c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java @@ -282,10 +282,13 @@ public class TenantSerializer { } private TenantBilling tenantInfoBillingContactFromSlime(Inspector billingObject) { + //TODO: Remove validity check once emailVerified has been written for all tenants + var emailVerified = billingObject.field("emailVerified").valid() ? + billingObject.field("emailVerified").asBool() : true; return TenantBilling.empty() .withContact(TenantContact.from( billingObject.field("name").asString(), - new Email(billingObject.field("email").asString(), true), + new Email(billingObject.field("email").asString(), emailVerified), billingObject.field("phone").asString())) .withAddress(tenantInfoAddressFromSlime(billingObject.field("address"))); } @@ -344,11 +347,12 @@ public class TenantSerializer { private void toSlime(TenantBilling billingContact, Cursor parentCursor) { if (billingContact.isEmpty()) return; - Cursor addressCursor = parentCursor.setObject("billingContact"); - addressCursor.setString("name", billingContact.contact().name()); - addressCursor.setString("email", billingContact.contact().email().getEmailAddress()); - addressCursor.setString("phone", billingContact.contact().phone()); - toSlime(billingContact.address(), addressCursor); + Cursor billingCursor = parentCursor.setObject("billingContact"); + billingCursor.setString("name", billingContact.contact().name()); + billingCursor.setString("email", billingContact.contact().email().getEmailAddress()); + billingCursor.setBool("emailVerified", billingContact.contact().email().isVerified()); + billingCursor.setString("phone", billingContact.contact().phone()); + toSlime(billingContact.address(), billingCursor); } private void toSlime(List<TenantSecretStore> tenantSecretStores, Cursor parentCursor) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TrialNotifications.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TrialNotifications.java new file mode 100644 index 00000000000..a205e6c4173 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TrialNotifications.java @@ -0,0 +1,57 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.config.provision.TenantName; +import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.logging.Logger; + +/** + * @author bjorncs + */ +public record TrialNotifications(List<Status> tenants) { + private static final Logger log = Logger.getLogger(TrialNotifications.class.getName()); + + public TrialNotifications { tenants = List.copyOf(tenants); } + + public record Status(TenantName tenant, State state, Instant lastUpdate) {} + public enum State { SIGNED_UP, MID_CHECK_IN, EXPIRES_SOON, EXPIRES_IMMEDIATELY, EXPIRED, UNKNOWN } + + public Slime toSlime() { + var slime = new Slime(); + var rootCursor = slime.setObject(); + var tenantsCursor = rootCursor.setArray("tenants"); + for (Status t : tenants) { + var tenantCursor = tenantsCursor.addObject(); + tenantCursor.setString("tenant", t.tenant().value()); + tenantCursor.setString("state", t.state().name()); + tenantCursor.setString("lastUpdate", t.lastUpdate().toString()); + } + log.fine(() -> "Generated json '%s' from '%s'".formatted(SlimeUtils.toJson(slime), this)); + return slime; + } + + public static TrialNotifications fromSlime(Slime slime) { + var rootCursor = slime.get(); + var tenantsCursor = rootCursor.field("tenants"); + var tenants = new ArrayList<Status>(); + for (int i = 0; i < tenantsCursor.entries(); i++) { + var tenantCursor = tenantsCursor.entry(i); + var name = TenantName.from(tenantCursor.field("tenant").asString()); + var stateStr = tenantCursor.field("state").asString(); + var state = Arrays.stream(State.values()) + .filter(s -> s.name().equals(stateStr)).findFirst().orElse(State.UNKNOWN); + var lastUpdate = Instant.parse(tenantCursor.field("lastUpdate").asString()); + tenants.add(new Status(name, state, lastUpdate)); + } + var tn = new TrialNotifications(tenants); + log.fine(() -> "Parsed '%s' from '%s'".formatted(tn, SlimeUtils.toJson(slime))); + return tn; + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index d274d59c417..fdde87074e9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -705,7 +705,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { var contact = info.billingContact().contact(); var address = info.billingContact().address(); - var mergedContact = updateTenantInfoContact(inspector.field("contact"), cloudTenant.name(), contact, false); + var mergedContact = updateBillingContact(inspector.field("contact"), cloudTenant.name(), contact); var mergedAddress = updateTenantInfoAddress(inspector.field("address"), info.billingContact().address()); var mergedBilling = info.billingContact() @@ -779,11 +779,12 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { private void toSlime(TenantBilling billingContact, Cursor parentCursor) { if (billingContact.isEmpty()) return; - Cursor addressCursor = parentCursor.setObject("billingContact"); - addressCursor.setString("name", billingContact.contact().name()); - addressCursor.setString("email", billingContact.contact().email().getEmailAddress()); - addressCursor.setString("phone", billingContact.contact().phone()); - toSlime(billingContact.address(), addressCursor); + Cursor billingCursor = parentCursor.setObject("billingContact"); + billingCursor.setString("name", billingContact.contact().name()); + billingCursor.setString("email", billingContact.contact().email().getEmailAddress()); + billingCursor.setBool("emailVerified", billingContact.contact().email().isVerified()); + billingCursor.setString("phone", billingContact.contact().phone()); + toSlime(billingContact.address(), billingCursor); } private void toSlime(TenantContacts contacts, Cursor parentCursor) { @@ -892,15 +893,13 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { throw new IllegalArgumentException("All address fields must be set"); } - private TenantContact updateTenantInfoContact(Inspector insp, TenantName tenantName, TenantContact oldContact, boolean isBillingContact) { + private TenantContact updateBillingContact(Inspector insp, TenantName tenantName, TenantContact oldContact) { if (!insp.valid()) return oldContact; var mergedEmail = optional("email", insp) .filter(address -> !address.equals(oldContact.email().getEmailAddress())) .map(address -> { - if (isBillingContact) - return new Email(address, true); - controller.mailVerifier().sendMailVerification(tenantName, address, PendingMailVerification.MailType.TENANT_CONTACT); + controller.mailVerifier().sendMailVerification(tenantName, address, PendingMailVerification.MailType.BILLING); return new Email(address, false); }) .orElse(oldContact.email()); @@ -915,7 +914,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (!insp.valid()) return oldContact; return TenantBilling.empty() - .withContact(updateTenantInfoContact(insp, tenantName, oldContact.contact(), true)) + .withContact(updateBillingContact(insp, tenantName, oldContact.contact())) .withAddress(updateTenantInfoAddress(insp.field("address"), oldContact.address())); } @@ -1071,6 +1070,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { case deployment: yield "deployment"; case feedBlock: yield "feedBlock"; case reindex: yield "reindex"; + case account: yield "account"; }; } @@ -1700,6 +1700,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { var mailType = switch (type) { case "contact" -> PendingMailVerification.MailType.TENANT_CONTACT; case "notifications" -> PendingMailVerification.MailType.NOTIFICATIONS; + case "billing" -> PendingMailVerification.MailType.BILLING; default -> throw new IllegalArgumentException("Unknown mail type " + type); }; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java index ac3a8f2ee23..696f759d16e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java @@ -425,7 +425,7 @@ public class BillingApiHandler extends ThreadedHttpRequestHandler { cursor.setLong("majorVersion", lineItem.getMajorVersion()); if (! lineItem.getCloudAccount().isUnspecified()) - cursor.setString("cloudAccount", lineItem.getCloudAccount().account()); + cursor.setString("cloudAccount", lineItem.getCloudAccount().value()); lineItem.getCpuHours().ifPresent(cpuHours -> cursor.setString("cpuHours", cpuHours.toString()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java index da83073609d..718320c02ca 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2.java @@ -12,6 +12,7 @@ import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; +import com.yahoo.slime.SlimeUtils; import com.yahoo.slime.Type; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; @@ -89,6 +90,14 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler .addRoute(RestApi.route("/billing/v2/accountant/preview/tenant/{tenant}") .get(self::previewBill) .post(Slime.class, self::createBill)) + .addRoute(RestApi.route("/billing/v2/accountant/tenant/{tenant}/preview") + .get(self::previewBill) + .post(Slime.class, self::createBill)) + .addRoute(RestApi.route("/billing/v2/accountant/tenant/{tenant}/items") + .get(self::additionalItems) + .post(Slime.class, self::newAdditionalItem)) + .addRoute(RestApi.route("/billing/v2/accountant/tenant/{tenant}/item/{item}") + .delete(self::deleteAdditionalItem)) .addRoute(RestApi.route("/billing/v2/accountant/bill/{invoice}/export") .put(Slime.class, self::putAccountantInvoiceExport)) .addRoute(RestApi.route("/billing/v2/accountant/plans") @@ -301,6 +310,57 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler return new SlimeJsonResponse(responseSlime); } + private MessageResponse deleteAdditionalItem(RestApi.RequestContext requestContext) { + var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); + var tenant = tenants.get(tenantName).orElseThrow(() -> new RestApiException.NotFound("No such tenant: " + tenantName)); + + var itemId = requestContext.pathParameters().getStringOrThrow("item"); + + var items = billing.getUnusedLineItems(tenant.name()); + var candidate = items.stream().filter(item -> item.id().equals(itemId)).findAny(); + + if (candidate.isEmpty()) { + throw new RestApiException.NotFound("Could not find item with ID " + itemId); + } + + billing.deleteLineItem(itemId);; + + return new MessageResponse("Successfully deleted line item " + itemId); + } + + private MessageResponse newAdditionalItem(RestApi.RequestContext requestContext, Slime body) { + var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); + var tenant = tenants.get(tenantName).orElseThrow(() -> new RestApiException.NotFound("No such tenant: " + tenantName)); + + var inspector = body.get(); + + var billId = SlimeUtils.optionalString(inspector.field("billId")).map(Bill.Id::of); + + billing.addLineItem( + tenant.name(), + getInspectorFieldOrThrow(inspector, "description"), + new BigDecimal(getInspectorFieldOrThrow(inspector, "amount")), + billId, + requestContext.userPrincipalOrThrow().getName()); + + return new MessageResponse("Added line item for tenant " + tenantName); + } + + private Slime additionalItems(RestApi.RequestContext requestContext) { + var tenantName = TenantName.from(requestContext.pathParameters().getStringOrThrow("tenant")); + var tenant = tenants.get(tenantName).orElseThrow(() -> new RestApiException.NotFound("No such tenant: " + tenantName)); + + var slime = new Slime(); + var items = slime.setObject().setArray("items"); + + billing.getUnusedLineItems(tenant.name()).forEach(item -> { + var itemCursor = items.addObject(); + toSlime(itemCursor, item); + }); + + return slime; + } + // --------- INVOICE RENDERING ---------- private void invoicesSummaryToSlime(Cursor slime, List<Bill> bills) { @@ -352,7 +412,7 @@ public class BillingApiHandlerV2 extends RestApiRequestHandler<BillingApiHandler item.getArchitecture().ifPresent(arch -> slime.setString("architecture", arch.name())); slime.setLong("majorVersion", item.getMajorVersion()); if (! item.getCloudAccount().isUnspecified()) - slime.setString("cloudAccount", item.getCloudAccount().account()); + slime.setString("cloudAccount", item.getCloudAccount().value()); item.applicationId().ifPresent(appId -> { slime.setString("application", appId.application().value()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandler.java index 6ef247c5b41..0a43ec599d5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandler.java @@ -204,7 +204,7 @@ public class PricingApiHandler extends ThreadedHttpRequestHandler { applicationPrices(applicationsArray, prices.priceInformationApplications(), priceParameters); var priceInfoArray = cursor.setArray("priceInfo"); - addItem(priceInfoArray, "Enclave", prices.totalPriceInformation().enclaveDiscount()); + addItem(priceInfoArray, "Enclave (minimum $10k per month)", prices.totalPriceInformation().enclaveDiscount()); addItem(priceInfoArray, "Committed spend", prices.totalPriceInformation().committedAmountDiscount()); setBigDecimal(cursor, "totalAmount", prices.totalPriceInformation().totalAmount()); diff --git a/controller-server/src/main/resources/mail/cloud-trial-notification.vm b/controller-server/src/main/resources/mail/cloud-trial-notification.vm new file mode 100644 index 00000000000..27bc9b1ad1b --- /dev/null +++ b/controller-server/src/main/resources/mail/cloud-trial-notification.vm @@ -0,0 +1,3 @@ +<p> + $esc.html($cloudTrialMessage): +</p>
\ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/MailVerifierTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/MailVerifierTest.java index e641b332a72..ca71b912feb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/MailVerifierTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/MailVerifierTest.java @@ -3,12 +3,15 @@ package com.yahoo.vespa.hosted.controller; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.application.MailVerifier; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.Email; import com.yahoo.vespa.hosted.controller.tenant.PendingMailVerification; import com.yahoo.vespa.hosted.controller.tenant.Tenant; +import com.yahoo.vespa.hosted.controller.tenant.TenantBilling; +import com.yahoo.vespa.hosted.controller.tenant.TenantContact; import com.yahoo.vespa.hosted.controller.tenant.TenantContacts; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -100,4 +103,29 @@ class MailVerifierTest { assertTrue(tester.curator().getPendingMailVerification(resentVerification.get().getVerificationCode()).isPresent()); } + @Test + public void test_billing_mail_verification() { + var billingMail = "billing@foo.bar"; + tester.controller().tenants().lockOrThrow(tenantName, LockedTenant.Cloud.class, lockedTenant -> { + var tenantBilling = TenantBilling.empty().withContact(TenantContact.empty().withEmail(new Email(billingMail, false))); + lockedTenant = lockedTenant.withInfo(lockedTenant.get().info().withBilling(tenantBilling)); + tester.controller().tenants().store(lockedTenant); + }); + mailVerifier.sendMailVerification(tenantName, billingMail, PendingMailVerification.MailType.BILLING); + + // Assert written verification data + var writtenMailVerification = tester.curator().listPendingMailVerifications().get(0); + assertEquals(PendingMailVerification.MailType.BILLING, writtenMailVerification.getMailType()); + assertEquals(tenantName, writtenMailVerification.getTenantName()); + assertEquals(tester.clock().instant().plus(Duration.ofDays(7)), writtenMailVerification.getVerificationDeadline()); + assertEquals(billingMail, writtenMailVerification.getMailAddress()); + + // Assert mail is verified + mailVerifier.verifyMail(writtenMailVerification.getVerificationCode()); + assertTrue(tester.curator().listPendingMailVerifications().isEmpty()); + var tenant = tester.controller().tenants().require(tenantName, CloudTenant.class); + var expectedBillingContact = TenantContact.empty().withEmail(new Email(billingMail, true)); + assertEquals(expectedBillingContact, tenant.info().billingContact().contact()); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index e8e3de697dc..67f77b9a872 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -89,7 +89,6 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final MockEnclaveAccessService mockAMIService = new MockEnclaveAccessService(); private final MockResourceTagger mockResourceTagger = new MockResourceTagger(); private final MockRoleService roleService = new MockRoleService(); - private final MockBillingController billingController = new MockBillingController(clock); private final ArtifactRegistryMock containerRegistry = new ArtifactRegistryMock(); private final NoopTenantSecretService tenantSecretService = new NoopTenantSecretService(); private final NoopEndpointSecretManager secretManager = new NoopEndpointSecretManager(); @@ -100,6 +99,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final PlanRegistry planRegistry = new PlanRegistryMock(); private final ResourceDatabaseClient resourceDb = new ResourceDatabaseClientMock(planRegistry); private final BillingDatabaseClient billingDb = new BillingDatabaseClientMock(clock, planRegistry); + private final MockBillingController billingController = new MockBillingController(clock, billingDb); private final RoleMaintainerMock roleMaintainer = new RoleMaintainerMock(); private final MockPricingController pricingController = new MockPricingController(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainerTest.java index 9e1aa64beae..0a57cf51f2e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/BillingReportMaintainerTest.java @@ -4,12 +4,15 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.billing.InvoiceUpdate; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistryMock; import com.yahoo.vespa.hosted.controller.tenant.BillingReference; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import org.junit.jupiter.api.Test; import java.time.Duration; +import java.time.LocalDate; +import java.time.ZoneOffset; import java.util.Optional; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -39,8 +42,34 @@ public class BillingReportMaintainerTest { assertNotNull(b1.orElseThrow().reference()); } + @Test + void only_bills_with_exported_id_are_maintained() { + var t1 = tester.createTenant("t1"); + var billingController = tester.controller().serviceRegistry().billingController(); + var billingDb = tester.controller().serviceRegistry().billingDatabase(); + + var start = LocalDate.of(2020, 5, 23).atStartOfDay(ZoneOffset.UTC); + var end = start.toLocalDate().plusDays(6).atStartOfDay(ZoneOffset.UTC); + var bill1 = billingDb.createBill(t1, start, end, "non-exported"); + var bill2 = billingDb.createBill(t1, start, end, "exported"); + + billingController.setPlan(t1, PlanRegistryMock.paidPlan.id(), false, true); + + billingController.exportBill(billingDb.readBill(bill2).get(), "FOO", cloudTenant(t1)); + var updates = maintainer.maintainInvoices(); + assertEquals(new InvoiceUpdate(0, 0, 1), updates); + + var exportedBill = billingDb.readBill(bill2).get(); + assertEquals("EXT-ID-123", exportedBill.getExportedId().get()); + assertTrue(billingDb.readBill(bill1).get().getExportedId().isEmpty()); + } + + private CloudTenant cloudTenant(TenantName tenantName) { + return tester.controller().tenants().require(tenantName, CloudTenant.class); + } + private Optional<BillingReference> billingReference(TenantName tenantName) { - var t = tester.controller().tenants().require(tenantName, CloudTenant.class); - return t.billingReference(); + return cloudTenant(tenantName).billingReference(); } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java index 95cffae8728..b595c8a8be3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CloudTrialExpirerTest.java @@ -3,12 +3,16 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; +import com.yahoo.test.ManualClock; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanId; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.notification.Notification; +import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import org.junit.jupiter.api.Test; @@ -89,6 +93,33 @@ public class CloudTrialExpirerTest { assertPlan("active", "none"); } + @Test + void queues_trial_notification_based_on_account_age() { + var clock = (ManualClock)tester.controller().clock(); + var tenant = TenantName.from("trial-tenant"); + ((InMemoryFlagSource) tester.controller().flagSource()) + .withBooleanFlag(Flags.CLOUD_TRIAL_NOTIFICATIONS.id(), true); + registerTenant(tenant.value(), "trial", Duration.ZERO); + assertEquals(0.0, expirer.maintain()); + assertEquals("Welcome to Vespa Cloud", lastAccountLevelNotificationTitle(tenant)); + + clock.advance(Duration.ofDays(7)); + assertEquals(0.0, expirer.maintain()); + assertEquals("How is your Vespa Cloud trial going?", lastAccountLevelNotificationTitle(tenant)); + + clock.advance(Duration.ofDays(5)); + assertEquals(0.0, expirer.maintain()); + assertEquals("Your Vespa Cloud trial expires in 2 days", lastAccountLevelNotificationTitle(tenant)); + + clock.advance(Duration.ofDays(1)); + assertEquals(0.0, expirer.maintain()); + assertEquals("Your Vespa Cloud trial expires tomorrow", lastAccountLevelNotificationTitle(tenant)); + + clock.advance(Duration.ofDays(2)); + assertEquals(0.0, expirer.maintain()); + assertEquals("Your Vespa Cloud trial has expired", lastAccountLevelNotificationTitle(tenant)); + } + private void registerTenant(String tenantName, String plan, Duration timeSinceLastLogin) { var name = TenantName.from(tenantName); tester.createTenant(tenantName, Tenant.Type.cloud); @@ -111,4 +142,11 @@ public class CloudTrialExpirerTest { assertEquals(planId, tester.serviceRegistry().billingController().getPlan(TenantName.from(tenant)).value()); } + private String lastAccountLevelNotificationTitle(TenantName tenant) { + return tester.controller().notificationsDb() + .listNotifications(NotificationSource.from(tenant), false).stream() + .filter(n -> n.type() == Notification.Type.account).map(n -> n.messages().get(0)) + .findFirst().orElseThrow(); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java index 8f114a7255c..cb867c76f1f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java @@ -229,7 +229,7 @@ public class TenantSerializerTest { .withCode("3510") .withRegion("Viken")) .withBilling(TenantBilling.empty() - .withContact(TenantContact.from("Thomas The Tank Engine", new Email("ceo@mycomp.any", true), "NA")) + .withContact(TenantContact.from("Thomas The Tank Engine", new Email("ceo@mycomp.any", false), "NA")) .withAddress(TenantAddress.empty() .withCity("Suddery") .withCountry("Sodor") diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java index e89c913ab7d..bf908069c1e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java @@ -151,10 +151,10 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { tester.assertResponse(postPartialContacts, "{\"message\":\"Tenant info updated\"}", 200); // Read back the updated info - tester.assertResponse(infoRequest, "{\"name\":\"\",\"email\":\"\",\"website\":\"\",\"contactName\":\"newName\",\"contactEmail\":\"foo@example.com\",\"contactEmailVerified\":false,\"billingContact\":{\"name\":\"billingName\",\"email\":\"\",\"phone\":\"\"},\"contacts\":[{\"audiences\":[\"tenant\"],\"email\":\"contact1@example.com\",\"emailVerified\":false}]}", 200); + tester.assertResponse(infoRequest, "{\"name\":\"\",\"email\":\"\",\"website\":\"\",\"contactName\":\"newName\",\"contactEmail\":\"foo@example.com\",\"contactEmailVerified\":false,\"billingContact\":{\"name\":\"billingName\",\"email\":\"\",\"emailVerified\":true,\"phone\":\"\"},\"contacts\":[{\"audiences\":[\"tenant\"],\"email\":\"contact1@example.com\",\"emailVerified\":false}]}", 200); String fullAddress = "{\"addressLines\":\"addressLines\",\"postalCodeOrZip\":\"postalCodeOrZip\",\"city\":\"city\",\"stateRegionProvince\":\"stateRegionProvince\",\"country\":\"country\"}"; - String fullBillingContact = "{\"name\":\"name\",\"email\":\"foo@example\",\"phone\":\"phone\",\"address\":" + fullAddress + "}"; + String fullBillingContact = "{\"name\":\"name\",\"email\":\"foo@example\",\"emailVerified\":false,\"phone\":\"phone\",\"address\":" + fullAddress + "}"; String fullContacts = "[{\"audiences\":[\"tenant\"],\"email\":\"contact1@example.com\",\"emailVerified\":false},{\"audiences\":[\"notifications\"],\"email\":\"contact2@example.com\",\"emailVerified\":false},{\"audiences\":[\"tenant\",\"notifications\"],\"email\":\"contact3@example.com\",\"emailVerified\":false}]"; String fullInfo = "{\"name\":\"name\",\"email\":\"foo@example\",\"website\":\"https://yahoo.com\",\"contactName\":\"contactName\",\"contactEmail\":\"contact@example.com\",\"contactEmailVerified\":false,\"address\":" + fullAddress + ",\"billingContact\":" + fullBillingContact + ",\"contacts\":" + fullContacts + "}"; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java index 78bbb006467..a2290f1f664 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandlerV2Test.java @@ -130,13 +130,13 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest { @Test void require_accountant_tenant_preview() { - var accountantRequest = request("/billing/v2/accountant/preview/tenant/tenant1").roles(Role.hostedAccountant()); + var accountantRequest = request("/billing/v2/accountant/tenant/tenant1/preview").roles(Role.hostedAccountant()); tester.assertResponse(accountantRequest, "{\"id\":\"empty\",\"from\":\"2021-04-13\",\"to\":\"2021-04-12\",\"total\":\"0.00\",\"status\":\"OPEN\",\"statusHistory\":[{\"at\":\"2021-04-13T00:00:00Z\",\"status\":\"OPEN\"}],\"items\":[]}"); } @Test void require_accountant_tenant_bill() { - var accountantRequest = request("/billing/v2/accountant/preview/tenant/tenant1", Request.Method.POST) + var accountantRequest = request("/billing/v2/accountant/tenant/tenant1/preview", Request.Method.POST) .roles(Role.hostedAccountant()) .data("{\"from\": \"2020-05-01\",\"to\": \"2020-06-01\"}"); tester.assertResponse(accountantRequest, "{\"message\":\"Created bill id-123\"}"); @@ -148,4 +148,41 @@ public class BillingApiHandlerV2Test extends ControllerContainerCloudTest { .roles(Role.hostedAccountant()); tester.assertResponse(accountantRequest, "{\"plans\":[{\"id\":\"trial\",\"name\":\"Free Trial - for testing purposes\"},{\"id\":\"paid\",\"name\":\"Paid Plan - for testing purposes\"},{\"id\":\"none\",\"name\":\"None Plan - for testing purposes\"}]}"); } + + @Test + void require_additional_items_empty() { + var accountantRequest = request("/billing/v2/accountant/tenant/tenant1/items") + .roles(Role.hostedAccountant()); + tester.assertResponse(accountantRequest, """ + {"items":[]}"""); + } + + @Test + void require_additional_items_with_content() { + { + var accountantRequest = request("/billing/v2/accountant/tenant/tenant1/items", Request.Method.POST) + .roles(Role.hostedAccountant()) + .data(""" + { + "description": "Additional support costs", + "amount": "123.45" + }"""); + tester.assertResponse(accountantRequest, """ + {"message":"Added line item for tenant tenant1"}"""); + } + + { + var accountantRequest = request("/billing/v2/accountant/tenant/tenant1/items") + .roles(Role.hostedAccountant()); + tester.assertResponse(accountantRequest, """ + {"items":[{"id":"line-item-id","description":"Additional support costs","amount":"123.45","plan":{"id":"paid","name":"Paid Plan - for testing purposes"},"majorVersion":0,"cpu":{},"memory":{},"disk":{}}]}"""); + } + + { + var accountantRequest = request("/billing/v2/accountant/tenant/tenant1/item/line-item-id", Request.Method.DELETE) + .roles(Role.hostedAccountant()); + tester.assertResponse(accountantRequest, """ + {"message":"Successfully deleted line item line-item-id"}"""); + } + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants.json index d761439667a..85c34edf7db 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/billing-all-tenants.json @@ -27,8 +27,8 @@ "id": "line-item-id", "description": "support", "amount": "42.00", - "plan": "some-plan", - "planName": "Plan with id: some-plan", + "plan": "paid", + "planName": "Plan with id: paid", "majorVersion": 0 } ] diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/line-item-list.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/line-item-list.json index fbfc5ce09ee..8b69ea78754 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/line-item-list.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/billing/responses/line-item-list.json @@ -4,8 +4,8 @@ "id": "line-item-id", "description": "some description", "amount": "123.45", - "plan": "some-plan", - "planName": "Plan with id: some-plan", + "plan": "paid", + "planName": "Plan with id: paid", "majorVersion": 0 } ] diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandlerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandlerTest.java index 63636b3ff20..c4b5a771725 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandlerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/pricing/PricingApiHandlerTest.java @@ -27,15 +27,15 @@ public class PricingApiHandlerTest extends ControllerContainerCloudTest { "applications": [ { "priceInfo": [ - {"description": "Basic support unit price", "amount": "2240.00"}, - {"description": "Volume discount", "amount": "-5.64"} + {"description": "Basic support unit price", "amount": "4.30"}, + {"description": "Volume discount", "amount": "-0.10"} ] } ], "priceInfo": [ - {"description": "Committed spend", "amount": "-1.23"} + {"description": "Committed spend", "amount": "-0.20"} ], - "totalAmount": "2233.13" + "totalAmount": "4.00" } """, 200); @@ -49,16 +49,17 @@ public class PricingApiHandlerTest extends ControllerContainerCloudTest { "applications": [ { "priceInfo": [ - {"description": "Basic support unit price", "amount": "2240.00"}, - {"description": "Enclave", "amount": "-15.12"}, - {"description": "Volume discount", "amount": "-5.64"} + {"description": "Basic support unit price", "amount": "4.30"}, + {"description": "Enclave", "amount": "-0.15"}, + {"description": "Volume discount", "amount": "-0.10"} ] } ], "priceInfo": [ - {"description": "Committed spend", "amount": "-1.23"} + {"description": "Enclave (minimum $10k per month)", "amount": "10.15"}, + {"description": "Committed spend", "amount": "-0.20"} ], - "totalAmount": "2218.00" + "totalAmount": "3.85" } """, 200); @@ -72,16 +73,17 @@ public class PricingApiHandlerTest extends ControllerContainerCloudTest { "applications": [ { "priceInfo": [ - {"description": "Commercial support unit price", "amount": "3200.00"}, - {"description": "Enclave", "amount": "-15.12"}, - {"description": "Volume discount", "amount": "-5.64"} + {"description": "Commercial support unit price", "amount": "13.30"}, + {"description": "Enclave", "amount": "-0.15"}, + {"description": "Volume discount", "amount": "-0.10"} ] } ], "priceInfo": [ - {"description": "Committed spend", "amount": "-1.23"} + {"description": "Enclave (minimum $10k per month)", "amount": "1.15"}, + {"description": "Committed spend", "amount": "-0.20"} ], - "totalAmount": "3178.00" + "totalAmount": "12.85" } """, 200); @@ -95,23 +97,23 @@ public class PricingApiHandlerTest extends ControllerContainerCloudTest { "applications": [ { "priceInfo": [ - {"description": "Commercial support unit price", "amount": "2000.00"}, - {"description": "Enclave", "amount": "-15.12"}, - {"description": "Volume discount", "amount": "-5.64"} + {"description": "Commercial support unit price", "amount": "13.30"}, + {"description": "Enclave", "amount": "-0.15"}, + {"description": "Volume discount", "amount": "-0.10"} ] }, { "priceInfo": [ - {"description": "Commercial support unit price", "amount": "2000.00"}, - {"description": "Enclave", "amount": "-15.12"}, - {"description": "Volume discount", "amount": "-5.64"} + {"description": "Commercial support unit price", "amount": "13.30"}, + {"description": "Enclave", "amount": "-0.15"}, + {"description": "Volume discount", "amount": "-0.10"} ] } ], "priceInfo": [ - {"description": "Committed spend", "amount": "-1.23"} + {"description": "Committed spend", "amount": "-0.20"} ], - "totalAmount": "3957.24" + "totalAmount": "25.90" } """, 200); @@ -151,31 +153,31 @@ public class PricingApiHandlerTest extends ControllerContainerCloudTest { /** * 1 app, with 2 clusters (with total resources for all clusters with each having - * 1 node, with 1 vcpu, 1 Gb memory, 10 Gb disk and no GPU, + * 1 node, with 4 vcpu, 8 Gb memory, 100 Gb disk and no GPU, * price will be 20000 + 2000 + 200 */ String urlEncodedPriceInformation1App(PricingInfo.SupportLevel supportLevel) { - return "application=" + URLEncoder.encode("vcpu=2,memoryGb=2,diskGb=20,gpuMemoryGb=0", UTF_8) + - "&supportLevel=" + supportLevel.name().toLowerCase() + "&committedSpend=100"; + return "application=" + URLEncoder.encode("vcpu=4,memoryGb=8,diskGb=100,gpuMemoryGb=0", UTF_8) + + "&supportLevel=" + supportLevel.name().toLowerCase() + "&committedSpend=20"; } /** * 1 app, with 2 clusters (with total resources for all clusters with each having - * 1 node, with 1 vcpu, 1 Gb memory, 10 Gb disk and no GPU, + * 1 node, with 4 vcpu, 8 Gb memory, 100 Gb disk and no GPU, * price will be 20000 + 2000 + 200 */ String urlEncodedPriceInformation1AppEnclave(PricingInfo.SupportLevel supportLevel) { - return "application=" + URLEncoder.encode("enclaveVcpu=2,enclaveMemoryGb=2,enclaveDiskGb=20,enclaveGpuMemoryGb=0", UTF_8) + - "&supportLevel=" + supportLevel.name().toLowerCase() + "&committedSpend=100"; + return "application=" + URLEncoder.encode("enclaveVcpu=4,enclaveMemoryGb=8,enclaveDiskGb=100,enclaveGpuMemoryGb=0", UTF_8) + + "&supportLevel=" + supportLevel.name().toLowerCase() + "&committedSpend=20"; } /** * 2 apps, with 1 cluster (with total resources for all clusters with each having - * 1 node, with 1 vcpu, 1 Gb memory, 10 Gb disk and no GPU + * 1 node, with 4 vcpu, 8 Gb memory, 100 Gb disk and no GPU, */ String urlEncodedPriceInformation2AppsEnclave(PricingInfo.SupportLevel supportLevel) { - return "application=" + URLEncoder.encode("enclaveVcpu=1,enclaveMemoryGb=1,enclaveDiskGb=10,enclaveGpuMemoryGb=0", UTF_8) + - "&application=" + URLEncoder.encode("enclaveVcpu=1,enclaveMemoryGb=1,enclaveDiskGb=10,enclaveGpuMemoryGb=0", UTF_8) + + return "application=" + URLEncoder.encode("enclaveVcpu=4,enclaveMemoryGb=8,enclaveDiskGb=100,enclaveGpuMemoryGb=0", UTF_8) + + "&application=" + URLEncoder.encode("enclaveVcpu=4,enclaveMemoryGb=8,enclaveDiskGb=100,enclaveGpuMemoryGb=0", UTF_8) + "&supportLevel=" + supportLevel.name().toLowerCase() + "&committedSpend=0"; } diff --git a/default_build_settings.cmake b/default_build_settings.cmake index 1e2981a2780..2da6af61211 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -130,16 +130,12 @@ function(vespa_use_default_build_settings) message("-- CMAKE_SYSTEM_PROCESSOR = ${CMAKE_SYSTEM_PROCESSOR}") if(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") if(APPLE AND (("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") OR ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "AppleClang"))) - elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") - # Require haswell cpu or newer when compiling with clang on linux. - set(DEFAULT_VESPA_CPU_ARCH_FLAGS "-march=haswell -mtune=skylake") + elseif(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 13.0) + # Temporary workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108599 + set(DEFAULT_VESPA_CPU_ARCH_FLAGS "-march=ivybridge") else() - if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 13.0) - # Temporary workaround for https://gcc.gnu.org/bugzilla/show_bug.cgi?id=108599 - set(DEFAULT_VESPA_CPU_ARCH_FLAGS "-march=ivybridge") - else() - set(DEFAULT_VESPA_CPU_ARCH_FLAGS "-msse3 -mcx16 -mtune=intel") - endif() + # Default to haswell cpu or newer + set(DEFAULT_VESPA_CPU_ARCH_FLAGS "-march=haswell -mtune=skylake") endif() elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64") set(DEFAULT_VESPA_CPU_ARCH_FLAGS "-march=armv8.2-a+fp16+rcpc+dotprod+crypto -mtune=neoverse-n1") diff --git a/document/src/vespa/document/bucket/bucketid.cpp b/document/src/vespa/document/bucket/bucketid.cpp index c077d2dd4f6..dee818b407e 100644 --- a/document/src/vespa/document/bucket/bucketid.cpp +++ b/document/src/vespa/document/bucket/bucketid.cpp @@ -8,7 +8,7 @@ #include <vespa/vespalib/stllike/hash_set.hpp> #include <vespa/vespalib/util/stringfmt.h> #include <limits> -#include <xxh3.h> +#include <xxhash.h> using vespalib::nbostream; using vespalib::asciistream; @@ -80,7 +80,28 @@ void BucketId::initialize() noexcept { uint64_t BucketId::hash::operator () (const BucketId& bucketId) const noexcept { const uint64_t raw_id = bucketId.getId(); - return XXH3_64bits(&raw_id, sizeof(uint64_t)); + /* + * This is a workaround for gcc 12 and on that produces incorrect warning when compiled with -march=haswell or newer + * /home/balder/git/vespa/document/src/vespa/document/bucket/bucketid.cpp: In member function ‘uint64_t document::BucketId::hash::operator()(const document::BucketId&) const’: + * /home/balder/git/vespa/document/src/vespa/document/bucket/bucketid.cpp:83:23: error: ‘raw_id’ may be used uninitialized [-Werror=maybe-uninitialized] + * 83 | return XXH3_64bits(&raw_id, sizeof(uint64_t)); + * | ^ + * In file included from /usr/include/xxh3.h:55, + * from /home/balder/git/vespa/document/src/vespa/document/bucket/bucketid.cpp:11: + * /usr/include/xxhash.h:5719:29: note: by argument 1 of type ‘const void*’ to ‘XXH64_hash_t XXH_INLINE_XXH3_64bits(const void*, size_t)’ declared here + * 5719 | XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length) + * | ^~~~~~~~~~~ + * /home/balder/git/vespa/document/src/vespa/document/bucket/bucketid.cpp:82:14: note: ‘raw_id’ declared here + * 82 | uint64_t raw_id = bucketId.getId(); + * | ^~~~~~ + * cc1plus: all warnings being treated as errors + * + * Same issue in storage/src/vespa/storage/persistence/filestorhandlerimpl.cpp:FileStorHandlerImpl::dispersed_bucket_bits + */ + uint8_t raw_as_bytes[sizeof(raw_id)]; + memcpy(raw_as_bytes, &raw_id, sizeof(raw_id)); + + return XXH3_64bits(raw_as_bytes, sizeof(raw_as_bytes)); } vespalib::string diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 5baefefb72b..2b3fe84ec84 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -305,12 +305,12 @@ public class Flags { INSTANCE_ID); public static final UnboundBooleanFlag ENABLE_CROWDSTRIKE = defineFeatureFlag( - "enable-crowdstrike", true, List.of("andreer"), "2023-04-13", "2023-10-14", + "enable-crowdstrike", true, List.of("andreer"), "2023-04-13", "2023-11-14", "Whether to enable CrowdStrike.", "Takes effect on next host admin tick", HOSTNAME); public static final UnboundBooleanFlag RANDOMIZED_ENDPOINT_NAMES = defineFeatureFlag( - "randomized-endpoint-names", false, List.of("andreer"), "2023-04-26", "2023-10-14", + "randomized-endpoint-names", false, List.of("andreer"), "2023-04-26", "2023-11-14", "Whether to use randomized endpoint names", "Takes effect on application deployment", INSTANCE_ID, APPLICATION_ID, TENANT_ID); @@ -354,21 +354,21 @@ public class Flags { public static final UnboundBooleanFlag MORE_WIREGUARD = defineFeatureFlag( "more-wireguard", false, - List.of("andreer"), "2023-08-21", "2023-10-14", + List.of("andreer"), "2023-08-21", "2023-11-14", "Use wireguard in INternal enCLAVES", "Takes effect on next host-admin run", HOSTNAME, CLOUD_ACCOUNT); public static final UnboundBooleanFlag IPV6_AWS_TARGET_GROUPS = defineFeatureFlag( "ipv6-aws-target-groups", false, - List.of("andreer"), "2023-08-28", "2023-10-14", + List.of("andreer"), "2023-08-28", "2023-11-14", "Always use IPv6 target groups for load balancers in aws", "Takes effect on next load-balancer provisioning", HOSTNAME, CLOUD_ACCOUNT); public static final UnboundBooleanFlag PROVISION_IPV6_ONLY_AWS = defineFeatureFlag( "provision-ipv6-only", false, - List.of("andreer"), "2023-08-28", "2023-10-14", + List.of("andreer"), "2023-08-28", "2023-11-14", "Provision without private IPv4 addresses in INternal enCLAVES in AWS", "Takes effect on next host provisioning / run of host-admin", HOSTNAME, CLOUD_ACCOUNT); @@ -436,6 +436,12 @@ public class Flags { "Takes effect on next deployment through controller", TENANT_ID, APPLICATION_ID, INSTANCE_ID); + public static final UnboundBooleanFlag CLOUD_TRIAL_NOTIFICATIONS = defineFeatureFlag( + "cloud-trial-notifications", false, + List.of("bjorncs", "oyving"), "2023-10-13", "2024-03-01", + "Whether to send cloud trial email notifications", + "Takes effect immediately"); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, diff --git a/screwdriver.yaml b/screwdriver.yaml index d2cb3b3dd51..6efb9145b09 100644 --- a/screwdriver.yaml +++ b/screwdriver.yaml @@ -117,6 +117,7 @@ jobs: (got VESPA_VERSION=$VESPA_VERSION, VESPA_REF=$VESPA_REF, SYSTEM_TEST_REF=$SYSTEM_TEST_REF)." exit 1 fi + meta set vespa.version $VESPA_VERSION - install-dependencies: | dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo dnf -y install docker-ce docker-ce-cli containerd.io @@ -237,6 +238,7 @@ jobs: - DOCKER_IMAGE_DEPLOY_KEY - DOCKER_HUB_DEPLOY_KEY - GHCR_DEPLOY_KEY + - ANN_BENCHMARK_DEPLOY_KEY - SVC_OKTA_VESPA_FACTORY_TOKEN environment: @@ -264,6 +266,8 @@ jobs: screwdriver/release-rpms.sh $VESPA_VERSION $VESPA_REF - release-container-image: | screwdriver/release-container-image-docker.sh $VESPA_VERSION + - release-ann-benchmark: | + screwdriver/release-ann-benchmark.sh $VESPA_VERSION - update-sample-apps: | screwdriver/update-vespa-version-in-sample-apps.sh $VESPA_VERSION - update-released-time: | diff --git a/screwdriver/build-vespa.sh b/screwdriver/build-vespa.sh index 1f43770b871..72b26e1032e 100755 --- a/screwdriver/build-vespa.sh +++ b/screwdriver/build-vespa.sh @@ -24,8 +24,7 @@ fi build_cpp() { cat /proc/cpuinfo | grep "model name" | head -1 cat /proc/cpuinfo | grep "flags" | head -1 - # TODO This will only build for x86_64 architecture, and is used for pull request builds. - cmake3 -DVESPA_UNPRIVILEGED=no -DDEFAULT_VESPA_CPU_ARCH_FLAGS="-march=skylake" $1 + cmake3 -DVESPA_UNPRIVILEGED=no $1 time make -j ${NUM_THREADS} time ctest3 --output-on-failure -j ${NUM_THREADS} ccache --show-stats diff --git a/screwdriver/release-ann-benchmark.sh b/screwdriver/release-ann-benchmark.sh new file mode 100755 index 00000000000..7ef7e4df68c --- /dev/null +++ b/screwdriver/release-ann-benchmark.sh @@ -0,0 +1,32 @@ +#!/usr/bin/ssh-agent /bin/bash +# Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "Usage: $0 <Vespa version>" + exit 1 +fi + +readonly VESPA_VERSION=$1 + +if [[ -z "$ANN_BENCHMARK_DEPLOY_KEY" ]]; then + echo "Environment variable ANN_BENCHMARK_DEPLOY_KEY must be set, but is empty." + exit 1 +fi + +BUILD_DIR=$(mktemp -d) +trap "rm -rf $BUILD_DIR" EXIT +cd $BUILD_DIR + +ssh-add -D +ssh-add <(echo $ANN_BENCHMARK_DEPLOY_KEY | base64 -d) +git clone git@github.com:vespa-engine/vespa-ann-benchmark +cd vespa-ann-benchmark + +RELEASE_TAG="v$VESPA_VERSION" +if ! git rev-parse $RELEASE_TAG &> /dev/null; then + git tag -a "$RELEASE_TAG" -m "Release version $VESPA_VERSION" + git push origin "$RELEASE_TAG" +fi + diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java index 7df08d7d356..093e65b2e4d 100755 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/ExpressionFunction.java @@ -212,7 +212,7 @@ public class ExpressionFunction { public String toString() { return "function '" + name + "'"; } - + /** * An instance of a serialization of this function, using a particular serialization context (by {@link * ExpressionFunction#expand}) diff --git a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java index d88bd03b7d4..d1cb77fb1b4 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/rankingexpression/rule/SerializationContext.java @@ -21,7 +21,7 @@ import java.util.Optional; * @author bratseth */ public class SerializationContext extends FunctionReferenceContext { - + /** Serialized form of functions indexed by name */ private final Map<String, String> serializedFunctions; diff --git a/searchlib/src/main/java/com/yahoo/searchlib/tensor/EvaluateTensorConformance.java b/searchlib/src/main/java/com/yahoo/searchlib/tensor/EvaluateTensorConformance.java index dcdf2f532e4..69c8a091cce 100644 --- a/searchlib/src/main/java/com/yahoo/searchlib/tensor/EvaluateTensorConformance.java +++ b/searchlib/src/main/java/com/yahoo/searchlib/tensor/EvaluateTensorConformance.java @@ -71,7 +71,7 @@ public class EvaluateTensorConformance { System.exit(1); } } - + private boolean testCase(String test, int count) { boolean wasOk = false; try { diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java index e9b1e3bda0d..8af77ec1cdd 100755 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/RankingExpressionTestCase.java @@ -61,7 +61,7 @@ public class RankingExpressionTestCase { assertParse("query(var1) + query(var2) - query(var3) * (query(var4) / query(var5))", " $var1 + $var2 - $var3 *($var4 / $var5)"); assertParse("if (if (f1.out < query(p1), 0, 1) < if (f2.out < query(p2), 0, 1), f3.out, query(p3))", "if(if(f1.out<$p1,0,1)<if(f2.out<$p2,0,1),f3.out,$p3)"); } - + @Test public void testProgrammaticBuilding() throws ParseException { ReferenceNode input = new ReferenceNode("input"); @@ -147,14 +147,14 @@ public class RankingExpressionTestCase { "10 + 8 * 1977"), "cox", functions ); } - + @Test public void testTensorSerialization() { - assertSerialization("map(constant(tensor0), f(a)(cos(a)))", + assertSerialization("map(constant(tensor0), f(a)(cos(a)))", "map(constant(tensor0), f(a)(cos(a)))"); - assertSerialization("map(constant(tensor0), f(a)(cos(a))) + join(attribute(tensor1), map(reduce(map(attribute(tensor1), f(a)(a * a)), sum, x), f(a)(sqrt(a))), f(a,b)(a / b))", + assertSerialization("map(constant(tensor0), f(a)(cos(a))) + join(attribute(tensor1), map(reduce(map(attribute(tensor1), f(a)(a * a)), sum, x), f(a)(sqrt(a))), f(a,b)(a / b))", "map(constant(tensor0), f(a)(cos(a))) + l2_normalize(attribute(tensor1), x)"); - assertSerialization("join(reduce(join(reduce(join(constant(tensor0), attribute(tensor1), f(a,b)(a * b)), sum, x), attribute(tensor1), f(a,b)(a * b)), sum, y), query(tensor2), f(a,b)(a + b))", + assertSerialization("join(reduce(join(reduce(join(constant(tensor0), attribute(tensor1), f(a,b)(a * b)), sum, x), attribute(tensor1), f(a,b)(a * b)), sum, y), query(tensor2), f(a,b)(a + b))", "xw_plus_b(matmul(constant(tensor0), attribute(tensor1), x), attribute(tensor1), query(tensor2), y)"); assertSerialization("tensor(x{}):{{x:a}:(1 + 2 + 3),{x:b}:(if (1 > 2, 3, 4)),{x:c}:(reduce(tensor0 * tensor1, sum))}", "tensor(x{}):{ {x:a}:1+2+3, {x:b}:if(1>2,3,4), {x:c}:sum(tensor0*tensor1) }"); @@ -378,7 +378,7 @@ public class RankingExpressionTestCase { // (but not the same one due to primitivization) RankingExpression reparsedExpression = new RankingExpression(serializedExpression); // Serializing the primitivized expression should yield the same expression again - String reserializedExpression = + String reserializedExpression = reparsedExpression.getRankProperties(new SerializationContext()).values().iterator().next(); assertEquals(expectedSerialization, reserializedExpression); } diff --git a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationBenchmark.java b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationBenchmark.java index 955ca05ce37..f0ce613e27f 100644 --- a/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationBenchmark.java +++ b/searchlib/src/test/java/com/yahoo/searchlib/rankingexpression/evaluation/EvaluationBenchmark.java @@ -209,7 +209,7 @@ public class EvaluationBenchmark { if (Math.abs(a-b) >= Math.abs((a+b)/100000000) ) throw new RuntimeException("Expected value " + a + " but optimized evaluation produced " + b); } - + private final String gbdt = "if (LW_NEWS_SEARCHES_RATIO < 1.72971, 0.0697159, if (LW_USERS < 0.10496, if (SEARCHES < 0.0329127, 0.151257, 0.117501), if (SUGG_OVERLAP < 18.5, 0.0897622, 0.0756903))) + \n" + "if (LW_NEWS_SEARCHES_RATIO < 1.73156, if (NEWS_USERS < 0.0737993, -0.00481646, 0.00110018), if (LW_USERS < 0.0844616, 0.0488919, if (SUGG_OVERLAP < 32.5, 0.0136917, 9.85328E-4))) + \n" + diff --git a/searchlib/src/vespa/searchlib/bitcompression/README.md b/searchlib/src/vespa/searchlib/bitcompression/README.md new file mode 100644 index 00000000000..e387ec099fb --- /dev/null +++ b/searchlib/src/vespa/searchlib/bitcompression/README.md @@ -0,0 +1,56 @@ +<!-- Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> + +## About the disk dictionary format + +The designs of the disk index dictionary formats were incremental, due +to changing requirements over the years. + +### 1st generation + +Patricia tree in memory. + +### 2nd generation, 1998-09-04 + +Problem: Dictionary did not fit in memory (machine had 512 MB ram) +when indexing 5 million web documents on a single machine. + +Changed format to variable length records on disk, with a sparse +version of the dictionary in memory (each 256th word) to limit disk +access for binary search. + +### 3rd generation, 2000-03-09 + +Problem: Too many disk read operations and too many bytes read from disk +(limited PCI bandwidth). + +Changed format to a "paged" dictionary where a dictionary lookup would +use 1 disk read, reading 4 kiB of data. Data was not compressed. Could +not memory map whole dictionary. The sparse files were read into +memory and used to determine the page to use for further lookup. +Binary search within the pages read from disk. + +### 4th generation, 2002-08-16 + +Problem: Dictionary used too much disk space. + +Changed format to compressed format. Decompression could not contain +much state, thus delta values were compressed using exp golomb coding. + +Two levels of skip lists within each page, where skip list on a level +contained enough information to skip on all levels below within the +same page. + +Start of word was replaced by a byte telling how many bytes is +ommitted due to the prefix being common with previous words (word +before in dictionary and word before in the lookup order). + +### 5th generation, 2010-08-21 + +Payload ("value") changed when skip information was added for large +posting lists. Added overflow handling for long words / huge payloads. +Added another level of pages ("sparse pages") to improve compression. + +### 6th generation, 2015-05-12 + +Started using a separate dictionary for each index field instead of a +shared dictionary across all index fields. Minor changes. diff --git a/searchsummary/src/tests/docsummary/linguistics_tokens_converter/linguistics_tokens_converter_test.cpp b/searchsummary/src/tests/docsummary/linguistics_tokens_converter/linguistics_tokens_converter_test.cpp index c8d959361ae..beaa43c7af8 100644 --- a/searchsummary/src/tests/docsummary/linguistics_tokens_converter/linguistics_tokens_converter_test.cpp +++ b/searchsummary/src/tests/docsummary/linguistics_tokens_converter/linguistics_tokens_converter_test.cpp @@ -9,6 +9,7 @@ #include <vespa/document/repo/configbuilder.h> #include <vespa/document/repo/fixedtyperepo.h> #include <vespa/searchlib/util/linguisticsannotation.h> +#include <vespa/searchlib/util/token_extractor.h> #include <vespa/searchsummary/docsummary/linguistics_tokens_converter.h> #include <vespa/vespalib/data/simple_buffer.h> #include <vespa/vespalib/data/slime/json_format.h> @@ -25,6 +26,7 @@ using document::SpanTree; using document::StringFieldValue; using search::docsummary::LinguisticsTokensConverter; using search::linguistics::SPANTREE_NAME; +using search::linguistics::TokenExtractor; using vespalib::SimpleBuffer; using vespalib::Slime; using vespalib::slime::JsonFormat; @@ -59,6 +61,8 @@ protected: std::shared_ptr<const DocumentTypeRepo> _repo; const DocumentType* _document_type; document::FixedTypeRepo _fixed_repo; + vespalib::string _dummy_field_name; + TokenExtractor _token_extractor; LinguisticsTokensConverterTest(); ~LinguisticsTokensConverterTest() override; @@ -73,7 +77,9 @@ LinguisticsTokensConverterTest::LinguisticsTokensConverterTest() : testing::Test(), _repo(std::make_unique<DocumentTypeRepo>(get_document_types_config())), _document_type(_repo->getDocumentType("indexingdocument")), - _fixed_repo(*_repo, *_document_type) + _fixed_repo(*_repo, *_document_type), + _dummy_field_name(), + _token_extractor(_dummy_field_name, 100) { } @@ -127,7 +133,7 @@ LinguisticsTokensConverterTest::make_exp_annotated_chinese_string_tokens() vespalib::string LinguisticsTokensConverterTest::convert(const StringFieldValue& fv) { - LinguisticsTokensConverter converter; + LinguisticsTokensConverter converter(_token_extractor); Slime slime; SlimeInserter inserter(slime); converter.convert(fv, inserter); diff --git a/searchsummary/src/tests/docsummary/slime_filler/slime_filler_test.cpp b/searchsummary/src/tests/docsummary/slime_filler/slime_filler_test.cpp index 10aedc6d9d0..c20f9570ef8 100644 --- a/searchsummary/src/tests/docsummary/slime_filler/slime_filler_test.cpp +++ b/searchsummary/src/tests/docsummary/slime_filler/slime_filler_test.cpp @@ -68,6 +68,7 @@ using search::docsummary::IStringFieldConverter; using search::docsummary::ResultConfig; using search::docsummary::SlimeFiller; using search::docsummary::SlimeFillerFilter; +using vespalib::Memory; using vespalib::SimpleBuffer; using vespalib::Slime; using vespalib::eval::SimpleValue; @@ -146,17 +147,27 @@ get_document_types_config() class MockStringFieldConverter : public IStringFieldConverter { std::vector<vespalib::string> _result; + bool _render_wset_as_array; + bool _insert; public: - MockStringFieldConverter() + MockStringFieldConverter(bool render_wset_as_array, bool insert) : IStringFieldConverter(), - _result() + _result(), + _render_wset_as_array(render_wset_as_array), + _insert(insert) { } ~MockStringFieldConverter() override = default; - void convert(const document::StringFieldValue& input, vespalib::slime::Inserter&) override { + void convert(const document::StringFieldValue& input, vespalib::slime::Inserter& inserter) override { _result.emplace_back(input.getValueRef()); + if (_insert) { + inserter.insertString(Memory(input.getValueRef())); + } } const std::vector<vespalib::string>& get_result() const noexcept { return _result; } + bool render_weighted_set_as_array() const override { + return _render_wset_as_array; + } }; } @@ -188,6 +199,7 @@ protected: void expect_insert_summary_field_with_filter(const vespalib::string& exp, const FieldValue& fv, const std::vector<uint32_t>& matching_elems); void expect_insert_summary_field_with_field_filter(const vespalib::string& exp, const FieldValue& fv, const SlimeFillerFilter* filter); void expect_insert_juniper_field(const std::vector<vespalib::string>& exp, const vespalib::string& exp_slime, const FieldValue& fv); + void expect_insert_summary_field_with_converter(const std::vector<vespalib::string>& exp, const vespalib::string& exp_slime, const FieldValue& fv, MockStringFieldConverter& converter); }; SlimeFillerTest::SlimeFillerTest() @@ -317,7 +329,7 @@ SlimeFillerTest::expect_insert_callback(const std::vector<vespalib::string>& exp { Slime slime; SlimeInserter inserter(slime); - MockStringFieldConverter converter; + MockStringFieldConverter converter(false, false); SlimeFiller filler(inserter, &converter, SlimeFillerFilter::all()); fv.accept(filler); auto act_null = slime_to_string(slime); @@ -361,7 +373,7 @@ SlimeFillerTest::expect_insert_juniper_field(const std::vector<vespalib::string> { Slime slime; SlimeInserter inserter(slime); - MockStringFieldConverter converter; + MockStringFieldConverter converter(false, false); SlimeFiller::insert_juniper_field(fv, inserter, converter); auto act_slime = slime_to_string(slime); EXPECT_EQ(exp_slime, act_slime); @@ -369,6 +381,18 @@ SlimeFillerTest::expect_insert_juniper_field(const std::vector<vespalib::string> EXPECT_EQ(exp, act); } +void +SlimeFillerTest::expect_insert_summary_field_with_converter(const std::vector<vespalib::string>& exp, const vespalib::string& exp_slime, const FieldValue& fv, MockStringFieldConverter& converter) +{ + Slime slime; + SlimeInserter inserter(slime); + SlimeFiller::insert_summary_field(fv, inserter, &converter); + auto act_slime = slime_to_string(slime); + EXPECT_EQ(exp_slime, act_slime); + auto act = converter.get_result(); + EXPECT_EQ(exp, act); +} + TEST_F(SlimeFillerTest, insert_primitive_values) { { @@ -625,4 +649,16 @@ TEST_F(SlimeFillerTest, insert_juniper_field) expect_insert_juniper_field({}, "null", make_empty_array()); } +TEST_F(SlimeFillerTest, string_field_is_not_converted_for_weighted_set_rendering) +{ + MockStringFieldConverter cvt_as_wset(false, true); + expect_insert_summary_field_with_converter({}, R"([{"item":"foo","weight":2},{"item":"bar","weight":4},{"item":"baz","weight":6}])", make_weighted_set(), cvt_as_wset); +} + +TEST_F(SlimeFillerTest, weighted_set_can_be_rendered_as_array) +{ + MockStringFieldConverter cvt_as_array(true, true); + expect_insert_summary_field_with_converter({"foo","bar","baz"}, R"(["foo","bar","baz"])", make_weighted_set(), cvt_as_array); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt index e5ae47593e5..57b6004fb61 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt +++ b/searchsummary/src/vespa/searchsummary/docsummary/CMakeLists.txt @@ -24,6 +24,7 @@ vespa_add_library(searchsummary_docsummary OBJECT juniper_query_adapter.cpp juniperproperties.cpp linguistics_tokens_converter.cpp + linguistics_tokens_dfw.cpp matched_elements_filter_dfw.cpp positionsdfw.cpp query_term_filter.cpp diff --git a/searchsummary/src/vespa/searchsummary/docsummary/annotation_converter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/annotation_converter.cpp index bf267ab9e27..77724305220 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/annotation_converter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/annotation_converter.cpp @@ -109,4 +109,10 @@ AnnotationConverter::convert(const StringFieldValue &input, vespalib::slime::Ins _juniper_converter.convert(_out.str(), inserter); } +bool +AnnotationConverter::render_weighted_set_as_array() const +{ + return false; +} + } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/annotation_converter.h b/searchsummary/src/vespa/searchsummary/docsummary/annotation_converter.h index b6430b35f29..b082269eb7e 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/annotation_converter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/annotation_converter.h @@ -33,6 +33,7 @@ public: AnnotationConverter(IJuniperConverter& juniper_converter); ~AnnotationConverter() override; void convert(const document::StringFieldValue &input, vespalib::slime::Inserter& inserter) override; + bool render_weighted_set_as_array() const override; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.cpp index 2ce809e1cbe..c4823f6beeb 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.cpp @@ -12,6 +12,7 @@ const vespalib::string documentid("documentid"); const vespalib::string dynamic_teaser("dynamicteaser"); const vespalib::string empty("empty"); const vespalib::string geo_position("geopos"); +const vespalib::string linguistics_tokens("linguistics-tokens"); const vespalib::string matched_attribute_elements_filter("matchedattributeelementsfilter"); const vespalib::string matched_elements_filter("matchedelementsfilter"); const vespalib::string positions("positions"); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.h b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.h index 26bc33e7e3c..2d0b8c23855 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_commands.h @@ -18,6 +18,7 @@ extern const vespalib::string documentid; extern const vespalib::string dynamic_teaser; extern const vespalib::string empty; extern const vespalib::string geo_position; +extern const vespalib::string linguistics_tokens; extern const vespalib::string matched_attribute_elements_filter; extern const vespalib::string matched_elements_filter; extern const vespalib::string positions; diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp index 9b7391dd1ab..d19d2994104 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsum_field_writer_factory.cpp @@ -9,6 +9,7 @@ #include "geoposdfw.h" #include "idocsumenvironment.h" #include "juniperdfw.h" +#include "linguistics_tokens_dfw.h" #include "matched_elements_filter_dfw.h" #include "positionsdfw.h" #include "rankfeaturesdfw.h" @@ -84,6 +85,12 @@ DocsumFieldWriterFactory::create_docsum_field_writer(const vespalib::string& fie } else { throw_missing_source(command); } + } else if (command == command::linguistics_tokens) { + if (!source.empty()) { + fieldWriter = std::make_unique<LinguisticsTokensDFW>(source); + } else { + throw_missing_source(command); + } } else if (command == command::abs_distance) { if (has_attribute_manager()) { fieldWriter = AbsDistanceDFW::create(source.c_str(), getEnvironment().getAttributeManager()); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/i_string_field_converter.h b/searchsummary/src/vespa/searchsummary/docsummary/i_string_field_converter.h index 3b36455d09d..805b5cf3508 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/i_string_field_converter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/i_string_field_converter.h @@ -17,6 +17,7 @@ class IStringFieldConverter public: virtual ~IStringFieldConverter() = default; virtual void convert(const document::StringFieldValue &input, vespalib::slime::Inserter& inserter) = 0; + virtual bool render_weighted_set_as_array() const = 0; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_converter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_converter.cpp index 838b0234cdb..b9b9d7c4c97 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_converter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_converter.cpp @@ -2,14 +2,11 @@ #include "linguistics_tokens_converter.h" #include <vespa/document/fieldvalue/stringfieldvalue.h> -#include <vespa/searchlib/memoryindex/field_inverter.h> -#include <vespa/searchlib/util/linguisticsannotation.h> #include <vespa/searchlib/util/token_extractor.h> #include <vespa/vespalib/data/slime/slime.h> using document::StringFieldValue; using search::linguistics::TokenExtractor; -using search::memoryindex::FieldInverter; using vespalib::Memory; using vespalib::slime::ArrayInserter; using vespalib::slime::Cursor; @@ -17,14 +14,9 @@ using vespalib::slime::Inserter; namespace search::docsummary { -namespace { - -vespalib::string dummy_field_name; - -} - -LinguisticsTokensConverter::LinguisticsTokensConverter() +LinguisticsTokensConverter::LinguisticsTokensConverter(const TokenExtractor& token_extractor) : IStringFieldConverter(), + _token_extractor(token_extractor), _text() { } @@ -56,8 +48,7 @@ LinguisticsTokensConverter::handle_indexing_terms(const StringFieldValue& value, using SpanTerm = TokenExtractor::SpanTerm; std::vector<SpanTerm> terms; auto span_trees = value.getSpanTrees(); - TokenExtractor token_extractor(dummy_field_name, FieldInverter::max_word_len); - token_extractor.extract(terms, span_trees, _text, nullptr); + _token_extractor.extract(terms, span_trees, _text, nullptr); auto it = terms.begin(); auto ite = terms.end(); auto itn = it; @@ -78,4 +69,10 @@ LinguisticsTokensConverter::convert(const StringFieldValue &input, vespalib::sli handle_indexing_terms(input, inserter); } +bool +LinguisticsTokensConverter::render_weighted_set_as_array() const +{ + return true; +} + } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_converter.h b/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_converter.h index cba3937c822..d752fe89ed9 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_converter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_converter.h @@ -4,6 +4,8 @@ #include "i_string_field_converter.h" +namespace search::linguistics { class TokenExtractor; } + namespace search::docsummary { /* @@ -13,16 +15,18 @@ namespace search::docsummary { */ class LinguisticsTokensConverter : public IStringFieldConverter { - vespalib::stringref _text; + const linguistics::TokenExtractor& _token_extractor; + vespalib::stringref _text; template <typename ForwardIt> void handle_alternative_index_terms(ForwardIt it, ForwardIt last, vespalib::slime::Inserter& inserter); void handle_index_term(vespalib::stringref word, vespalib::slime::Inserter& inserter); void handle_indexing_terms(const document::StringFieldValue& value, vespalib::slime::Inserter& inserter); public: - LinguisticsTokensConverter(); + LinguisticsTokensConverter(const linguistics::TokenExtractor& token_extractor); ~LinguisticsTokensConverter() override; void convert(const document::StringFieldValue &input, vespalib::slime::Inserter& inserter) override; + bool render_weighted_set_as_array() const override; }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_dfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_dfw.cpp new file mode 100644 index 00000000000..5e94e270c53 --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_dfw.cpp @@ -0,0 +1,36 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "linguistics_tokens_dfw.h" +#include "i_docsum_store_document.h" +#include "linguistics_tokens_converter.h" +#include <vespa/searchlib/memoryindex/field_inverter.h> + +using search::memoryindex::FieldInverter; + +namespace search::docsummary { + +LinguisticsTokensDFW::LinguisticsTokensDFW(const vespalib::string& input_field_name) + : DocsumFieldWriter(), + _input_field_name(input_field_name), + _token_extractor(_input_field_name, FieldInverter::max_word_len) +{ +} + +LinguisticsTokensDFW::~LinguisticsTokensDFW() = default; + +bool +LinguisticsTokensDFW::isGenerated() const +{ + return false; +} + +void +LinguisticsTokensDFW::insertField(uint32_t, const IDocsumStoreDocument* doc, GetDocsumsState&, vespalib::slime::Inserter& target) const +{ + if (doc != nullptr) { + LinguisticsTokensConverter converter(_token_extractor); + doc->insert_summary_field(_input_field_name, target, &converter); + } +} + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_dfw.h b/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_dfw.h new file mode 100644 index 00000000000..9c6955b322e --- /dev/null +++ b/searchsummary/src/vespa/searchsummary/docsummary/linguistics_tokens_dfw.h @@ -0,0 +1,28 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "docsum_field_writer.h" +#include <vespa/searchlib/util/token_extractor.h> +#include <memory> + +namespace search::docsummary { + +/* + * Class for writing annotated string field values from document as + * arrays containing the indexing terms. + */ +class LinguisticsTokensDFW : public DocsumFieldWriter +{ +private: + vespalib::string _input_field_name; + linguistics::TokenExtractor _token_extractor; + +public: + explicit LinguisticsTokensDFW(const vespalib::string& input_field_name); + ~LinguisticsTokensDFW() override; + bool isGenerated() const override; + void insertField(uint32_t docid, const IDocsumStoreDocument* doc, GetDocsumsState& state, vespalib::slime::Inserter& target) const override; +}; + +} diff --git a/searchsummary/src/vespa/searchsummary/docsummary/slime_filler.cpp b/searchsummary/src/vespa/searchsummary/docsummary/slime_filler.cpp index 7266642b18b..080129fe780 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/slime_filler.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/slime_filler.cpp @@ -285,6 +285,7 @@ SlimeFiller::visit(const WeightedSetFieldValue& value) if (empty_or_empty_after_filtering(value)) { return; } + bool render_as_array = _string_converter != nullptr && _string_converter->render_weighted_set_as_array(); Cursor& a = _inserter.insertArray(); Symbol isym = a.resolve("item"); Symbol wsym = a.resolve("weight"); @@ -305,12 +306,18 @@ SlimeFiller::visit(const WeightedSetFieldValue& value) } ++matching_elements_itr; } - Cursor& o = a.addObject(); - ObjectSymbolInserter ki(o, isym); - SlimeFiller conv(ki); - entry.first->accept(conv); - int weight = static_cast<const IntFieldValue&>(*entry.second).getValue(); - o.setLong(wsym, weight); + if (render_as_array) { + ArrayInserter ai(a); + SlimeFiller conv(ai, _string_converter, SlimeFillerFilter::all()); + entry.first->accept(conv); + } else { + Cursor& o = a.addObject(); + ObjectSymbolInserter ki(o, isym); + SlimeFiller conv(ki); + entry.first->accept(conv); + int weight = static_cast<const IntFieldValue&>(*entry.second).getValue(); + o.setLong(wsym, weight); + } ++idx; } } diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index 871cdaddb53..582b67a4dbc 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -897,7 +897,13 @@ FileStorHandlerImpl::flush() uint64_t FileStorHandlerImpl::dispersed_bucket_bits(const document::Bucket& bucket) noexcept { const uint64_t raw_id = bucket.getBucketId().getId(); - return XXH3_64bits(&raw_id, sizeof(uint64_t)); + /* + * This is a workaround for gcc 12 and on that produces incorrect warning when compiled with -march=haswell or newer + * See document/src/vespa/document/bucket/bucketid.cpp: In member function ‘uint64_t document::BucketId::hash::operator()(const document::BucketId&) const + */ + uint8_t raw_as_bytes[sizeof(raw_id)]; + memcpy(raw_as_bytes, &raw_id, sizeof(raw_id)); + return XXH3_64bits(&raw_as_bytes, sizeof(raw_id)); } FileStorHandlerImpl::Stripe::Stripe(const FileStorHandlerImpl & owner, MessageSender & messageSender) diff --git a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp index b48f556f4be..b94de154a35 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp @@ -132,6 +132,7 @@ public: } ~SnippetModifierJuniperConverter() override = default; void convert(const document::StringFieldValue &input, vespalib::slime::Inserter& inserter) override; + bool render_weighted_set_as_array() const override; }; void @@ -147,6 +148,12 @@ SnippetModifierJuniperConverter::convert(const document::StringFieldValue &input } } +bool +SnippetModifierJuniperConverter::render_weighted_set_as_array() const +{ + return false; +} + /** * Class providing access to a document retrieved from an IDocsumStore * (vsm::DocsumFilter). VSM specific transforms might be applied when diff --git a/vespalib/src/vespa/vespalib/fuzzy/sparse_state.h b/vespalib/src/vespa/vespalib/fuzzy/sparse_state.h index 0f58853170e..7e381468fbe 100644 --- a/vespalib/src/vespa/vespalib/fuzzy/sparse_state.h +++ b/vespalib/src/vespa/vespalib/fuzzy/sparse_state.h @@ -8,7 +8,7 @@ #include <cstdint> #include <ostream> #include <span> -#include <xxh3.h> // TODO factor out? +#include <xxhash.h> // TODO factor out? namespace vespalib::fuzzy { |