diff options
111 files changed, 1339 insertions, 312 deletions
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 50301066145..d5c611680dd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,7 +33,7 @@ We only seek to accept code that you are authorized to contribute to the project ## Versioning Vespa uses semantic versioning - see -[vespa versions](http://docs.vespa.ai/documentation/vespa-versions.html). +[vespa versions](http://docs.vespa.ai/en/vespa-versions.html). Notice in particular that any Java API in a package having a @PublicAPI annotation in the package-info file cannot be changed in an incompatible way between major versions: Existing types and method signatures must be preserved diff --git a/Code-map.md b/Code-map.md index 6eae723f31d..3c54be35b9d 100644 --- a/Code-map.md +++ b/Code-map.md @@ -12,7 +12,7 @@ thing we haven't done is to create a module structure friendly to newcomers - th simply organized in a flat structure of about 150 modules. This document aims to provide a map from the -[functional elements](https://docs.vespa.ai/documentation/overview.html) +[functional elements](https://docs.vespa.ai/en/overview.html) of Vespa to the most important modules in the flat module structure in the [code base on GitHub](https://github.com/vespa-engine/vespa). diff --git a/ERRATA.md b/ERRATA.md index 3107216f527..9278f3c39cf 100644 --- a/ERRATA.md +++ b/ERRATA.md @@ -31,7 +31,7 @@ This bug was introduced in Vespa-7.277.38, fixed in Vespa-7.292.82. The following needs to happen to trigger the bug: * visibility-delay is non-zero. Note that the default is zero, so for this to trigger, - [visibility-delay](https://docs.vespa.ai/documentation/reference/services-content.html#visibility-delay) + [visibility-delay](https://docs.vespa.ai/en/reference/services-content.html#visibility-delay) must have been set. * A new config change is deployed that contains changes to proton. This config snapshot is stored in the transaction log on the content node. diff --git a/README.md b/README.md index 3056f619982..280d3ae8ee8 100644 --- a/README.md +++ b/README.md @@ -33,18 +33,18 @@ Vespa per second. To get started using Vespa pick one of the quick start documents: -- [Run on a Mac or Linux machine using Docker](https://docs.vespa.ai/documentation/vespa-quick-start.html) -- [Run on a Windows machine using Docker](https://docs.vespa.ai/documentation/vespa-quick-start-windows.html) -- [Run on a Mac or Linux machine using VirtualBox+Vagrant](https://docs.vespa.ai/documentation/vespa-quick-start-centos.html) -- [Multinode install on AWS EC2](https://docs.vespa.ai/documentation/vespa-quick-start-multinode-aws.html) -- [Multinode install on AWS ECS](https://docs.vespa.ai/documentation/vespa-quick-start-multinode-aws-ecs.html) +- [Run on a Mac or Linux machine using Docker](https://docs.vespa.ai/en/vespa-quick-start.html) +- [Run on a Windows machine using Docker](https://docs.vespa.ai/en/vespa-quick-start-windows.html) +- [Run on a Mac or Linux machine using VirtualBox+Vagrant](https://docs.vespa.ai/en/vespa-quick-start-centos.html) +- [Multinode install on AWS EC2](https://docs.vespa.ai/en/vespa-quick-start-multinode-aws.html) +- [Multinode install on AWS ECS](https://docs.vespa.ai/en/vespa-quick-start-multinode-aws-ecs.html) ## Usage -- The application created in the quickstart is fully functional and production ready, but you may want to [add more nodes](https://docs.vespa.ai/documentation/multinode-systems.html) for redundancy. -- Try the [Blog search and recommendation tutorial](https://docs.vespa.ai/documentation/tutorials/blog-search.html) to learn more about using Vespa -- See [developing applications](https://docs.vespa.ai/documentation/jdisc/developing-applications.html) on adding your own Java components to your Vespa application. -- [Vespa APIs](https://docs.vespa.ai/documentation/api.html) is useful to understand how to interface with Vespa +- The application created in the quickstart is fully functional and production ready, but you may want to [add more nodes](https://docs.vespa.ai/en/multinode-systems.html) for redundancy. +- Try the [Blog search and recommendation tutorial](https://docs.vespa.ai/en/tutorials/blog-search.html) to learn more about using Vespa +- See [developing applications](https://docs.vespa.ai/en/jdisc/developing-applications.html) on adding your own Java components to your Vespa application. +- [Vespa APIs](https://docs.vespa.ai/en/api.html) is useful to understand how to interface with Vespa - Explore the [sample applications](https://github.com/vespa-engine/sample-apps/tree/master) Full documentation is available on [https://docs.vespa.ai](https://docs.vespa.ai). diff --git a/client/README.md b/client/README.md index d08b19a1a12..39d9bd6b669 100644 --- a/client/README.md +++ b/client/README.md @@ -1,7 +1,7 @@ # vespa_query_dsl This lib is used for composing vespa YQL queries -Reference: https://docs.vespa.ai/documentation/reference/query-language-reference.html +Reference: https://docs.vespa.ai/en/reference/query-language-reference.html # usage Please refer to the unit test: @@ -9,7 +9,7 @@ Please refer to the unit test: https://github.com/vespa-engine/vespa/tree/master/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy # todos -- [ ] support `predicate` (https://docs.vespa.ai/documentation/predicate-fields.html) +- [ ] support `predicate` (https://docs.vespa.ai/en/predicate-fields.html) - [ ] support methods for checking positive/negative conditions for specific field - [X] support order by annotation - [X] support order by diff --git a/client/src/main/java/ai/vespa/client/dsl/A.java b/client/src/main/java/ai/vespa/client/dsl/A.java index a691d1288e8..c1f94a08d25 100644 --- a/client/src/main/java/ai/vespa/client/dsl/A.java +++ b/client/src/main/java/ai/vespa/client/dsl/A.java @@ -8,7 +8,7 @@ import java.util.stream.Stream; /** * Helper class for generating Annotation - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#annotations + * https://docs.vespa.ai/en/reference/query-language-reference.html#annotations */ public final class A { diff --git a/client/src/main/java/ai/vespa/client/dsl/EndQuery.java b/client/src/main/java/ai/vespa/client/dsl/EndQuery.java index 2af1e0bb49d..d07d3536563 100644 --- a/client/src/main/java/ai/vespa/client/dsl/EndQuery.java +++ b/client/src/main/java/ai/vespa/client/dsl/EndQuery.java @@ -44,7 +44,7 @@ public class EndQuery { /** * Offset. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#limit-offset + * https://docs.vespa.ai/en/reference/query-language-reference.html#limit-offset * * @param offset the offset * @return the end query @@ -55,7 +55,7 @@ public class EndQuery { /** * Timeout. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#timeout + * https://docs.vespa.ai/en/reference/query-language-reference.html#timeout * * @param timeout the timeout * @return the end query @@ -66,7 +66,7 @@ public class EndQuery { /** * Limit. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#limit-offset + * https://docs.vespa.ai/en/reference/query-language-reference.html#limit-offset * * @param limit the limit * @return the end query @@ -87,7 +87,7 @@ public class EndQuery { /** * Group. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#grouping + * https://docs.vespa.ai/en/reference/query-language-reference.html#grouping * * @param group the group * @return the end query @@ -99,7 +99,7 @@ public class EndQuery { /** * Group. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#grouping + * https://docs.vespa.ai/en/reference/query-language-reference.html#grouping * * @param groupQueryStr the group str * @return the end query @@ -111,7 +111,7 @@ public class EndQuery { /** * Order by asc. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#order-by + * https://docs.vespa.ai/en/reference/query-language-reference.html#order-by * * @param annotation the annotation * @param fieldName the field name @@ -124,7 +124,7 @@ public class EndQuery { /** * Order by asc. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#order-by + * https://docs.vespa.ai/en/reference/query-language-reference.html#order-by * * @param fieldName the field name * @return the end query @@ -136,7 +136,7 @@ public class EndQuery { /** * Order by desc. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#order-by + * https://docs.vespa.ai/en/reference/query-language-reference.html#order-by * * @param annotation the annotation * @param fieldName the field name @@ -149,7 +149,7 @@ public class EndQuery { /** * Order by desc. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#order-by + * https://docs.vespa.ai/en/reference/query-language-reference.html#order-by * * @param fieldName the field name * @return the end query diff --git a/client/src/main/java/ai/vespa/client/dsl/Field.java b/client/src/main/java/ai/vespa/client/dsl/Field.java index 0cbe03a8fb9..b12ae51787d 100644 --- a/client/src/main/java/ai/vespa/client/dsl/Field.java +++ b/client/src/main/java/ai/vespa/client/dsl/Field.java @@ -27,7 +27,7 @@ public class Field extends QueryChain { /** * Contains query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param value the value * @return the query @@ -38,7 +38,7 @@ public class Field extends QueryChain { /** * Contains query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param annotation the annotation * @param value the value @@ -50,7 +50,7 @@ public class Field extends QueryChain { /** * Contains phrase query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param value the value * @param others the others @@ -62,7 +62,7 @@ public class Field extends QueryChain { /** * Contains phrase query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param values the values * @return the query @@ -77,7 +77,7 @@ public class Field extends QueryChain { /** * Contains near query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param value the value * @param others the others @@ -89,7 +89,7 @@ public class Field extends QueryChain { /** * Contains near query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param values the values * @return the query @@ -104,7 +104,7 @@ public class Field extends QueryChain { /** * Contains near query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param annotation the annotation * @param value the value @@ -117,7 +117,7 @@ public class Field extends QueryChain { /** * Contains near query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param annotation the annotation * @param values the values @@ -133,7 +133,7 @@ public class Field extends QueryChain { /** * Contains onear query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param value the value * @param others the others @@ -145,7 +145,7 @@ public class Field extends QueryChain { /** * Contains onear query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param values the values * @return the query @@ -160,7 +160,7 @@ public class Field extends QueryChain { /** * Contains onear query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param annotation the annotation * @param value the value @@ -173,7 +173,7 @@ public class Field extends QueryChain { /** * Contains onear query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param annotation the annotation * @param values the values @@ -189,7 +189,7 @@ public class Field extends QueryChain { /** * Contains same element query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param andQuery the and query * @return the query @@ -200,7 +200,7 @@ public class Field extends QueryChain { /** * Contains equiv query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param value the value * @param others the others @@ -212,7 +212,7 @@ public class Field extends QueryChain { /** * Contains equiv query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param values the values * @return the query @@ -230,7 +230,7 @@ public class Field extends QueryChain { /** * Contains uri query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param value the value * @return the query @@ -241,7 +241,7 @@ public class Field extends QueryChain { /** * Contains uri query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param annotation the annotation * @param value the value @@ -253,7 +253,7 @@ public class Field extends QueryChain { /** * Matches query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#matches + * https://docs.vespa.ai/en/reference/query-language-reference.html#matches * * @param str the str * @return the query @@ -264,7 +264,7 @@ public class Field extends QueryChain { /** * Equals query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#numeric + * https://docs.vespa.ai/en/reference/query-language-reference.html#numeric * * @param t the t * @return the query @@ -275,7 +275,7 @@ public class Field extends QueryChain { /** * Greater than or equal to query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#numeric + * https://docs.vespa.ai/en/reference/query-language-reference.html#numeric * * @param t the t * @return the query @@ -286,7 +286,7 @@ public class Field extends QueryChain { /** * Greater than query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#numeric + * https://docs.vespa.ai/en/reference/query-language-reference.html#numeric * * @param t the t * @return the query @@ -297,7 +297,7 @@ public class Field extends QueryChain { /** * Less than or equal to query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#numeric + * https://docs.vespa.ai/en/reference/query-language-reference.html#numeric * * @param t the t * @return the query @@ -308,7 +308,7 @@ public class Field extends QueryChain { /** * Less than query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#numeric + * https://docs.vespa.ai/en/reference/query-language-reference.html#numeric * * @param t the t * @return the query @@ -319,7 +319,7 @@ public class Field extends QueryChain { /** * In range query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#numeric + * https://docs.vespa.ai/en/reference/query-language-reference.html#numeric * * @param l the l * @param m the m @@ -331,7 +331,7 @@ public class Field extends QueryChain { /** * Equal to query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#numeric + * https://docs.vespa.ai/en/reference/query-language-reference.html#numeric * * @param t the t * @return the query @@ -342,7 +342,7 @@ public class Field extends QueryChain { /** * Greater than or equal to query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#numeric + * https://docs.vespa.ai/en/reference/query-language-reference.html#numeric * * @param t the t * @return the query @@ -353,7 +353,7 @@ public class Field extends QueryChain { /** * Greater than query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#numeric + * https://docs.vespa.ai/en/reference/query-language-reference.html#numeric * * @param t the t * @return the query @@ -364,7 +364,7 @@ public class Field extends QueryChain { /** * Less than or equal to query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#numeric + * https://docs.vespa.ai/en/reference/query-language-reference.html#numeric * * @param t the t * @return the query @@ -375,7 +375,7 @@ public class Field extends QueryChain { /** * Less than query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#numeric + * https://docs.vespa.ai/en/reference/query-language-reference.html#numeric * * @param t the t * @return the query @@ -386,7 +386,7 @@ public class Field extends QueryChain { /** * In range query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#numeric + * https://docs.vespa.ai/en/reference/query-language-reference.html#numeric * * @param l the l * @param m the m @@ -399,7 +399,7 @@ public class Field extends QueryChain { /** * Is true query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#boolean + * https://docs.vespa.ai/en/reference/query-language-reference.html#boolean * * @return the query */ @@ -409,7 +409,7 @@ public class Field extends QueryChain { /** * Is false query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#boolean + * https://docs.vespa.ai/en/reference/query-language-reference.html#boolean * * @return the query */ diff --git a/client/src/main/java/ai/vespa/client/dsl/FixedQuery.java b/client/src/main/java/ai/vespa/client/dsl/FixedQuery.java index b41dd2c546b..94c571dba5a 100644 --- a/client/src/main/java/ai/vespa/client/dsl/FixedQuery.java +++ b/client/src/main/java/ai/vespa/client/dsl/FixedQuery.java @@ -12,7 +12,7 @@ import java.util.stream.Collectors; /** * FixedQuery contains a 'Query' which is terminated by a 'semicolon' * This object holds vespa or user defined parameters - * https://docs.vespa.ai/documentation/reference/search-api-reference.html + * https://docs.vespa.ai/en/reference/search-api-reference.html */ public class FixedQuery { diff --git a/client/src/main/java/ai/vespa/client/dsl/G.java b/client/src/main/java/ai/vespa/client/dsl/G.java index 4a82658d3e0..c65bc982a88 100644 --- a/client/src/main/java/ai/vespa/client/dsl/G.java +++ b/client/src/main/java/ai/vespa/client/dsl/G.java @@ -4,7 +4,7 @@ package ai.vespa.client.dsl; /** * Helper class for generating group syntax - * https://docs.vespa.ai/documentation/reference/grouping-syntax.html + * https://docs.vespa.ai/en/reference/grouping-syntax.html * * basically the syntax is exactly the same as Vespa group syntax. * The only exception "max" in the Vespa group syntax which represents 'max returned documents', diff --git a/client/src/main/java/ai/vespa/client/dsl/Q.java b/client/src/main/java/ai/vespa/client/dsl/Q.java index 4048e6b8869..7637f76f095 100644 --- a/client/src/main/java/ai/vespa/client/dsl/Q.java +++ b/client/src/main/java/ai/vespa/client/dsl/Q.java @@ -8,7 +8,7 @@ import java.util.Map; /** * Helper class for generating Vespa search queries - * https://docs.vespa.ai/documentation/reference/query-language-reference.html + * https://docs.vespa.ai/en/reference/query-language-reference.html */ public final class Q { @@ -54,7 +54,7 @@ public final class Q { /** * Rank rank. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#rank + * https://docs.vespa.ai/en/reference/query-language-reference.html#rank * * @param query the query * @param ranks the ranks @@ -66,7 +66,7 @@ public final class Q { /** * UI represents "userInput". - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#userinput + * https://docs.vespa.ai/en/reference/query-language-reference.html#userinput * * @param value the value * @return the user input query @@ -77,7 +77,7 @@ public final class Q { /** * userInput with an annotation. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#userinput + * https://docs.vespa.ai/en/reference/query-language-reference.html#userinput * * @param a the a * @param value the value @@ -89,7 +89,7 @@ public final class Q { /** * A convenience method to generate userInput with default index annotation. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#userinput + * https://docs.vespa.ai/en/reference/query-language-reference.html#userinput * * @param index the index * @param value the value @@ -101,7 +101,7 @@ public final class Q { /** * dotPdt represents "dotProduct". - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#dotproduct + * https://docs.vespa.ai/en/reference/query-language-reference.html#dotproduct * * @param field the field * @param weightedSet the weighted set @@ -113,7 +113,7 @@ public final class Q { /** * wtdSet represents "weightedSet". - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#weightedset + * https://docs.vespa.ai/en/reference/query-language-reference.html#weightedset * * @param field the field * @param weightedSet the weighted set @@ -125,7 +125,7 @@ public final class Q { /** * NonEmpty non empty. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#nonempty + * https://docs.vespa.ai/en/reference/query-language-reference.html#nonempty * * @param query the query * @return the non empty query @@ -136,7 +136,7 @@ public final class Q { /** * Wand wand. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#wand + * https://docs.vespa.ai/en/reference/query-language-reference.html#wand * * @param field the field * @param weightedSet the weighted set @@ -148,7 +148,7 @@ public final class Q { /** * Wand wand. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#wand + * https://docs.vespa.ai/en/reference/query-language-reference.html#wand * * @param field the field * @param numericRange the numeric range @@ -160,7 +160,7 @@ public final class Q { /** * Weakand weak and. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#weakand + * https://docs.vespa.ai/en/reference/query-language-reference.html#weakand * * @param field the field * @param query the query diff --git a/client/src/main/java/ai/vespa/client/dsl/Query.java b/client/src/main/java/ai/vespa/client/dsl/Query.java index 62f92c51bee..9339733195a 100644 --- a/client/src/main/java/ai/vespa/client/dsl/Query.java +++ b/client/src/main/java/ai/vespa/client/dsl/Query.java @@ -90,7 +90,7 @@ public class Query extends QueryChain { /** * And. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#and + * https://docs.vespa.ai/en/reference/query-language-reference.html#and * * @param fieldName the field name * @return the field @@ -105,7 +105,7 @@ public class Query extends QueryChain { /** * Andnot. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#andnot + * https://docs.vespa.ai/en/reference/query-language-reference.html#andnot * * @param fieldName the field name * @return the field @@ -120,7 +120,7 @@ public class Query extends QueryChain { /** * Or. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#or + * https://docs.vespa.ai/en/reference/query-language-reference.html#or * * @param fieldName the field name * @return the field @@ -135,7 +135,7 @@ public class Query extends QueryChain { /** * And. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#and + * https://docs.vespa.ai/en/reference/query-language-reference.html#and * * @param query the query * @return the query @@ -149,7 +149,7 @@ public class Query extends QueryChain { /** * Andnot. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#andnot + * https://docs.vespa.ai/en/reference/query-language-reference.html#andnot * * @param query the query * @return the query @@ -163,7 +163,7 @@ public class Query extends QueryChain { /** * Or. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#or + * https://docs.vespa.ai/en/reference/query-language-reference.html#or * * @param query the query * @return the query @@ -177,7 +177,7 @@ public class Query extends QueryChain { /** * Annotate a query (sub-expression). - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#annotations-of-sub-expressions + * https://docs.vespa.ai/en/reference/query-language-reference.html#annotations-of-sub-expressions * * @param annotation the annotation * @return the query @@ -189,7 +189,7 @@ public class Query extends QueryChain { /** * Offset. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#limit-offset + * https://docs.vespa.ai/en/reference/query-language-reference.html#limit-offset * * @param offset the offset * @return the end query @@ -200,7 +200,7 @@ public class Query extends QueryChain { /** * Limit. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#limit-offset + * https://docs.vespa.ai/en/reference/query-language-reference.html#limit-offset * * @param hits the hits * @return the end query @@ -211,7 +211,7 @@ public class Query extends QueryChain { /** * Timeout. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#timeout + * https://docs.vespa.ai/en/reference/query-language-reference.html#timeout * * @param timeout the timeout * @return the end query @@ -222,7 +222,7 @@ public class Query extends QueryChain { /** * Group. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#grouping + * https://docs.vespa.ai/en/reference/query-language-reference.html#grouping * * @param group the group * @return the end query @@ -233,7 +233,7 @@ public class Query extends QueryChain { /** * Group. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#grouping + * https://docs.vespa.ai/en/reference/query-language-reference.html#grouping * * @param groupStr the group str * @return the end query @@ -244,7 +244,7 @@ public class Query extends QueryChain { /** * Order by asc. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#order-by + * https://docs.vespa.ai/en/reference/query-language-reference.html#order-by * * @param fieldName the field name * @return the end query @@ -255,7 +255,7 @@ public class Query extends QueryChain { /** * Order by asc. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#order-by + * https://docs.vespa.ai/en/reference/query-language-reference.html#order-by * * @param annotation the annotation * @param fieldName the field name @@ -267,7 +267,7 @@ public class Query extends QueryChain { /** * Order by desc. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#order-by + * https://docs.vespa.ai/en/reference/query-language-reference.html#order-by * * @param fieldName the field name * @return the end query @@ -279,7 +279,7 @@ public class Query extends QueryChain { /** * Order by desc. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#order-by + * https://docs.vespa.ai/en/reference/query-language-reference.html#order-by * * @param annotation the annotation * @param fieldName the field name diff --git a/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy b/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy index 19c87d6aecd..3353ef6f827 100644 --- a/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy +++ b/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy @@ -439,7 +439,7 @@ class QTest extends Specification { def "basic group syntax"() { /* example from vespa document: - https://docs.vespa.ai/documentation/grouping.html + https://docs.vespa.ai/en/grouping.html all( group(a) max(5) each(output(count()) all(max(1) each(output(summary()))) all(group(b) each(output(count()) @@ -470,7 +470,7 @@ class QTest extends Specification { def "set group syntax string directly"() { /* example from vespa document: - https://docs.vespa.ai/documentation/grouping.html + https://docs.vespa.ai/en/grouping.html all( group(a) max(5) each(output(count()) all(max(1) each(output(summary()))) all(group(b) each(output(count()) diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java index f22cf0d8c47..8845431c71b 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java @@ -87,7 +87,7 @@ public class ValidationOverrides { public static String toAllowMessage(ValidationId id) { return "To allow this add <allow until='yyyy-mm-dd'>" + id + "</allow> to validation-overrides.xml" + - ", see https://docs.vespa.ai/documentation/reference/validation-overrides.html"; + ", see https://docs.vespa.ai/en/reference/validation-overrides.html"; } /** diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 50676577d3d..42ecf7e6bf3 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -72,6 +72,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Select sequencer type use while feeding") default String feedSequencerType() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default String responseSequencerType() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default int defaultNumResponseThreads() { return 2; } + @ModelFeatureFlag(owners = {"baldersheim"}) default int maxPendingMoveOps() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean skipCommunicationManagerThread() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean skipMbusRequestThread() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean skipMbusReplyThread() { throw new UnsupportedOperationException("TODO specify default value"); } @@ -79,6 +80,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useAsyncMessageHandlingOnSchedule() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double feedConcurrency() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useBucketExecutorForLidSpaceCompact() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useBucketExecutorForBucketMove() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"musum", "mpolden"}, comment = "Revisit in February 2021") default boolean reconfigurableZookeeperServer() { return false; } @ModelFeatureFlag(owners = {"bjorncs", "tokle"}, removeAfter = "7.357") default boolean enableJdiscConnectionLog() { return true; } @ModelFeatureFlag(owners = {"bjorncs", "tokle", "baldersheim"}, removeAfter = "7.357") default boolean enableZstdCompressionAccessLog() { return true; } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 046eef65eb5..8befbb48016 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -156,10 +156,11 @@ public class DeployState implements ConfigDefinitionStore { } public static HostProvisioner getDefaultModelHostProvisioner(ApplicationPackage applicationPackage) { - if (applicationPackage.getHosts() == null) { - return new SingleNodeProvisioner(); - } else { - return new HostsXmlProvisioner(applicationPackage.getHosts()); + try (Reader hostsReader = applicationPackage.getHosts()) { + return hostsReader == null ? new SingleNodeProvisioner() : new HostsXmlProvisioner(hostsReader); + } + catch (IOException e) { + throw new IllegalStateException("Could not read hosts.xml", e); } } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 397b11661e5..d0e8afa5616 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -41,6 +41,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private String sequencerType = "LATENCY"; private String responseSequencerType = "ADAPTIVE"; private int responseNumThreads = 2; + private int maxPendingMoveOps = 10; private Optional<EndpointCertificateSecrets> endpointCertificateSecrets = Optional.empty(); private AthenzDomain athenzDomain; private ApplicationRoles applicationRoles; @@ -51,6 +52,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean enableAutomaticReindexing = false; private boolean reconfigurableZookeeperServer = false; private boolean useBucketExecutorForLidSpaceCompact; + private boolean useBucketExecutorForBucketMove; private boolean enableFeedBlockInDistributor = false; private double maxDeadBytesRatio = 0.2; private int clusterControllerMaxHeapSizeInMb = 512; @@ -77,6 +79,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public Optional<ApplicationRoles> applicationRoles() { return Optional.ofNullable(applicationRoles); } @Override public String responseSequencerType() { return responseSequencerType; } @Override public int defaultNumResponseThreads() { return responseNumThreads; } + @Override public int maxPendingMoveOps() { return maxPendingMoveOps; } @Override public boolean skipCommunicationManagerThread() { return false; } @Override public boolean skipMbusRequestThread() { return false; } @Override public boolean skipMbusReplyThread() { return false; } @@ -87,6 +90,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean enableAutomaticReindexing() { return enableAutomaticReindexing; } @Override public boolean reconfigurableZookeeperServer() { return reconfigurableZookeeperServer; } @Override public boolean useBucketExecutorForLidSpaceCompact() { return useBucketExecutorForLidSpaceCompact; } + @Override public boolean useBucketExecutorForBucketMove() { return useBucketExecutorForBucketMove; } @Override public boolean enableFeedBlockInDistributor() { return enableFeedBlockInDistributor; } @Override public double maxDeadBytesRatio() { return maxDeadBytesRatio; } @Override public int clusterControllerMaxHeapSizeInMb() { return clusterControllerMaxHeapSizeInMb; } @@ -117,6 +121,10 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea responseNumThreads = numThreads; return this; } + public TestProperties setMaxPendingMoveOps(int moveOps) { + maxPendingMoveOps = moveOps; + return this; + } public TestProperties setDefaultTermwiseLimit(double limit) { defaultTermwiseLimit = limit; return this; @@ -194,6 +202,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties useBucketExecutorForBucketMove(boolean enabled) { + useBucketExecutorForBucketMove = enabled; + return this; + } + public TestProperties enableFeedBlockInDistributor(boolean enabled) { enableFeedBlockInDistributor = enabled; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java index 04660f2b990..6e16b1a120e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java @@ -194,16 +194,14 @@ public abstract class AbstractService extends AbstractConfigProducer<AbstractCon } /** - * Gets the ports metainfo object. The service implementation - * must populate this object in the constructor. + * Gets the ports metainfo object. The service implementation must populate this object in the constructor. */ public PortsMeta getPortsMeta() { return portsMeta; } /** - * Computes and returns the i'th port for this service, based on - * this Service's baseport. + * Computes and returns the i'th port for this service, based on this Service's baseport. * * @param i the offset from 'basePort' of the port to return * @return the i'th port relative to the base port. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java index 27fb0f444ef..78d9dd473b3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/HostResource.java @@ -10,6 +10,7 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; @@ -67,9 +68,11 @@ public class HostResource implements Comparable<HostResource> { List<Integer> allocateService(DeployLogger deployLogger, AbstractService service, int wantedPort) { ports().useLogger(deployLogger); List<Integer> ports = hostPorts.allocatePorts(service, wantedPort); - assert (getService(service.getServiceName()) == null) : - ("There is already a service with name '" + service.getServiceName() + "' registered on " + this + - ". Most likely a programming error - all service classes must have unique names, even in different packages!"); + if (getService(service.getServiceName()) != null) + throw new IllegalStateException("There is already a service with name '" + service.getServiceName() + + "' registered on " + this + ". " + + "Most likely a programming error - " + + "all service classes must have unique names, even in different packages!"); services.put(service.getServiceName(), service); return ports; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/PortAllocBridge.java b/config-model/src/main/java/com/yahoo/vespa/model/PortAllocBridge.java index ac518b3a7d9..87db852e682 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/PortAllocBridge.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/PortAllocBridge.java @@ -10,13 +10,14 @@ import java.util.List; * This class acts as a bridge between NetworkPortRequestor and HostPorts * for a single call to allocatePorts(), gathering the resulting port * allocations in a list of integers. + * * @author arnej */ public class PortAllocBridge { - private HostPorts host; - private NetworkPortRequestor service; - private List<Integer> ports = new ArrayList<>(); + private final HostPorts host; + private final NetworkPortRequestor service; + private final List<Integer> ports = new ArrayList<>(); public PortAllocBridge(HostPorts host, NetworkPortRequestor service) { this.host = host; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/PortFinder.java b/config-model/src/main/java/com/yahoo/vespa/model/PortFinder.java index 3d5cbcd0e6e..968f8399e4b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/PortFinder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/PortFinder.java @@ -41,7 +41,7 @@ public class PortFinder { String key = request.key(); if (byKeys.containsKey(key)) { int port = byKeys.get(key).port; - log.log(Level.FINE, "Re-using port "+port+" for allocation "+request+" on "+host); + log.log(Level.FINE, "Re-using port "+port+" for allocation " + request + " on " + host); return port; } int port = request.port; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index 660c47d037d..44bade4cd84 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -418,7 +418,6 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri * @param configId the config id for the config client * @return the builder if a producer was found, and it did apply config, null otherwise */ - @SuppressWarnings("unchecked") @Override public ConfigInstance.Builder getConfig(ConfigInstance.Builder builder, String configId) { checkId(configId); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java index 3879df52390..e37f558c2ce 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -134,7 +134,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { return cfgs; } - private List<Slobrok> getSlobroks(DeployState deployState, AbstractConfigProducer parent, Element slobroksE) { + private List<Slobrok> getSlobroks(DeployState deployState, AbstractConfigProducer<?> parent, Element slobroksE) { List<Slobrok> slobs = new ArrayList<>(); if (slobroksE != null) { slobs = getExplicitSlobrokSetup(deployState, parent, slobroksE); @@ -142,16 +142,12 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { return slobs; } - private List<Slobrok> getExplicitSlobrokSetup(DeployState deployState, AbstractConfigProducer parent, Element slobroksE) { - List<Slobrok> slobs = new ArrayList<>(); - List<Element> slobsE = XML.getChildren(slobroksE, "slobrok"); + private List<Slobrok> getExplicitSlobrokSetup(DeployState deployState, AbstractConfigProducer<?> parent, Element slobroksE) { + List<Slobrok> slobroks = new ArrayList<>(); int i = 0; - for (Element e : slobsE) { - Slobrok slob = new SlobrokBuilder(i).build(deployState, parent, e); - slobs.add(slob); - i++; - } - return slobs; + for (Element e : XML.getChildren(slobroksE, "slobrok")) + slobroks.add(new SlobrokBuilder(i++).build(deployState, parent, e)); + return slobroks; } private static class LogserverBuilder extends DomConfigProducerBuilder<Logserver> { @@ -184,6 +180,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { } private static class SlobrokBuilder extends DomConfigProducerBuilder<Slobrok> { + int i; public SlobrokBuilder(int i) { @@ -194,6 +191,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { protected Slobrok doBuild(DeployState deployState, AbstractConfigProducer parent, Element spec) { return new Slobrok(parent, i); } + } private static class ClusterControllerBuilder extends DomConfigProducerBuilder<ClusterControllerContainer> { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 42af94b1d8e..a1db3c43f1c 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -187,6 +187,7 @@ public abstract class ContainerCluster<CONTAINER extends Container> addSimpleComponent("com.yahoo.container.jdisc.ContainerThreadFactory"); addSimpleComponent("com.yahoo.container.handler.VipStatus"); addSimpleComponent(com.yahoo.container.handler.ClustersStatus.class.getName()); + addSimpleComponent("com.yahoo.container.jdisc.DisabledConnectionLogProvider"); addJaxProviders(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java index 4736a207659..a001d66bc7b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java @@ -70,7 +70,7 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer ? "" : "In particular, the tensors (" + String.join(", ", tensorFields) + ") will be interpreted as strings, not tensors if sent in requests. ") + - "See https://docs.vespa.ai/documentation/query-profiles.html"); + "See https://docs.vespa.ai/en/query-profiles.html"); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index d9543c2e917..0c0c3f59e88 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -31,7 +31,6 @@ import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Zone; import com.yahoo.container.logging.FileConnectionLog; -import com.yahoo.jdisc.http.server.jetty.VoidConnectionLog; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.search.rendering.RendererRegistry; import com.yahoo.searchdefinition.derived.RankProfileList; @@ -350,8 +349,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { // Add connection log if access log is configured if (cluster.getAllComponents().stream().anyMatch(component -> component instanceof AccessLogComponent)) { cluster.addComponent(new ConnectionLogComponent(cluster, FileConnectionLog.class, "qrs")); - } else { - cluster.addComponent(new ConnectionLogComponent(cluster, VoidConnectionLog.class, "qrs")); } } @@ -387,7 +384,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { if (deployState.endpointCertificateSecrets().isPresent()) { boolean authorizeClient = deployState.zone().system().isPublic(); if (authorizeClient && deployState.tlsClientAuthority().isEmpty()) { - throw new RuntimeException("Client certificate authority security/clients.pem is missing - see: https://cloud.vespa.ai/security-model#data-plane"); + throw new RuntimeException("Client certificate authority security/clients.pem is missing - see: https://cloud.vespa.ai/en/security-model#data-plane"); } EndpointCertificateSecrets endpointCertificateSecrets = deployState.endpointCertificateSecrets().get(); @@ -725,7 +722,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { var hosts = hostSystem.allocateHosts(clusterSpec, capacity, log); return createNodesFromHosts(log, hosts, cluster); } - return singleHostContainerCluster(cluster, hostSystem.getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC), context); + else { + return singleHostContainerCluster(cluster, hostSystem.getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC), context); + } } private List<ApplicationContainer> singleHostContainerCluster(ApplicationContainerCluster cluster, HostResource host, ConfigModelContext context) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java index 705d2dc668d..37a2ef224fc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentNode.java @@ -14,8 +14,6 @@ import com.yahoo.vespa.model.application.validation.RestartConfigs; /** * Common class for config producers for storage and distributor nodes. - * - * TODO: Author */ @RestartConfigs({StorCommunicationmanagerConfig.class, StorStatusConfig.class, StorServerConfig.class, LoadTypeConfig.class, MetricsmanagerConfig.class}) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 0fdd39e70cf..0de111e459e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -12,7 +12,6 @@ import com.yahoo.vespa.model.builder.xml.dom.DomSearchTuningBuilder; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; import com.yahoo.vespa.model.content.cluster.ContentCluster; -import com.yahoo.vespa.model.content.cluster.DomResourceLimitsBuilder; import com.yahoo.vespa.model.search.AbstractSearchCluster; import com.yahoo.vespa.model.search.IndexedSearchCluster; import com.yahoo.vespa.model.search.NamedSchema; @@ -64,8 +63,10 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private final Map<StorageGroup, NodeSpec> groupToSpecMap = new LinkedHashMap<>(); private Optional<ResourceLimits> resourceLimits = Optional.empty(); private final ProtonConfig.Indexing.Optimize.Enum feedSequencerType; + private final int maxPendingMoveOps; private final double defaultFeedConcurrency; private final boolean useBucketExecutorForLidSpaceCompact; + private final boolean useBucketExecutorForBucketMove; private final double defaultMaxDeadBytesRatio; /** Whether the nodes of this cluster also hosts a container cluster in a hosted system */ @@ -204,9 +205,11 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> this.globallyDistributedDocuments = globallyDistributedDocuments; this.flushOnShutdown = flushOnShutdown; this.combined = combined; + maxPendingMoveOps = featureFlags.maxPendingMoveOps(); feedSequencerType = convertFeedSequencerType(featureFlags.feedSequencerType()); defaultFeedConcurrency = featureFlags.feedConcurrency(); useBucketExecutorForLidSpaceCompact = featureFlags.useBucketExecutorForLidSpaceCompact(); + useBucketExecutorForBucketMove = featureFlags.useBucketExecutorForBucketMove(); defaultMaxDeadBytesRatio = featureFlags.maxDeadBytesRatio(); } @@ -263,7 +266,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> } public void addSearchNode(DeployState deployState, ContentNode node, StorageGroup parentGroup, ModelElement element) { - AbstractConfigProducer parent = hasIndexedCluster() ? getIndexed() : this; + AbstractConfigProducer<?> parent = hasIndexedCluster() ? getIndexed() : this; NodeSpec spec = getNextSearchNodeSpec(parentGroup); SearchNode searchNode; @@ -429,7 +432,9 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> } else { builder.indexing.optimize(feedSequencerType); } + builder.maintenancejobs.maxoutstandingmoveops(maxPendingMoveOps); builder.lidspacecompaction.usebucketexecutor(useBucketExecutorForLidSpaceCompact); + builder.bucketmove.usebucketexecutor(useBucketExecutorForBucketMove); } private boolean isGloballyDistributed(NewDocumentType docType) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java index 0f9eb5341ab..3b694f8986c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.model.content; /** * Tuning of dispatching to content nodes, see the - * <a href="https://docs.vespa.ai/documentation/reference/services-content.html#dispatch-tuning">dispatch tuning documentation</a>. + * <a href="https://docs.vespa.ai/en/reference/services-content.html#dispatch-tuning">dispatch tuning documentation</a>. * * @author Simon Thoresen Hult */ diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 37619d3afc6..5a71f83e468 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -14,6 +14,7 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.container.core.ApplicationMetadataConfig; import com.yahoo.search.config.QrStartConfig; +import com.yahoo.vespa.config.content.core.StorStatusConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.HostSystem; @@ -1509,6 +1510,7 @@ public class ModelProvisioningTest { assertEquals("Nodes in container cluster", 1, model.getContainerClusters().get("container1").getContainers().size()); assertEquals("Nodes in content cluster (downscaled)", 1, model.getContentClusters().get("content").getRootGroup().getNodes().size()); + model.getConfig(new StorStatusConfig.Builder(), "default"); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java index ba9dfcdc388..d5c7fb78c89 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java @@ -37,7 +37,7 @@ public class IndexingModeChangeValidatorTest { catch (ValidationException e) { assertEquals("indexing-mode-change:\n" + "\tDocument type 'music' in cluster 'default' changed indexing mode from 'indexed' to 'streaming'\n" + - "To allow this add <allow until='yyyy-mm-dd'>indexing-mode-change</allow> to validation-overrides.xml, see https://docs.vespa.ai/documentation/reference/validation-overrides.html", + "To allow this add <allow until='yyyy-mm-dd'>indexing-mode-change</allow> to validation-overrides.xml, see https://docs.vespa.ai/en/reference/validation-overrides.html", e.getMessage()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 55234a3a96e..3a3dde0cf87 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -835,6 +835,34 @@ public class ContentBuilderTest extends DomBuilderTest { assertEquals(flag, config.lidspacecompaction().usebucketexecutor()); } + private void verifyThatFeatureFlagControlsUseBucketExecutorForBucketMove(boolean flag) { + DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties().useBucketExecutorForBucketMove(flag)); + VespaModel model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() + .withServices(singleNodeContentXml()) + .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .build()) + .create(deployStateBuilder); + ProtonConfig config = getProtonConfig(model.getContentClusters().values().iterator().next()); + assertEquals(flag, config.bucketmove().usebucketexecutor()); + } + + private void verifyThatFeatureFlagControlsMaxpendingMoveOps(int moveOps) { + DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties().setMaxPendingMoveOps(moveOps)); + VespaModel model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() + .withServices(singleNodeContentXml()) + .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .build()) + .create(deployStateBuilder); + ProtonConfig config = getProtonConfig(model.getContentClusters().values().iterator().next()); + assertEquals(moveOps, config.maintenancejobs().maxoutstandingmoveops()); + } + + @Test + public void verifyMaxPendingMoveOps() { + verifyThatFeatureFlagControlsMaxpendingMoveOps(13); + verifyThatFeatureFlagControlsMaxpendingMoveOps(107); + } + @Test public void verifyUseBucketExecutorForLidSpaceCompact() { verifyThatFeatureFlagControlsUseBucketExecutorForLidSpaceCompact(true); @@ -842,6 +870,12 @@ public class ContentBuilderTest extends DomBuilderTest { } @Test + public void verifyUseBucketExecutorForBucketMove() { + verifyThatFeatureFlagControlsUseBucketExecutorForBucketMove(true); + verifyThatFeatureFlagControlsUseBucketExecutorForBucketMove(false); + } + + @Test public void failWhenNoDocumentsElementSpecified() { expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage("The specified content engine requires the <documents> element to be specified."); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/QueryProfilesTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/QueryProfilesTestCase.java index d1d32df5b56..fc6c5aeb387 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/QueryProfilesTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/QueryProfilesTestCase.java @@ -151,7 +151,7 @@ public class QueryProfilesTestCase { assertEquals(1, logger.entries.size()); assertEquals("This application define query profile types, but has no query profiles referencing them " + "so they have no effect. " + - "See https://docs.vespa.ai/documentation/query-profiles.html", + "See https://docs.vespa.ai/en/query-profiles.html", logger.entries.get(0).message); } @@ -171,7 +171,7 @@ public class QueryProfilesTestCase { assertEquals("This application define query profile types, but has no query profiles referencing them " + "so they have no effect. " + "In particular, the tensors (vector, matrix) will be interpreted as strings, not tensors if sent in requests. " + - "See https://docs.vespa.ai/documentation/query-profiles.html", + "See https://docs.vespa.ai/en/query-profiles.html", logger.entries.get(0).message); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java index 56f09eefe82..f3f3b2b1076 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/AccessLogTest.java @@ -8,16 +8,15 @@ import com.yahoo.container.logging.ConnectionLogConfig; import com.yahoo.container.logging.FileConnectionLog; import com.yahoo.container.logging.JSONAccessLog; import com.yahoo.container.logging.VespaAccessLog; -import com.yahoo.jdisc.http.server.jetty.VoidConnectionLog; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.component.Component; import org.junit.Test; import org.w3c.dom.Element; import static com.yahoo.text.StringUtilities.quote; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertEquals; /** * @author gjoranv @@ -127,8 +126,6 @@ public class AccessLogTest extends ContainerModelBuilderTestBase { nodesXml, "</container>" ); createModel(root, clusterElem); - Component<?, ?> voidConnectionLogComponent = getContainerComponent("default", VoidConnectionLog.class.getName()); - assertNotNull(voidConnectionLogComponent); Component<?, ?> fileConnectionLogComponent = getContainerComponent("default", FileConnectionLog.class.getName()); assertNull(fileConnectionLogComponent); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 35257686a5a..e0e4cbf19c0 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -728,7 +728,8 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { .build(); createModel(root, state, null, clusterElem); } catch (RuntimeException e) { - assertEquals(e.getMessage(), "Client certificate authority security/clients.pem is missing - see: https://cloud.vespa.ai/security-model#data-plane"); + assertEquals("Client certificate authority security/clients.pem is missing - see: https://cloud.vespa.ai/en/security-model#data-plane", + e.getMessage()); return; } fail(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index 7484095c5bc..ce8d2c193b9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -6,12 +6,14 @@ import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.ConfigModelRegistry; import com.yahoo.config.model.NullConfigModelRegistry; import com.yahoo.config.model.api.HostProvisioner; +import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.provision.Host; import com.yahoo.config.model.provision.Hosts; import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.provision.SingleNodeProvisioner; +import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java index 57eac2a1a7b..96dc334ba45 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/Application.java @@ -96,7 +96,6 @@ public class Application implements ModelResult { /** * Gets a config from ZK. Returns null if not found. */ - @SuppressWarnings("deprecation") public ConfigResponse resolveConfig(GetConfigRequest req, ConfigResponseFactory responseFactory) { long start = System.currentTimeMillis(); metricUpdater.incrementRequests(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 490ceb48c08..4dd384bf649 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -152,6 +152,7 @@ public class ModelContextImpl implements ModelContext { private final String feedSequencer; private final String responseSequencer; private final int numResponseThreads; + private final int maxPendingMoveOps; private final boolean skipCommunicationManagerThread; private final boolean skipMbusRequestThread; private final boolean skipMbusReplyThread; @@ -160,6 +161,7 @@ public class ModelContextImpl implements ModelContext { private final double feedConcurrency; private final boolean reconfigurableZookeeperServer; private final boolean useBucketExecutorForLidSpaceCompact; + private final boolean useBucketExecutorForBucketMove; private final boolean enableFeedBlockInDistributor; private final double maxDeadBytesRatio; private final int clusterControllerMaxHeapSizeInMb; @@ -171,6 +173,7 @@ public class ModelContextImpl implements ModelContext { this.feedSequencer = flagValue(source, appId, Flags.FEED_SEQUENCER_TYPE); this.responseSequencer = flagValue(source, appId, Flags.RESPONSE_SEQUENCER_TYPE); this.numResponseThreads = flagValue(source, appId, Flags.RESPONSE_NUM_THREADS); + this.maxPendingMoveOps = flagValue(source, appId, Flags.MAX_PENDING_MOVE_OPS); this.skipCommunicationManagerThread = flagValue(source, appId, Flags.SKIP_COMMUNICATIONMANAGER_THREAD); this.skipMbusRequestThread = flagValue(source, appId, Flags.SKIP_MBUS_REQUEST_THREAD); this.skipMbusReplyThread = flagValue(source, appId, Flags.SKIP_MBUS_REPLY_THREAD); @@ -179,6 +182,7 @@ public class ModelContextImpl implements ModelContext { this.feedConcurrency = flagValue(source, appId, Flags.FEED_CONCURRENCY); this.reconfigurableZookeeperServer = flagValue(source, appId, Flags.RECONFIGURABLE_ZOOKEEPER_SERVER_FOR_CLUSTER_CONTROLLER); this.useBucketExecutorForLidSpaceCompact = flagValue(source, appId, Flags.USE_BUCKET_EXECUTOR_FOR_LID_SPACE_COMPACT); + this.useBucketExecutorForBucketMove = flagValue(source, appId, Flags.USE_BUCKET_EXECUTOR_FOR_BUCKET_MOVE); this.enableFeedBlockInDistributor = flagValue(source, appId, Flags.ENABLE_FEED_BLOCK_IN_DISTRIBUTOR); this.maxDeadBytesRatio = flagValue(source, appId, Flags.MAX_DEAD_BYTES_RATIO); this.clusterControllerMaxHeapSizeInMb = flagValue(source, appId, Flags.CLUSTER_CONTROLLER_MAX_HEAP_SIZE_IN_MB); @@ -190,6 +194,7 @@ public class ModelContextImpl implements ModelContext { @Override public String feedSequencerType() { return feedSequencer; } @Override public String responseSequencerType() { return responseSequencer; } @Override public int defaultNumResponseThreads() { return numResponseThreads; } + @Override public int maxPendingMoveOps() { return maxPendingMoveOps; } @Override public boolean skipCommunicationManagerThread() { return skipCommunicationManagerThread; } @Override public boolean skipMbusRequestThread() { return skipMbusRequestThread; } @Override public boolean skipMbusReplyThread() { return skipMbusReplyThread; } @@ -198,6 +203,7 @@ public class ModelContextImpl implements ModelContext { @Override public double feedConcurrency() { return feedConcurrency; } @Override public boolean reconfigurableZookeeperServer() { return reconfigurableZookeeperServer; } @Override public boolean useBucketExecutorForLidSpaceCompact() { return useBucketExecutorForLidSpaceCompact; } + @Override public boolean useBucketExecutorForBucketMove() { return useBucketExecutorForBucketMove; } @Override public boolean enableFeedBlockInDistributor() { return enableFeedBlockInDistributor; } @Override public double maxDeadBytesRatio() { return maxDeadBytesRatio; } @Override public int clusterControllerMaxHeapSizeInMb() { return clusterControllerMaxHeapSizeInMb; } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java index 79d70f1cb4e..9e232a6533f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java @@ -33,9 +33,9 @@ public class ProvisionerAdapter implements HostProvisioner { @Override public HostSpec allocateHost(String alias) { - // Wow. Such mess. TODO: Actually support polymorphy or stop pretending to, see also ModelContextImpl.getHostProvisioner + // TODO: Remove this method since hosted/non-hosted needs different interfaces. See also ModelContextImpl.getHostProvisioner throw new UnsupportedOperationException("Allocating hosts using <node> tags is not supported in hosted environments, " + - "use <nodes count='N'> instead, see https://cloud.vespa.ai/reference/services"); + "use <nodes count='N'> instead, see https://cloud.vespa.ai/en/reference/services"); } @Override diff --git a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java index 26650cb93a8..236602b7767 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java @@ -16,11 +16,9 @@ import com.yahoo.container.di.config.SubscriberFactory; import com.yahoo.container.di.osgi.BundleClasses; import com.yahoo.container.di.osgi.OsgiUtil; import com.yahoo.container.logging.AccessLog; -import com.yahoo.container.logging.ConnectionLog; import com.yahoo.filedistribution.fileacquirer.FileAcquirer; import com.yahoo.jdisc.application.OsgiFramework; import com.yahoo.jdisc.handler.RequestHandler; -import com.yahoo.jdisc.http.server.jetty.VoidConnectionLog; import com.yahoo.jdisc.service.ClientProvider; import com.yahoo.jdisc.service.ServerProvider; import com.yahoo.osgi.OsgiImpl; @@ -50,6 +48,9 @@ public class HandlersConfigurerDi { private static final Logger log = Logger.getLogger(HandlersConfigurerDi.class.getName()); + private static final Executor fallbackExecutor = Executors.newCachedThreadPool( + ThreadFactoryFactory.getThreadFactory("HandlersConfigurerDI")); + private final com.yahoo.container.Container vespaContainer; private final Container container; @@ -140,11 +141,12 @@ public class HandlersConfigurerDi { return discInjector.createChildInjector(new AbstractModule() { @Override protected void configure() { + // Provide a singleton instance for all component fallbacks, + // otherwise fallback injection may lead to a cascade of components requiring reconstruction. bind(com.yahoo.container.Container.class).toInstance(vespaContainer); bind(com.yahoo.statistics.Statistics.class).toInstance(Statistics.nullImplementation); - bind(AccessLog.class).toInstance(new AccessLog(new ComponentRegistry<>())); - bind(Executor.class).toInstance(Executors.newCachedThreadPool(ThreadFactoryFactory.getThreadFactory("HandlersConfigurerDI"))); - bind(ConnectionLog.class).toInstance(new VoidConnectionLog()); + bind(AccessLog.class).toInstance(AccessLog.NONE_INSTANCE); + bind(Executor.class).toInstance(fallbackExecutor); if (vespaContainer.getFileAcquirer() != null) bind(com.yahoo.filedistribution.fileacquirer.FileAcquirer.class).toInstance(vespaContainer.getFileAcquirer()); } diff --git a/container-disc/src/main/java/com/yahoo/container/jdisc/DisabledConnectionLogProvider.java b/container-disc/src/main/java/com/yahoo/container/jdisc/DisabledConnectionLogProvider.java new file mode 100644 index 00000000000..53e479652ab --- /dev/null +++ b/container-disc/src/main/java/com/yahoo/container/jdisc/DisabledConnectionLogProvider.java @@ -0,0 +1,18 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.container.jdisc; + +import com.yahoo.container.di.componentgraph.Provider; +import com.yahoo.container.logging.ConnectionLog; +import com.yahoo.jdisc.http.server.jetty.VoidConnectionLog; + +/** + * @author bjorncs + */ +public class DisabledConnectionLogProvider implements Provider<ConnectionLog> { + + private static final ConnectionLog INSTANCE = new VoidConnectionLog(); + + @Override public ConnectionLog get() { return INSTANCE; } + @Override public void deconstruct() {} + +} diff --git a/container-search-gui/pom.xml b/container-search-gui/pom.xml index 867f2464b8b..64c53a1f453 100644 --- a/container-search-gui/pom.xml +++ b/container-search-gui/pom.xml @@ -72,7 +72,7 @@ <goal>wget</goal> </goals> <configuration> - <url>https://docs.vespa.ai/documentation/reference/search-api-reference.html</url> + <url>https://docs.vespa.ai/en/reference/search-api-reference.html</url> <outputFileName>search-api-reference.html</outputFileName> <outputDirectory>${project.build.outputDirectory}/gui/_includes/</outputDirectory> </configuration> diff --git a/container-search-gui/src/main/resources/gui/_includes/index.html b/container-search-gui/src/main/resources/gui/_includes/index.html index a441e302720..5eaa2b790d5 100644 --- a/container-search-gui/src/main/resources/gui/_includes/index.html +++ b/container-search-gui/src/main/resources/gui/_includes/index.html @@ -64,11 +64,11 @@ <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> <ul class="nav navbar-nav navbar-right"> <li class="hidden"><a href="#page-top"></a> - <li><a href="http://blog.vespa.ai/">Blog</a> + <li><a href="https://blog.vespa.ai/">Blog</a> <li><a href="https://twitter.com/vespaengine">Twitter</a> - <li><a href="http://docs.vespa.ai">Docs</a> + <li><a href="https://docs.vespa.ai">Docs</a> <li><a href="https://github.com/vespa-engine">GitHub</a> - <li><a href="http://docs.vespa.ai/documentation/vespa-quick-start.html">Get Started Now</a> + <li><a href="https://docs.vespa.ai/en/vespa-quick-start.html">Get Started Now</a> </ul> </div> </div> @@ -128,7 +128,7 @@ <span> â—‹ Autocompletion of YQL-syntax</span> </br> <span> â—‹ Drop-down lists of all valid parameters</span> </br> <span> â—‹ Sending both POST and GET-requests to <i>Vespa</i></span> </br> - <span> â—‹ Easy access to the <a href="https://docs.vespa.ai/documentation/reference/search-api-reference.html">documentation</a> of each parameter</span> </br> + <span> â—‹ Easy access to the <a href="https://docs.vespa.ai/en/reference/search-api-reference.html">documentation</a> of each parameter</span> </br> <span> â—‹ Conversion of POST- to GET-query</span> </br> <span> â—‹ Pasting already built JSON-query</span> </br> <span> â—‹ View and copy the response of queries</span> </br> @@ -151,8 +151,8 @@ <div class="col-xs-2 quicklink-section"> <div class="footer-title">Resources</div> <ul class="quicklinks"> - <li><a href="http://docs.vespa.ai/documentation/vespa-quick-start.html">Getting Started</a> - <li><a href="http://docs.vespa.ai">Documentation</a> + <li><a href="https://docs.vespa.ai/en/vespa-quick-start.html">Getting Started</a> + <li><a href="https://docs.vespa.ai">Documentation</a> <li><a href="https://github.com/vespa-engine/vespa">Open source</a> </ul> </div> diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index 663cb44215d..a639f2368a1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -195,7 +195,7 @@ public class FastHit extends Hit { /** * Returns values for the features listed in - * <a href="https://docs.vespa.ai/documentation/reference/schema-reference.html#summary-features">summary-features</a> + * <a href="https://docs.vespa.ai/en/reference/schema-reference.html#summary-features">summary-features</a> * in the rank profile specified in the query producing this. */ public FeatureData features() { diff --git a/container-search/src/main/java/com/yahoo/search/query/Select.java b/container-search/src/main/java/com/yahoo/search/query/Select.java index 53628978db3..d90550084eb 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Select.java +++ b/container-search/src/main/java/com/yahoo/search/query/Select.java @@ -79,7 +79,7 @@ public class Select implements Cloneable { * Sets the document selection criterion of the query. * * @param where the documents to select as a JSON string on the format specified in - * <a href="https://docs.vespa.ai/documentation/reference/select-reference.html">the select reference doc</a> + * <a href="https://docs.vespa.ai/en/reference/select-reference.html">the select reference doc</a> */ public void setWhereString(String where) { this.where = where; @@ -96,7 +96,7 @@ public class Select implements Cloneable { * Sets the grouping operation of the query. * * @param grouping the grouping to perform as a JSON string on the format specified in - * <a href="https://docs.vespa.ai/documentation/reference/select-reference.html">the select reference doc</a> + * <a href="https://docs.vespa.ai/en/reference/select-reference.html">the select reference doc</a> */ public void setGroupingString(String grouping) { groupingRequests.clear(); diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java index 058f6b93ae3..d40e1cfc4a6 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java @@ -41,7 +41,7 @@ public class WeakAndReplacementSearcher extends Searcher { * Recursively iterates over an Item to replace all instances of OrItems with WeakAndItems * @param item the current item in the replacement iteration * @param hits the wand.hits property from the request which is assigned to the N value of the new WeakAndItem - * @return The original item or a WeakAndItem replacement of an OrItem + * @return the original item or a WeakAndItem replacement of an OrItem */ private Item replaceOrItems(Item item, int hits) { if (!(item instanceof CompositeItem)) { @@ -63,4 +63,5 @@ public class WeakAndReplacementSearcher extends Searcher { } return compositeItem; } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java index 7100baea5a0..c02943c7774 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ServiceRegistry.java @@ -23,6 +23,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMoni import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumer; import com.yahoo.vespa.hosted.controller.api.integration.resource.MeteringClient; import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService; +import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretService; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import java.time.Clock; @@ -85,4 +86,6 @@ public interface ServiceRegistry { ContainerRegistry containerRegistry(); + TenantSecretService tenantSecretService(); + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java index 4d2cafa3e48..0bf3f0440d2 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMetadata.java @@ -110,6 +110,19 @@ public class EndpointCertificateMetadata { Optional.of(lastRefreshed)); } + public EndpointCertificateMetadata withRequestId(String requestId) { + return new EndpointCertificateMetadata( + this.keyName, + this.certName, + this.version, + this.lastRequested, + requestId, + this.requestedDnsSans, + this.issuer, + this.expiry, + lastRefreshed); + } + @Override public String toString() { return "EndpointCertificateMetadata{" + diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/secrets/NoopTenantSecretService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/secrets/NoopTenantSecretService.java new file mode 100644 index 00000000000..f1721185899 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/secrets/NoopTenantSecretService.java @@ -0,0 +1,12 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.secrets; + +/** + * @author olaa + */ +public class NoopTenantSecretService implements TenantSecretService { + + @Override + public void addSecretStore(TenantSecretStore tenantSecretStore, String externalId) {} + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/secrets/TenantSecretService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/secrets/TenantSecretService.java new file mode 100644 index 00000000000..42df4796189 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/secrets/TenantSecretService.java @@ -0,0 +1,11 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.secrets; + +/** + * @author olaa + */ +public interface TenantSecretService { + + void addSecretStore(TenantSecretStore tenantSecretStore, String externalId); + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/secrets/TenantSecretStore.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/secrets/TenantSecretStore.java new file mode 100644 index 00000000000..b77de3cc9b8 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/secrets/TenantSecretStore.java @@ -0,0 +1,62 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.secrets; + +import java.util.Objects; + +/** + * @author olaa + */ +public class TenantSecretStore { + + private final String name; + private final String awsId; + private final String role; + + public TenantSecretStore(String name, String awsId, String role) { + this.name = name; + this.awsId = awsId; + this.role = role; + } + + public String getName() { + return name; + } + + public String getAwsId() { + return awsId; + } + + public String getRole() { + return role; + } + + public boolean isValid() { + return !name.isBlank() && + !awsId.isBlank() && + !role.isBlank(); + } + + @Override + public String toString() { + return "TenantSecretStore{" + + "name='" + name + '\'' + + ", awsId='" + awsId + '\'' + + ", role='" + role + '\'' + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TenantSecretStore that = (TenantSecretStore) o; + return name.equals(that.name) && + awsId.equals(that.awsId) && + role.equals(that.role); + } + + @Override + public int hashCode() { + return Objects.hash(name, awsId, role); + } +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/secrets/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/secrets/package-info.java new file mode 100644 index 00000000000..25f09e45cdd --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/secrets/package-info.java @@ -0,0 +1,5 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.integration.secrets; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index 12df0a5e0a7..2a8dc34ea72 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -36,7 +36,8 @@ enum PathGroup { "/zone/v2/{*}", "/routing/v1/", "/routing/v1/status/environment/{*}", - "/routing/v1/inactive/environment/{*}"), + "/routing/v1/inactive/environment/{*}", + "/state/v1/{*}"), /** Paths used for creating and reading user resources. */ user(PathPrefix.api, @@ -57,6 +58,7 @@ enum PathGroup { tenantInfo(Matcher.tenant, PathPrefix.api, "/application/v4/tenant/{tenant}/application/", + "/application/v4/tenant/{tenant}/secret-store/", "/application/v4/tenant/{tenant}/info/", "/routing/v1/status/tenant/{tenant}/{*}"), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java index e45bda0708e..519c08b66f5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedTenant.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; +import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import com.yahoo.vespa.hosted.controller.tenant.LastLoginInfo; @@ -20,6 +21,8 @@ import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; import java.security.Principal; import java.security.PublicKey; import java.time.Instant; +import java.util.ArrayList; +import java.util.List; import java.util.Optional; import static java.util.Objects.requireNonNull; @@ -117,21 +120,23 @@ public abstract class LockedTenant { private final Optional<Principal> creator; private final BiMap<PublicKey, Principal> developerKeys; private final TenantInfo info; + private final List<TenantSecretStore> tenantSecretStores; - private Cloud(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Optional<Principal> creator, BiMap<PublicKey, Principal> developerKeys, TenantInfo info) { + private Cloud(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Optional<Principal> creator, BiMap<PublicKey, Principal> developerKeys, TenantInfo info, List<TenantSecretStore> tenantSecretStores) { super(name, createdAt, lastLoginInfo); this.developerKeys = ImmutableBiMap.copyOf(developerKeys); this.creator = creator; this.info = info; + this.tenantSecretStores = tenantSecretStores; } private Cloud(CloudTenant tenant) { - this(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), Optional.empty(), tenant.developerKeys(), tenant.info()); + this(tenant.name(), tenant.createdAt(), tenant.lastLoginInfo(), Optional.empty(), tenant.developerKeys(), tenant.info(), tenant.tenantSecretStores()); } @Override public CloudTenant get() { - return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info); + return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores); } public Cloud withDeveloperKey(PublicKey key, Principal principal) { @@ -139,22 +144,28 @@ public abstract class LockedTenant { if (keys.containsKey(key)) throw new IllegalArgumentException("Key " + KeyUtils.toPem(key) + " is already owned by " + keys.get(key)); keys.put(key, principal); - return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info); + return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores); } public Cloud withoutDeveloperKey(PublicKey key) { BiMap<PublicKey, Principal> keys = HashBiMap.create(developerKeys); keys.remove(key); - return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info); + return new Cloud(name, createdAt, lastLoginInfo, creator, keys, info, tenantSecretStores); } public Cloud withInfo(TenantInfo newInfo) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, newInfo); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, newInfo, tenantSecretStores); } @Override public LockedTenant with(LastLoginInfo lastLoginInfo) { - return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores); + } + + public Cloud withSecretStore(TenantSecretStore tenantSecretStore) { + ArrayList<TenantSecretStore> secretStores = new ArrayList<>(tenantSecretStores); + secretStores.add(tenantSecretStore); + return new Cloud(name, createdAt, lastLoginInfo, creator, developerKeys, info, secretStores); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java index 86ba25b7ad7..f071fe86002 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java @@ -89,7 +89,9 @@ public class EndpointCertificateManager { // Re-provision certificate if it is missing SANs for the zone we are deploying to var requiredSansForZone = dnsNamesOf(instance.id(), zone); if (!currentCertificateMetadata.get().requestedDnsSans().containsAll(requiredSansForZone)) { - var reprovisionedCertificateMetadata = provisionEndpointCertificate(instance, currentCertificateMetadata, zone, instanceSpec); + var reprovisionedCertificateMetadata = + provisionEndpointCertificate(instance, currentCertificateMetadata, zone, instanceSpec) + .withRequestId(currentCertificateMetadata.get().request_id()); // We're required to keep the original request_id curator.writeEndpointCertificateMetadata(instance.id(), reprovisionedCertificateMetadata); // Verification is unlikely to succeed in this case, as certificate must be available first - controller will retry endpointCertificateValidator.validate(reprovisionedCertificateMetadata, instance.id().serializedForm(), zone, requiredSansForZone); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java index cc2d7c207e5..beaf546930f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java @@ -13,6 +13,7 @@ import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; +import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingInfo; @@ -69,6 +70,9 @@ public class TenantSerializer { private static final String pemDeveloperKeysField = "pemDeveloperKeys"; private static final String tenantInfoField = "info"; private static final String lastLoginInfoField = "lastLoginInfo"; + private static final String secretStoresField = "secretStores"; + private static final String awsIdField = "awsId"; + private static final String roleField = "role"; public Slime toSlime(Tenant tenant) { Slime slime = new Slime(); @@ -105,6 +109,7 @@ public class TenantSerializer { developerKeysToSlime(tenant.developerKeys(), root.setArray(pemDeveloperKeysField)); toSlime(legacyBillingInfo, root.setObject(billingInfoField)); toSlime(tenant.info(), root); + toSlime(tenant.tenantSecretStores(), root); } private void developerKeysToSlime(BiMap<PublicKey, Principal> keys, Cursor array) { @@ -156,7 +161,8 @@ public class TenantSerializer { Optional<Principal> creator = SlimeUtils.optionalString(tenantObject.field(creatorField)).map(SimplePrincipal::new); BiMap<PublicKey, Principal> developerKeys = developerKeysFromSlime(tenantObject.field(pemDeveloperKeysField)); TenantInfo info = tenantInfoFromSlime(tenantObject.field(tenantInfoField)); - return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info); + List<TenantSecretStore> tenantSecretStores = secretStoresFromSlime(tenantObject.field(secretStoresField)); + return new CloudTenant(name, createdAt, lastLoginInfo, creator, developerKeys, info, tenantSecretStores); } private BiMap<PublicKey, Principal> developerKeysFromSlime(Inspector array) { @@ -199,6 +205,22 @@ public class TenantSerializer { .withAddress(tenantInfoAddressFromSlime(billingObject.field("address"))); } + private List<TenantSecretStore> secretStoresFromSlime(Inspector secretStoresObject) { + List<TenantSecretStore> secretStores = new ArrayList<>(); + if (!secretStoresObject.valid()) return secretStores; + + secretStoresObject.traverse((ArrayTraverser) (index, inspector) -> { + secretStores.add( + new TenantSecretStore( + inspector.field(nameField).asString(), + inspector.field(awsIdField).asString(), + inspector.field(roleField).asString() + ) + ); + }); + return secretStores; + } + private LastLoginInfo lastLoginInfoFromSlime(Inspector lastLoginInfoObject) { Map<LastLoginInfo.UserLevel, Instant> lastLoginByUserLevel = new HashMap<>(); lastLoginInfoObject.traverse((String name, Inspector value) -> @@ -240,6 +262,19 @@ public class TenantSerializer { toSlime(billingContact.address(), addressCursor); } + private void toSlime(List<TenantSecretStore> tenantSecretStores, Cursor parentCursor) { + if (tenantSecretStores.isEmpty()) return; + + Cursor secretStoresCursor = parentCursor.setArray(secretStoresField); + tenantSecretStores.forEach(tenantSecretStore -> { + Cursor secretStoreCursor = secretStoresCursor.addObject(); + secretStoreCursor.setString(nameField, tenantSecretStore.getName()); + secretStoreCursor.setString(awsIdField, tenantSecretStore.getAwsId()); + secretStoreCursor.setString(roleField, tenantSecretStore.getRole()); + }); + + } + private Optional<Contact> contactFrom(Inspector object) { if ( ! object.valid()) return Optional.empty(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 431a694fcd8..f9dc62d7f34 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -98,6 +98,7 @@ import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.vespa.hosted.controller.tenant.TenantInfo; import com.yahoo.vespa.hosted.controller.tenant.TenantInfoAddress; import com.yahoo.vespa.hosted.controller.tenant.TenantInfoBillingContact; +import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.serviceview.bindings.ApplicationView; @@ -117,6 +118,7 @@ import java.security.PublicKey; import java.time.DayOfWeek; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; import java.util.Arrays; import java.util.Base64; import java.util.Comparator; @@ -267,6 +269,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse handlePUT(Path path, HttpRequest request) { if (path.matches("/application/v4/tenant/{tenant}")) return updateTenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/info")) return updateTenantInfo(path.get("tenant"), request); + if (path.matches("/application/v4/tenant/{tenant}/secret-store")) return addSecretStore(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false, request); return ErrorResponse.notFoundError("Nothing at " + path); @@ -631,6 +634,36 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(root); } + private HttpResponse addSecretStore(String tenantName, HttpRequest request) { + if (controller.tenants().require(TenantName.from(tenantName)).type() != Tenant.Type.cloud) + throw new IllegalArgumentException("Tenant '" + tenantName + "' is not a cloud tenant"); + + var data = toSlime(request.getData()).get(); + var awsId = mandatory("awsId", data).asString(); + var externalId = mandatory("externalId", data).asString(); + var name = mandatory("name", data).asString(); + var role = mandatory("role", data).asString(); + + var tenant = (CloudTenant) controller.tenants().require(TenantName.from(tenantName)); + var tenantSecretStore = new TenantSecretStore(name, awsId, role); + + if (!tenantSecretStore.isValid()) { + return ErrorResponse.badRequest(String.format("Secret store " + tenantSecretStore + " is invalid")); + } + if (tenant.tenantSecretStores().contains(tenantSecretStore)) { + return ErrorResponse.badRequest(String.format("Secret store " + tenantSecretStore + " is already configured")); + } + + controller.serviceRegistry().roleService().createTenantPolicy(TenantName.from(tenantName), name, awsId, role); + controller.serviceRegistry().tenantSecretService().addSecretStore(tenantSecretStore, externalId); + // Store changes + controller.tenants().lockOrThrow(tenant.name(), LockedTenant.Cloud.class, lockedTenant -> { + lockedTenant = lockedTenant.withSecretStore(tenantSecretStore); + controller.tenants().store(lockedTenant); + }); + return new MessageResponse("Configured secret store: " + tenantSecretStore); + } + private HttpResponse patchApplication(String tenantName, String applicationName, HttpRequest request) { Inspector requestObject = toSlime(request.getData()).get(); StringJoiner messageBuilder = new StringJoiner("\n").setEmptyValue("No applicable changes."); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java index 5d0bb780c81..888bfd7fe42 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/CloudTenant.java @@ -4,10 +4,12 @@ package com.yahoo.vespa.hosted.controller.tenant; import com.google.common.collect.BiMap; import com.google.common.collect.ImmutableBiMap; import com.yahoo.config.provision.TenantName; +import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; import java.security.Principal; import java.security.PublicKey; import java.time.Instant; +import java.util.List; import java.util.Objects; import java.util.Optional; @@ -21,14 +23,17 @@ public class CloudTenant extends Tenant { private final Optional<Principal> creator; private final BiMap<PublicKey, Principal> developerKeys; private final TenantInfo info; + private final List<TenantSecretStore> tenantSecretStores; + /** Public for the serialization layer — do not use! */ public CloudTenant(TenantName name, Instant createdAt, LastLoginInfo lastLoginInfo, Optional<Principal> creator, - BiMap<PublicKey, Principal> developerKeys, TenantInfo info) { + BiMap<PublicKey, Principal> developerKeys, TenantInfo info, List<TenantSecretStore> tenantSecretStores) { super(name, createdAt, lastLoginInfo, Optional.empty()); this.creator = creator; this.developerKeys = developerKeys; this.info = Objects.requireNonNull(info); + this.tenantSecretStores = tenantSecretStores; } /** Creates a tenant with the given name, provided it passes validation. */ @@ -37,7 +42,7 @@ public class CloudTenant extends Tenant { createdAt, LastLoginInfo.EMPTY, Optional.ofNullable(creator), - ImmutableBiMap.of(), TenantInfo.EMPTY); + ImmutableBiMap.of(), TenantInfo.EMPTY, List.of()); } /** The user that created the tenant */ @@ -53,6 +58,11 @@ public class CloudTenant extends Tenant { /** Returns the set of developer keys and their corresponding developers for this tenant. */ public BiMap<PublicKey, Principal> developerKeys() { return developerKeys; } + /** List of configured secret stores */ + public List<TenantSecretStore> tenantSecretStores() { + return tenantSecretStores; + } + @Override public Type type() { return Type.cloud; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfo.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfo.java index 81c08e1083b..a20477d7aab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfo.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tenant/TenantInfo.java @@ -1,6 +1,10 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.tenant; +import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; + +import java.util.ArrayList; +import java.util.List; import java.util.Objects; /** diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 63452f40dbb..07c0a6ca122 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -930,7 +930,7 @@ public class ControllerTest { "[endpoint 'default' (cluster foo) -> us-east-3, us-west-1] and add " + "[endpoint 'default' (cluster bar) -> us-east-3, us-west-1]. To allow this add " + "<allow until='yyyy-mm-dd'>global-endpoint-change</allow> to validation-overrides.xml, see " + - "https://docs.vespa.ai/documentation/reference/validation-overrides.html", e.getMessage()); + "https://docs.vespa.ai/en/reference/validation-overrides.html", e.getMessage()); } // Redeploy with override succeeds diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java index be77f355e35..fa0af070e1e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java @@ -161,7 +161,7 @@ public class EndpointCertificateManagerTest { public void reprovisions_certificate_with_added_sans_when_deploying_to_new_zone() { ZoneId testZone = zoneRegistryMock.zones().directlyRouted().in(Environment.prod).zones().stream().skip(1).findFirst().orElseThrow().getId(); - mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, -1, 0, "uuid", expectedSans, "mockCa", Optional.empty(), Optional.empty())); + mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, -1, 0, "original-request-uuid", expectedSans, "mockCa", Optional.empty(), Optional.empty())); secretStore.setSecret("vespa.tls.default.default.default-key", KeyUtils.toPem(testKeyPair.getPrivate()), -1); secretStore.setSecret("vespa.tls.default.default.default-cert", X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), -1); @@ -172,6 +172,7 @@ public class EndpointCertificateManagerTest { assertTrue(endpointCertificateMetadata.isPresent()); assertEquals(0, endpointCertificateMetadata.get().version()); assertEquals(endpointCertificateMetadata, mockCuratorDb.readEndpointCertificateMetadata(testInstance.id())); + assertEquals("original-request-uuid", endpointCertificateMetadata.get().request_id()); assertEquals(Set.copyOf(expectedCombinedSans), Set.copyOf(endpointCertificateMetadata.get().requestedDnsSans())); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java index 1b7970f6fcb..69960ed393b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ServiceRegistryMock.java @@ -24,6 +24,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.MockIssueH import com.yahoo.vespa.hosted.controller.api.integration.resource.CostReportConsumerMock; import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService; import com.yahoo.vespa.hosted.controller.api.integration.routing.MemoryGlobalRoutingService; +import com.yahoo.vespa.hosted.controller.api.integration.secrets.NoopTenantSecretService; +import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretService; import com.yahoo.vespa.hosted.controller.api.integration.stubs.DummyOwnershipIssues; import com.yahoo.vespa.hosted.controller.api.integration.stubs.DummySystemMonitor; import com.yahoo.vespa.hosted.controller.api.integration.stubs.LoggingDeploymentIssues; @@ -64,6 +66,7 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg private final RoleService roleService = new NoopRoleService(); private final BillingController billingController = new MockBillingController(); private final ContainerRegistryMock containerRegistry = new ContainerRegistryMock(); + private final NoopTenantSecretService tenantSecretService = new NoopTenantSecretService(); public ServiceRegistryMock(SystemName system) { this.zoneRegistryMock = new ZoneRegistryMock(system); @@ -205,6 +208,11 @@ public class ServiceRegistryMock extends AbstractComponent implements ServiceReg return containerRegistry; } + @Override + public TenantSecretService tenantSecretService() { + return tenantSecretService; + } + public ConfigServerMock configServerMock() { return configServerMock; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java index 9d187d1d76a..1467c1f6392 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializerTest.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; +import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore; import com.yahoo.vespa.hosted.controller.api.role.SimplePrincipal; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; @@ -95,7 +96,9 @@ public class TenantSerializerTest { Optional.of(new SimplePrincipal("foobar-user")), ImmutableBiMap.of(publicKey, new SimplePrincipal("joe"), otherPublicKey, new SimplePrincipal("jane")), - TenantInfo.EMPTY); + TenantInfo.EMPTY, + List.of() + ); CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant)); assertEquals(tenant.name(), serialized.name()); assertEquals(tenant.creator(), serialized.creator()); @@ -111,9 +114,15 @@ public class TenantSerializerTest { Optional.of(new SimplePrincipal("foobar-user")), ImmutableBiMap.of(publicKey, new SimplePrincipal("joe"), otherPublicKey, new SimplePrincipal("jane")), - TenantInfo.EMPTY.withName("Ofni Tnanet")); + TenantInfo.EMPTY.withName("Ofni Tnanet"), + List.of( + new TenantSecretStore("ss1", "123", "role1"), + new TenantSecretStore("ss2", "124", "role2") + ) + ); CloudTenant serialized = (CloudTenant) serializer.tenantFrom(serializer.toSlime(tenant)); assertEquals(tenant.info(), serialized.info()); + assertEquals(tenant.tenantSecretStores(), serialized.tenantSecretStores()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java index 10bb722a605..8d6b4f1f629 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiCloudTest.java @@ -114,6 +114,38 @@ public class ApplicationApiCloudTest extends ControllerContainerCloudTest { } } + @Test + public void test_secret_store_configuration() { + var secretStoreRequest = + request("/application/v4/tenant/scoober/secret-store", PUT) + .data("{" + + "\"awsId\": \"123\"," + + "\"name\": \"some-name\"," + + "\"role\": \"role-id\"," + + "\"externalId\": \"321\"" + + "}") + .roles(Set.of(Role.administrator(tenantName))); + tester.assertResponse(secretStoreRequest, "{\"message\":\"Configured secret store: TenantSecretStore{name='some-name', awsId='123', role='role-id'}\"}", 200); + tester.assertResponse(secretStoreRequest, "{" + + "\"error-code\":\"BAD_REQUEST\"," + + "\"message\":\"Secret store TenantSecretStore{name='some-name', awsId='123', role='role-id'} is already configured\"" + + "}", 400); + + secretStoreRequest = + request("/application/v4/tenant/scoober/secret-store", PUT) + .data("{" + + "\"awsId\": \"123\"," + + "\"name\": \" \"," + + "\"role\": \"role-id\"," + + "\"externalId\": \"321\"" + + "}") + .roles(Set.of(Role.administrator(tenantName))); + tester.assertResponse(secretStoreRequest, "{" + + "\"error-code\":\"BAD_REQUEST\"," + + "\"message\":\"Secret store TenantSecretStore{name=' ', awsId='123', role='role-id'} is invalid\"" + + "}", 400); + } + private ApplicationPackageBuilder prodBuilder() { return new ApplicationPackageBuilder() .instances("default") diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java index 390823271b4..85aa01a11be 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/filter/SignatureFilterTest.java @@ -29,6 +29,7 @@ import java.net.http.HttpRequest; import java.security.PrivateKey; import java.security.PublicKey; import java.time.Instant; +import java.util.List; import java.util.Optional; import java.util.Set; @@ -73,7 +74,8 @@ public class SignatureFilterTest { LastLoginInfo.EMPTY, Optional.empty(), ImmutableBiMap.of(), - TenantInfo.EMPTY)); + TenantInfo.EMPTY, + List.of())); tester.curator().writeApplication(new Application(appId, tester.clock().instant())); } @@ -117,7 +119,8 @@ public class SignatureFilterTest { LastLoginInfo.EMPTY, Optional.empty(), ImmutableBiMap.of(publicKey, () -> "user"), - TenantInfo.EMPTY)); + TenantInfo.EMPTY, + List.of())); verifySecurityContext(requestOf(signer.signed(request.copy(), Method.POST, () -> new ByteArrayInputStream(hiBytes)), hiBytes), new SecurityContext(new SimplePrincipal("user"), Set.of(Role.reader(id.tenant()), diff --git a/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java b/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java index 51ed93bb6ee..e2beafa66f2 100644 --- a/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java +++ b/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java @@ -60,7 +60,7 @@ import static com.yahoo.document.json.JsonSerializationHelper.*; /** * The DocumentUpdateJsonSerializer utility class is used to serialize a DocumentUpdate instance using the JSON format described in - * <a href="https://docs.vespa.ai/documentation/reference/document-json-format.html#update">Document JSON Format: The Update Structure</a> + * <a href="https://docs.vespa.ai/en/reference/document-json-format.html#update">Document JSON Format: The Update Structure</a> * * @see #serialize(com.yahoo.document.DocumentUpdate) * @author Vegard Sjonfjell diff --git a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java index ffc276fc94c..ad016a40fca 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java @@ -14,7 +14,7 @@ import static com.yahoo.document.json.readers.JsonParserHelpers.*; /** * Reads the tensor format defined at - * See <a href="https://docs.vespa.ai/documentation/reference/document-json-format.html">https://docs.vespa.ai/documentation/reference/document-json-format.html</a> + * See <a href="https://docs.vespa.ai/en/reference/document-json-format.html">https://docs.vespa.ai/en/reference/document-json-format.html</a> * * @author geirst * @author bratseth diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/NodeState.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/NodeState.java index d23a0e580a0..0102d6cba36 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/NodeState.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/NodeState.java @@ -22,7 +22,7 @@ public class NodeState { /** A location string that expresses the use of THIS node. */ public static final String NODE_CURRENT = "."; - private static Logger log = Logger.getLogger(NodeState.class.getName()); + private static final Logger log = Logger.getLogger(NodeState.class.getName()); private final Map<String, NodeState> children = new LinkedHashMap<String, NodeState>(); private final Map<String, String> state = new LinkedHashMap<String, String>(); private NodeState parent = null; diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index 0cba519bf88..22750a27f20 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -67,6 +67,7 @@ vespa_define_module( src/tests/instruction/remove_trivial_dimension_optimizer src/tests/instruction/sparse_dot_product_function src/tests/instruction/sparse_merge_function + src/tests/instruction/sparse_no_overlap_join_function src/tests/instruction/sum_max_dot_product_function src/tests/instruction/vector_from_doubles_function src/tests/streamed/value diff --git a/eval/src/tests/instruction/sparse_no_overlap_join_function/CMakeLists.txt b/eval/src/tests/instruction/sparse_no_overlap_join_function/CMakeLists.txt new file mode 100644 index 00000000000..87b58360366 --- /dev/null +++ b/eval/src/tests/instruction/sparse_no_overlap_join_function/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(eval_sparse_no_overlap_join_function_test_app TEST + SOURCES + sparse_no_overlap_join_function_test.cpp + DEPENDS + vespaeval + GTest::GTest +) +vespa_add_test(NAME eval_sparse_no_overlap_join_function_test_app COMMAND eval_sparse_no_overlap_join_function_test_app) diff --git a/eval/src/tests/instruction/sparse_no_overlap_join_function/sparse_no_overlap_join_function_test.cpp b/eval/src/tests/instruction/sparse_no_overlap_join_function/sparse_no_overlap_join_function_test.cpp new file mode 100644 index 00000000000..60755b50c1d --- /dev/null +++ b/eval/src/tests/instruction/sparse_no_overlap_join_function/sparse_no_overlap_join_function_test.cpp @@ -0,0 +1,92 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/instruction/sparse_no_overlap_join_function.h> +#include <vespa/eval/eval/test/eval_fixture.h> +#include <vespa/vespalib/gtest/gtest.h> + +using namespace vespalib::eval; +using namespace vespalib::eval::test; + +const ValueBuilderFactory &prod_factory = FastValueBuilderFactory::get(); +const ValueBuilderFactory &test_factory = SimpleValueBuilderFactory::get(); + +//----------------------------------------------------------------------------- + +EvalFixture::ParamRepo make_params() { + return EvalFixture::ParamRepo() + .add_variants("v1_a", GenSpec(3.0).map("a", 8, 1)) + .add_variants("v2_b", GenSpec(7.0).map("b", 4, 2)) + .add_variants("v2_b_trivial", GenSpec(7.0).map("b", 4, 2).idx("c", 1).idx("d", 1)) + .add("m1_ac", GenSpec(3.0).map("a", 8, 1).map("c", 8, 1)) + .add("m2_bd", GenSpec(17.0).map("b", 4, 2).map("d", 4, 2)) + .add("scalar", GenSpec(1.0)) + .add("dense_b", GenSpec().idx("b", 5)) + .add("mixed_bc", GenSpec().map("b", 5, 1).idx("c", 5)); +} +EvalFixture::ParamRepo param_repo = make_params(); + +void assert_optimized(const vespalib::string &expr) { + EvalFixture fast_fixture(prod_factory, expr, param_repo, true); + EvalFixture test_fixture(test_factory, expr, param_repo, true); + EvalFixture slow_fixture(prod_factory, expr, param_repo, false); + EXPECT_EQ(fast_fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQ(test_fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQ(slow_fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQ(fast_fixture.find_all<SparseNoOverlapJoinFunction>().size(), 1u); + EXPECT_EQ(test_fixture.find_all<SparseNoOverlapJoinFunction>().size(), 1u); + EXPECT_EQ(slow_fixture.find_all<SparseNoOverlapJoinFunction>().size(), 0u); +} + +void assert_not_optimized(const vespalib::string &expr) { + EvalFixture fast_fixture(prod_factory, expr, param_repo, true); + EXPECT_EQ(fast_fixture.result(), EvalFixture::ref(expr, param_repo)); + EXPECT_EQ(fast_fixture.find_all<SparseNoOverlapJoinFunction>().size(), 0u); +} + +//----------------------------------------------------------------------------- + +TEST(SparseNoOverlapJoin, expression_can_be_optimized) +{ + assert_optimized("v1_a*v2_b"); + assert_optimized("v2_b*v1_a"); + assert_optimized("m1_ac*m2_bd"); + assert_optimized("m2_bd*m1_ac"); + assert_optimized("m1_ac*v2_b"); + assert_optimized("m2_bd*v1_a"); + assert_optimized("join(v1_a,v2_b,f(x,y)(max(x,y)))"); +} + +TEST(SparseNoOverlapJoin, trivial_dimensions_are_ignored) +{ + assert_optimized("v1_a*v2_b_trivial"); + assert_optimized("v2_b_trivial*v1_a"); +} + +TEST(SparseNoOverlapJoin, overlapping_dimensions_are_not_optimized) +{ + assert_not_optimized("v1_a*v1_a"); + assert_not_optimized("v1_a*m1_ac"); + assert_not_optimized("m1_ac*v1_a"); +} + +TEST(SparseNoOverlapJoin, both_values_must_be_sparse_tensors) +{ + assert_not_optimized("v1_a*scalar"); + assert_not_optimized("scalar*v1_a"); + assert_not_optimized("v1_a*dense_b"); + assert_not_optimized("dense_b*v1_a"); + assert_not_optimized("v1_a*mixed_bc"); + assert_not_optimized("mixed_bc*v1_a"); +} + +TEST(SparseNoOverlapJoin, mixed_cell_types_are_not_optimized) +{ + assert_not_optimized("v1_a*v2_b_f"); + assert_not_optimized("v1_a_f*v2_b"); +} + +//----------------------------------------------------------------------------- + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/eval/src/vespa/eval/eval/fast_value.hpp b/eval/src/vespa/eval/eval/fast_value.hpp index 6673494ccd2..d5cfc9c6368 100644 --- a/eval/src/vespa/eval/eval/fast_value.hpp +++ b/eval/src/vespa/eval/eval/fast_value.hpp @@ -135,8 +135,6 @@ struct FastIterateView : public Value::Index::View { //----------------------------------------------------------------------------- -using JoinAddrSource = instruction::SparseJoinPlan::Source; - // This is the class instructions will look for when optimizing sparse // operations by calling inline functions directly. struct FastValueIndex final : Value::Index { @@ -149,12 +147,6 @@ struct FastValueIndex final : Value::Index { const FastValueIndex &lhs, const FastValueIndex &rhs, ConstArrayRef<LCT> lhs_cells, ConstArrayRef<RCT> rhs_cells, Stash &stash); - template <typename LCT, typename RCT, typename OCT, typename Fun> - static const Value &sparse_no_overlap_join(const ValueType &res_type, const Fun &fun, - const FastValueIndex &lhs, const FastValueIndex &rhs, - const std::vector<JoinAddrSource> &addr_sources, - ConstArrayRef<LCT> lhs_cells, ConstArrayRef<RCT> rhs_cells, Stash &stash); - size_t size() const override { return map.size(); } std::unique_ptr<View> create_view(const std::vector<size_t> &dims) const override; }; @@ -373,54 +365,4 @@ FastValueIndex::sparse_full_overlap_join(const ValueType &res_type, const Fun &f //----------------------------------------------------------------------------- -template <typename LCT, typename RCT, typename OCT, typename Fun> -const Value & -FastValueIndex::sparse_no_overlap_join(const ValueType &res_type, const Fun &fun, - const FastValueIndex &lhs, const FastValueIndex &rhs, - const std::vector<JoinAddrSource> &addr_sources, - ConstArrayRef<LCT> lhs_cells, ConstArrayRef<RCT> rhs_cells, Stash &stash) -{ - size_t num_mapped_dims = addr_sources.size(); - auto &result = stash.create<FastValue<OCT,true>>(res_type, num_mapped_dims, 1, lhs.map.size()*rhs.map.size()); - std::vector<string_id> output_addr(num_mapped_dims); - std::vector<size_t> store_lhs_idx; - std::vector<size_t> store_rhs_idx; - size_t out_idx = 0; - for (JoinAddrSource source : addr_sources) { - switch (source) { - case JoinAddrSource::LHS: - store_lhs_idx.push_back(out_idx++); - break; - case JoinAddrSource::RHS: - store_rhs_idx.push_back(out_idx++); - break; - default: - abort(); - } - } - assert(out_idx == output_addr.size()); - for (size_t lhs_subspace = 0; lhs_subspace < lhs.map.size(); ++lhs_subspace) { - auto l_addr = lhs.map.get_addr(lhs_subspace); - assert(l_addr.size() == store_lhs_idx.size()); - for (size_t i = 0; i < store_lhs_idx.size(); ++i) { - size_t addr_idx = store_lhs_idx[i]; - output_addr[addr_idx] = l_addr[i]; - } - for (size_t rhs_subspace = 0; rhs_subspace < rhs.map.size(); ++rhs_subspace) { - auto r_addr = rhs.map.get_addr(rhs_subspace); - assert(r_addr.size() == store_rhs_idx.size()); - for (size_t i = 0; i < store_rhs_idx.size(); ++i) { - size_t addr_idx = store_rhs_idx[i]; - output_addr[addr_idx] = r_addr[i]; - } - result.add_mapping(ConstArrayRef(output_addr)); - auto cell_value = fun(lhs_cells[lhs_subspace], rhs_cells[rhs_subspace]); - result.my_cells.push_back_fast(cell_value); - } - } - return result; -} - -//----------------------------------------------------------------------------- - } diff --git a/eval/src/vespa/eval/eval/optimize_tensor_function.cpp b/eval/src/vespa/eval/eval/optimize_tensor_function.cpp index 196e8a98679..aef49a2c75b 100644 --- a/eval/src/vespa/eval/eval/optimize_tensor_function.cpp +++ b/eval/src/vespa/eval/eval/optimize_tensor_function.cpp @@ -7,6 +7,7 @@ #include <vespa/eval/instruction/dense_dot_product_function.h> #include <vespa/eval/instruction/sparse_dot_product_function.h> #include <vespa/eval/instruction/sparse_merge_function.h> +#include <vespa/eval/instruction/sparse_no_overlap_join_function.h> #include <vespa/eval/instruction/mixed_inner_product_function.h> #include <vespa/eval/instruction/sum_max_dot_product_function.h> #include <vespa/eval/instruction/dense_xw_product_function.h> @@ -74,6 +75,7 @@ const TensorFunction &optimize_for_factory(const ValueBuilderFactory &, const Te child.set(JoinWithNumberFunction::optimize(child.get(), stash)); child.set(DenseSingleReduceFunction::optimize(child.get(), stash)); child.set(SparseMergeFunction::optimize(child.get(), stash)); + child.set(SparseNoOverlapJoinFunction::optimize(child.get(), stash)); nodes.pop_back(); } } diff --git a/eval/src/vespa/eval/instruction/CMakeLists.txt b/eval/src/vespa/eval/instruction/CMakeLists.txt index 3def8907ac8..50f7dbe7005 100644 --- a/eval/src/vespa/eval/instruction/CMakeLists.txt +++ b/eval/src/vespa/eval/instruction/CMakeLists.txt @@ -34,6 +34,7 @@ vespa_add_library(eval_instruction OBJECT replace_type_function.cpp sparse_dot_product_function.cpp sparse_merge_function.cpp + sparse_no_overlap_join_function.cpp sum_max_dot_product_function.cpp vector_from_doubles_function.cpp ) diff --git a/eval/src/vespa/eval/instruction/generic_join.cpp b/eval/src/vespa/eval/instruction/generic_join.cpp index 6d6f86b7c4d..4b3755509c7 100644 --- a/eval/src/vespa/eval/instruction/generic_join.cpp +++ b/eval/src/vespa/eval/instruction/generic_join.cpp @@ -19,8 +19,6 @@ using operation::SwapArgs2; using State = InterpretedFunction::State; using Instruction = InterpretedFunction::Instruction; -namespace { - //----------------------------------------------------------------------------- template <typename LCT, typename RCT, typename OCT, typename Fun> @@ -56,6 +54,8 @@ generic_mixed_join(const Value &lhs, const Value &rhs, const JoinParam ¶m) return builder->build(std::move(builder)); }; +namespace { + template <typename LCT, typename RCT, typename OCT, typename Fun> void my_mixed_join_op(State &state, uint64_t param_in) { const auto ¶m = unwrap_param<JoinParam>(param_in); @@ -70,48 +70,6 @@ void my_mixed_join_op(State &state, uint64_t param_in) { //----------------------------------------------------------------------------- template <typename LCT, typename RCT, typename OCT, typename Fun> -void my_sparse_no_overlap_join_op(State &state, uint64_t param_in) { - const auto ¶m = unwrap_param<JoinParam>(param_in); - const Value &lhs = state.peek(1); - const Value &rhs = state.peek(0); - auto lhs_cells = lhs.cells().typify<LCT>(); - auto rhs_cells = rhs.cells().typify<RCT>(); - const Value::Index &lhs_index = lhs.index(); - const Value::Index &rhs_index = rhs.index(); - if (auto indexes = detect_type<FastValueIndex>(lhs_index, rhs_index)) { - const auto &lhs_fast = indexes.get<0>(); - const auto &rhs_fast = indexes.get<1>(); - return state.pop_pop_push( - FastValueIndex::sparse_no_overlap_join<LCT,RCT,OCT,Fun> - (param.res_type, Fun(param.function), - lhs_fast, rhs_fast, - param.sparse_plan.sources, - lhs_cells, rhs_cells, state.stash)); - } - Fun fun(param.function); - SparseJoinState sparse(param.sparse_plan, lhs.index(), rhs.index()); - auto guess = lhs.index().size() * rhs.index().size(); - assert(param.dense_plan.out_size == 1); - auto builder = param.factory.create_transient_value_builder<OCT>(param.res_type, param.sparse_plan.sources.size(), 1, guess); - auto outer = sparse.first_index.create_view({}); - assert(sparse.second_view_dims.empty()); - auto inner = sparse.second_index.create_view({}); - outer->lookup({}); - while (outer->next_result(sparse.first_address, sparse.first_subspace)) { - inner->lookup({}); - while (inner->next_result(sparse.second_only_address, sparse.second_subspace)) { - auto cell_value = fun(lhs_cells[sparse.lhs_subspace], rhs_cells[sparse.rhs_subspace]); - builder->add_subspace(sparse.full_address)[0] = cell_value; - } - } - auto &result = state.stash.create<std::unique_ptr<Value>>(builder->build(std::move(builder))); - const Value &result_ref = *(result.get()); - state.pop_pop_push(result_ref); -}; - -//----------------------------------------------------------------------------- - -template <typename LCT, typename RCT, typename OCT, typename Fun> void my_sparse_full_overlap_join_op(State &state, uint64_t param_in) { const auto ¶m = unwrap_param<JoinParam>(param_in); const Value &lhs = state.peek(1); @@ -222,12 +180,6 @@ struct SelectGenericJoinOp { { return my_sparse_full_overlap_join_op<LCT,RCT,OCT,Fun>; } - if ((param.dense_plan.out_size == 1) && - (param.sparse_plan.lhs_overlap.size() == 0) && - (param.sparse_plan.rhs_overlap.size() == 0)) - { - return my_sparse_no_overlap_join_op<LCT,RCT,OCT,Fun>; - } return my_mixed_join_op<LCT,RCT,OCT,Fun>; } }; diff --git a/eval/src/vespa/eval/instruction/generic_join.h b/eval/src/vespa/eval/instruction/generic_join.h index 026a2938971..d55718a0ff1 100644 --- a/eval/src/vespa/eval/instruction/generic_join.h +++ b/eval/src/vespa/eval/instruction/generic_join.h @@ -16,6 +16,11 @@ using join_fun_t = operation::op2_t; //----------------------------------------------------------------------------- +struct JoinParam; + +template <typename LCT, typename RCT, typename OCT, typename Fun> +Value::UP generic_mixed_join(const Value &lhs, const Value &rhs, const JoinParam ¶m); + struct GenericJoin { static InterpretedFunction::Instruction make_instruction(const ValueType &lhs_type, const ValueType &rhs_type, diff --git a/eval/src/vespa/eval/instruction/sparse_no_overlap_join_function.cpp b/eval/src/vespa/eval/instruction/sparse_no_overlap_join_function.cpp new file mode 100644 index 00000000000..a9f68c7314d --- /dev/null +++ b/eval/src/vespa/eval/instruction/sparse_no_overlap_join_function.cpp @@ -0,0 +1,137 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "sparse_no_overlap_join_function.h" +#include "generic_join.h" +#include <vespa/eval/eval/fast_value.hpp> +#include <vespa/vespalib/util/typify.h> + +namespace vespalib::eval { + +using namespace tensor_function; +using namespace operation; +using namespace instruction; + +namespace { + +template <typename CT, typename Fun> +const Value &my_fast_no_overlap_sparse_join(const FastAddrMap &lhs_map, const FastAddrMap &rhs_map, + const CT *lhs_cells, const CT *rhs_cells, + const JoinParam ¶m, Stash &stash) +{ + Fun fun(param.function); + const auto &addr_sources = param.sparse_plan.sources; + size_t num_mapped_dims = addr_sources.size(); + auto &result = stash.create<FastValue<CT,true>>(param.res_type, num_mapped_dims, 1, lhs_map.size() * rhs_map.size()); + std::vector<string_id> output_addr(num_mapped_dims); + std::vector<size_t> store_lhs_idx; + std::vector<size_t> store_rhs_idx; + size_t out_idx = 0; + for (auto source: addr_sources) { + switch (source) { + case SparseJoinPlan::Source::LHS: + store_lhs_idx.push_back(out_idx++); + break; + case SparseJoinPlan::Source::RHS: + store_rhs_idx.push_back(out_idx++); + break; + default: abort(); + } + } + assert(out_idx == output_addr.size()); + for (size_t lhs_subspace = 0; lhs_subspace < lhs_map.size(); ++lhs_subspace) { + auto l_addr = lhs_map.get_addr(lhs_subspace); + assert(l_addr.size() == store_lhs_idx.size()); + for (size_t i = 0; i < store_lhs_idx.size(); ++i) { + size_t addr_idx = store_lhs_idx[i]; + output_addr[addr_idx] = l_addr[i]; + } + for (size_t rhs_subspace = 0; rhs_subspace < rhs_map.size(); ++rhs_subspace) { + auto r_addr = rhs_map.get_addr(rhs_subspace); + assert(r_addr.size() == store_rhs_idx.size()); + for (size_t i = 0; i < store_rhs_idx.size(); ++i) { + size_t addr_idx = store_rhs_idx[i]; + output_addr[addr_idx] = r_addr[i]; + } + result.add_mapping(ConstArrayRef(output_addr)); + CT cell_value = fun(lhs_cells[lhs_subspace], rhs_cells[rhs_subspace]); + result.my_cells.push_back_fast(cell_value); + } + } + return result; +} + +template <typename CT, typename Fun> +void my_sparse_no_overlap_join_op(InterpretedFunction::State &state, uint64_t param_in) { + const auto ¶m = unwrap_param<JoinParam>(param_in); + const Value &lhs = state.peek(1); + const Value &rhs = state.peek(0); + const auto &lhs_idx = lhs.index(); + const auto &rhs_idx = rhs.index(); + if (__builtin_expect(are_fast(lhs_idx, rhs_idx), true)) { + const Value &res = my_fast_no_overlap_sparse_join<CT,Fun>(as_fast(lhs_idx).map, as_fast(rhs_idx).map, + lhs.cells().typify<CT>().cbegin(), rhs.cells().typify<CT>().cbegin(), param, state.stash); + state.pop_pop_push(res); + } else { + auto res = generic_mixed_join<CT,CT,CT,Fun>(lhs, rhs, param); + state.pop_pop_push(*state.stash.create<std::unique_ptr<Value>>(std::move(res))); + } +} + +struct SelectSparseNoOverlapJoinOp { + template <typename CT, typename Fun> + static auto invoke() { return my_sparse_no_overlap_join_op<CT,Fun>; } +}; + +using MyTypify = TypifyValue<TypifyCellType,operation::TypifyOp2>; + +bool is_sparse_like(const ValueType &type) { + return ((type.count_mapped_dimensions() > 0) && (type.dense_subspace_size() == 1)); +} + +} // namespace <unnamed> + +SparseNoOverlapJoinFunction::SparseNoOverlapJoinFunction(const tensor_function::Join &original) + : tensor_function::Join(original.result_type(), + original.lhs(), + original.rhs(), + original.function()) +{ + assert(compatible_types(result_type(), lhs().result_type(), rhs().result_type())); +} + +InterpretedFunction::Instruction +SparseNoOverlapJoinFunction::compile_self(const ValueBuilderFactory &factory, Stash &stash) const +{ + const auto ¶m = stash.create<JoinParam>(lhs().result_type(), rhs().result_type(), function(), factory); + auto op = typify_invoke<2,MyTypify,SelectSparseNoOverlapJoinOp>(result_type().cell_type(), function()); + return InterpretedFunction::Instruction(op, wrap_param<JoinParam>(param)); +} + +bool +SparseNoOverlapJoinFunction::compatible_types(const ValueType &res, const ValueType &lhs, const ValueType &rhs) +{ + if ((lhs.cell_type() == rhs.cell_type()) && + is_sparse_like(lhs) && is_sparse_like(rhs) && + (res.count_mapped_dimensions() == (lhs.count_mapped_dimensions() + rhs.count_mapped_dimensions()))) + { + assert(is_sparse_like(res)); + assert(res.cell_type() == lhs.cell_type()); + return true; + } + return false; +} + +const TensorFunction & +SparseNoOverlapJoinFunction::optimize(const TensorFunction &expr, Stash &stash) +{ + if (auto join = as<Join>(expr)) { + const TensorFunction &lhs = join->lhs(); + const TensorFunction &rhs = join->rhs(); + if (compatible_types(expr.result_type(), lhs.result_type(), rhs.result_type())) { + return stash.create<SparseNoOverlapJoinFunction>(*join); + } + } + return expr; +} + +} // namespace diff --git a/eval/src/vespa/eval/instruction/sparse_no_overlap_join_function.h b/eval/src/vespa/eval/instruction/sparse_no_overlap_join_function.h new file mode 100644 index 00000000000..962e6dc1361 --- /dev/null +++ b/eval/src/vespa/eval/instruction/sparse_no_overlap_join_function.h @@ -0,0 +1,23 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/eval/eval/tensor_function.h> + +namespace vespalib::eval { + +/** + * Tensor function for joining sparse tensors with no overlapping + * dimensions. + */ +class SparseNoOverlapJoinFunction : public tensor_function::Join +{ +public: + SparseNoOverlapJoinFunction(const tensor_function::Join &original); + InterpretedFunction::Instruction compile_self(const ValueBuilderFactory &factory, Stash &stash) const override; + bool result_is_mutable() const override { return true; } + static bool compatible_types(const ValueType &res, const ValueType &lhs, const ValueType &rhs); + static const TensorFunction &optimize(const TensorFunction &expr, Stash &stash); +}; + +} // namespace diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 8d7fc494dcd..aca3da1ffe8 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -12,7 +12,12 @@ import java.util.List; import java.util.Optional; import java.util.TreeMap; -import static com.yahoo.vespa.flags.FetchVector.Dimension.*; +import static com.yahoo.vespa.flags.FetchVector.Dimension.APPLICATION_ID; +import static com.yahoo.vespa.flags.FetchVector.Dimension.HOSTNAME; +import static com.yahoo.vespa.flags.FetchVector.Dimension.NODE_TYPE; +import static com.yahoo.vespa.flags.FetchVector.Dimension.TENANT_ID; +import static com.yahoo.vespa.flags.FetchVector.Dimension.VESPA_VERSION; +import static com.yahoo.vespa.flags.FetchVector.Dimension.ZONE_ID; /** * Definitions of feature flags. @@ -165,6 +170,13 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundIntFlag MAX_PENDING_MOVE_OPS = defineIntFlag( + "max-pending-move-ops", 10, + List.of("baldersheim"), "2021-02-15", "2021-04-01", + "Max number of move operations inflight", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundDoubleFlag FEED_CONCURRENCY = defineDoubleFlag( "feed-concurrency", 0.5, List.of("baldersheim"), "2020-12-02", "2022-01-01", @@ -179,6 +191,13 @@ public class Flags { "Takes effect on next internal redeployment", APPLICATION_ID); + public static final UnboundBooleanFlag USE_BUCKET_EXECUTOR_FOR_BUCKET_MOVE = defineFeatureFlag( + "use-bucket-executor-for-bucket-move", false, + List.of("baldersheim"), "2021-02-15", "2021-04-01", + "Wheter to use content-level bucket executor or legacy frozen buckets", + "Takes effect on next internal redeployment", + APPLICATION_ID); + public static final UnboundBooleanFlag USE_POWER_OF_TWO_CHOICES_LOAD_BALANCING = defineFeatureFlag( "use-power-of-two-choices-load-balancing", false, List.of("tokle"), "2020-12-02", "2021-02-15", @@ -218,7 +237,7 @@ public class Flags { "sync-host-logs-to-s3-bucket", "", List.of("andreer", "valerijf"), "2021-02-10", "2021-03-01", "Host-admin should sync host logs to an S3 bucket named by this flag. If left empty, sync is disabled", "Takes effect on next run of S3 log sync task in host-admin", - APPLICATION_ID); + APPLICATION_ID, NODE_TYPE); public static final UnboundIntFlag CLUSTER_CONTROLLER_MAX_HEAP_SIZE_IN_MB = defineIntFlag( "cluster-controller-max-heap-size-in-mb", 512, diff --git a/jdisc_http_service/src/main/java/com/yahoo/container/logging/AccessLog.java b/jdisc_http_service/src/main/java/com/yahoo/container/logging/AccessLog.java index cdb4febb775..2d46c53bca7 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/container/logging/AccessLog.java +++ b/jdisc_http_service/src/main/java/com/yahoo/container/logging/AccessLog.java @@ -13,6 +13,8 @@ import com.yahoo.component.provider.ComponentRegistry; */ public class AccessLog implements RequestLog { + public static final AccessLog NONE_INSTANCE = new AccessLog(new ComponentRegistry<>()); + private final ComponentRegistry<RequestLogHandler> implementers; @Inject @@ -21,7 +23,7 @@ public class AccessLog implements RequestLog { } public static AccessLog voidAccessLog() { - return new AccessLog(new ComponentRegistry<>()); + return NONE_INSTANCE; } @Override diff --git a/jdisc_http_service/src/main/java/com/yahoo/container/logging/JSONFormatter.java b/jdisc_http_service/src/main/java/com/yahoo/container/logging/JSONFormatter.java index 441e139bc67..680ee5acbd9 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/container/logging/JSONFormatter.java +++ b/jdisc_http_service/src/main/java/com/yahoo/container/logging/JSONFormatter.java @@ -15,8 +15,6 @@ import java.util.Objects; import java.util.logging.Level; import java.util.logging.Logger; -import static com.yahoo.container.logging.FormatUtil.writeSecondsField; - /** * Formatting of an {@link AccessLogEntry} in the Vespa JSON access log format. * @@ -64,9 +62,14 @@ public class JSONFormatter implements LogWriter<RequestLogEntry> { generator.writeStringField("connection", connectionId); } - Principal principal = entry.userPrincipal().orElse(null); - if (principal != null) { - generator.writeStringField("user-principal", principal.getName()); + Principal userPrincipal = entry.userPrincipal().orElse(null); + if (userPrincipal != null) { + generator.writeStringField("user-principal", userPrincipal.getName()); + } + + Principal sslPrincipal = entry.sslPrincipal().orElse(null); + if (sslPrincipal != null) { + generator.writeStringField("ssl-principal", sslPrincipal.getName()); } String remoteAddress = entry.remoteAddress().orElse(null); diff --git a/jdisc_http_service/src/main/java/com/yahoo/container/logging/RequestLogEntry.java b/jdisc_http_service/src/main/java/com/yahoo/container/logging/RequestLogEntry.java index b771ea11ed0..819907fc9f1 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/container/logging/RequestLogEntry.java +++ b/jdisc_http_service/src/main/java/com/yahoo/container/logging/RequestLogEntry.java @@ -43,6 +43,7 @@ public class RequestLogEntry { private final String rawPath; private final String rawQuery; private final Principal userPrincipal; + private final Principal sslPrincipal; private final HitCounts hitCounts; private final TraceNode traceNode; private final Map<String, Collection<String>> extraAttributes; @@ -67,6 +68,7 @@ public class RequestLogEntry { this.rawPath = builder.rawPath; this.rawQuery = builder.rawQuery; this.userPrincipal = builder.userPrincipal; + this.sslPrincipal = builder.sslPrincipal; this.hitCounts = builder.hitCounts; this.traceNode = builder.traceNode; this.extraAttributes = copyExtraAttributes(builder.extraAttributes); @@ -91,6 +93,7 @@ public class RequestLogEntry { public Optional<String> rawPath() { return Optional.ofNullable(rawPath); } public Optional<String> rawQuery() { return Optional.ofNullable(rawQuery); } public Optional<Principal> userPrincipal() { return Optional.ofNullable(userPrincipal); } + public Optional<Principal> sslPrincipal() { return Optional.ofNullable(sslPrincipal); } public Optional<HitCounts> hitCounts() { return Optional.ofNullable(hitCounts); } public Optional<TraceNode> traceNode() { return Optional.ofNullable(traceNode); } public Collection<String> extraAttributeKeys() { return Collections.unmodifiableCollection(extraAttributes.keySet()); } @@ -135,6 +138,7 @@ public class RequestLogEntry { private Principal userPrincipal; private HitCounts hitCounts; private TraceNode traceNode; + private Principal sslPrincipal; private final Map<String, Collection<String>> extraAttributes = new HashMap<>(); public Builder connectionId(String connectionId) { this.connectionId = requireNonNull(connectionId); return this; } @@ -156,6 +160,7 @@ public class RequestLogEntry { public Builder rawPath(String rawPath) { this.rawPath = requireNonNull(rawPath); return this; } public Builder rawQuery(String rawQuery) { this.rawQuery = requireNonNull(rawQuery); return this; } public Builder userPrincipal(Principal userPrincipal) { this.userPrincipal = requireNonNull(userPrincipal); return this; } + public Builder sslPrincipal(Principal sslPrincipal) { this.sslPrincipal = requireNonNull(sslPrincipal); return this; } public Builder hitCounts(HitCounts hitCounts) { this.hitCounts = requireNonNull(hitCounts); return this; } public Builder traceNode(TraceNode traceNode) { this.traceNode = requireNonNull(traceNode); return this; } public Builder addExtraAttribute(String key, String value) { diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java index b38b9dcdfb2..4b023f427a4 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java @@ -14,6 +14,7 @@ import org.eclipse.jetty.util.component.AbstractLifeCycle; import javax.servlet.http.HttpServletRequest; import java.security.Principal; +import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; import java.util.List; @@ -101,6 +102,10 @@ class AccessLogRequestLog extends AbstractLifeCycle implements org.eclipse.jetty builder.addExtraAttribute(header, value); } }); + X509Certificate[] clientCert = (X509Certificate[]) request.getAttribute(ServletRequest.SERVLET_REQUEST_X509CERT); + if (clientCert != null && clientCert.length > 0) { + builder.sslPrincipal(clientCert[0].getSubjectX500Principal()); + } AccessLogEntry accessLogEntry = (AccessLogEntry) request.getAttribute(JDiscHttpServlet.ATTRIBUTE_NAME_ACCESS_LOG_ENTRY); if (accessLogEntry != null) { diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/tensorflow/VariableConverter.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/tensorflow/VariableConverter.java index 31cb60b5509..c0db65c5c99 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/tensorflow/VariableConverter.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/tensorflow/VariableConverter.java @@ -37,7 +37,7 @@ class VariableConverter { if ( args.length != 3) { System.out.println("Converts a TensorFlow variable into Vespa tensor document field value JSON:"); System.out.println("A JSON map containing a 'cells' array, see"); - System.out.println("http://docs.vespa.ai/documentation/reference/document-json-put-format.html#tensor)"); + System.out.println("https://docs.vespa.ai/en/reference/document-json-put-format.html#tensor)"); System.out.println(""); System.out.println("Arguments: modelDirectory tensorFlowVariableName orderedTypeSpec"); System.out.println(" - modelDirectory: The directory of the TensorFlow SavedModel"); diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncClient.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncClient.java new file mode 100644 index 00000000000..64684a4b3e7 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncClient.java @@ -0,0 +1,22 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.sync; + +import com.yahoo.vespa.hosted.node.admin.component.TaskContext; + +import java.util.List; + +/** + * @author freva + */ +public interface SyncClient { + + /** + * Syncs the given files, will only upload each file once. + * + * @param context context used to log which files were synced + * @param syncFileInfos list of files and their metadata to sync + * @param limit max number of files to upload for this invocation, to avoid blocking for too long + * @return true iff any files were uploaded + */ + boolean sync(TaskContext context, List<SyncFileInfo> syncFileInfos, int limit); +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java new file mode 100644 index 00000000000..3b88d28613c --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfo.java @@ -0,0 +1,83 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.sync; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostName; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** + * @author freva + */ +public class SyncFileInfo { + + private final String bucketName; + private final Path srcPath; + private final Path destPath; + private final boolean compressWithZstd; + + private SyncFileInfo(String bucketName, Path srcPath, Path destPath, boolean compressWithZstd) { + this.bucketName = bucketName; + this.srcPath = srcPath; + this.destPath = destPath; + this.compressWithZstd = compressWithZstd; + } + + public String bucketName() { + return bucketName; + } + + public Path srcPath() { + return srcPath; + } + + public Path destPath() { + return destPath; + } + + public InputStream inputStream() throws IOException { + InputStream is = Files.newInputStream(srcPath); + if (compressWithZstd) return new ZstdCompressingInputStream(is, 4 << 20); + return is; + } + + + public static SyncFileInfo tenantVespaLog(String bucketName, ApplicationId applicationId, HostName hostName, Path vespaLogFile) { + return new SyncFileInfo(bucketName, vespaLogFile, destination(applicationId, hostName, "logs/vespa", vespaLogFile, ".zst"), true); + } + + public static SyncFileInfo tenantAccessLog(String bucketName, ApplicationId applicationId, HostName hostName, Path accessLogFile) { + return new SyncFileInfo(bucketName, accessLogFile, destination(applicationId, hostName, "logs/access", accessLogFile, null), false); + } + + public static SyncFileInfo infrastructureVespaLog(String bucketName, HostName hostName, Path vespaLogFile) { + return new SyncFileInfo(bucketName, vespaLogFile, destination(null, hostName, "logs/vespa", vespaLogFile, ".zst"), true); + } + + public static SyncFileInfo infrastructureAccessLog(String bucketName, HostName hostName, Path accessLogFile) { + return new SyncFileInfo(bucketName, accessLogFile, destination(null, hostName, "logs/access", accessLogFile, null), false); + } + + private static Path destination(ApplicationId app, HostName hostName, String dir, Path filename, String extension) { + StringBuilder sb = new StringBuilder(100).append('/'); + + if (app == null) sb.append("infrastructure"); + else sb.append(app.tenant().value()).append('.').append(app.application().value()).append('.').append(app.instance().value()); + + sb.append('/'); + for (char c: hostName.value().toCharArray()) { + if (c == '.') break; + sb.append(c); + } + + sb.append('/').append(dir).append('/').append(filename.getFileName().toString()); + + if (extension != null) sb.append(extension); + + return Paths.get(sb.toString()); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStream.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStream.java new file mode 100644 index 00000000000..0cf8eaa9983 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStream.java @@ -0,0 +1,75 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.sync; + +import com.yahoo.compress.ZstdCompressor; + +import java.io.IOException; +import java.io.InputStream; + +/** + * InputStream that outputs given InputStream compressed with the ZStandard. + * + * @author freva + */ +public class ZstdCompressingInputStream extends InputStream { + + public static final int DEFAULT_INPUT_BUFFER_SIZE = 8 * 1024; + static final ZstdCompressor compressor = new ZstdCompressor(); + + private final InputStream is; + private final byte[] inputBuffer; + private final byte[] outputBuffer; + + private int outputPosition = 0; + private int outputLength = 0; + private boolean isClosed = false; + + public ZstdCompressingInputStream(InputStream is, int inputBufferSize) { + this.is = is; + this.inputBuffer = new byte[inputBufferSize]; + this.outputBuffer = new byte[ZstdCompressor.getMaxCompressedLength(inputBufferSize)]; + } + + public ZstdCompressingInputStream(InputStream is) { + this(is, DEFAULT_INPUT_BUFFER_SIZE); + } + + @Override + public int read() throws IOException { + throwIfClosed(); + + if (outputPosition >= outputLength) { + int readLength = is.read(inputBuffer); + if (readLength == -1) + return -1; + + outputLength = compressor.compress(inputBuffer, 0, readLength, outputBuffer, 0, outputBuffer.length); + outputPosition = 0; + } + + return Byte.toUnsignedInt(outputBuffer[outputPosition++]); + } + + @Override + public int read(byte[] b, int off, int len) throws IOException { + int first = read(); + if (first == -1) return -1; + + b[off++] = (byte) first; + len = Math.min(Math.min(len, outputLength - outputPosition), b.length - off); + System.arraycopy(outputBuffer, outputPosition, b, off, len); + outputPosition += len; + return len + 1; + } + + @Override + public void close() throws IOException { + throwIfClosed(); + is.close(); + isClosed = true; + } + + private void throwIfClosed() { + if (isClosed) throw new IllegalArgumentException("Input stream is already closed"); + } +} diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/package-info.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/package-info.java new file mode 100644 index 00000000000..fc197450492 --- /dev/null +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/package-info.java @@ -0,0 +1,8 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author freva + */ +@ExportPackage +package com.yahoo.vespa.hosted.node.admin.maintenance.sync; + +import com.yahoo.osgi.annotation.ExportPackage;
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java new file mode 100644 index 00000000000..0d596a46d77 --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java @@ -0,0 +1,79 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.node.admin.maintenance.sync; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostName; +import com.yahoo.vespa.test.file.TestFileSystem; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.FileSystem; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; + +/** + * @author freva + */ +public class SyncFileInfoTest { + + private static final FileSystem fileSystem = TestFileSystem.create(); + + private static final String bucket = "logs-region-acdf21"; + private static final ApplicationId application = ApplicationId.from("tenant", "application", "instance"); + private static final HostName hostname = HostName.from("h12352a.env.region-1.vespa.domain.example"); + private static final Path accessLogPath = fileSystem.getPath("/opt/vespa/logs/qrs/access.json-20210212.zst"); + private static final Path vespaLogPath = fileSystem.getPath("/opt/vespa/logs/vespa.log-2021-02-12"); + + @Test + public void tenant_access_log() { + SyncFileInfo sfi = SyncFileInfo.tenantAccessLog(bucket, application, hostname, accessLogPath); + assertEquals(Paths.get("/tenant.application.instance/h12352a/logs/access/access.json-20210212.zst"), sfi.destPath()); + assertEquals(bucket, sfi.bucketName()); + assertNotEquals(ZstdCompressingInputStream.class, getInputStreamType(sfi)); + } + + @Test + public void tenant_vespa_log() { + SyncFileInfo sfi = SyncFileInfo.tenantVespaLog(bucket, application, hostname, vespaLogPath); + assertEquals(Paths.get("/tenant.application.instance/h12352a/logs/vespa/vespa.log-2021-02-12.zst"), sfi.destPath()); + assertEquals(ZstdCompressingInputStream.class, getInputStreamType(sfi)); + } + + @Test + public void infra_access_log() { + SyncFileInfo sfi = SyncFileInfo.infrastructureAccessLog(bucket, hostname, accessLogPath); + assertEquals(Paths.get("/infrastructure/h12352a/logs/access/access.json-20210212.zst"), sfi.destPath()); + assertNotEquals(ZstdCompressingInputStream.class, getInputStreamType(sfi)); + } + + @Test + public void infra_vespa_log() { + SyncFileInfo sfi = SyncFileInfo.infrastructureVespaLog(bucket, hostname, vespaLogPath); + assertEquals(Paths.get("/infrastructure/h12352a/logs/vespa/vespa.log-2021-02-12.zst"), sfi.destPath()); + assertEquals(ZstdCompressingInputStream.class, getInputStreamType(sfi)); + } + + @BeforeClass + public static void setup() throws IOException { + Files.createDirectories(vespaLogPath.getParent()); + Files.createFile(vespaLogPath); + Files.createDirectories(accessLogPath.getParent()); + Files.createFile(accessLogPath); + } + + private static Class<? extends InputStream> getInputStreamType(SyncFileInfo syncFileInfo) { + try (InputStream inputStream = syncFileInfo.inputStream()) { + return inputStream.getClass(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + +}
\ No newline at end of file diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java new file mode 100644 index 00000000000..be0df437b7f --- /dev/null +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/ZstdCompressingInputStreamTest.java @@ -0,0 +1,45 @@ +package com.yahoo.vespa.hosted.node.admin.maintenance.sync; + +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.Random; + +import static com.yahoo.vespa.hosted.node.admin.maintenance.sync.ZstdCompressingInputStream.compressor; +import static org.junit.Assert.assertArrayEquals; + +/** + * @author freva + */ +public class ZstdCompressingInputStreamTest { + + @Test + public void compression_test() throws Exception { + Random rnd = new Random(); + byte[] data = new byte[(int) (100_000 * (10 + rnd.nextDouble()))]; + rnd.nextBytes(data); + assertCompression(data, 1 << 14); + } + + private static void assertCompression(byte[] data, int bufferSize) { + ByteArrayInputStream bais = new ByteArrayInputStream(data); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ZstdCompressingInputStream zcis = new ZstdCompressingInputStream(bais, bufferSize)) { + byte[] buffer = new byte[bufferSize]; + for (int nRead; (nRead = zcis.read(buffer, 0, buffer.length)) != -1; ) + baos.write(buffer, 0, nRead); + baos.flush(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + + byte[] compressedData = baos.toByteArray(); + byte[] decompressedData = new byte[data.length]; + compressor.decompress(compressedData, 0, compressedData.length, decompressedData, 0, decompressedData.length); + + assertArrayEquals(data, decompressedData); + } +} diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index f1d910f2635..8efccf9b51b 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -26,6 +26,7 @@ #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/test/insertion_operators.h> #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/util/mmap_file_allocator_factory.h> #include <vespa/searchlib/util/bufferwriter.h> #include <vespa/log/log.h> @@ -256,6 +257,7 @@ struct FixtureTraits { bool use_direct_tensor_attribute = false; bool enable_hnsw_index = false; bool use_mock_index = false; + bool use_mmap_file_allocator = false; FixtureTraits dense() && { use_dense_tensor_attribute = true; @@ -263,6 +265,11 @@ struct FixtureTraits { return *this; } + FixtureTraits mmap_file_allocator() && { + use_mmap_file_allocator = true; + return *this; + } + FixtureTraits hnsw() && { use_dense_tensor_attribute = true; enable_hnsw_index = true; @@ -327,6 +334,9 @@ struct Fixture { if (_cfg.tensorType().is_dense()) { _denseTensors = true; } + if (_traits.use_mmap_file_allocator) { + _cfg.setHuge(true); + } if (_traits.use_mock_index) { _index_factory = std::make_unique<MockNearestNeighborIndexFactory>(); } else { @@ -998,4 +1008,17 @@ TEST_F("NN blueprint handles strong filter triggering brute force search", Neare EXPECT_FALSE(bp->may_approximate()); } +TEST("Dense tensor attribute with huge flag uses mmap file allocator") +{ + vespalib::string basedir("mmap-file-allocator-factory-dir"); + vespalib::alloc::MmapFileAllocatorFactory::instance().setup(basedir); + { + Fixture f(vec_2d_spec, FixtureTraits().dense().mmap_file_allocator()); + vespalib::string allocator_dir(basedir + "/0.my_attr"); + EXPECT_TRUE(vespalib::isDirectory(allocator_dir)); + } + vespalib::alloc::MmapFileAllocatorFactory::instance().setup(""); + vespalib::rmdir(basedir, true); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp b/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp index b5d4fb3130f..75e231a815c 100644 --- a/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp +++ b/searchlib/src/tests/tensor/dense_tensor_store/dense_tensor_store_test.cpp @@ -2,6 +2,7 @@ #include <vespa/log/log.h> LOG_SETUP("dense_tensor_store_test"); #include <vespa/vespalib/testkit/test_kit.h> +#include <vespa/vespalib/util/memory_allocator.h> #include <vespa/searchlib/tensor/dense_tensor_store.h> #include <vespa/eval/eval/simple_value.h> #include <vespa/eval/eval/tensor_spec.h> @@ -27,7 +28,7 @@ struct Fixture { DenseTensorStore store; Fixture(const vespalib::string &tensorType) - : store(ValueType::from_spec(tensorType)) + : store(ValueType::from_spec(tensorType), {}) {} void assertSetAndGetTensor(const TensorSpec &tensorSpec) { Value::UP expTensor = makeTensor(tensorSpec); diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp index be127a8be3e..0f39816257f 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_attribute.cpp @@ -10,6 +10,8 @@ #include <vespa/searchlib/attribute/load_utils.h> #include <vespa/searchlib/attribute/readerbase.h> #include <vespa/vespalib/data/slime/inserter.h> +#include <vespa/vespalib/util/memory_allocator.h> +#include <vespa/vespalib/util/mmap_file_allocator_factory.h> #include <vespa/log/log.h> LOG_SETUP(".searchlib.tensor.dense_tensor_attribute"); @@ -72,6 +74,15 @@ can_use_index_save_file(const search::attribute::Config &config, const search::a return true; } +std::unique_ptr<vespalib::alloc::MemoryAllocator> +make_memory_allocator(const vespalib::string& name, bool huge) +{ + if (huge) { + return vespalib::alloc::MmapFileAllocatorFactory::instance().make_memory_allocator(name); + } + return {}; +} + } void @@ -104,7 +115,7 @@ DenseTensorAttribute::memory_usage() const DenseTensorAttribute::DenseTensorAttribute(vespalib::stringref baseFileName, const Config& cfg, const NearestNeighborIndexFactory& index_factory) : TensorAttribute(baseFileName, cfg, _denseTensorStore), - _denseTensorStore(cfg.tensorType()), + _denseTensorStore(cfg.tensorType(), make_memory_allocator(getName(), cfg.huge())), _index() { if (cfg.hnsw_index_params().has_value()) { diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp index bbffc7ec4a5..36a803aa806 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.cpp @@ -3,6 +3,7 @@ #include "dense_tensor_store.h" #include <vespa/eval/eval/value.h> #include <vespa/vespalib/datastore/datastore.hpp> +#include <vespa/vespalib/util/memory_allocator.h> using vespalib::datastore::Handle; using vespalib::eval::Value; @@ -46,8 +47,9 @@ DenseTensorStore::TensorSizeCalc::alignedSize() const return my_align(bufSize(), DENSE_TENSOR_ALIGNMENT); } -DenseTensorStore::BufferType::BufferType(const TensorSizeCalc &tensorSizeCalc) - : vespalib::datastore::BufferType<char>(tensorSizeCalc.alignedSize(), MIN_BUFFER_ARRAYS, RefType::offsetSize()) +DenseTensorStore::BufferType::BufferType(const TensorSizeCalc &tensorSizeCalc, std::unique_ptr<vespalib::alloc::MemoryAllocator> allocator) + : vespalib::datastore::BufferType<char>(tensorSizeCalc.alignedSize(), MIN_BUFFER_ARRAYS, RefType::offsetSize()), + _allocator(std::move(allocator)) {} DenseTensorStore::BufferType::~BufferType() = default; @@ -59,11 +61,17 @@ DenseTensorStore::BufferType::cleanHold(void *buffer, size_t offset, memset(static_cast<char *>(buffer) + offset, 0, numElems); } -DenseTensorStore::DenseTensorStore(const ValueType &type) +const vespalib::alloc::MemoryAllocator* +DenseTensorStore::BufferType::get_memory_allocator() const +{ + return _allocator.get(); +} + +DenseTensorStore::DenseTensorStore(const ValueType &type, std::unique_ptr<vespalib::alloc::MemoryAllocator> allocator) : TensorStore(_concreteStore), _concreteStore(), _tensorSizeCalc(type), - _bufferType(_tensorSizeCalc), + _bufferType(_tensorSizeCalc, std::move(allocator)), _type(type), _emptySpace() { diff --git a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h index 3f27b4c5218..aa5a7993eaf 100644 --- a/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h +++ b/searchlib/src/vespa/searchlib/tensor/dense_tensor_store.h @@ -34,10 +34,12 @@ public: class BufferType : public vespalib::datastore::BufferType<char> { using CleanContext = vespalib::datastore::BufferType<char>::CleanContext; + std::unique_ptr<vespalib::alloc::MemoryAllocator> _allocator; public: - BufferType(const TensorSizeCalc &tensorSizeCalc); + BufferType(const TensorSizeCalc &tensorSizeCalc, std::unique_ptr<vespalib::alloc::MemoryAllocator> allocator); ~BufferType() override; void cleanHold(void *buffer, size_t offset, ElemCount numElems, CleanContext cleanCtx) override; + const vespalib::alloc::MemoryAllocator* get_memory_allocator() const override; }; private: DataStoreType _concreteStore; @@ -53,7 +55,7 @@ private: setDenseTensor(const TensorType &tensor); public: - DenseTensorStore(const ValueType &type); + DenseTensorStore(const ValueType &type, std::unique_ptr<vespalib::alloc::MemoryAllocator> allocator); ~DenseTensorStore() override; const ValueType &type() const { return _type; } diff --git a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/ProductionTest.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/ProductionTest.java index b9054689b00..100626ee091 100644 --- a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/ProductionTest.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/ProductionTest.java @@ -14,7 +14,7 @@ import static java.lang.annotation.RetentionPolicy.RUNTIME; * Tests that verify the health of production deployments of Vespa applications. * * Test classes annotated with this annotation are run during declared production tests. - * See <a href="https://cloud.vespa.ai/automated-deployments.html">Vespa cloud documentation</a>. + * See <a href="https://cloud.vespa.ai/en/automated-deployments">Vespa cloud documentation</a>. * * @author jonmv */ diff --git a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/StagingSetup.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/StagingSetup.java index bef3eabcef6..e7000933b9e 100644 --- a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/StagingSetup.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/StagingSetup.java @@ -15,7 +15,7 @@ import static java.lang.annotation.RetentionPolicy.RUNTIME; * * Test classes annotated with this annotation are run in the first phase of automated staging tests, * to make the initial deployment similar to a production one. - * See <a href="https://cloud.vespa.ai/automated-deployments.html">Vespa cloud documentation</a>. + * See <a href="https://cloud.vespa.ai/en/automated-deployments">Vespa cloud documentation</a>. * * @author jonmv */ diff --git a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/StagingTest.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/StagingTest.java index 59360b2753c..711c95a12fb 100644 --- a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/StagingTest.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/StagingTest.java @@ -16,7 +16,7 @@ import static java.lang.annotation.RetentionPolicy.RUNTIME; * * Test classes annotated with this annotation are run in the second phase of automated staging tests, * to verify the upgraded deployment. - * See <a href="https://cloud.vespa.ai/automated-deployments.html">Vespa cloud documentation</a>. + * See <a href="https://cloud.vespa.ai/en/automated-deployments">Vespa cloud documentation</a>. * * @author jonmv */ diff --git a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/SystemTest.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/SystemTest.java index f01f2ca6c90..525771d2f55 100644 --- a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/SystemTest.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/SystemTest.java @@ -17,7 +17,7 @@ import static java.lang.annotation.RetentionPolicy.RUNTIME; * Tests that compare the behaviour of a Vespa application deployment against a fixed specification. * * Test classes annotated with this annotation are run against a fresh deployment during automated system tests. - * See <a href="https://cloud.vespa.ai/automated-deployments.html">Vespa cloud documentation</a>. + * See <a href="https://cloud.vespa.ai/en/automated-deployments">Vespa cloud documentation</a>. * * @author jonmv */ diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java index b07c54399af..45fb4e54f0a 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java @@ -69,7 +69,7 @@ public class CommandLineOptions { options.addOption(Option.builder("f") .hasArg(true) - .desc("Retrieve the specified fields only (see https://docs.vespa.ai/documentation/documents.html#fieldsets) (default '" + AllFields.NAME + "')") + .desc("Retrieve the specified fields only (see https://docs.vespa.ai/en/documents.html#fieldsets) (default '" + AllFields.NAME + "')") .longOpt(FIELDSET_OPTION) .argName("fieldset").build()); diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java index b1f91e44e5c..adfbb246a9e 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java @@ -194,7 +194,7 @@ public class VdsVisit { .longOpt("fieldset") .hasArg(true) .argName("fieldset") - .desc("Retrieve the specified fields only (see https://docs.vespa.ai/documentation/documents.html#fieldsets). Default is [all].") + .desc("Retrieve the specified fields only (see https://docs.vespa.ai/en/documents.html#fieldsets). Default is [all].") .build()); options.addOption(Option.builder() diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java index 0fba2ca4875..fbf5bc35129 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java @@ -297,7 +297,7 @@ public interface Tensor { /** * Returns this tensor on the - * <a href="https://docs.vespa.ai/documentation/reference/tensor.html#tensor-literal-form">tensor literal form</a> + * <a href="https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form">tensor literal form</a> * with type included. */ @Override @@ -305,7 +305,7 @@ public interface Tensor { /** * Call this from toString in implementations to return this tensor on the - * <a href="https://docs.vespa.ai/documentation/reference/tensor.html#tensor-literal-form">tensor literal form</a>. + * <a href="https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form">tensor literal form</a>. * (toString cannot be a default method because default methods cannot override super methods). * * @param tensor the tensor to return the standard string format of @@ -377,7 +377,7 @@ public interface Tensor { /** * Returns a tensor instance containing the given data on the - * <a href="https://docs.vespa.ai/documentation/reference/tensor.html#tensor-literal-form">tensor literal form</a>. + * <a href="https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form">tensor literal form</a>. * * @param type the type of the tensor to return * @param tensorString the tensor on the standard tensor string format @@ -388,7 +388,7 @@ public interface Tensor { /** * Returns a tensor instance containing the given data on the - * <a href="https://docs.vespa.ai/documentation/reference/tensor.html#tensor-literal-form">tensor literal form</a>. + * <a href="https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form">tensor literal form</a>. * * @param tensorType the type of the tensor to return, as a string on the tensor type format, given in * {@link TensorType#fromSpec} @@ -400,7 +400,7 @@ public interface Tensor { /** * Returns a tensor instance containing the given data on the - * <a href="https://docs.vespa.ai/documentation/reference/tensor.html#tensor-literal-form">tensor literal form</a>. + * <a href="https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form">tensor literal form</a>. */ static Tensor from(String tensorString) { return TensorParser.tensorFrom(tensorString, Optional.empty()); diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java index 790743c745c..236e9d31c39 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java @@ -112,7 +112,7 @@ public class TensorType { /** * Returns a tensor type instance from a - * <a href="https://docs.vespa.ai/documentation/reference/tensor.html#tensor-type-spec">tensor type spec</a>: + * <a href="https://docs.vespa.ai/en/reference/tensor.html#tensor-type-spec">tensor type spec</a>: * <code>tensor(dimension1, dimension2, ...)</code> * where each dimension is either * <ul> diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java index c78be98e11e..fa2094e9d2a 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java @@ -20,8 +20,8 @@ import java.util.Iterator; /** * Writes tensors on the JSON format used in Vespa tensor document fields: * A JSON map containing a 'cells' or 'values' array. - * See <a href="https://docs.vespa.ai/documentation/reference/document-json-format.html"> - * https://docs.vespa.ai/documentation/reference/document-json-format.html</a> + * See <a href="https://docs.vespa.ai/en/reference/document-json-format.html"> + * https://docs.vespa.ai/en/reference/document-json-format.html</a> * * @author bratseth */ diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 92bb4f7622b..5bdeecfcb7f 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -132,6 +132,7 @@ vespa_define_module( src/tests/util/generationhandler src/tests/util/generationhandler_stress src/tests/util/md5 + src/tests/util/mmap_file_allocator_factory src/tests/util/rcuvector src/tests/util/reusable_set src/tests/valgrind diff --git a/vespalib/src/tests/util/mmap_file_allocator_factory/.gitignore b/vespalib/src/tests/util/mmap_file_allocator_factory/.gitignore new file mode 100644 index 00000000000..a18e9aac589 --- /dev/null +++ b/vespalib/src/tests/util/mmap_file_allocator_factory/.gitignore @@ -0,0 +1 @@ +/mmap-file-allocator-factory-dir diff --git a/vespalib/src/tests/util/mmap_file_allocator_factory/CMakeLists.txt b/vespalib/src/tests/util/mmap_file_allocator_factory/CMakeLists.txt new file mode 100644 index 00000000000..c6122659a22 --- /dev/null +++ b/vespalib/src/tests/util/mmap_file_allocator_factory/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_mmap_file_allocator_factory_test_app TEST + SOURCES + mmap_file_allocator_factory_test.cpp + DEPENDS + vespalib + GTest::GTest +) +vespa_add_test(NAME vespalib_mmap_file_allocator_factory_test_app COMMAND vespalib_mmap_file_allocator_factory_test_app) diff --git a/vespalib/src/tests/util/mmap_file_allocator_factory/mmap_file_allocator_factory_test.cpp b/vespalib/src/tests/util/mmap_file_allocator_factory/mmap_file_allocator_factory_test.cpp new file mode 100644 index 00000000000..f5c0317f623 --- /dev/null +++ b/vespalib/src/tests/util/mmap_file_allocator_factory/mmap_file_allocator_factory_test.cpp @@ -0,0 +1,49 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/util/mmap_file_allocator_factory.h> +#include <vespa/vespalib/util/mmap_file_allocator.h> +#include <vespa/vespalib/util/memory_allocator.h> +#include <vespa/vespalib/gtest/gtest.h> + +using vespalib::alloc::MemoryAllocator; +using vespalib::alloc::MmapFileAllocator; +using vespalib::alloc::MmapFileAllocatorFactory; + +namespace { + +vespalib::string basedir("mmap-file-allocator-factory-dir"); + +bool is_mmap_file_allocator(const MemoryAllocator *allocator) +{ + return dynamic_cast<const MmapFileAllocator *>(allocator) != nullptr; +} + +} + +TEST(MmapFileAllocatorFactoryTest, empty_dir_gives_no_allocator) +{ + MmapFileAllocatorFactory::instance().setup(""); + auto allocator = MmapFileAllocatorFactory::instance().make_memory_allocator("foo"); + EXPECT_FALSE(allocator); +} + +TEST(MmapFileAllocatorFactoryTest, nonempty_dir_gives_allocator) +{ + MmapFileAllocatorFactory::instance().setup(basedir); + auto allocator0 = MmapFileAllocatorFactory::instance().make_memory_allocator("foo"); + auto allocator1 = MmapFileAllocatorFactory::instance().make_memory_allocator("bar"); + EXPECT_TRUE(is_mmap_file_allocator(allocator0.get())); + EXPECT_TRUE(is_mmap_file_allocator(allocator1.get())); + vespalib::string allocator0_dir(basedir + "/0.foo"); + vespalib::string allocator1_dir(basedir + "/1.bar"); + EXPECT_TRUE(isDirectory(allocator0_dir)); + EXPECT_TRUE(isDirectory(allocator1_dir)); + allocator0.reset(); + EXPECT_FALSE(isDirectory(allocator0_dir)); + allocator1.reset(); + EXPECT_FALSE(isDirectory(allocator1_dir)); + MmapFileAllocatorFactory::instance().setup(""); + rmdir(basedir, true); +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index fcac7cde412..d934a7b38ca 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -32,6 +32,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT lz4compressor.cpp md5.c mmap_file_allocator.cpp + mmap_file_allocator_factory.cpp printable.cpp priority_queue.cpp random.cpp diff --git a/vespalib/src/vespa/vespalib/util/mmap_file_allocator_factory.cpp b/vespalib/src/vespa/vespalib/util/mmap_file_allocator_factory.cpp new file mode 100644 index 00000000000..a575c08309d --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/mmap_file_allocator_factory.cpp @@ -0,0 +1,44 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "mmap_file_allocator_factory.h" +#include "mmap_file_allocator.h" +#include <vespa/vespalib/stllike/asciistream.h> + +namespace vespalib::alloc { + +MmapFileAllocatorFactory::MmapFileAllocatorFactory() + : _dir_name(), + _generation(0) +{ +} + +MmapFileAllocatorFactory::~MmapFileAllocatorFactory() +{ +} + +void +MmapFileAllocatorFactory::setup(const vespalib::string& dir_name) +{ + _dir_name = dir_name; + _generation = 0; +} + +std::unique_ptr<MemoryAllocator> +MmapFileAllocatorFactory::make_memory_allocator(const vespalib::string& name) +{ + if (_dir_name.empty()) { + return {}; + } + vespalib::asciistream os; + os << _dir_name << "/" << _generation.fetch_add(1) << "." << name; + return std::make_unique<MmapFileAllocator>(os.str()); +}; + +MmapFileAllocatorFactory& +MmapFileAllocatorFactory::instance() +{ + static MmapFileAllocatorFactory instance; + return instance; +} + +} diff --git a/vespalib/src/vespa/vespalib/util/mmap_file_allocator_factory.h b/vespalib/src/vespa/vespalib/util/mmap_file_allocator_factory.h new file mode 100644 index 00000000000..b2586787845 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/mmap_file_allocator_factory.h @@ -0,0 +1,31 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/vespalib/stllike/string.h> +#include <memory> +#include <atomic> + +namespace vespalib::alloc { + +class MemoryAllocator; + +/* + * Class for creating an mmap file allocator on demand. + */ +class MmapFileAllocatorFactory { + vespalib::string _dir_name; + std::atomic<uint64_t> _generation; + + MmapFileAllocatorFactory(); + ~MmapFileAllocatorFactory(); + MmapFileAllocatorFactory(const MmapFileAllocatorFactory &) = delete; + MmapFileAllocatorFactory& operator=(const MmapFileAllocatorFactory &) = delete; +public: + void setup(const vespalib::string &dir_name); + std::unique_ptr<MemoryAllocator> make_memory_allocator(const vespalib::string& name); + + static MmapFileAllocatorFactory& instance(); +}; + +} |