diff options
66 files changed, 364 insertions, 231 deletions
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 50301066145..d5c611680dd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,7 +33,7 @@ We only seek to accept code that you are authorized to contribute to the project ## Versioning Vespa uses semantic versioning - see -[vespa versions](http://docs.vespa.ai/documentation/vespa-versions.html). +[vespa versions](http://docs.vespa.ai/en/vespa-versions.html). Notice in particular that any Java API in a package having a @PublicAPI annotation in the package-info file cannot be changed in an incompatible way between major versions: Existing types and method signatures must be preserved diff --git a/Code-map.md b/Code-map.md index 6eae723f31d..3c54be35b9d 100644 --- a/Code-map.md +++ b/Code-map.md @@ -12,7 +12,7 @@ thing we haven't done is to create a module structure friendly to newcomers - th simply organized in a flat structure of about 150 modules. This document aims to provide a map from the -[functional elements](https://docs.vespa.ai/documentation/overview.html) +[functional elements](https://docs.vespa.ai/en/overview.html) of Vespa to the most important modules in the flat module structure in the [code base on GitHub](https://github.com/vespa-engine/vespa). diff --git a/ERRATA.md b/ERRATA.md index 3107216f527..9278f3c39cf 100644 --- a/ERRATA.md +++ b/ERRATA.md @@ -31,7 +31,7 @@ This bug was introduced in Vespa-7.277.38, fixed in Vespa-7.292.82. The following needs to happen to trigger the bug: * visibility-delay is non-zero. Note that the default is zero, so for this to trigger, - [visibility-delay](https://docs.vespa.ai/documentation/reference/services-content.html#visibility-delay) + [visibility-delay](https://docs.vespa.ai/en/reference/services-content.html#visibility-delay) must have been set. * A new config change is deployed that contains changes to proton. This config snapshot is stored in the transaction log on the content node. diff --git a/README.md b/README.md index 3056f619982..280d3ae8ee8 100644 --- a/README.md +++ b/README.md @@ -33,18 +33,18 @@ Vespa per second. To get started using Vespa pick one of the quick start documents: -- [Run on a Mac or Linux machine using Docker](https://docs.vespa.ai/documentation/vespa-quick-start.html) -- [Run on a Windows machine using Docker](https://docs.vespa.ai/documentation/vespa-quick-start-windows.html) -- [Run on a Mac or Linux machine using VirtualBox+Vagrant](https://docs.vespa.ai/documentation/vespa-quick-start-centos.html) -- [Multinode install on AWS EC2](https://docs.vespa.ai/documentation/vespa-quick-start-multinode-aws.html) -- [Multinode install on AWS ECS](https://docs.vespa.ai/documentation/vespa-quick-start-multinode-aws-ecs.html) +- [Run on a Mac or Linux machine using Docker](https://docs.vespa.ai/en/vespa-quick-start.html) +- [Run on a Windows machine using Docker](https://docs.vespa.ai/en/vespa-quick-start-windows.html) +- [Run on a Mac or Linux machine using VirtualBox+Vagrant](https://docs.vespa.ai/en/vespa-quick-start-centos.html) +- [Multinode install on AWS EC2](https://docs.vespa.ai/en/vespa-quick-start-multinode-aws.html) +- [Multinode install on AWS ECS](https://docs.vespa.ai/en/vespa-quick-start-multinode-aws-ecs.html) ## Usage -- The application created in the quickstart is fully functional and production ready, but you may want to [add more nodes](https://docs.vespa.ai/documentation/multinode-systems.html) for redundancy. -- Try the [Blog search and recommendation tutorial](https://docs.vespa.ai/documentation/tutorials/blog-search.html) to learn more about using Vespa -- See [developing applications](https://docs.vespa.ai/documentation/jdisc/developing-applications.html) on adding your own Java components to your Vespa application. -- [Vespa APIs](https://docs.vespa.ai/documentation/api.html) is useful to understand how to interface with Vespa +- The application created in the quickstart is fully functional and production ready, but you may want to [add more nodes](https://docs.vespa.ai/en/multinode-systems.html) for redundancy. +- Try the [Blog search and recommendation tutorial](https://docs.vespa.ai/en/tutorials/blog-search.html) to learn more about using Vespa +- See [developing applications](https://docs.vespa.ai/en/jdisc/developing-applications.html) on adding your own Java components to your Vespa application. +- [Vespa APIs](https://docs.vespa.ai/en/api.html) is useful to understand how to interface with Vespa - Explore the [sample applications](https://github.com/vespa-engine/sample-apps/tree/master) Full documentation is available on [https://docs.vespa.ai](https://docs.vespa.ai). diff --git a/client/README.md b/client/README.md index d08b19a1a12..39d9bd6b669 100644 --- a/client/README.md +++ b/client/README.md @@ -1,7 +1,7 @@ # vespa_query_dsl This lib is used for composing vespa YQL queries -Reference: https://docs.vespa.ai/documentation/reference/query-language-reference.html +Reference: https://docs.vespa.ai/en/reference/query-language-reference.html # usage Please refer to the unit test: @@ -9,7 +9,7 @@ Please refer to the unit test: https://github.com/vespa-engine/vespa/tree/master/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy # todos -- [ ] support `predicate` (https://docs.vespa.ai/documentation/predicate-fields.html) +- [ ] support `predicate` (https://docs.vespa.ai/en/predicate-fields.html) - [ ] support methods for checking positive/negative conditions for specific field - [X] support order by annotation - [X] support order by diff --git a/client/src/main/java/ai/vespa/client/dsl/A.java b/client/src/main/java/ai/vespa/client/dsl/A.java index a691d1288e8..c1f94a08d25 100644 --- a/client/src/main/java/ai/vespa/client/dsl/A.java +++ b/client/src/main/java/ai/vespa/client/dsl/A.java @@ -8,7 +8,7 @@ import java.util.stream.Stream; /** * Helper class for generating Annotation - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#annotations + * https://docs.vespa.ai/en/reference/query-language-reference.html#annotations */ public final class A { diff --git a/client/src/main/java/ai/vespa/client/dsl/EndQuery.java b/client/src/main/java/ai/vespa/client/dsl/EndQuery.java index 2af1e0bb49d..d07d3536563 100644 --- a/client/src/main/java/ai/vespa/client/dsl/EndQuery.java +++ b/client/src/main/java/ai/vespa/client/dsl/EndQuery.java @@ -44,7 +44,7 @@ public class EndQuery { /** * Offset. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#limit-offset + * https://docs.vespa.ai/en/reference/query-language-reference.html#limit-offset * * @param offset the offset * @return the end query @@ -55,7 +55,7 @@ public class EndQuery { /** * Timeout. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#timeout + * https://docs.vespa.ai/en/reference/query-language-reference.html#timeout * * @param timeout the timeout * @return the end query @@ -66,7 +66,7 @@ public class EndQuery { /** * Limit. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#limit-offset + * https://docs.vespa.ai/en/reference/query-language-reference.html#limit-offset * * @param limit the limit * @return the end query @@ -87,7 +87,7 @@ public class EndQuery { /** * Group. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#grouping + * https://docs.vespa.ai/en/reference/query-language-reference.html#grouping * * @param group the group * @return the end query @@ -99,7 +99,7 @@ public class EndQuery { /** * Group. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#grouping + * https://docs.vespa.ai/en/reference/query-language-reference.html#grouping * * @param groupQueryStr the group str * @return the end query @@ -111,7 +111,7 @@ public class EndQuery { /** * Order by asc. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#order-by + * https://docs.vespa.ai/en/reference/query-language-reference.html#order-by * * @param annotation the annotation * @param fieldName the field name @@ -124,7 +124,7 @@ public class EndQuery { /** * Order by asc. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#order-by + * https://docs.vespa.ai/en/reference/query-language-reference.html#order-by * * @param fieldName the field name * @return the end query @@ -136,7 +136,7 @@ public class EndQuery { /** * Order by desc. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#order-by + * https://docs.vespa.ai/en/reference/query-language-reference.html#order-by * * @param annotation the annotation * @param fieldName the field name @@ -149,7 +149,7 @@ public class EndQuery { /** * Order by desc. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#order-by + * https://docs.vespa.ai/en/reference/query-language-reference.html#order-by * * @param fieldName the field name * @return the end query diff --git a/client/src/main/java/ai/vespa/client/dsl/Field.java b/client/src/main/java/ai/vespa/client/dsl/Field.java index 0cbe03a8fb9..b12ae51787d 100644 --- a/client/src/main/java/ai/vespa/client/dsl/Field.java +++ b/client/src/main/java/ai/vespa/client/dsl/Field.java @@ -27,7 +27,7 @@ public class Field extends QueryChain { /** * Contains query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param value the value * @return the query @@ -38,7 +38,7 @@ public class Field extends QueryChain { /** * Contains query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param annotation the annotation * @param value the value @@ -50,7 +50,7 @@ public class Field extends QueryChain { /** * Contains phrase query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param value the value * @param others the others @@ -62,7 +62,7 @@ public class Field extends QueryChain { /** * Contains phrase query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param values the values * @return the query @@ -77,7 +77,7 @@ public class Field extends QueryChain { /** * Contains near query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param value the value * @param others the others @@ -89,7 +89,7 @@ public class Field extends QueryChain { /** * Contains near query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param values the values * @return the query @@ -104,7 +104,7 @@ public class Field extends QueryChain { /** * Contains near query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param annotation the annotation * @param value the value @@ -117,7 +117,7 @@ public class Field extends QueryChain { /** * Contains near query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param annotation the annotation * @param values the values @@ -133,7 +133,7 @@ public class Field extends QueryChain { /** * Contains onear query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param value the value * @param others the others @@ -145,7 +145,7 @@ public class Field extends QueryChain { /** * Contains onear query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param values the values * @return the query @@ -160,7 +160,7 @@ public class Field extends QueryChain { /** * Contains onear query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param annotation the annotation * @param value the value @@ -173,7 +173,7 @@ public class Field extends QueryChain { /** * Contains onear query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param annotation the annotation * @param values the values @@ -189,7 +189,7 @@ public class Field extends QueryChain { /** * Contains same element query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param andQuery the and query * @return the query @@ -200,7 +200,7 @@ public class Field extends QueryChain { /** * Contains equiv query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param value the value * @param others the others @@ -212,7 +212,7 @@ public class Field extends QueryChain { /** * Contains equiv query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param values the values * @return the query @@ -230,7 +230,7 @@ public class Field extends QueryChain { /** * Contains uri query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param value the value * @return the query @@ -241,7 +241,7 @@ public class Field extends QueryChain { /** * Contains uri query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#contains + * https://docs.vespa.ai/en/reference/query-language-reference.html#contains * * @param annotation the annotation * @param value the value @@ -253,7 +253,7 @@ public class Field extends QueryChain { /** * Matches query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#matches + * https://docs.vespa.ai/en/reference/query-language-reference.html#matches * * @param str the str * @return the query @@ -264,7 +264,7 @@ public class Field extends QueryChain { /** * Equals query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#numeric + * https://docs.vespa.ai/en/reference/query-language-reference.html#numeric * * @param t the t * @return the query @@ -275,7 +275,7 @@ public class Field extends QueryChain { /** * Greater than or equal to query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#numeric + * https://docs.vespa.ai/en/reference/query-language-reference.html#numeric * * @param t the t * @return the query @@ -286,7 +286,7 @@ public class Field extends QueryChain { /** * Greater than query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#numeric + * https://docs.vespa.ai/en/reference/query-language-reference.html#numeric * * @param t the t * @return the query @@ -297,7 +297,7 @@ public class Field extends QueryChain { /** * Less than or equal to query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#numeric + * https://docs.vespa.ai/en/reference/query-language-reference.html#numeric * * @param t the t * @return the query @@ -308,7 +308,7 @@ public class Field extends QueryChain { /** * Less than query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#numeric + * https://docs.vespa.ai/en/reference/query-language-reference.html#numeric * * @param t the t * @return the query @@ -319,7 +319,7 @@ public class Field extends QueryChain { /** * In range query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#numeric + * https://docs.vespa.ai/en/reference/query-language-reference.html#numeric * * @param l the l * @param m the m @@ -331,7 +331,7 @@ public class Field extends QueryChain { /** * Equal to query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#numeric + * https://docs.vespa.ai/en/reference/query-language-reference.html#numeric * * @param t the t * @return the query @@ -342,7 +342,7 @@ public class Field extends QueryChain { /** * Greater than or equal to query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#numeric + * https://docs.vespa.ai/en/reference/query-language-reference.html#numeric * * @param t the t * @return the query @@ -353,7 +353,7 @@ public class Field extends QueryChain { /** * Greater than query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#numeric + * https://docs.vespa.ai/en/reference/query-language-reference.html#numeric * * @param t the t * @return the query @@ -364,7 +364,7 @@ public class Field extends QueryChain { /** * Less than or equal to query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#numeric + * https://docs.vespa.ai/en/reference/query-language-reference.html#numeric * * @param t the t * @return the query @@ -375,7 +375,7 @@ public class Field extends QueryChain { /** * Less than query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#numeric + * https://docs.vespa.ai/en/reference/query-language-reference.html#numeric * * @param t the t * @return the query @@ -386,7 +386,7 @@ public class Field extends QueryChain { /** * In range query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#numeric + * https://docs.vespa.ai/en/reference/query-language-reference.html#numeric * * @param l the l * @param m the m @@ -399,7 +399,7 @@ public class Field extends QueryChain { /** * Is true query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#boolean + * https://docs.vespa.ai/en/reference/query-language-reference.html#boolean * * @return the query */ @@ -409,7 +409,7 @@ public class Field extends QueryChain { /** * Is false query. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#boolean + * https://docs.vespa.ai/en/reference/query-language-reference.html#boolean * * @return the query */ diff --git a/client/src/main/java/ai/vespa/client/dsl/FixedQuery.java b/client/src/main/java/ai/vespa/client/dsl/FixedQuery.java index b41dd2c546b..94c571dba5a 100644 --- a/client/src/main/java/ai/vespa/client/dsl/FixedQuery.java +++ b/client/src/main/java/ai/vespa/client/dsl/FixedQuery.java @@ -12,7 +12,7 @@ import java.util.stream.Collectors; /** * FixedQuery contains a 'Query' which is terminated by a 'semicolon' * This object holds vespa or user defined parameters - * https://docs.vespa.ai/documentation/reference/search-api-reference.html + * https://docs.vespa.ai/en/reference/search-api-reference.html */ public class FixedQuery { diff --git a/client/src/main/java/ai/vespa/client/dsl/G.java b/client/src/main/java/ai/vespa/client/dsl/G.java index 4a82658d3e0..c65bc982a88 100644 --- a/client/src/main/java/ai/vespa/client/dsl/G.java +++ b/client/src/main/java/ai/vespa/client/dsl/G.java @@ -4,7 +4,7 @@ package ai.vespa.client.dsl; /** * Helper class for generating group syntax - * https://docs.vespa.ai/documentation/reference/grouping-syntax.html + * https://docs.vespa.ai/en/reference/grouping-syntax.html * * basically the syntax is exactly the same as Vespa group syntax. * The only exception "max" in the Vespa group syntax which represents 'max returned documents', diff --git a/client/src/main/java/ai/vespa/client/dsl/Q.java b/client/src/main/java/ai/vespa/client/dsl/Q.java index 4048e6b8869..7637f76f095 100644 --- a/client/src/main/java/ai/vespa/client/dsl/Q.java +++ b/client/src/main/java/ai/vespa/client/dsl/Q.java @@ -8,7 +8,7 @@ import java.util.Map; /** * Helper class for generating Vespa search queries - * https://docs.vespa.ai/documentation/reference/query-language-reference.html + * https://docs.vespa.ai/en/reference/query-language-reference.html */ public final class Q { @@ -54,7 +54,7 @@ public final class Q { /** * Rank rank. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#rank + * https://docs.vespa.ai/en/reference/query-language-reference.html#rank * * @param query the query * @param ranks the ranks @@ -66,7 +66,7 @@ public final class Q { /** * UI represents "userInput". - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#userinput + * https://docs.vespa.ai/en/reference/query-language-reference.html#userinput * * @param value the value * @return the user input query @@ -77,7 +77,7 @@ public final class Q { /** * userInput with an annotation. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#userinput + * https://docs.vespa.ai/en/reference/query-language-reference.html#userinput * * @param a the a * @param value the value @@ -89,7 +89,7 @@ public final class Q { /** * A convenience method to generate userInput with default index annotation. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#userinput + * https://docs.vespa.ai/en/reference/query-language-reference.html#userinput * * @param index the index * @param value the value @@ -101,7 +101,7 @@ public final class Q { /** * dotPdt represents "dotProduct". - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#dotproduct + * https://docs.vespa.ai/en/reference/query-language-reference.html#dotproduct * * @param field the field * @param weightedSet the weighted set @@ -113,7 +113,7 @@ public final class Q { /** * wtdSet represents "weightedSet". - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#weightedset + * https://docs.vespa.ai/en/reference/query-language-reference.html#weightedset * * @param field the field * @param weightedSet the weighted set @@ -125,7 +125,7 @@ public final class Q { /** * NonEmpty non empty. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#nonempty + * https://docs.vespa.ai/en/reference/query-language-reference.html#nonempty * * @param query the query * @return the non empty query @@ -136,7 +136,7 @@ public final class Q { /** * Wand wand. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#wand + * https://docs.vespa.ai/en/reference/query-language-reference.html#wand * * @param field the field * @param weightedSet the weighted set @@ -148,7 +148,7 @@ public final class Q { /** * Wand wand. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#wand + * https://docs.vespa.ai/en/reference/query-language-reference.html#wand * * @param field the field * @param numericRange the numeric range @@ -160,7 +160,7 @@ public final class Q { /** * Weakand weak and. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#weakand + * https://docs.vespa.ai/en/reference/query-language-reference.html#weakand * * @param field the field * @param query the query diff --git a/client/src/main/java/ai/vespa/client/dsl/Query.java b/client/src/main/java/ai/vespa/client/dsl/Query.java index 62f92c51bee..9339733195a 100644 --- a/client/src/main/java/ai/vespa/client/dsl/Query.java +++ b/client/src/main/java/ai/vespa/client/dsl/Query.java @@ -90,7 +90,7 @@ public class Query extends QueryChain { /** * And. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#and + * https://docs.vespa.ai/en/reference/query-language-reference.html#and * * @param fieldName the field name * @return the field @@ -105,7 +105,7 @@ public class Query extends QueryChain { /** * Andnot. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#andnot + * https://docs.vespa.ai/en/reference/query-language-reference.html#andnot * * @param fieldName the field name * @return the field @@ -120,7 +120,7 @@ public class Query extends QueryChain { /** * Or. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#or + * https://docs.vespa.ai/en/reference/query-language-reference.html#or * * @param fieldName the field name * @return the field @@ -135,7 +135,7 @@ public class Query extends QueryChain { /** * And. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#and + * https://docs.vespa.ai/en/reference/query-language-reference.html#and * * @param query the query * @return the query @@ -149,7 +149,7 @@ public class Query extends QueryChain { /** * Andnot. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#andnot + * https://docs.vespa.ai/en/reference/query-language-reference.html#andnot * * @param query the query * @return the query @@ -163,7 +163,7 @@ public class Query extends QueryChain { /** * Or. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#or + * https://docs.vespa.ai/en/reference/query-language-reference.html#or * * @param query the query * @return the query @@ -177,7 +177,7 @@ public class Query extends QueryChain { /** * Annotate a query (sub-expression). - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#annotations-of-sub-expressions + * https://docs.vespa.ai/en/reference/query-language-reference.html#annotations-of-sub-expressions * * @param annotation the annotation * @return the query @@ -189,7 +189,7 @@ public class Query extends QueryChain { /** * Offset. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#limit-offset + * https://docs.vespa.ai/en/reference/query-language-reference.html#limit-offset * * @param offset the offset * @return the end query @@ -200,7 +200,7 @@ public class Query extends QueryChain { /** * Limit. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#limit-offset + * https://docs.vespa.ai/en/reference/query-language-reference.html#limit-offset * * @param hits the hits * @return the end query @@ -211,7 +211,7 @@ public class Query extends QueryChain { /** * Timeout. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#timeout + * https://docs.vespa.ai/en/reference/query-language-reference.html#timeout * * @param timeout the timeout * @return the end query @@ -222,7 +222,7 @@ public class Query extends QueryChain { /** * Group. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#grouping + * https://docs.vespa.ai/en/reference/query-language-reference.html#grouping * * @param group the group * @return the end query @@ -233,7 +233,7 @@ public class Query extends QueryChain { /** * Group. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#grouping + * https://docs.vespa.ai/en/reference/query-language-reference.html#grouping * * @param groupStr the group str * @return the end query @@ -244,7 +244,7 @@ public class Query extends QueryChain { /** * Order by asc. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#order-by + * https://docs.vespa.ai/en/reference/query-language-reference.html#order-by * * @param fieldName the field name * @return the end query @@ -255,7 +255,7 @@ public class Query extends QueryChain { /** * Order by asc. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#order-by + * https://docs.vespa.ai/en/reference/query-language-reference.html#order-by * * @param annotation the annotation * @param fieldName the field name @@ -267,7 +267,7 @@ public class Query extends QueryChain { /** * Order by desc. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#order-by + * https://docs.vespa.ai/en/reference/query-language-reference.html#order-by * * @param fieldName the field name * @return the end query @@ -279,7 +279,7 @@ public class Query extends QueryChain { /** * Order by desc. - * https://docs.vespa.ai/documentation/reference/query-language-reference.html#order-by + * https://docs.vespa.ai/en/reference/query-language-reference.html#order-by * * @param annotation the annotation * @param fieldName the field name diff --git a/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy b/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy index 19c87d6aecd..3353ef6f827 100644 --- a/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy +++ b/client/src/test/groovy/ai/vespa/client/dsl/QTest.groovy @@ -439,7 +439,7 @@ class QTest extends Specification { def "basic group syntax"() { /* example from vespa document: - https://docs.vespa.ai/documentation/grouping.html + https://docs.vespa.ai/en/grouping.html all( group(a) max(5) each(output(count()) all(max(1) each(output(summary()))) all(group(b) each(output(count()) @@ -470,7 +470,7 @@ class QTest extends Specification { def "set group syntax string directly"() { /* example from vespa document: - https://docs.vespa.ai/documentation/grouping.html + https://docs.vespa.ai/en/grouping.html all( group(a) max(5) each(output(count()) all(max(1) each(output(summary()))) all(group(b) each(output(count()) diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java index f22cf0d8c47..8845431c71b 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ValidationOverrides.java @@ -87,7 +87,7 @@ public class ValidationOverrides { public static String toAllowMessage(ValidationId id) { return "To allow this add <allow until='yyyy-mm-dd'>" + id + "</allow> to validation-overrides.xml" + - ", see https://docs.vespa.ai/documentation/reference/validation-overrides.html"; + ", see https://docs.vespa.ai/en/reference/validation-overrides.html"; } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java index 4736a207659..a001d66bc7b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/QueryProfiles.java @@ -70,7 +70,7 @@ public class QueryProfiles implements Serializable, QueryProfilesConfig.Producer ? "" : "In particular, the tensors (" + String.join(", ", tensorFields) + ") will be interpreted as strings, not tensors if sent in requests. ") + - "See https://docs.vespa.ai/documentation/query-profiles.html"); + "See https://docs.vespa.ai/en/query-profiles.html"); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index c1c9058472d..0c0c3f59e88 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -384,7 +384,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { if (deployState.endpointCertificateSecrets().isPresent()) { boolean authorizeClient = deployState.zone().system().isPublic(); if (authorizeClient && deployState.tlsClientAuthority().isEmpty()) { - throw new RuntimeException("Client certificate authority security/clients.pem is missing - see: https://cloud.vespa.ai/security-model#data-plane"); + throw new RuntimeException("Client certificate authority security/clients.pem is missing - see: https://cloud.vespa.ai/en/security-model#data-plane"); } EndpointCertificateSecrets endpointCertificateSecrets = deployState.endpointCertificateSecrets().get(); @@ -722,7 +722,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { var hosts = hostSystem.allocateHosts(clusterSpec, capacity, log); return createNodesFromHosts(log, hosts, cluster); } - return singleHostContainerCluster(cluster, hostSystem.getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC), context); + else { + return singleHostContainerCluster(cluster, hostSystem.getHost(Container.SINGLENODE_CONTAINER_SERVICESPEC), context); + } } private List<ApplicationContainer> singleHostContainerCluster(ApplicationContainerCluster cluster, HostResource host, ConfigModelContext context) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java index 0f9eb5341ab..3b694f8986c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.model.content; /** * Tuning of dispatching to content nodes, see the - * <a href="https://docs.vespa.ai/documentation/reference/services-content.html#dispatch-tuning">dispatch tuning documentation</a>. + * <a href="https://docs.vespa.ai/en/reference/services-content.html#dispatch-tuning">dispatch tuning documentation</a>. * * @author Simon Thoresen Hult */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java index c36d3c1dabb..91ad413adb3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java @@ -15,13 +15,16 @@ import static java.lang.Long.min; public class NodeResourcesTuning implements ProtonConfig.Producer { final static long MB = 1024 * 1024; - final static long GB = MB * 1024; + public final static long GB = MB * 1024; private final NodeResources resources; private final int redundancy; private final int searchableCopies; private final int threadsPerSearch; private final boolean combined; + // "Reserve" 1GB of memory for other processes running on the content node (config-proxy, cluster-controller, metrics-proxy). + public static final double reservedMemoryGb = 1; + public NodeResourcesTuning(NodeResources resources, int redundancy, int searchableCopies, @@ -45,7 +48,6 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { tuneSummaryReadIo(builder.summary.read); tuneSummaryCache(builder.summary.cache); tuneSearchReadIo(builder.search.mmap); - tuneWriteFilter(builder.writefilter); for (ProtonConfig.Documentdb.Builder dbb : builder.documentdb) { getConfig(dbb); } @@ -121,20 +123,15 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { builder.numthreadspersearch(threadsPerSearch); } - private void tuneWriteFilter(ProtonConfig.Writefilter.Builder builder) { - // "Reserve" 1GB of memory for other processes running on the content node (config-proxy, cluster-controller, metrics-proxy) - double reservedMemoryGb = 1; - double defaultMemoryLimit = new ProtonConfig.Writefilter(new ProtonConfig.Writefilter.Builder()).memorylimit(); - double scaledMemoryLimit = ((usableMemoryGb() - reservedMemoryGb) * defaultMemoryLimit) / usableMemoryGb(); - builder.memorylimit(scaledMemoryLimit); - } - /** Returns the memory we can expect will be available for the content node processes */ private double usableMemoryGb() { - if ( ! combined ) return resources.memoryGb(); + double usableMemoryGb = resources.memoryGb() - reservedMemoryGb; + if (!combined) { + return usableMemoryGb; + } double fractionTakenByContainer = (double)ApplicationContainerCluster.heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster / 100; - return resources.memoryGb() * (1 - fractionTakenByContainer); + return usableMemoryGb * (1 - fractionTakenByContainer); } } diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 5a71f83e468..3e6a3332f9a 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -48,6 +48,8 @@ import java.util.stream.Collectors; import static com.yahoo.config.model.test.TestUtil.joinLines; import static com.yahoo.vespa.defaults.Defaults.getDefaults; +import static com.yahoo.vespa.model.search.NodeResourcesTuning.GB; +import static com.yahoo.vespa.model.search.NodeResourcesTuning.reservedMemoryGb; import static org.hamcrest.CoreMatchers.is; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -269,7 +271,7 @@ public class ModelProvisioningTest { assertEquals("Heap size is lowered with combined clusters", 17, physicalMemoryPercentage(model.getContainerClusters().get("container1"))); assertEquals("Memory for proton is lowered to account for the jvm heap", - (long)(3 * (Math.pow(1024, 3)) * (1 - 0.17)), protonMemorySize(model.getContentClusters().get("content1"))); + (long)((3 - reservedMemoryGb) * (Math.pow(1024, 3)) * (1 - 0.17)), protonMemorySize(model.getContentClusters().get("content1"))); assertProvisioned(0, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model); assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Id.from("container1"), ClusterSpec.Type.combined, model); } @@ -305,7 +307,7 @@ public class ModelProvisioningTest { assertEquals("Heap size is normal", 60, physicalMemoryPercentage(model.getContainerClusters().get("container1"))); assertEquals("Memory for proton is normal", - (long)(3 * (Math.pow(1024, 3))), protonMemorySize(model.getContentClusters().get("content1"))); + (long)((3 - reservedMemoryGb) * (Math.pow(1024, 3))), protonMemorySize(model.getContentClusters().get("content1"))); } } @@ -2002,12 +2004,10 @@ public class ModelProvisioningTest { ProtonConfig cfg = getProtonConfig(model, cluster.getSearchNodes().get(0).getConfigId()); assertEquals(2000, cfg.flush().memory().maxtlssize()); // from config override assertEquals(1000, cfg.flush().memory().maxmemory()); // from explicit tuning - assertEquals((long) 16 * GB, cfg.flush().memory().each().maxmemory()); // from default node flavor tuning + assertEquals((long) (128 - reservedMemoryGb) * GB / 8, cfg.flush().memory().each().maxmemory()); // from default node flavor tuning assertEquals(0.92, cfg.writefilter().memorylimit(), 0.0001); // from explicit resource-limits } - private static long GB = 1024 * 1024 * 1024; - private static ProtonConfig getProtonConfig(VespaModel model, String configId) { ProtonConfig.Builder builder = new ProtonConfig.Builder(); model.getConfig(builder, configId); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java index ba9dfcdc388..d5c7fb78c89 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/IndexingModeChangeValidatorTest.java @@ -37,7 +37,7 @@ public class IndexingModeChangeValidatorTest { catch (ValidationException e) { assertEquals("indexing-mode-change:\n" + "\tDocument type 'music' in cluster 'default' changed indexing mode from 'indexed' to 'streaming'\n" + - "To allow this add <allow until='yyyy-mm-dd'>indexing-mode-change</allow> to validation-overrides.xml, see https://docs.vespa.ai/documentation/reference/validation-overrides.html", + "To allow this add <allow until='yyyy-mm-dd'>indexing-mode-change</allow> to validation-overrides.xml, see https://docs.vespa.ai/en/reference/validation-overrides.html", e.getMessage()); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/QueryProfilesTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/QueryProfilesTestCase.java index d1d32df5b56..fc6c5aeb387 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/QueryProfilesTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/QueryProfilesTestCase.java @@ -151,7 +151,7 @@ public class QueryProfilesTestCase { assertEquals(1, logger.entries.size()); assertEquals("This application define query profile types, but has no query profiles referencing them " + "so they have no effect. " + - "See https://docs.vespa.ai/documentation/query-profiles.html", + "See https://docs.vespa.ai/en/query-profiles.html", logger.entries.get(0).message); } @@ -171,7 +171,7 @@ public class QueryProfilesTestCase { assertEquals("This application define query profile types, but has no query profiles referencing them " + "so they have no effect. " + "In particular, the tensors (vector, matrix) will be interpreted as strings, not tensors if sent in requests. " + - "See https://docs.vespa.ai/documentation/query-profiles.html", + "See https://docs.vespa.ai/en/query-profiles.html", logger.entries.get(0).message); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 35257686a5a..e0e4cbf19c0 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -728,7 +728,8 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { .build(); createModel(root, state, null, clusterElem); } catch (RuntimeException e) { - assertEquals(e.getMessage(), "Client certificate authority security/clients.pem is missing - see: https://cloud.vespa.ai/security-model#data-plane"); + assertEquals("Client certificate authority security/clients.pem is missing - see: https://cloud.vespa.ai/en/security-model#data-plane", + e.getMessage()); return; } fail(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java index 5df522e1b9d..1f9c9b1e07a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java @@ -20,6 +20,8 @@ import static com.yahoo.vespa.model.search.NodeResourcesTuning.GB; public class NodeResourcesTuningTest { private static double delta = 0.00001; + private static double combinedFactor = 1 - 17.0/100; + private static int reservedMemoryGb = (int)NodeResourcesTuning.reservedMemoryGb; @Test public void require_that_hwinfo_disk_size_is_set() { @@ -29,9 +31,13 @@ public class NodeResourcesTuningTest { @Test public void require_that_hwinfo_memory_size_is_set() { - double combinedFactor = 1 - 17.0/100; - assertEquals(24 * GB, configFromMemorySetting(24, false).hwinfo().memory().size()); - assertEquals(combinedFactor * 24 * GB, configFromMemorySetting(24, true).hwinfo().memory().size(), 1000); + assertEquals(24 * GB, configFromMemorySetting(24 + reservedMemoryGb, false).hwinfo().memory().size()); + assertEquals(combinedFactor * 24 * GB, configFromMemorySetting(24 + reservedMemoryGb, true).hwinfo().memory().size(), 1000); + } + + @Test + public void reserved_memory_on_content_node_is_1_gb() { + assertEquals(1.0, NodeResourcesTuning.reservedMemoryGb, delta); } private ProtonConfig getProtonMemoryConfig(List<Pair<String, String>> sdAndMode, int gb, int redundancy, int searchableCopies) { @@ -47,13 +53,13 @@ public class NodeResourcesTuningTest { private void verify_that_initial_numdocs_is_dependent_of_mode(int redundancy, int searchablecopies) { int divisor = Math.max(redundancy, searchablecopies); - ProtonConfig cfg = getProtonMemoryConfig(Arrays.asList(new Pair<>("a", "INDEX"), new Pair<>("b", "STREAMING"), new Pair<>("c", "STORE_ONLY")), 24, redundancy, searchablecopies); + ProtonConfig cfg = getProtonMemoryConfig(Arrays.asList(new Pair<>("a", "INDEX"), new Pair<>("b", "STREAMING"), new Pair<>("c", "STORE_ONLY")), 24 + reservedMemoryGb, redundancy, searchablecopies); assertEquals(3, cfg.documentdb().size()); assertEquals(1024, cfg.documentdb(0).allocation().initialnumdocs()); assertEquals("a", cfg.documentdb(0).inputdoctypename()); - assertEquals(402653184/divisor, cfg.documentdb(1).allocation().initialnumdocs()); + assertEquals(24 * GB / 64 / divisor, cfg.documentdb(1).allocation().initialnumdocs()); assertEquals("b", cfg.documentdb(1).inputdoctypename()); - assertEquals(402653184/divisor, cfg.documentdb(2).allocation().initialnumdocs()); + assertEquals(24 * GB / 64 / divisor, cfg.documentdb(2).allocation().initialnumdocs()); assertEquals("c", cfg.documentdb(2).inputdoctypename()); } @@ -148,15 +154,14 @@ public class NodeResourcesTuningTest { @Test public void require_that_summary_cache_max_bytes_is_set_based_on_memory() { - assertEquals(1*GB / 20, configFromMemorySetting(1, false).summary().cache().maxbytes()); - assertEquals(256*GB / 20, configFromMemorySetting(256, false).summary().cache().maxbytes()); + assertEquals(1*GB / 20, configFromMemorySetting(1 + reservedMemoryGb, false).summary().cache().maxbytes()); + assertEquals(256*GB / 20, configFromMemorySetting(256 + reservedMemoryGb, false).summary().cache().maxbytes()); } @Test public void require_that_summary_cache_memory_is_reduced_with_combined_cluster() { - double combinedFactor = 1 - 17.0/100; - assertEquals(combinedFactor * 1*GB / 20, configFromMemorySetting(1, true).summary().cache().maxbytes(), 1000); - assertEquals(combinedFactor * 256*GB / 20, configFromMemorySetting(256, true).summary().cache().maxbytes(), 1000); + assertEquals(combinedFactor * 1*GB / 20, configFromMemorySetting(1 + reservedMemoryGb, true).summary().cache().maxbytes(), 1000); + assertEquals(combinedFactor * 256*GB / 20, configFromMemorySetting(256 + reservedMemoryGb, true).summary().cache().maxbytes(), 1000); } @Test @@ -164,21 +169,13 @@ public class NodeResourcesTuningTest { assertSharedDisk(true, true); } - @Test - public void require_that_write_filter_memory_limit_is_scaled() { - assertWriteFilter(0.7, 8); - assertWriteFilter(0.75, 16); - assertWriteFilter(0.775, 32); - assertWriteFilter(0.7875, 64); - } - - private static void assertDocumentStoreMaxFileSize(long expFileSizeBytes, int memoryGb) { - assertEquals(expFileSizeBytes, configFromMemorySetting(memoryGb, false).summary().log().maxfilesize()); + private static void assertDocumentStoreMaxFileSize(long expFileSizeBytes, int wantedMemoryGb) { + assertEquals(expFileSizeBytes, configFromMemorySetting(wantedMemoryGb + reservedMemoryGb, false).summary().log().maxfilesize()); } - private static void assertFlushStrategyMemory(long expMemoryBytes, int memoryGb) { - assertEquals(expMemoryBytes, configFromMemorySetting(memoryGb, false).flush().memory().maxmemory()); - assertEquals(expMemoryBytes, configFromMemorySetting(memoryGb, false).flush().memory().each().maxmemory()); + private static void assertFlushStrategyMemory(long expMemoryBytes, int wantedMemoryGb) { + assertEquals(expMemoryBytes, configFromMemorySetting(wantedMemoryGb + reservedMemoryGb, false).flush().memory().maxmemory()); + assertEquals(expMemoryBytes, configFromMemorySetting(wantedMemoryGb + reservedMemoryGb, false).flush().memory().each().maxmemory()); } private static void assertFlushStrategyTlsSize(long expTlsSizeBytes, int diskGb) { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java index 6b4a4f2a073..21c6cc8e795 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeResources.java @@ -220,6 +220,9 @@ public class NodeResources { } @Override public String toString() { + if (isUnspecified()) + return "unspecified resources"; + StringBuilder sb = new StringBuilder("[vcpu: "); appendDouble(sb, vcpu); sb.append(", memory: "); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java index 79d70f1cb4e..9e232a6533f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/provision/ProvisionerAdapter.java @@ -33,9 +33,9 @@ public class ProvisionerAdapter implements HostProvisioner { @Override public HostSpec allocateHost(String alias) { - // Wow. Such mess. TODO: Actually support polymorphy or stop pretending to, see also ModelContextImpl.getHostProvisioner + // TODO: Remove this method since hosted/non-hosted needs different interfaces. See also ModelContextImpl.getHostProvisioner throw new UnsupportedOperationException("Allocating hosts using <node> tags is not supported in hosted environments, " + - "use <nodes count='N'> instead, see https://cloud.vespa.ai/reference/services"); + "use <nodes count='N'> instead, see https://cloud.vespa.ai/en/reference/services"); } @Override diff --git a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java index 0fd31db67f6..236602b7767 100644 --- a/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java +++ b/container-core/src/main/java/com/yahoo/container/core/config/HandlersConfigurerDi.java @@ -48,6 +48,9 @@ public class HandlersConfigurerDi { private static final Logger log = Logger.getLogger(HandlersConfigurerDi.class.getName()); + private static final Executor fallbackExecutor = Executors.newCachedThreadPool( + ThreadFactoryFactory.getThreadFactory("HandlersConfigurerDI")); + private final com.yahoo.container.Container vespaContainer; private final Container container; @@ -138,10 +141,12 @@ public class HandlersConfigurerDi { return discInjector.createChildInjector(new AbstractModule() { @Override protected void configure() { + // Provide a singleton instance for all component fallbacks, + // otherwise fallback injection may lead to a cascade of components requiring reconstruction. bind(com.yahoo.container.Container.class).toInstance(vespaContainer); bind(com.yahoo.statistics.Statistics.class).toInstance(Statistics.nullImplementation); - bind(AccessLog.class).toInstance(new AccessLog(new ComponentRegistry<>())); - bind(Executor.class).toInstance(Executors.newCachedThreadPool(ThreadFactoryFactory.getThreadFactory("HandlersConfigurerDI"))); + bind(AccessLog.class).toInstance(AccessLog.NONE_INSTANCE); + bind(Executor.class).toInstance(fallbackExecutor); if (vespaContainer.getFileAcquirer() != null) bind(com.yahoo.filedistribution.fileacquirer.FileAcquirer.class).toInstance(vespaContainer.getFileAcquirer()); } diff --git a/container-search-gui/pom.xml b/container-search-gui/pom.xml index 867f2464b8b..64c53a1f453 100644 --- a/container-search-gui/pom.xml +++ b/container-search-gui/pom.xml @@ -72,7 +72,7 @@ <goal>wget</goal> </goals> <configuration> - <url>https://docs.vespa.ai/documentation/reference/search-api-reference.html</url> + <url>https://docs.vespa.ai/en/reference/search-api-reference.html</url> <outputFileName>search-api-reference.html</outputFileName> <outputDirectory>${project.build.outputDirectory}/gui/_includes/</outputDirectory> </configuration> diff --git a/container-search-gui/src/main/resources/gui/_includes/index.html b/container-search-gui/src/main/resources/gui/_includes/index.html index a441e302720..5eaa2b790d5 100644 --- a/container-search-gui/src/main/resources/gui/_includes/index.html +++ b/container-search-gui/src/main/resources/gui/_includes/index.html @@ -64,11 +64,11 @@ <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> <ul class="nav navbar-nav navbar-right"> <li class="hidden"><a href="#page-top"></a> - <li><a href="http://blog.vespa.ai/">Blog</a> + <li><a href="https://blog.vespa.ai/">Blog</a> <li><a href="https://twitter.com/vespaengine">Twitter</a> - <li><a href="http://docs.vespa.ai">Docs</a> + <li><a href="https://docs.vespa.ai">Docs</a> <li><a href="https://github.com/vespa-engine">GitHub</a> - <li><a href="http://docs.vespa.ai/documentation/vespa-quick-start.html">Get Started Now</a> + <li><a href="https://docs.vespa.ai/en/vespa-quick-start.html">Get Started Now</a> </ul> </div> </div> @@ -128,7 +128,7 @@ <span> â—‹ Autocompletion of YQL-syntax</span> </br> <span> â—‹ Drop-down lists of all valid parameters</span> </br> <span> â—‹ Sending both POST and GET-requests to <i>Vespa</i></span> </br> - <span> â—‹ Easy access to the <a href="https://docs.vespa.ai/documentation/reference/search-api-reference.html">documentation</a> of each parameter</span> </br> + <span> â—‹ Easy access to the <a href="https://docs.vespa.ai/en/reference/search-api-reference.html">documentation</a> of each parameter</span> </br> <span> â—‹ Conversion of POST- to GET-query</span> </br> <span> â—‹ Pasting already built JSON-query</span> </br> <span> â—‹ View and copy the response of queries</span> </br> @@ -151,8 +151,8 @@ <div class="col-xs-2 quicklink-section"> <div class="footer-title">Resources</div> <ul class="quicklinks"> - <li><a href="http://docs.vespa.ai/documentation/vespa-quick-start.html">Getting Started</a> - <li><a href="http://docs.vespa.ai">Documentation</a> + <li><a href="https://docs.vespa.ai/en/vespa-quick-start.html">Getting Started</a> + <li><a href="https://docs.vespa.ai">Documentation</a> <li><a href="https://github.com/vespa-engine/vespa">Open source</a> </ul> </div> diff --git a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java index 663cb44215d..a639f2368a1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java +++ b/container-search/src/main/java/com/yahoo/prelude/fastsearch/FastHit.java @@ -195,7 +195,7 @@ public class FastHit extends Hit { /** * Returns values for the features listed in - * <a href="https://docs.vespa.ai/documentation/reference/schema-reference.html#summary-features">summary-features</a> + * <a href="https://docs.vespa.ai/en/reference/schema-reference.html#summary-features">summary-features</a> * in the rank profile specified in the query producing this. */ public FeatureData features() { diff --git a/container-search/src/main/java/com/yahoo/search/query/Select.java b/container-search/src/main/java/com/yahoo/search/query/Select.java index 53628978db3..d90550084eb 100644 --- a/container-search/src/main/java/com/yahoo/search/query/Select.java +++ b/container-search/src/main/java/com/yahoo/search/query/Select.java @@ -79,7 +79,7 @@ public class Select implements Cloneable { * Sets the document selection criterion of the query. * * @param where the documents to select as a JSON string on the format specified in - * <a href="https://docs.vespa.ai/documentation/reference/select-reference.html">the select reference doc</a> + * <a href="https://docs.vespa.ai/en/reference/select-reference.html">the select reference doc</a> */ public void setWhereString(String where) { this.where = where; @@ -96,7 +96,7 @@ public class Select implements Cloneable { * Sets the grouping operation of the query. * * @param grouping the grouping to perform as a JSON string on the format specified in - * <a href="https://docs.vespa.ai/documentation/reference/select-reference.html">the select reference doc</a> + * <a href="https://docs.vespa.ai/en/reference/select-reference.html">the select reference doc</a> */ public void setGroupingString(String grouping) { groupingRequests.clear(); diff --git a/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java index 058f6b93ae3..d40e1cfc4a6 100644 --- a/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/querytransform/WeakAndReplacementSearcher.java @@ -41,7 +41,7 @@ public class WeakAndReplacementSearcher extends Searcher { * Recursively iterates over an Item to replace all instances of OrItems with WeakAndItems * @param item the current item in the replacement iteration * @param hits the wand.hits property from the request which is assigned to the N value of the new WeakAndItem - * @return The original item or a WeakAndItem replacement of an OrItem + * @return the original item or a WeakAndItem replacement of an OrItem */ private Item replaceOrItems(Item item, int hits) { if (!(item instanceof CompositeItem)) { @@ -63,4 +63,5 @@ public class WeakAndReplacementSearcher extends Searcher { } return compositeItem; } + } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index 76ac0611328..2a8dc34ea72 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -36,7 +36,8 @@ enum PathGroup { "/zone/v2/{*}", "/routing/v1/", "/routing/v1/status/environment/{*}", - "/routing/v1/inactive/environment/{*}"), + "/routing/v1/inactive/environment/{*}", + "/state/v1/{*}"), /** Paths used for creating and reading user resources. */ user(PathPrefix.api, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 63452f40dbb..07c0a6ca122 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -930,7 +930,7 @@ public class ControllerTest { "[endpoint 'default' (cluster foo) -> us-east-3, us-west-1] and add " + "[endpoint 'default' (cluster bar) -> us-east-3, us-west-1]. To allow this add " + "<allow until='yyyy-mm-dd'>global-endpoint-change</allow> to validation-overrides.xml, see " + - "https://docs.vespa.ai/documentation/reference/validation-overrides.html", e.getMessage()); + "https://docs.vespa.ai/en/reference/validation-overrides.html", e.getMessage()); } // Redeploy with override succeeds diff --git a/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java b/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java index 51ed93bb6ee..e2beafa66f2 100644 --- a/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java +++ b/document/src/main/java/com/yahoo/document/json/DocumentUpdateJsonSerializer.java @@ -60,7 +60,7 @@ import static com.yahoo.document.json.JsonSerializationHelper.*; /** * The DocumentUpdateJsonSerializer utility class is used to serialize a DocumentUpdate instance using the JSON format described in - * <a href="https://docs.vespa.ai/documentation/reference/document-json-format.html#update">Document JSON Format: The Update Structure</a> + * <a href="https://docs.vespa.ai/en/reference/document-json-format.html#update">Document JSON Format: The Update Structure</a> * * @see #serialize(com.yahoo.document.DocumentUpdate) * @author Vegard Sjonfjell diff --git a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java index ffc276fc94c..ad016a40fca 100644 --- a/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java +++ b/document/src/main/java/com/yahoo/document/json/readers/TensorReader.java @@ -14,7 +14,7 @@ import static com.yahoo.document.json.readers.JsonParserHelpers.*; /** * Reads the tensor format defined at - * See <a href="https://docs.vespa.ai/documentation/reference/document-json-format.html">https://docs.vespa.ai/documentation/reference/document-json-format.html</a> + * See <a href="https://docs.vespa.ai/en/reference/document-json-format.html">https://docs.vespa.ai/en/reference/document-json-format.html</a> * * @author geirst * @author bratseth diff --git a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/NodeState.java b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/NodeState.java index d23a0e580a0..0102d6cba36 100755 --- a/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/NodeState.java +++ b/documentapi/src/main/java/com/yahoo/documentapi/messagebus/systemstate/rule/NodeState.java @@ -22,7 +22,7 @@ public class NodeState { /** A location string that expresses the use of THIS node. */ public static final String NODE_CURRENT = "."; - private static Logger log = Logger.getLogger(NodeState.class.getName()); + private static final Logger log = Logger.getLogger(NodeState.class.getName()); private final Map<String, NodeState> children = new LinkedHashMap<String, NodeState>(); private final Map<String, String> state = new LinkedHashMap<String, String>(); private NodeState parent = null; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index aca3da1ffe8..484e3864b0f 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -200,7 +200,7 @@ public class Flags { public static final UnboundBooleanFlag USE_POWER_OF_TWO_CHOICES_LOAD_BALANCING = defineFeatureFlag( "use-power-of-two-choices-load-balancing", false, - List.of("tokle"), "2020-12-02", "2021-02-15", + List.of("tokle"), "2020-12-02", "2021-04-01", "Whether to use Power of two load balancing algorithm for application", "Takes effect on next internal redeployment", APPLICATION_ID); diff --git a/jdisc_http_service/src/main/java/com/yahoo/container/logging/AccessLog.java b/jdisc_http_service/src/main/java/com/yahoo/container/logging/AccessLog.java index cdb4febb775..2d46c53bca7 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/container/logging/AccessLog.java +++ b/jdisc_http_service/src/main/java/com/yahoo/container/logging/AccessLog.java @@ -13,6 +13,8 @@ import com.yahoo.component.provider.ComponentRegistry; */ public class AccessLog implements RequestLog { + public static final AccessLog NONE_INSTANCE = new AccessLog(new ComponentRegistry<>()); + private final ComponentRegistry<RequestLogHandler> implementers; @Inject @@ -21,7 +23,7 @@ public class AccessLog implements RequestLog { } public static AccessLog voidAccessLog() { - return new AccessLog(new ComponentRegistry<>()); + return NONE_INSTANCE; } @Override diff --git a/jdisc_http_service/src/main/java/com/yahoo/container/logging/JSONFormatter.java b/jdisc_http_service/src/main/java/com/yahoo/container/logging/JSONFormatter.java index 441e139bc67..680ee5acbd9 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/container/logging/JSONFormatter.java +++ b/jdisc_http_service/src/main/java/com/yahoo/container/logging/JSONFormatter.java @@ -15,8 +15,6 @@ import java.util.Objects; import java.util.logging.Level; import java.util.logging.Logger; -import static com.yahoo.container.logging.FormatUtil.writeSecondsField; - /** * Formatting of an {@link AccessLogEntry} in the Vespa JSON access log format. * @@ -64,9 +62,14 @@ public class JSONFormatter implements LogWriter<RequestLogEntry> { generator.writeStringField("connection", connectionId); } - Principal principal = entry.userPrincipal().orElse(null); - if (principal != null) { - generator.writeStringField("user-principal", principal.getName()); + Principal userPrincipal = entry.userPrincipal().orElse(null); + if (userPrincipal != null) { + generator.writeStringField("user-principal", userPrincipal.getName()); + } + + Principal sslPrincipal = entry.sslPrincipal().orElse(null); + if (sslPrincipal != null) { + generator.writeStringField("ssl-principal", sslPrincipal.getName()); } String remoteAddress = entry.remoteAddress().orElse(null); diff --git a/jdisc_http_service/src/main/java/com/yahoo/container/logging/RequestLogEntry.java b/jdisc_http_service/src/main/java/com/yahoo/container/logging/RequestLogEntry.java index b771ea11ed0..819907fc9f1 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/container/logging/RequestLogEntry.java +++ b/jdisc_http_service/src/main/java/com/yahoo/container/logging/RequestLogEntry.java @@ -43,6 +43,7 @@ public class RequestLogEntry { private final String rawPath; private final String rawQuery; private final Principal userPrincipal; + private final Principal sslPrincipal; private final HitCounts hitCounts; private final TraceNode traceNode; private final Map<String, Collection<String>> extraAttributes; @@ -67,6 +68,7 @@ public class RequestLogEntry { this.rawPath = builder.rawPath; this.rawQuery = builder.rawQuery; this.userPrincipal = builder.userPrincipal; + this.sslPrincipal = builder.sslPrincipal; this.hitCounts = builder.hitCounts; this.traceNode = builder.traceNode; this.extraAttributes = copyExtraAttributes(builder.extraAttributes); @@ -91,6 +93,7 @@ public class RequestLogEntry { public Optional<String> rawPath() { return Optional.ofNullable(rawPath); } public Optional<String> rawQuery() { return Optional.ofNullable(rawQuery); } public Optional<Principal> userPrincipal() { return Optional.ofNullable(userPrincipal); } + public Optional<Principal> sslPrincipal() { return Optional.ofNullable(sslPrincipal); } public Optional<HitCounts> hitCounts() { return Optional.ofNullable(hitCounts); } public Optional<TraceNode> traceNode() { return Optional.ofNullable(traceNode); } public Collection<String> extraAttributeKeys() { return Collections.unmodifiableCollection(extraAttributes.keySet()); } @@ -135,6 +138,7 @@ public class RequestLogEntry { private Principal userPrincipal; private HitCounts hitCounts; private TraceNode traceNode; + private Principal sslPrincipal; private final Map<String, Collection<String>> extraAttributes = new HashMap<>(); public Builder connectionId(String connectionId) { this.connectionId = requireNonNull(connectionId); return this; } @@ -156,6 +160,7 @@ public class RequestLogEntry { public Builder rawPath(String rawPath) { this.rawPath = requireNonNull(rawPath); return this; } public Builder rawQuery(String rawQuery) { this.rawQuery = requireNonNull(rawQuery); return this; } public Builder userPrincipal(Principal userPrincipal) { this.userPrincipal = requireNonNull(userPrincipal); return this; } + public Builder sslPrincipal(Principal sslPrincipal) { this.sslPrincipal = requireNonNull(sslPrincipal); return this; } public Builder hitCounts(HitCounts hitCounts) { this.hitCounts = requireNonNull(hitCounts); return this; } public Builder traceNode(TraceNode traceNode) { this.traceNode = requireNonNull(traceNode); return this; } public Builder addExtraAttribute(String key, String value) { diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java index b38b9dcdfb2..4b023f427a4 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/server/jetty/AccessLogRequestLog.java @@ -14,6 +14,7 @@ import org.eclipse.jetty.util.component.AbstractLifeCycle; import javax.servlet.http.HttpServletRequest; import java.security.Principal; +import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; import java.util.List; @@ -101,6 +102,10 @@ class AccessLogRequestLog extends AbstractLifeCycle implements org.eclipse.jetty builder.addExtraAttribute(header, value); } }); + X509Certificate[] clientCert = (X509Certificate[]) request.getAttribute(ServletRequest.SERVLET_REQUEST_X509CERT); + if (clientCert != null && clientCert.length > 0) { + builder.sslPrincipal(clientCert[0].getSubjectX500Principal()); + } AccessLogEntry accessLogEntry = (AccessLogEntry) request.getAttribute(JDiscHttpServlet.ATTRIBUTE_NAME_ACCESS_LOG_ENTRY); if (accessLogEntry != null) { diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/tensorflow/VariableConverter.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/tensorflow/VariableConverter.java index 31cb60b5509..c0db65c5c99 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/tensorflow/VariableConverter.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/tensorflow/VariableConverter.java @@ -37,7 +37,7 @@ class VariableConverter { if ( args.length != 3) { System.out.println("Converts a TensorFlow variable into Vespa tensor document field value JSON:"); System.out.println("A JSON map containing a 'cells' array, see"); - System.out.println("http://docs.vespa.ai/documentation/reference/document-json-put-format.html#tensor)"); + System.out.println("https://docs.vespa.ai/en/reference/document-json-put-format.html#tensor)"); System.out.println(""); System.out.println("Arguments: modelDirectory tensorFlowVariableName orderedTypeSpec"); System.out.println(" - modelDirectory: The directory of the TensorFlow SavedModel"); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index 1ca8b5782b8..dd6f10f616e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -8,8 +8,6 @@ import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.TenantName; -import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; -import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance; import com.yahoo.vespa.hosted.provision.lb.LoadBalancers; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.Allocation; @@ -22,12 +20,9 @@ import com.yahoo.vespa.hosted.provision.node.Status; import java.time.Instant; import java.util.Arrays; -import java.util.Comparator; -import java.util.LinkedHashSet; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.TreeSet; /** * A node in the node repository. The identity of a node is given by its id. @@ -212,6 +207,16 @@ public final class Node implements Nodelike { return withWantToRetire(wantToRetire, status.wantToDeprovision(), agent, at); } + /** Returns a copy of this node with preferToRetire set to given value and updated history */ + public Node withPreferToRetire(boolean preferToRetire, Agent agent, Instant at) { + if (preferToRetire == status.preferToRetire()) return this; + Node node = this.with(status.withPreferToRetire(preferToRetire)); + if (preferToRetire) { + node = node.with(history.with(new History.Event(History.Event.Type.preferToRetire, agent, at))); + } + return node; + } + /** * Returns a copy of this node which is retired. * If the node was already retired it is returned as-is. @@ -225,7 +230,7 @@ public final class Node implements Nodelike { /** Returns a copy of this node which is retired */ public Node retire(Instant retiredAt) { - if (status.wantToRetire()) + if (status.wantToRetire() || status.preferToRetire()) return retire(Agent.system, retiredAt); else return retire(Agent.application, retiredAt); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java index 84aafa77c27..bba4e93616e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeList.java @@ -168,9 +168,9 @@ public class NodeList extends AbstractFilteringList<Node, NodeList> { /** Returns the subset of nodes which have a record of being down */ public NodeList down() { return matching(Node::isDown); } - /** Returns the subset of nodes which wantToRetire set true */ - public NodeList wantToRetire() { - return matching(node -> node.status().wantToRetire()); + /** Returns the subset of nodes which have retirement requested */ + public NodeList retirementRequested() { + return matching(node -> node.status().wantToRetire() || node.status().preferToRetire()); } /** Returns the parent nodes of the given child nodes */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java index 860076dd111..2e5a772efd9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/MaintenanceDeployment.java @@ -122,7 +122,7 @@ class MaintenanceDeployment implements Closeable { @Override public void close() { - lock.ifPresent(l -> l.close()); + lock.ifPresent(Mutex::close); closed = true; } @@ -160,7 +160,7 @@ class MaintenanceDeployment implements Closeable { try (MaintenanceDeployment deployment = new MaintenanceDeployment(application, deployer, metric, nodeRepository)) { if ( ! deployment.isValid()) return false; - boolean couldMarkRetiredNow = markWantToRetire(node, true, agent, nodeRepository); + boolean couldMarkRetiredNow = markPreferToRetire(node, true, agent, nodeRepository); if ( ! couldMarkRetiredNow) return false; Optional<Node> expectedNewNode = Optional.empty(); @@ -182,7 +182,7 @@ class MaintenanceDeployment implements Closeable { return true; } finally { - markWantToRetire(node, false, agent, nodeRepository); // Necessary if this failed, no-op otherwise + markPreferToRetire(node, false, agent, nodeRepository); // Necessary if this failed, no-op otherwise // Immediately clean up if we reserved the node but could not activate or reserved a node on the wrong host expectedNewNode.flatMap(node -> nodeRepository.nodes().node(node.hostname(), Node.State.reserved)) @@ -191,17 +191,17 @@ class MaintenanceDeployment implements Closeable { } } - /** Returns true only if this operation changes the state of the wantToRetire flag */ - private boolean markWantToRetire(Node node, boolean wantToRetire, Agent agent, NodeRepository nodeRepository) { + /** Returns true only if this operation changes the state of the preferToRetire flag */ + private boolean markPreferToRetire(Node node, boolean preferToRetire, Agent agent, NodeRepository nodeRepository) { Optional<NodeMutex> nodeMutex = nodeRepository.nodes().lockAndGet(node); if (nodeMutex.isEmpty()) return false; try (var nodeLock = nodeMutex.get()) { if (nodeLock.node().state() != Node.State.active) return false; - if (nodeLock.node().status().wantToRetire() == wantToRetire) return false; + if (nodeLock.node().status().preferToRetire() == preferToRetire) return false; - nodeRepository.nodes().write(nodeLock.node().withWantToRetire(wantToRetire, agent, nodeRepository.clock().instant()), nodeLock); + nodeRepository.nodes().write(nodeLock.node().withPreferToRetire(preferToRetire, agent, nodeRepository.clock().instant()), nodeLock); return true; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMover.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMover.java index 58711e14d7f..609d1f74526 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMover.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/NodeMover.java @@ -69,10 +69,10 @@ public abstract class NodeMover<MOVE> extends NodeRepositoryMaintainer { /** Returns true if no active nodes are retiring or about to be retired */ static boolean zoneIsStable(NodeList allNodes) { - NodeList active = allNodes.state(Node.State.active); - if (active.stream().anyMatch(node -> node.allocation().get().membership().retired())) return false; - if (active.stream().anyMatch(node -> node.status().wantToRetire())) return false; - return true; + return allNodes.state(Node.State.active).stream() + .noneMatch(node -> node.allocation().get().membership().retired() || + node.status().wantToRetire() || + node.status().preferToRetire()); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java index 88e64331c82..1543506a78e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/Rebalancer.java @@ -68,7 +68,7 @@ public class Rebalancer extends NodeMover<Rebalancer.Move> { HostCapacity capacity = new HostCapacity(allNodes, nodeRepository().resourcesCalculator()); double totalSkew = 0; int hostCount = 0; - for (Node host : allNodes.nodeType((NodeType.host)).state(Node.State.active)) { + for (Node host : allNodes.nodeType(NodeType.host).state(Node.State.active)) { hostCount++; totalSkew += Node.skew(host.flavor().resources(), capacity.freeCapacityOf(host)); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java index ca580753fc8..79f3dad75d3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainer.java @@ -155,7 +155,7 @@ public class SpareCapacityMaintainer extends NodeRepositoryMaintainer { if (!NodeMover.zoneIsStable(allNodes)) return; - // Find an active node on a overcommited host and retire it + // Find an active node on a overcommitted host and retire it Optional<Node> nodeToRetire = overcommittedHosts.stream().flatMap(parent -> allNodes.childrenOf(parent).stream()) .filter(node -> node.state() == Node.State.active) .min(this::retireOvercomittedComparator); @@ -170,7 +170,7 @@ public class SpareCapacityMaintainer extends NodeRepositoryMaintainer { if (nodeWithWantToRetire.isEmpty()) return; nodeRepository().nodes().write(nodeWithWantToRetire.get(), deployment.applicationLock().get()); - log.log(Level.INFO, String.format("Redeploying %s to relocate %s from overcommited host", + log.log(Level.INFO, String.format("Redeploying %s to move %s from overcommitted host", application, nodeToRetire.get().hostname())); deployment.activate(); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java index 3c2541bac27..158ad88a968 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/History.java @@ -155,7 +155,9 @@ public class History { // The node was failed failed(false), // The node was breakfixed - breakfixed(false); + breakfixed(false), + // The node was scheduled to be moved + preferToRetire(false); private final boolean applicationLevel; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java index 71e60c92cb0..43fc07ea2c3 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Status.java @@ -21,6 +21,7 @@ public class Status { private final int failCount; private final boolean wantToRetire; private final boolean wantToDeprovision; + private final boolean preferToRetire; private final OsVersion osVersion; private final Optional<Instant> firmwareVerifiedAt; @@ -30,6 +31,7 @@ public class Status { int failCount, boolean wantToRetire, boolean wantToDeprovision, + boolean preferToRetire, OsVersion osVersion, Optional<Instant> firmwareVerifiedAt) { this.reboot = Objects.requireNonNull(generation, "Generation must be non-null"); @@ -41,45 +43,46 @@ public class Status { } this.wantToRetire = wantToRetire; this.wantToDeprovision = wantToDeprovision; + this.preferToRetire = preferToRetire; this.osVersion = Objects.requireNonNull(osVersion, "OS version must be non-null"); this.firmwareVerifiedAt = Objects.requireNonNull(firmwareVerifiedAt, "Firmware check instant must be non-null"); } /** Returns a copy of this with the reboot generation changed */ - public Status withReboot(Generation reboot) { return new Status(reboot, vespaVersion, containerImage, failCount, wantToRetire, wantToDeprovision, osVersion, firmwareVerifiedAt); } + public Status withReboot(Generation reboot) { return new Status(reboot, vespaVersion, containerImage, failCount, wantToRetire, wantToDeprovision, preferToRetire, osVersion, firmwareVerifiedAt); } /** Returns the reboot generation of this node */ public Generation reboot() { return reboot; } /** Returns a copy of this with the vespa version changed */ - public Status withVespaVersion(Version version) { return new Status(reboot, Optional.of(version), containerImage, failCount, wantToRetire, wantToDeprovision, osVersion, firmwareVerifiedAt); } + public Status withVespaVersion(Version version) { return new Status(reboot, Optional.of(version), containerImage, failCount, wantToRetire, wantToDeprovision, preferToRetire, osVersion, firmwareVerifiedAt); } /** Returns the Vespa version installed on the node, if known */ public Optional<Version> vespaVersion() { return vespaVersion; } /** Returns a copy of this with the container image changed */ - public Status withContainerImage(DockerImage containerImage) { return new Status(reboot, vespaVersion, Optional.of(containerImage), failCount, wantToRetire, wantToDeprovision, osVersion, firmwareVerifiedAt); } + public Status withContainerImage(DockerImage containerImage) { return new Status(reboot, vespaVersion, Optional.of(containerImage), failCount, wantToRetire, wantToDeprovision, preferToRetire, osVersion, firmwareVerifiedAt); } /** Returns the container image the node is running, if any */ public Optional<DockerImage> containerImage() { return containerImage; } - public Status withIncreasedFailCount() { return new Status(reboot, vespaVersion, containerImage, failCount + 1, wantToRetire, wantToDeprovision, osVersion, firmwareVerifiedAt); } + public Status withIncreasedFailCount() { return new Status(reboot, vespaVersion, containerImage, failCount + 1, wantToRetire, wantToDeprovision, preferToRetire, osVersion, firmwareVerifiedAt); } - public Status withDecreasedFailCount() { return new Status(reboot, vespaVersion, containerImage, failCount - 1, wantToRetire, wantToDeprovision, osVersion, firmwareVerifiedAt); } + public Status withDecreasedFailCount() { return new Status(reboot, vespaVersion, containerImage, failCount - 1, wantToRetire, wantToDeprovision, preferToRetire, osVersion, firmwareVerifiedAt); } - public Status withFailCount(int value) { return new Status(reboot, vespaVersion, containerImage, value, wantToRetire, wantToDeprovision, osVersion, firmwareVerifiedAt); } + public Status withFailCount(int value) { return new Status(reboot, vespaVersion, containerImage, value, wantToRetire, wantToDeprovision, preferToRetire, osVersion, firmwareVerifiedAt); } /** Returns how many times this node has been moved to the failed state. */ public int failCount() { return failCount; } /** Returns a copy of this with the want to retire/deprovision flags changed */ public Status withWantToRetire(boolean wantToRetire, boolean wantToDeprovision) { - return new Status(reboot, vespaVersion, containerImage, failCount, wantToRetire, wantToDeprovision, osVersion, firmwareVerifiedAt); + return new Status(reboot, vespaVersion, containerImage, failCount, wantToRetire, wantToDeprovision, preferToRetire, osVersion, firmwareVerifiedAt); } /** - * Returns whether this node should be retired at some point in the future. It does NOT indicate whether the node - * is actually retired. + * Returns whether this node is requested to retire. This is a hard request to retire, which allows any replacement + * to increase node skew in the cluster. */ public boolean wantToRetire() { return wantToRetire; @@ -92,9 +95,22 @@ public class Status { return wantToDeprovision; } + /** + * Returns whether this node is requested to retire. Unlike {@link Status#wantToRetire()}, this is a soft + * request to retire, which will not allow any replacement to increase node skew in the cluster. + */ + public boolean preferToRetire() { + return preferToRetire; + } + + /** Returns a copy of this with prefer-to-retire set to given value */ + public Status withPreferToRetire(boolean preferToRetire) { + return new Status(reboot, vespaVersion, containerImage, failCount, wantToRetire, wantToDeprovision, preferToRetire, osVersion, firmwareVerifiedAt); + } + /** Returns a copy of this with the OS version set to given version */ public Status withOsVersion(OsVersion version) { - return new Status(reboot, vespaVersion, containerImage, failCount, wantToRetire, wantToDeprovision, version, firmwareVerifiedAt); + return new Status(reboot, vespaVersion, containerImage, failCount, wantToRetire, wantToDeprovision, preferToRetire, version, firmwareVerifiedAt); } /** Returns the OS version of this node */ @@ -104,7 +120,7 @@ public class Status { /** Returns a copy of this with the firmwareVerifiedAt set to the given instant. */ public Status withFirmwareVerifiedAt(Instant instant) { - return new Status(reboot, vespaVersion, containerImage, failCount, wantToRetire, wantToDeprovision, osVersion, Optional.of(instant)); + return new Status(reboot, vespaVersion, containerImage, failCount, wantToRetire, wantToDeprovision, preferToRetire, osVersion, Optional.of(instant)); } /** Returns the last time this node had firmware that was verified to be up to date. */ @@ -115,7 +131,7 @@ public class Status { /** Returns the initial status of a newly provisioned node */ public static Status initial() { return new Status(Generation.initial(), Optional.empty(), Optional.empty(), 0, false, - false, OsVersion.EMPTY, Optional.empty()); + false, false, OsVersion.EMPTY, Optional.empty()); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 2d2ea05dc44..2d0ef78c9ba 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -82,6 +82,7 @@ public class NodeSerializer { private static final String nodeTypeKey = "type"; private static final String wantToRetireKey = "wantToRetire"; private static final String wantToDeprovisionKey = "wantToDeprovision"; + private static final String preferToRetireKey = "preferToRetire"; private static final String osVersionKey = "osVersion"; private static final String wantedOsVersionKey = "wantedOsVersion"; private static final String firmwareCheckKey = "firmwareCheck"; @@ -161,6 +162,7 @@ public class NodeSerializer { node.status().containerImage().ifPresent(image -> object.setString(currentContainerImageKey, image.asString())); object.setLong(failCountKey, node.status().failCount()); object.setBool(wantToRetireKey, node.status().wantToRetire()); + object.setBool(preferToRetireKey, node.status().preferToRetire()); object.setBool(wantToDeprovisionKey, node.status().wantToDeprovision()); node.allocation().ifPresent(allocation -> toSlime(allocation, object.setObject(instanceKey))); toSlime(node.history(), object.setArray(historyKey)); @@ -269,6 +271,7 @@ public class NodeSerializer { (int) object.field(failCountKey).asLong(), object.field(wantToRetireKey).asBool(), object.field(wantToDeprovisionKey).asBool(), + object.field(preferToRetireKey).asBool(), new OsVersion(versionFromSlime(object.field(osVersionKey)), versionFromSlime(object.field(wantedOsVersionKey))), instantFromSlime(object.field(firmwareCheckKey))); @@ -421,6 +424,7 @@ public class NodeSerializer { case "osUpgraded" : return History.Event.Type.osUpgraded; case "firmwareVerified" : return History.Event.Type.firmwareVerified; case "breakfixed" : return History.Event.Type.breakfixed; + case "preferToRetire" : return History.Event.Type.preferToRetire; } throw new IllegalArgumentException("Unknown node event type '" + eventTypeString + "'"); } @@ -444,6 +448,7 @@ public class NodeSerializer { case osUpgraded: return "osUpgraded"; case firmwareVerified: return "firmwareVerified"; case breakfixed: return "breakfixed"; + case preferToRetire: return "preferToRetire"; } throw new IllegalArgumentException("Serialized form of '" + nodeEventType + "' not defined"); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index bc164dc37e0..042d227f8f1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -115,7 +115,7 @@ class NodeAllocation { if ((! saturated() && hasCompatibleFlavor(candidate) && requestedNodes.acceptable(candidate)) || acceptToRetire) { candidate = candidate.withNode(); if (candidate.isValid()) - accepted.add(acceptNode(candidate, shouldRetire(candidate), resizeable)); + accepted.add(acceptNode(candidate, shouldRetire(candidate, nodesPrioritized), resizeable)); } } else if (! saturated() && hasCompatibleFlavor(candidate)) { @@ -146,7 +146,7 @@ class NodeAllocation { return accepted; } - private boolean shouldRetire(NodeCandidate candidate) { + private boolean shouldRetire(NodeCandidate candidate, List<NodeCandidate> candidates) { if ( ! requestedNodes.considerRetiring()) return candidate.allocation().map(a -> a.membership().retired()).orElse(false); // don't second-guess if already retired @@ -154,6 +154,7 @@ class NodeAllocation { if (violatesParentHostPolicy(candidate)) return true; if ( ! hasCompatibleFlavor(candidate)) return true; if (candidate.wantToRetire()) return true; + if (candidate.preferToRetire() && !candidate.replacementIncreasesSkew(candidates)) return true; if (violatesExclusivity(candidate)) return true; return false; } @@ -226,13 +227,13 @@ class NodeAllocation { return requestedNodes.isCompatible(candidate.flavor(), nodeRepository.flavors()) || candidate.isResizable; } - private Node acceptNode(NodeCandidate candidate, boolean wantToRetire, boolean resizeable) { + private Node acceptNode(NodeCandidate candidate, boolean shouldRetire, boolean resizeable) { Node node = candidate.toNode(); if (node.allocation().isPresent()) // Record the currently requested resources node = node.with(node.allocation().get().withRequestedResources(requestedNodes.resources().orElse(node.resources()))); - if (! wantToRetire) { + if (! shouldRetire) { accepted++; // We want to allocate new nodes rather than unretiring with resize, so count without those diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java index 460b7a821e6..aad65ef5646 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeCandidate.java @@ -77,6 +77,8 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat public abstract boolean wantToRetire(); + public abstract boolean preferToRetire(); + public abstract Flavor flavor(); public abstract NodeCandidate allocate(ApplicationId owner, ClusterMembership membership, NodeResources requestedResources, Instant at); @@ -97,6 +99,21 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat /** Returns whether this node can - as far as we know - be used to run the application workload */ public abstract boolean isValid(); + /** Returns whether replacing this with any of the reserved candidates will increase skew */ + public boolean replacementIncreasesSkew(List<NodeCandidate> candidates) { + return candidates.stream() + .filter(candidate -> candidate.state() == Node.State.reserved) + .allMatch(reserved -> { + int switchPriority = switchPriority(reserved); + if (switchPriority < 0) { + return true; + } else if (switchPriority > 0) { + return false; + } + return hostPriority(reserved) < 0; + }); + } + /** * Compare this candidate to another * @@ -117,8 +134,8 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat if (!other.isSurplus && this.isSurplus) return 1; // Prefer node on exclusive switch - if (this.exclusiveSwitch && !other.exclusiveSwitch) return -1; - if (other.exclusiveSwitch && !this.exclusiveSwitch) return 1; + int switchPriority = switchPriority(other); + if (switchPriority != 0) return switchPriority; // Choose reserved nodes from a previous allocation attempt (which exist in node repo) if (this.isInNodeRepoAndReserved() && ! other.isInNodeRepoAndReserved()) return -1; @@ -156,8 +173,7 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat if ( ! lessThanHalfTheHost(this) && lessThanHalfTheHost(other)) return 1; } - int hostPriority = Double.compare(this.skewWithThis() - this.skewWithoutThis(), - other.skewWithThis() - other.skewWithoutThis()); + int hostPriority = hostPriority(other); if (hostPriority != 0) return hostPriority; // Choose cheapest node @@ -188,6 +204,19 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat return new ConcreteNodeCandidate(node, freeParentCapacity, parent, violatesSpares, exclusiveSwitch, isSurplus, isNew, isResizable); } + /** Returns the switch priority, based on switch exclusivity, of this compared to other */ + private int switchPriority(NodeCandidate other) { + if (this.exclusiveSwitch && !other.exclusiveSwitch) return -1; + if (other.exclusiveSwitch && !this.exclusiveSwitch) return 1; + return 0; + } + + /** Returns the host priority, based on allocation skew, of this compared to other */ + private int hostPriority(NodeCandidate other) { + return Double.compare(this.skewWithThis() - this.skewWithoutThis(), + other.skewWithThis() - other.skewWithoutThis()); + } + private boolean lessThanHalfTheHost(NodeCandidate node) { var n = node.resources(); var h = node.parent.get().resources(); @@ -267,6 +296,9 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat public boolean wantToRetire() { return node.status().wantToRetire(); } @Override + public boolean preferToRetire() { return node.status().preferToRetire(); } + + @Override public Flavor flavor() { return node.flavor(); } @Override @@ -350,6 +382,9 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat public boolean wantToRetire() { return false; } @Override + public boolean preferToRetire() { return false; } + + @Override public Flavor flavor() { return new Flavor(resources); } @Override @@ -443,6 +478,9 @@ public abstract class NodeCandidate implements Nodelike, Comparable<NodeCandidat public boolean wantToRetire() { return false; } @Override + public boolean preferToRetire() { return false; } + + @Override public Flavor flavor() { return new Flavor(resources); } @Override diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java index c31ebbb2c11..cd9e32ea9d2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockDeployer.java @@ -195,7 +195,7 @@ public class MockDeployer implements Deployer { public long activate() { lastDeployTimes.put(applicationId, clock.instant()); - for (Node node : nodeRepository.nodes().list().owner(applicationId).state(Node.State.active).wantToRetire().asList()) { + for (Node node : nodeRepository.nodes().list().owner(applicationId).state(Node.State.active).retirementRequested()) { try (NodeMutex lock = nodeRepository.nodes().lockAndGetRequired(node)) { nodeRepository.nodes().write(lock.node().retire(nodeRepository.clock().instant()), lock); } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java index 9fddaab8b3b..f6db9a45a61 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/RebalancerTest.java @@ -23,9 +23,11 @@ import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.provisioning.FlavorConfigBuilder; import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester; import com.yahoo.vespa.hosted.provision.testutils.MockDeployer; +import com.yahoo.vespa.hosted.provision.testutils.MockDeployer.ApplicationContext; import org.junit.Test; import java.time.Duration; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -134,6 +136,30 @@ public class RebalancerTest { tester.getNodes(cpuApp, Node.State.active).stream().anyMatch(node -> node.hasParent(newCpuHost.hostname()))); } + @Test + public void testRebalancingDoesNotReduceSwitchExclusivity() { + Capacity capacity = Capacity.from(new ClusterResources(4, 1, RebalancerTester.cpuResources), true, false); + Map<ApplicationId, ApplicationContext> apps = Map.of(cpuApp, new ApplicationContext(cpuApp, RebalancerTester.clusterSpec("c"), capacity)); + RebalancerTester tester = new RebalancerTester(4, apps); + + // Application is deployed and balanced across exclusive switches + tester.deployApp(cpuApp); + NodeList allNodes = tester.nodeRepository().nodes().list(); + NodeList applicationNodes = allNodes.owner(cpuApp); + NodeList nodesOnExclusiveSwitch = applicationNodes.onExclusiveSwitch(allNodes.parentsOf(applicationNodes)); + assertEquals(4, nodesOnExclusiveSwitch.size()); + + // Rebalancer does nothing + tester.assertNoMovesAfter(waitTimeAfterPreviousDeployment, cpuApp); + + // Another host is provisioned on an existing switch, which can reduce skew + Node newCpuHost = tester.makeReadyNode("cpu"); + tester.tester.patchNode(newCpuHost, (host) -> host.withSwitchHostname("switch0")); + tester.activateTenantHosts(); + + // Rebalancer does not move to new host, as this would violate switch exclusivity + tester.assertNoMovesAfter(waitTimeAfterPreviousDeployment, cpuApp); + } static class RebalancerTester { @@ -150,12 +176,19 @@ public class RebalancerTest { private final Rebalancer rebalancer; RebalancerTester() { - Map<ApplicationId, MockDeployer.ApplicationContext> apps = Map.of( - cpuApp, new MockDeployer.ApplicationContext(cpuApp, clusterSpec("c"), Capacity.from(new ClusterResources(1, 1, cpuResources))), - memoryApp, new MockDeployer.ApplicationContext(memoryApp, clusterSpec("c"), Capacity.from(new ClusterResources(1, 1, memResources)))); + this(3, + Map.of(cpuApp, new ApplicationContext(cpuApp, clusterSpec("c"), Capacity.from(new ClusterResources(1, 1, cpuResources))), + memoryApp, new ApplicationContext(memoryApp, clusterSpec("c"), Capacity.from(new ClusterResources(1, 1, memResources))))); + } + + RebalancerTester(int hostCount, Map<ApplicationId, ApplicationContext> apps) { deployer = new MockDeployer(tester.provisioner(), tester.clock(), apps); rebalancer = new Rebalancer(deployer, tester.nodeRepository(), metric, Duration.ofMinutes(1)); - tester.makeReadyNodes(3, "flat", NodeType.host, 8); + List<Node> hosts = tester.makeReadyNodes(hostCount, "flat", NodeType.host, 8); + for (int i = 0; i < hosts.size(); i++) { + String switchHostname = "switch" + i; + tester.patchNode(hosts.get(i), (host) -> host.withSwitchHostname(switchHostname)); + } tester.activateTenantHosts(); } @@ -194,6 +227,17 @@ public class RebalancerTest { MockDeployer deployer() { return deployer; } + private void assertNoMovesAfter(Duration duration, ApplicationId app) { + tester.clock().advance(duration); + NodeList before = tester.nodeRepository().nodes().list(Node.State.active).owner(app) + .sortedBy(Comparator.comparing(Node::hostname)); + maintain(); + NodeList after = tester.nodeRepository().nodes().list(Node.State.active).owner(app) + .sortedBy(Comparator.comparing(Node::hostname)); + assertEquals("Node allocation is unchanged", before.asList(), after.asList()); + assertEquals("No nodes are retired", List.of(), after.retired().asList()); + } + private FlavorsConfig flavorsConfig() { FlavorConfigBuilder b = new FlavorConfigBuilder(); b.addFlavor("flat", 30, 30, 400, 3, Flavor.Type.BARE_METAL); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java index 1c7b24ba783..bb3788ba186 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/SpareCapacityMaintainerTest.java @@ -326,9 +326,9 @@ public class SpareCapacityMaintainerTest { } private void dumpState() { - for (Node host : nodeRepository.nodes().list().hosts().asList()) { + for (Node host : nodeRepository.nodes().list().hosts()) { System.out.println("Host " + host.hostname() + " " + host.resources()); - for (Node node : nodeRepository.nodes().list().childrenOf(host).asList()) + for (Node node : nodeRepository.nodes().list().childrenOf(host)) System.out.println(" Node " + node.hostname() + " " + node.resources() + " allocation " +node.allocation()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java index 124f7db569a..6e50c934047 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/NodeTypeProvisioningTest.java @@ -202,7 +202,7 @@ public class NodeTypeProvisioningTest { // The retiring node should be one of the nodes we marked for retirement currentyRetiringHostname = nodesCurrentlyRetiring.get(0).hostname(); - assertTrue(nodesToRetire.stream().map(Node::hostname).filter(hostname -> hostname.equals(currentyRetiringHostname)).count() == 1); + assertEquals(1, nodesToRetire.stream().map(Node::hostname).filter(hostname -> hostname.equals(currentyRetiringHostname)).count()); } { // Redeploying while the node is still retiring has no effect diff --git a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/ProductionTest.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/ProductionTest.java index b9054689b00..100626ee091 100644 --- a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/ProductionTest.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/ProductionTest.java @@ -14,7 +14,7 @@ import static java.lang.annotation.RetentionPolicy.RUNTIME; * Tests that verify the health of production deployments of Vespa applications. * * Test classes annotated with this annotation are run during declared production tests. - * See <a href="https://cloud.vespa.ai/automated-deployments.html">Vespa cloud documentation</a>. + * See <a href="https://cloud.vespa.ai/en/automated-deployments">Vespa cloud documentation</a>. * * @author jonmv */ diff --git a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/StagingSetup.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/StagingSetup.java index bef3eabcef6..e7000933b9e 100644 --- a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/StagingSetup.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/StagingSetup.java @@ -15,7 +15,7 @@ import static java.lang.annotation.RetentionPolicy.RUNTIME; * * Test classes annotated with this annotation are run in the first phase of automated staging tests, * to make the initial deployment similar to a production one. - * See <a href="https://cloud.vespa.ai/automated-deployments.html">Vespa cloud documentation</a>. + * See <a href="https://cloud.vespa.ai/en/automated-deployments">Vespa cloud documentation</a>. * * @author jonmv */ diff --git a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/StagingTest.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/StagingTest.java index 59360b2753c..711c95a12fb 100644 --- a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/StagingTest.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/StagingTest.java @@ -16,7 +16,7 @@ import static java.lang.annotation.RetentionPolicy.RUNTIME; * * Test classes annotated with this annotation are run in the second phase of automated staging tests, * to verify the upgraded deployment. - * See <a href="https://cloud.vespa.ai/automated-deployments.html">Vespa cloud documentation</a>. + * See <a href="https://cloud.vespa.ai/en/automated-deployments">Vespa cloud documentation</a>. * * @author jonmv */ diff --git a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/SystemTest.java b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/SystemTest.java index f01f2ca6c90..525771d2f55 100644 --- a/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/SystemTest.java +++ b/tenant-cd-api/src/main/java/ai/vespa/hosted/cd/SystemTest.java @@ -17,7 +17,7 @@ import static java.lang.annotation.RetentionPolicy.RUNTIME; * Tests that compare the behaviour of a Vespa application deployment against a fixed specification. * * Test classes annotated with this annotation are run against a fresh deployment during automated system tests. - * See <a href="https://cloud.vespa.ai/automated-deployments.html">Vespa cloud documentation</a>. + * See <a href="https://cloud.vespa.ai/en/automated-deployments">Vespa cloud documentation</a>. * * @author jonmv */ diff --git a/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java b/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java index b07c54399af..45fb4e54f0a 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespaget/CommandLineOptions.java @@ -69,7 +69,7 @@ public class CommandLineOptions { options.addOption(Option.builder("f") .hasArg(true) - .desc("Retrieve the specified fields only (see https://docs.vespa.ai/documentation/documents.html#fieldsets) (default '" + AllFields.NAME + "')") + .desc("Retrieve the specified fields only (see https://docs.vespa.ai/en/documents.html#fieldsets) (default '" + AllFields.NAME + "')") .longOpt(FIELDSET_OPTION) .argName("fieldset").build()); diff --git a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java index b1f91e44e5c..adfbb246a9e 100644 --- a/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java +++ b/vespaclient-java/src/main/java/com/yahoo/vespavisit/VdsVisit.java @@ -194,7 +194,7 @@ public class VdsVisit { .longOpt("fieldset") .hasArg(true) .argName("fieldset") - .desc("Retrieve the specified fields only (see https://docs.vespa.ai/documentation/documents.html#fieldsets). Default is [all].") + .desc("Retrieve the specified fields only (see https://docs.vespa.ai/en/documents.html#fieldsets). Default is [all].") .build()); options.addOption(Option.builder() diff --git a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java index 0fba2ca4875..fbf5bc35129 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/Tensor.java @@ -297,7 +297,7 @@ public interface Tensor { /** * Returns this tensor on the - * <a href="https://docs.vespa.ai/documentation/reference/tensor.html#tensor-literal-form">tensor literal form</a> + * <a href="https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form">tensor literal form</a> * with type included. */ @Override @@ -305,7 +305,7 @@ public interface Tensor { /** * Call this from toString in implementations to return this tensor on the - * <a href="https://docs.vespa.ai/documentation/reference/tensor.html#tensor-literal-form">tensor literal form</a>. + * <a href="https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form">tensor literal form</a>. * (toString cannot be a default method because default methods cannot override super methods). * * @param tensor the tensor to return the standard string format of @@ -377,7 +377,7 @@ public interface Tensor { /** * Returns a tensor instance containing the given data on the - * <a href="https://docs.vespa.ai/documentation/reference/tensor.html#tensor-literal-form">tensor literal form</a>. + * <a href="https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form">tensor literal form</a>. * * @param type the type of the tensor to return * @param tensorString the tensor on the standard tensor string format @@ -388,7 +388,7 @@ public interface Tensor { /** * Returns a tensor instance containing the given data on the - * <a href="https://docs.vespa.ai/documentation/reference/tensor.html#tensor-literal-form">tensor literal form</a>. + * <a href="https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form">tensor literal form</a>. * * @param tensorType the type of the tensor to return, as a string on the tensor type format, given in * {@link TensorType#fromSpec} @@ -400,7 +400,7 @@ public interface Tensor { /** * Returns a tensor instance containing the given data on the - * <a href="https://docs.vespa.ai/documentation/reference/tensor.html#tensor-literal-form">tensor literal form</a>. + * <a href="https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form">tensor literal form</a>. */ static Tensor from(String tensorString) { return TensorParser.tensorFrom(tensorString, Optional.empty()); diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java index 790743c745c..236e9d31c39 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorType.java @@ -112,7 +112,7 @@ public class TensorType { /** * Returns a tensor type instance from a - * <a href="https://docs.vespa.ai/documentation/reference/tensor.html#tensor-type-spec">tensor type spec</a>: + * <a href="https://docs.vespa.ai/en/reference/tensor.html#tensor-type-spec">tensor type spec</a>: * <code>tensor(dimension1, dimension2, ...)</code> * where each dimension is either * <ul> diff --git a/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java b/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java index c78be98e11e..fa2094e9d2a 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/serialization/JsonFormat.java @@ -20,8 +20,8 @@ import java.util.Iterator; /** * Writes tensors on the JSON format used in Vespa tensor document fields: * A JSON map containing a 'cells' or 'values' array. - * See <a href="https://docs.vespa.ai/documentation/reference/document-json-format.html"> - * https://docs.vespa.ai/documentation/reference/document-json-format.html</a> + * See <a href="https://docs.vespa.ai/en/reference/document-json-format.html"> + * https://docs.vespa.ai/en/reference/document-json-format.html</a> * * @author bratseth */ |