diff options
74 files changed, 481 insertions, 493 deletions
diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldLongOperation.java b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldLongOperation.java index 8661928699f..f776f7b1cec 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldLongOperation.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldLongOperation.java @@ -18,7 +18,6 @@ public class SummaryInFieldLongOperation extends SummaryInFieldOperation { private DataType type; private Boolean bold; private Set<String> destinations = new java.util.LinkedHashSet<>(); - private List<SummaryField.Property> properties = new ArrayList<>(); public SummaryInFieldLongOperation(String name) { super(name); @@ -44,11 +43,6 @@ public class SummaryInFieldLongOperation extends SummaryInFieldOperation { return destinations.iterator(); } - - public void addProperty(SummaryField.Property property) { - properties.add(property); - } - public void apply(SDField field) { if (type == null) { type = field.getDataType(); @@ -74,9 +68,5 @@ public class SummaryInFieldLongOperation extends SummaryInFieldOperation { for (String destination : destinations) { summary.addDestination(destination); } - - for (SummaryField.Property prop : properties) { - summary.addProperty(prop.getName(), prop.getValue()); - } } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java index 29dd36c1d25..6bde4759eb7 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java @@ -4,6 +4,12 @@ package com.yahoo.searchdefinition.parser; import java.util.ArrayList; import java.util.List; +/** + * This class holds the extracted information after parsing a + * "document" block in a schema (.sd) file, using simple data + * structures as far as possible. Do not put advanced logic here! + * @author arnej27959 + **/ public class ParsedDocument { private final String name; private final List<String> inherited = new ArrayList<>(); @@ -12,7 +18,7 @@ public class ParsedDocument { this.name = name; } - String getName() { return name; } + String name() { return name; } void inherit(String other) { inherited.add(other); } /* diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankFunction.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankFunction.java new file mode 100644 index 00000000000..9f843ca15a7 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankFunction.java @@ -0,0 +1,21 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.parser; + +/** + * This class holds the extracted information after parsing a + * "function" block in a rank-profile, using simple data structures as + * far as possible. Do not put advanced logic here! + * @author arnej27959 + **/ +class ParsedRankFunction { + + private final String name; + + ParsedRankFunction(String name) { + this.name = name; + } + + void addParameter(String param) {} + void setInline(boolean inline) {} + void setExpression(String expr) {} +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java new file mode 100644 index 00000000000..16b7b507121 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java @@ -0,0 +1,53 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.searchdefinition.parser; + +import com.yahoo.searchdefinition.RankProfile.MatchPhaseSettings; +import com.yahoo.searchdefinition.RankProfile.MutateOperation; +import com.yahoo.searchlib.rankingexpression.FeatureList; +import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; +import com.yahoo.searchlib.rankingexpression.evaluation.Value; + +/** + * This class holds the extracted information after parsing a + * rank-profile block in a schema (.sd) file, using simple data + * structures as far as possible. Do not put advanced logic here! + * @author arnej27959 + **/ +class ParsedRankProfile { + + private final String name; + + ParsedRankProfile(String name) { + this.name = name; + } + + public String name() { return name; } + + void addSummaryFeatures(FeatureList features) {} + void addMatchFeatures(FeatureList features) {} + void addRankFeatures(FeatureList features) {} + + void inherit(String other) {} + void setInheritedSummaryFeatures(String other) {} + + void addFieldRankType(String field, String type) {} + void addFieldRankWeight(String field, int weight) {} + void addFieldRankFilter(String field, boolean filter) {} + void setSecondPhaseRanking(String expression) {} + void setRerankCount(int count) {} + void setFirstPhaseRanking(String expression) {} + void setKeepRankCount(int count) {} + void setRankScoreDropLimit(double limit) {} + void setMatchPhaseSettings(MatchPhaseSettings settings) {} + void addFunction(ParsedRankFunction func) {} + void addMutateOperation(MutateOperation.Phase phase, String attrName, String operation) {} + void setIgnoreDefaultRankFeatures(boolean ignore) {} + void setNumThreadsPerSearch(int threads) {} + void setMinHitsPerThread(int minHits) {} + void setNumSearchPartitions(int numParts) {} + void setTermwiseLimit(double limit) {} + void setInheritedMatchFeatures(String other) {} + void addRankProperty(String key, String value) {} + void addConstant(String name, Value value) {} + void addConstantTensor(String name, TensorValue value) {} +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java index 6026cf55beb..85e20b2f714 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java +++ b/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java @@ -8,6 +8,12 @@ import com.yahoo.searchdefinition.document.Stemming; import java.util.ArrayList; import java.util.List; +/** + * This class holds the extracted information after parsing + * one schema (.sd) file, using simple data structures + * as far as possible. Do not put advanced logic here! + * @author arnej27959 + **/ public class ParsedSchema { private final String name; private final List<String> inherited = new ArrayList<>(); @@ -16,12 +22,13 @@ public class ParsedSchema { this.name = name; } - String getName() { return name; } + String name() { return name; } void addDocument(ParsedDocument document) {} void addImportedField(String asFieldName, String refFieldName, String foregnFieldName) {} void addOnnxModel(OnnxModel model) {} void addRankingConstant(RankingConstant constant) {} + void addRankProfile(ParsedRankProfile profile) {} void enableRawAsBase64(boolean value) {} void inherit(String other) { inherited.add(other); } void setStemming(Stemming value) {} @@ -35,7 +42,6 @@ public class ParsedSchema { void addStruct(ParsedStruct struct) {} void addFieldSet(ParsedFieldSet fieldSet) {} void addDocumentSummary(ParsedDocumentSummary docsum) {} - void addRankProfile(ParsedRankProfile profile) {} */ } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java index 75a2a808a89..da3070af55f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java @@ -53,30 +53,6 @@ public class SummaryField extends Field implements Cloneable, TypedKey { } - /** A name-value property (used for smart summary) */ - public static class Property implements Serializable { - private String name; - private String value; - public Property(String name, String value) { - this.name = name; - this.value = value; - } - public String getName() { return name; } - public String getValue() { return value; } - @Override - public int hashCode() { - return name.hashCode() + 17*value.hashCode(); - } - @Override - public boolean equals(Object obj) { - if (!(obj instanceof Property)) { - return false; - } - Property other = (Property)obj; - return name.equals(other.name) && value.equals(other.value); - } - } - /** The transform to perform on the stored source */ private SummaryTransform transform=SummaryTransform.NONE; @@ -96,9 +72,6 @@ public class SummaryField extends Field implements Cloneable, TypedKey { /** True if this field was defined implicitly */ private boolean implicit=false; - /** The list of properties for this summary field */ - private List<Property> properties = new ArrayList<>(); - /** Creates a summary field with NONE as transform */ public SummaryField(String name, DataType type) { this(name,type, SummaryTransform.NONE); @@ -298,15 +271,6 @@ public class SummaryField extends Field implements Cloneable, TypedKey { this.vsmCommand = vsmCommand; } - /** Adds a property to this summary field */ - public void addProperty(String name, String value) { - properties.add(new Property(name, value)); - } - - public List<Property> getProperties() { - return properties; - } - /** * The command used when using data from this SummaryField to generate StreamingSummary config (vsmsummary). * Not used for ordinary Summary config. diff --git a/config-model/src/main/javacc/IntermediateParser.jj b/config-model/src/main/javacc/IntermediateParser.jj index 6e69cc4624a..a6482ca0b81 100644 --- a/config-model/src/main/javacc/IntermediateParser.jj +++ b/config-model/src/main/javacc/IntermediateParser.jj @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -// Schema parser. -// -// NOTE: When this is changed, also change integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf +// Duplicate Schema parser - work in progress +// @author arnej27959 +// NOTE: When grammar is changed, also change integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf options { UNICODE_INPUT = true; @@ -441,11 +441,11 @@ void rootSchemaItem(ParsedSchema schema) : { } | importField(schema) | rankingConstant(schema) | useDocument(schema) + | rankProfile(schema) /* | documentSummary(schema) | field(null, schema) | index(schema, null) - | rankProfile(schema) | structOutside(schema) | annotationOutside(schema) | fieldSet(schema) @@ -496,7 +496,7 @@ void useDocument(ParsedSchema schema) : { } */ void document(ParsedSchema schema) : { - String name = schema.getName(); + String name = schema.name(); ParsedDocument document; } { @@ -536,9 +536,9 @@ void documentBody(ParsedDocument document) : { } { - ( compression(null) - | headercfg(null) - | bodycfg(null) + ( compression() + | headercfg() + | bodycfg() /* | annotation(schema, document) | structInside(document, schema) | field(document, schema) */ @@ -562,30 +562,24 @@ void rawAsBase64(ParsedSchema schema) : /** * Consumes a document head block. - * - * @param document The document type to modify. */ -void headercfg(SDDocumentType document) : { } +void headercfg() : { } { - <HEADER> lbrace() [compression(document) (<NL>)*] <RBRACE> + <HEADER> lbrace() [compression() (<NL>)*] <RBRACE> } /** * Consumes a document body block. - * - * @param document The document type to modify. */ -void bodycfg(SDDocumentType document) : { } +void bodycfg() : { } { - <BODY> lbrace() [compression(document) (<NL>)*] <RBRACE> + <BODY> lbrace() [compression() (<NL>)*] <RBRACE> } /** * Consumes a compression block. This can be set in both document header and -body block. - * - * @param document The document type to modify. */ -void compression(SDDocumentType document) : +void compression() : { deployLogger.logApplicationPackage(Level.WARNING, "'compression' for a document is deprecated and ignored"); } @@ -595,7 +589,6 @@ void compression(SDDocumentType document) : /** * Consumes the body of a compression block. - * */ void compressionItem() : { } @@ -1428,7 +1421,7 @@ void summaryProperty(SummaryInFieldLongOperation field) : } { name = identifierWithDash() <COLON> (value = identifierWithDash() | value = quotedString()) - { field.addProperty(new SummaryField.Property(name, value)); } + { deployLogger.logApplicationPackage(Level.WARNING, "Specifying summary properties is deprecated and has no effect."); } } /** @@ -1947,37 +1940,30 @@ String rankingConstantErrorMessage(String name) : {} * * @param schema the schema object to add content to */ -void rankProfile(Schema schema) : +void rankProfile(ParsedSchema schema) : { String name; - RankProfile profile; + ParsedRankProfile profile; } { ( ( <MODEL> | <RANKPROFILE> ) name = identifierWithDash() { - if (documentsOnly) { - profile = new DocumentsOnlyRankProfile(name, schema, rankProfileRegistry, schema.rankingConstants()); - } - else if ("default".equals(name)) { - profile = rankProfileRegistry.get(schema, "default"); - } else { - profile = new RankProfile(name, schema, rankProfileRegistry, schema.rankingConstants()); - } + profile = new ParsedRankProfile(name); } [inheritsRankProfile(profile)] lbrace() (rankProfileItem(profile) (<NL>)*)* <RBRACE> ) { - if (documentsOnly) return; - rankProfileRegistry.add(profile); + schema.addRankProfile(profile); } } + /** * This rule consumes a single statement for a rank-profile block. * * @param profile The rank profile to modify. */ -void rankProfileItem(RankProfile profile) : { } +void rankProfileItem(ParsedRankProfile profile) : { } { ( fieldRankType(profile) | fieldWeight(profile) @@ -1994,7 +1980,7 @@ void rankProfileItem(RankProfile profile) : { } | rankFeatures(profile) | rankProperties(profile) | secondPhase(profile) - | rankDegradation(profile) + | rankDegradation() | constants(profile) | matchFeatures(profile) | summaryFeatures(profile) ) @@ -2005,13 +1991,13 @@ void rankProfileItem(RankProfile profile) : { } * * @param profile the profile to modify */ -void inheritsRankProfile(RankProfile profile) : +void inheritsRankProfile(ParsedRankProfile profile) : { String name; } { <INHERITS> name = identifierWithDash() { profile.inherit(name); } - ( <COMMA> name = identifierWithDash() { profile.inherit(name);; } )* + ( <COMMA> name = identifierWithDash() { profile.inherit(name); } )* } /** @@ -2019,15 +2005,12 @@ void inheritsRankProfile(RankProfile profile) : * * @param profile The profile to modify. */ -void mutate(RankProfile profile) : -{ -} +void mutate(ParsedRankProfile profile) : { } { <MUTATE> lbrace() (mutate_operation(profile) <NL>)+ <RBRACE> - { } } -void mutate_operation(RankProfile profile) : +void mutate_operation(ParsedRankProfile profile) : { String attribute, operation; RankProfile.MutateOperation.Phase phase; @@ -2057,20 +2040,25 @@ String mutate_expr() : * * @param profile The profile to modify. */ -void function(RankProfile profile) : +void function(ParsedRankProfile profile) : { String name, expression, parameter; - List parameters = new ArrayList(); boolean inline = false; + ParsedRankFunction func; } { ( ( <FUNCTION> | <MACRO> ) inline = inline() name = identifier() [ "$" { name = name + token.image; } ] "(" - [ parameter = identifier() { parameters.add(parameter); } - ( <COMMA> parameter = identifier() { parameters.add(parameter); } )* ] + { func = new ParsedRankFunction(name); } + [ parameter = identifier() { func.addParameter(parameter); } + ( <COMMA> parameter = identifier() { func.addParameter(parameter); } )* ] ")" lbrace() expression = expression() (<NL>)* <RBRACE> ) - { profile.addFunction(name, parameters, expression, inline); } + { + func.setExpression(expression); + func.setInline(inline); + profile.addFunction(func); + } } boolean inline() : @@ -2086,7 +2074,7 @@ boolean inline() : * * @param profile The rank profile to modify. */ -void matchPhase(RankProfile profile) : +void matchPhase(ParsedRankProfile profile) : { MatchPhaseSettings settings = new MatchPhaseSettings(); } @@ -2094,7 +2082,7 @@ void matchPhase(RankProfile profile) : <MATCHPHASE> lbrace() (matchPhaseItem(settings) (<NL>)*)* <RBRACE> { settings.checkValid(); - profile.setMatchPhaseSettings(settings); + profile.setMatchPhaseSettings(settings); } } @@ -2159,7 +2147,7 @@ void diversityItem(DiversitySettings settings) : * * @param profile The rank profile to modify. */ -void firstPhase(RankProfile profile) : +void firstPhase(ParsedRankProfile profile) : { String exp; } @@ -2167,16 +2155,16 @@ void firstPhase(RankProfile profile) : <FIRSTPHASE> lbrace() (firstPhaseItem(profile) (<NL>)*)* <RBRACE> } -void firstPhaseItem(RankProfile profile) : +void firstPhaseItem(ParsedRankProfile profile) : { - String expression; - int rerankCount; - double dropLimit; + String expression; + int keepRankCount; + double dropLimit; } { - ( expression = expression() { profile.setFirstPhaseRanking(expression); } - | (<KEEPRANKCOUNT> <COLON> rerankCount = integer()) { profile.setKeepRankCount(rerankCount); } - | (<RANKSCOREDROPLIMIT> <COLON> dropLimit = consumeFloat()) { profile.setRankScoreDropLimit(dropLimit); } + ( expression = expression() { profile.setFirstPhaseRanking(expression); } + | (<KEEPRANKCOUNT> <COLON> keepRankCount = integer()) { profile.setKeepRankCount(keepRankCount); } + | (<RANKSCOREDROPLIMIT> <COLON> dropLimit = consumeFloat()) { profile.setRankScoreDropLimit(dropLimit); } ) } @@ -2185,7 +2173,7 @@ void firstPhaseItem(RankProfile profile) : * * @param profile The rank profile to modify. */ -void secondPhase(RankProfile profile) : { } +void secondPhase(ParsedRankProfile profile) : { } { <SECONDPHASE> lbrace() (secondPhaseItem(profile) (<NL>)*)* <RBRACE> } @@ -2195,23 +2183,24 @@ void secondPhase(RankProfile profile) : { } * * @param profile The rank profile to modify. */ -void secondPhaseItem(RankProfile profile) : +void secondPhaseItem(ParsedRankProfile profile) : { String expression; int rerankCount; } { - ( expression = expression() { profile.setSecondPhaseRanking(expression); } - | (<RERANKCOUNT> <COLON> rerankCount = integer()) { profile.setRerankCount(rerankCount); } + ( expression = expression() { profile.setSecondPhaseRanking(expression); } + | (<RERANKCOUNT> <COLON> rerankCount = integer()) { profile.setRerankCount(rerankCount); } ) } + /** * This rule consumes a summary-features block of a rank profile. * * @param profile The rank profile to modify. */ -void summaryFeatures(RankProfile profile) : +void summaryFeatures(ParsedRankProfile profile) : { String features; String inherited = null; @@ -2237,7 +2226,7 @@ void summaryFeatures(RankProfile profile) : * * @param profile The rank profile to modify. */ -void matchFeatures(RankProfile profile) : +void matchFeatures(ParsedRankProfile profile) : { String features; } @@ -2258,7 +2247,7 @@ void matchFeatures(RankProfile profile) : } /** Consumes a rank-features block of a rank profile */ -void rankFeatures(RankProfile profile) : +void rankFeatures(ParsedRankProfile profile) : { String features; } @@ -2276,7 +2265,7 @@ void rankFeatures(RankProfile profile) : * * @param profile The rank profile to modify. */ -void ignoreRankFeatures(RankProfile profile) : { } +void ignoreRankFeatures(ParsedRankProfile profile) : { } { <IGNOREDEFAULTRANKFEATURES> { profile.setIgnoreDefaultRankFeatures(true); } } @@ -2286,7 +2275,7 @@ void ignoreRankFeatures(RankProfile profile) : { } * * @param profile The rank profile to modify. */ -void numThreadsPerSearch(RankProfile profile) : +void numThreadsPerSearch(ParsedRankProfile profile) : { int num; } @@ -2299,7 +2288,7 @@ void numThreadsPerSearch(RankProfile profile) : * * @param profile The rank profile to modify. */ -void minHitsPerThread(RankProfile profile) : +void minHitsPerThread(ParsedRankProfile profile) : { int num; } @@ -2312,7 +2301,7 @@ void minHitsPerThread(RankProfile profile) : * * @param profile the rank profile to modify */ -void numSearchPartitions(RankProfile profile) : +void numSearchPartitions(ParsedRankProfile profile) : { int num; } @@ -2321,24 +2310,26 @@ void numSearchPartitions(RankProfile profile) : } /** - * This rule consumes a num-threads-per-search statement for a rank profile. + * This rule consumes a termwise-limit statement for a rank profile. * * @param profile the rank profile to modify */ -void termwiseLimit(RankProfile profile) : +void termwiseLimit(ParsedRankProfile profile) : { double num; } { (<TERMWISELIMIT> <COLON> num = consumeFloat()) { profile.setTermwiseLimit(num); } } + /** - * This rule consumes a rank-properties block of a rank profile. There is a little trick within this rule to allow the - * final rank property to skip the terminating newline token. + * This rule consumes a rank-properties block of a rank profile. There + * is a little trick within this rule to allow the final rank property + * to skip the terminating newline token. * * @param profile the rank profile to modify */ -void rankProperties(RankProfile profile) : { } +void rankProperties(ParsedRankProfile profile) : { } { <RANKPROPERTIES> lbrace() (LOOKAHEAD(rankPropertyItem() <COLON> rankPropertyItem() <NL>) rankProperty(profile) (<NL>)+)* [rankProperty(profile)] <RBRACE> @@ -2349,7 +2340,7 @@ void rankProperties(RankProfile profile) : { } * * @param profile the rank profile to modify */ -void rankProperty(RankProfile profile) : +void rankProperty(ParsedRankProfile profile) : { String key, val; } @@ -2358,7 +2349,6 @@ void rankProperty(RankProfile profile) : { profile.addRankProperty(key, val); } } - /** * This rule consumes a single rank property for a rank-properties block. * @@ -2380,14 +2370,14 @@ String rankPropertyItem() : * * @param profile The rank profile to modify. */ -void fieldWeight(RankProfile profile) : +void fieldWeight(ParsedRankProfile profile) : { Integer num; String name; } { <WEIGHT> name = identifier() <COLON> num = integer() - { profile.addRankSetting(name, RankProfile.RankSetting.Type.WEIGHT, num); } + { profile.addFieldRankWeight(name, num); } } /** @@ -2395,14 +2385,14 @@ void fieldWeight(RankProfile profile) : * * @param profile The rank profile to modify. */ -void fieldRankType(RankProfile profile) : +void fieldRankType(ParsedRankProfile profile) : { String name; String type; } { <RANKTYPE> name = identifier() <COLON> type = identifier() - { profile.addRankSetting(name, RankProfile.RankSetting.Type.RANKTYPE, RankType.fromString(type)); } + { profile.addFieldRankType(name, type); } } /** @@ -2410,13 +2400,13 @@ void fieldRankType(RankProfile profile) : * * @param profile The rank profile to modify. */ -void fieldRankFilter(RankProfile profile) : +void fieldRankFilter(ParsedRankProfile profile) : { String name; } { <RANK> name = identifier() <COLON> <FILTER> - { profile.addRankSetting(name, RankProfile.RankSetting.Type.PREFERBITVECTOR, Boolean.TRUE); } + { profile.addFieldRankFilter(name, true); } } /** @@ -2472,10 +2462,8 @@ void rankDegradationItem() : /** * This rule consumes a rank-degradation statement of a rank profile. - * - * @param profile The rank profile to modify. */ -void rankDegradation(RankProfile profile) : +void rankDegradation() : { double freq; } @@ -2489,7 +2477,7 @@ void rankDegradation(RankProfile profile) : /** * Consumes a set of constants available in ranking expressions in the enclosing profile. */ -void constants(RankProfile profile) : +void constants(ParsedRankProfile profile) : { String name; } @@ -2500,7 +2488,7 @@ void constants(RankProfile profile) : <RBRACE> } -void constantValue(RankProfile profile, String name) : +void constantValue(ParsedRankProfile profile, String name) : { String value; } @@ -2508,7 +2496,7 @@ void constantValue(RankProfile profile, String name) : <COLON> value = identifier() { profile.addConstant(name, Value.parse(value)); } } -void constantTensor(RankProfile profile, String name) : +void constantTensor(ParsedRankProfile profile, String name) : { String tensorString = ""; TensorType tensorType = null; diff --git a/config-model/src/main/javacc/SDParser.jj b/config-model/src/main/javacc/SDParser.jj index 38622cbbcdc..5da20acf486 100644 --- a/config-model/src/main/javacc/SDParser.jj +++ b/config-model/src/main/javacc/SDParser.jj @@ -2,6 +2,10 @@ // Schema parser. // +// NOTE: A full rewrite of parsing is in progress; avoid changing this +// file if possible, coordinate with IntermediateParser.jj if you must +// change something. +// // NOTE: When this is changed, also change integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf options { @@ -1410,7 +1414,7 @@ void summaryProperty(SummaryInFieldLongOperation field) : } { name = identifierWithDash() <COLON> (value = identifierWithDash() | value = quotedString()) - { field.addProperty(new SummaryField.Property(name, value)); } + { deployLogger.logApplicationPackage(Level.WARNING, "Specifying summary properties is deprecated and has no effect."); } } /** diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/NodeAllocationException.java b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeAllocationException.java new file mode 100644 index 00000000000..d568a61fc69 --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/NodeAllocationException.java @@ -0,0 +1,16 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.provision; + +/** + * + * Exception thrown when we are unable to fulfill a node allocation request. + * + * @author hmusum + */ +public class NodeAllocationException extends RuntimeException { + + public NodeAllocationException(String message) { + super(message); + } + +} diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/OutOfCapacityException.java b/config-provisioning/src/main/java/com/yahoo/config/provision/OutOfCapacityException.java deleted file mode 100644 index 177b3f6e198..00000000000 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/OutOfCapacityException.java +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.provision; - -/** - * - * Exception thrown when we are unable to fulfill the request due to - * having too few nodes (of the specified flavor) - * - * @author hmusum - */ -public class OutOfCapacityException extends RuntimeException { - - public OutOfCapacityException(String message) { - super(message); - } - -} diff --git a/config/src/tests/file_subscription/file_subscription.cpp b/config/src/tests/file_subscription/file_subscription.cpp index b4a2b890bb5..6b900876bda 100644 --- a/config/src/tests/file_subscription/file_subscription.cpp +++ b/config/src/tests/file_subscription/file_subscription.cpp @@ -58,7 +58,7 @@ TEST("requireThatFileSpecGivesCorrectSource") { writeFile("my.cfg", "foobar"); FileSpec spec("my.cfg"); - SourceFactory::UP factory(spec.createSourceFactory(TimingValues())); + auto factory = spec.createSourceFactory(TimingValues()); ASSERT_TRUE(factory); auto holder = std::make_shared<ConfigHolder>(); std::unique_ptr<Source> src = factory->createSource(holder, ConfigKey("my", "my", "bar", "foo")); diff --git a/config/src/tests/frtconnectionpool/frtconnectionpool.cpp b/config/src/tests/frtconnectionpool/frtconnectionpool.cpp index c2ed60370e0..46e1f698091 100644 --- a/config/src/tests/frtconnectionpool/frtconnectionpool.cpp +++ b/config/src/tests/frtconnectionpool/frtconnectionpool.cpp @@ -3,6 +3,9 @@ #include <vespa/vespalib/testkit/testapp.h> #include <vespa/config/frt/frtconnectionpool.h> #include <vespa/fnet/frt/error.h> +#include <vespa/fnet/transport.h> +#include <vespa/fastos/thread.h> +#include <vespa/vespalib/util/size_literals.h> #include <sstream> #include <set> #include <unistd.h> @@ -12,8 +15,12 @@ using namespace config; class Test : public vespalib::TestApp { private: static ServerSpec::HostSpecList _sources; + FastOS_ThreadPool _threadPool; + FNET_Transport _transport; void verifyAllSourcesInRotation(FRTConnectionPool& sourcePool); public: + Test(); + ~Test() override; int Main() override; void testBasicRoundRobin(); void testBasicHashBasedSelection(); @@ -25,6 +32,18 @@ public: void testManySources(); }; +Test::Test() + : vespalib::TestApp(), + _threadPool(64_Ki), + _transport() +{ + _transport.Start(&_threadPool); +} + +Test::~Test() { + _transport.ShutDown(true); +} + TEST_APPHOOK(Test); ServerSpec::HostSpecList Test::_sources; @@ -79,7 +98,7 @@ void Test::verifyAllSourcesInRotation(FRTConnectionPool& sourcePool) { */ void Test::testBasicRoundRobin() { const ServerSpec spec(_sources); - FRTConnectionPool sourcePool(spec, timingValues); + FRTConnectionPool sourcePool(_transport, spec, timingValues); for (int i = 0; i < 9; i++) { int j = i % _sources.size(); std::stringstream s; @@ -93,7 +112,7 @@ void Test::testBasicRoundRobin() { */ void Test::testBasicHashBasedSelection() { const ServerSpec spec(_sources); - FRTConnectionPool sourcePool(spec, timingValues); + FRTConnectionPool sourcePool(_transport, spec, timingValues); sourcePool.setHostname("a.b.com"); for (int i = 0; i < 9; i++) { EXPECT_EQUAL("host1", sourcePool.getNextHashBased()->getAddress()); @@ -108,7 +127,7 @@ void Test::testBasicHashBasedSelection() { hostnames.push_back("stroustrup-02.example.yahoo.com"); hostnames.push_back("alexandrescu-03.example.yahoo.com"); const ServerSpec spec2(hostnames); - FRTConnectionPool sourcePool2(spec2, timingValues); + FRTConnectionPool sourcePool2(_transport, spec2, timingValues); sourcePool2.setHostname("sutter-01.example.yahoo.com"); EXPECT_EQUAL("stroustrup-02.example.yahoo.com", sourcePool2.getNextHashBased()->getAddress()); sourcePool2.setHostname("stroustrup-02.example.yahoo.com"); @@ -123,7 +142,7 @@ void Test::testBasicHashBasedSelection() { */ void Test::testSetErrorRoundRobin() { const ServerSpec spec(_sources); - FRTConnectionPool sourcePool(spec, timingValues); + FRTConnectionPool sourcePool(_transport, spec, timingValues); FRTConnection* source = sourcePool.getNextRoundRobin(); source->setError(FRTE_RPC_CONNECTION); for (int i = 0; i < 9; i++) { @@ -138,7 +157,7 @@ void Test::testSetErrorRoundRobin() { */ void Test::testSetErrorAllRoundRobin() { const ServerSpec spec(_sources); - FRTConnectionPool sourcePool(spec, timingValues); + FRTConnectionPool sourcePool(_transport, spec, timingValues); for (int i = 0; i < (int)_sources.size(); i++) { FRTConnection* source = sourcePool.getNextRoundRobin(); source->setError(FRTE_RPC_CONNECTION); @@ -152,7 +171,7 @@ void Test::testSetErrorAllRoundRobin() { */ void Test::testSetErrorHashBased() { const ServerSpec spec(_sources); - FRTConnectionPool sourcePool(spec, timingValues); + FRTConnectionPool sourcePool(_transport, spec, timingValues); FRTConnection* source = sourcePool.getNextHashBased(); source->setError(FRTE_RPC_CONNECTION); for (int i = 0; i < (int)_sources.size(); i++) { @@ -167,7 +186,7 @@ void Test::testSetErrorHashBased() { */ void Test::testSetErrorAllHashBased() { const ServerSpec spec(_sources); - FRTConnectionPool sourcePool(spec, timingValues); + FRTConnectionPool sourcePool(_transport, spec, timingValues); FRTConnection* firstSource = sourcePool.getNextHashBased(); auto readySources = sourcePool.getReadySources(); for (int i = 0; i < (int)readySources.size(); i++) { @@ -199,7 +218,7 @@ void Test::testSetErrorAllHashBased() { */ void Test::testSuspensionTimeout() { const ServerSpec spec(_sources); - FRTConnectionPool sourcePool(spec, timingValues); + FRTConnectionPool sourcePool(_transport, spec, timingValues); Connection* source = sourcePool.getCurrent(); source->setTransientDelay(1s); source->setError(FRTE_RPC_CONNECTION); @@ -227,7 +246,7 @@ void Test::testManySources() { twoSources.push_back("host1"); const ServerSpec spec(twoSources); - FRTConnectionPool sourcePool(spec, timingValues); + FRTConnectionPool sourcePool(_transport, spec, timingValues); for (int i = 0; i < (int)hostnames.size(); i++) { sourcePool.setHostname(hostnames[i]); diff --git a/config/src/tests/subscriber/subscriber.cpp b/config/src/tests/subscriber/subscriber.cpp index 548e54e1d18..d7320bb6a2f 100644 --- a/config/src/tests/subscriber/subscriber.cpp +++ b/config/src/tests/subscriber/subscriber.cpp @@ -85,7 +85,7 @@ namespace { return std::make_shared<ConfigSubscription>(0, key, holder, std::make_unique<MySource>()); } - void unsubscribe(const ConfigSubscription::SP & subscription) override { + void unsubscribe(const ConfigSubscription & subscription) override { (void) subscription; numCancel++; } diff --git a/config/src/vespa/config/common/cancelhandler.h b/config/src/vespa/config/common/cancelhandler.h index 8dde83ba7d9..694f0185e7c 100644 --- a/config/src/vespa/config/common/cancelhandler.h +++ b/config/src/vespa/config/common/cancelhandler.h @@ -13,7 +13,7 @@ struct CancelHandler * * @param subscription ConfigSubscription to cancel */ - virtual void unsubscribe(const std::shared_ptr<ConfigSubscription> & subscription) = 0; + virtual void unsubscribe(const ConfigSubscription & subscription) = 0; virtual ~CancelHandler() = default; }; diff --git a/config/src/vespa/config/common/configcontext.cpp b/config/src/vespa/config/common/configcontext.cpp index 2c8c981862d..eb56202bf06 100644 --- a/config/src/vespa/config/common/configcontext.cpp +++ b/config/src/vespa/config/common/configcontext.cpp @@ -6,9 +6,7 @@ namespace config { ConfigContext::ConfigContext(const SourceSpec & spec) - : _timingValues(), - _generation(1), - _manager(std::make_unique<ConfigManager>(spec.createSourceFactory(_timingValues), _generation)) + : ConfigContext(TimingValues(), spec) { } ConfigContext::ConfigContext(const TimingValues & timingValues, const SourceSpec & spec) diff --git a/config/src/vespa/config/common/configmanager.cpp b/config/src/vespa/config/common/configmanager.cpp index 983cccb0b0b..1417aed79ae 100644 --- a/config/src/vespa/config/common/configmanager.cpp +++ b/config/src/vespa/config/common/configmanager.cpp @@ -11,7 +11,7 @@ LOG_SETUP(".config.common.configmanager"); namespace config { -ConfigManager::ConfigManager(SourceFactory::UP sourceFactory, int64_t initialGeneration) +ConfigManager::ConfigManager(std::unique_ptr<SourceFactory> sourceFactory, int64_t initialGeneration) : _idGenerator(0), _sourceFactory(std::move(sourceFactory)), _generation(initialGeneration), @@ -53,10 +53,10 @@ ConfigManager::subscribe(const ConfigKey & key, vespalib::duration timeout) } void -ConfigManager::unsubscribe(const ConfigSubscription::SP & subscription) +ConfigManager::unsubscribe(const ConfigSubscription & subscription) { std::lock_guard guard(_lock); - const SubscriptionId id(subscription->getSubscriptionId()); + const SubscriptionId id(subscription.getSubscriptionId()); if (_subscriptionMap.find(id) != _subscriptionMap.end()) _subscriptionMap.erase(id); } diff --git a/config/src/vespa/config/common/configmanager.h b/config/src/vespa/config/common/configmanager.h index 09d263d9c25..a622bce469e 100644 --- a/config/src/vespa/config/common/configmanager.h +++ b/config/src/vespa/config/common/configmanager.h @@ -21,19 +21,19 @@ struct TimingValues; class ConfigManager : public IConfigManager { public: - ConfigManager(SourceFactory::UP sourceFactory, int64_t initialGeneration); + ConfigManager(std::unique_ptr<SourceFactory> sourceFactory, int64_t initialGeneration); ~ConfigManager() override; ConfigSubscription::SP subscribe(const ConfigKey & key, vespalib::duration timeout) override; - void unsubscribe(const ConfigSubscription::SP & subscription) override; + void unsubscribe(const ConfigSubscription & subscription) override; void reload(int64_t generation) override; private: - std::atomic<SubscriptionId> _idGenerator; - SourceFactory::UP _sourceFactory; - int64_t _generation; + std::atomic<SubscriptionId> _idGenerator; + std::unique_ptr<SourceFactory> _sourceFactory; + int64_t _generation; - typedef std::map<SubscriptionId, ConfigSubscription::SP> SubscriptionMap; + using SubscriptionMap = std::map<SubscriptionId, ConfigSubscription::SP>; SubscriptionMap _subscriptionMap; std::mutex _lock; }; diff --git a/config/src/vespa/config/common/reloadhandler.h b/config/src/vespa/config/common/reloadhandler.h index 73016a2f883..cd329e8d525 100644 --- a/config/src/vespa/config/common/reloadhandler.h +++ b/config/src/vespa/config/common/reloadhandler.h @@ -9,7 +9,7 @@ struct ReloadHandler * Reload any configs with a given generation. */ virtual void reload(int64_t generation) = 0; - virtual ~ReloadHandler() { } + virtual ~ReloadHandler() = default; }; } diff --git a/config/src/vespa/config/common/sourcefactory.h b/config/src/vespa/config/common/sourcefactory.h index 0236bea2802..f11a070fd95 100644 --- a/config/src/vespa/config/common/sourcefactory.h +++ b/config/src/vespa/config/common/sourcefactory.h @@ -13,7 +13,6 @@ class IConfigHolder; */ class SourceFactory { public: - typedef std::unique_ptr<SourceFactory> UP; virtual std::unique_ptr<Source> createSource(std::shared_ptr<IConfigHolder> holder, const ConfigKey & key) const = 0; virtual ~SourceFactory() = default; }; diff --git a/config/src/vespa/config/frt/connectionfactory.h b/config/src/vespa/config/frt/connectionfactory.h index dc1eadf3cf6..eb7ca10fa0d 100644 --- a/config/src/vespa/config/frt/connectionfactory.h +++ b/config/src/vespa/config/frt/connectionfactory.h @@ -12,12 +12,10 @@ class Connection; class ConnectionFactory { public: - typedef std::unique_ptr <ConnectionFactory> UP; - typedef std::shared_ptr<ConnectionFactory> SP; virtual Connection * getCurrent() = 0; virtual void syncTransport() = 0; virtual FNET_Scheduler * getScheduler() = 0; - virtual ~ConnectionFactory() { } + virtual ~ConnectionFactory() = default; }; } diff --git a/config/src/vespa/config/frt/frtconnectionpool.cpp b/config/src/vespa/config/frt/frtconnectionpool.cpp index 629baf67cf8..f416b0e1a57 100644 --- a/config/src/vespa/config/frt/frtconnectionpool.cpp +++ b/config/src/vespa/config/frt/frtconnectionpool.cpp @@ -27,10 +27,8 @@ FRTConnectionPool::FRTConnectionKey::operator==(const FRTConnectionKey& right) c return _hostname == right._hostname; } -FRTConnectionPool::FRTConnectionPool(const ServerSpec & spec, const TimingValues & timingValues) - : _threadPool(std::make_unique<FastOS_ThreadPool>(60_Ki)), - _transport(std::make_unique<FNET_Transport>()), - _supervisor(std::make_unique<FRT_Supervisor>(_transport.get())), +FRTConnectionPool::FRTConnectionPool(FNET_Transport & transport, const ServerSpec & spec, const TimingValues & timingValues) + : _supervisor(std::make_unique<FRT_Supervisor>(& transport)), _selectIdx(0), _hostname("") { @@ -39,13 +37,9 @@ FRTConnectionPool::FRTConnectionPool(const ServerSpec & spec, const TimingValues _connections[key] = std::make_shared<FRTConnection>(spec.getHost(i), *_supervisor, timingValues); } setHostname(); - _transport->Start(_threadPool.get()); } -FRTConnectionPool::~FRTConnectionPool() -{ - _transport->ShutDown(true); -} +FRTConnectionPool::~FRTConnectionPool() = default; void FRTConnectionPool::syncTransport() @@ -82,6 +76,27 @@ FRTConnectionPool::getNextRoundRobin() return nextFRTConnection; } +namespace { +/** + * Implementation of the Java hashCode function for the String class. + * + * Ensures that the same hostname maps to the same configserver/proxy + * for both language implementations. + * + * @param s the string to compute the hash from + * @return the hash value + */ +int hashCode(const vespalib::string & s) { + int hashval = 0; + + for (int i = 0; i < (int) s.length(); i++) { + hashval = 31 * hashval + s[i]; + } + return hashval; +} + +} + FRTConnection * FRTConnectionPool::getNextHashBased() { @@ -129,20 +144,10 @@ FRTConnectionPool::getSuspendedSources() const return suspendedSources; } -int FRTConnectionPool::hashCode(const vespalib::string & s) -{ - int hashval = 0; - - for (int i = 0; i < (int)s.length(); i++) { - hashval = 31 * hashval + s[i]; - } - return hashval; -} - void FRTConnectionPool::setHostname() { - _hostname = vespalib::HostName::get(); + setHostname(vespalib::HostName::get()); } FNET_Scheduler * @@ -150,4 +155,20 @@ FRTConnectionPool::getScheduler() { return _supervisor->GetScheduler(); } +FRTConnectionPoolWithTransport::FRTConnectionPoolWithTransport(std::unique_ptr<FastOS_ThreadPool> threadPool, + std::unique_ptr<FNET_Transport> transport, + const ServerSpec & spec, const TimingValues & timingValues) + : _threadPool(std::move(threadPool)), + _transport(std::move(transport)), + _connectionPool(std::make_unique<FRTConnectionPool>(*_transport, spec, timingValues)) +{ + _transport->Start(_threadPool.get()); +} + +FRTConnectionPoolWithTransport::~FRTConnectionPoolWithTransport() +{ + syncTransport(); + _transport->ShutDown(true); +} + } diff --git a/config/src/vespa/config/frt/frtconnectionpool.h b/config/src/vespa/config/frt/frtconnectionpool.h index 6b387fbf67f..564c6506159 100644 --- a/config/src/vespa/config/frt/frtconnectionpool.h +++ b/config/src/vespa/config/frt/frtconnectionpool.h @@ -13,7 +13,6 @@ class FastOS_ThreadPool; namespace config { class FRTConnectionPool : public ConnectionFactory { - private: /** @@ -32,8 +31,6 @@ private: int operator==(const FRTConnectionKey& right) const; }; - std::unique_ptr<FastOS_ThreadPool> _threadPool; - std::unique_ptr<FNET_Transport> _transport; std::unique_ptr<FRT_Supervisor> _supervisor; int _selectIdx; vespalib::string _hostname; @@ -41,7 +38,7 @@ private: ConnectionMap _connections; public: - FRTConnectionPool(const ServerSpec & spec, const TimingValues & timingValues); + FRTConnectionPool(FNET_Transport & transport, const ServerSpec & spec, const TimingValues & timingValues); FRTConnectionPool(const FRTConnectionPool&) = delete; FRTConnectionPool& operator=(const FRTConnectionPool&) = delete; ~FRTConnectionPool() override; @@ -63,21 +60,6 @@ public: FNET_Scheduler * getScheduler() override; /** - * Gets the hostname. - * - * @return the hostname - */ - vespalib::string & getHostname() { return _hostname; } - - /** - * Trim away leading and trailing spaces. - * - * @param s the string to trim away spaces from - * @return string without leading or trailing spaces - */ - vespalib::string trim(vespalib::string s); - - /** * Returns the current FRTConnection instance, taken from the list of error-free sources. * If no sources are error-free, an instance from the list of sources with errors * is returned. @@ -117,17 +99,23 @@ public: * @param suspendedSources is list of FRTConnection pointers */ std::vector<FRTConnection*> getSuspendedSources() const; +}; - /** - * Implementation of the Java hashCode function for the String class. - * - * Ensures that the same hostname maps to the same configserver/proxy - * for both language implementations. - * - * @param s the string to compute the hash from - * @return the hash value - */ - static int hashCode(const vespalib::string & s); +class FRTConnectionPoolWithTransport : public ConnectionFactory { +public: + FRTConnectionPoolWithTransport(std::unique_ptr<FastOS_ThreadPool> threadPool, + std::unique_ptr<FNET_Transport> transport, + const ServerSpec & spec, const TimingValues & timingValues); + FRTConnectionPoolWithTransport(const FRTConnectionPoolWithTransport&) = delete; + FRTConnectionPoolWithTransport& operator=(const FRTConnectionPoolWithTransport&) = delete; + ~FRTConnectionPoolWithTransport() override; + FNET_Scheduler * getScheduler() override { return _connectionPool->getScheduler(); } + void syncTransport() override { _connectionPool->syncTransport(); } + Connection* getCurrent() override { return _connectionPool->getCurrent(); } +private: + std::unique_ptr<FastOS_ThreadPool> _threadPool; + std::unique_ptr<FNET_Transport> _transport; + std::unique_ptr<FRTConnectionPool> _connectionPool; }; } // namespace config diff --git a/config/src/vespa/config/frt/frtsourcefactory.cpp b/config/src/vespa/config/frt/frtsourcefactory.cpp index 9706e3bbd5e..1c740534730 100644 --- a/config/src/vespa/config/frt/frtsourcefactory.cpp +++ b/config/src/vespa/config/frt/frtsourcefactory.cpp @@ -1,17 +1,21 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + #include "frtsourcefactory.h" #include "frtsource.h" #include "frtconfigagent.h" +#include "connectionfactory.h" namespace config { -FRTSourceFactory::FRTSourceFactory(ConnectionFactory::UP connectionFactory, const TimingValues & timingValues, int traceLevel, const VespaVersion & vespaVersion, const CompressionType & compressionType) +FRTSourceFactory::FRTSourceFactory(std::unique_ptr<ConnectionFactory> connectionFactory, const TimingValues & timingValues, int traceLevel, const VespaVersion & vespaVersion, const CompressionType & compressionType) : _connectionFactory(std::move(connectionFactory)), _requestFactory(traceLevel, vespaVersion, compressionType), _timingValues(timingValues) { } +FRTSourceFactory::~FRTSourceFactory() = default; + std::unique_ptr<Source> FRTSourceFactory::createSource(std::shared_ptr<IConfigHolder> holder, const ConfigKey & key) const { diff --git a/config/src/vespa/config/frt/frtsourcefactory.h b/config/src/vespa/config/frt/frtsourcefactory.h index 4331c6411dc..19cf6241481 100644 --- a/config/src/vespa/config/frt/frtsourcefactory.h +++ b/config/src/vespa/config/frt/frtsourcefactory.h @@ -1,28 +1,32 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include "connectionfactory.h" #include "frtconfigrequestfactory.h" #include <vespa/config/common/sourcefactory.h> #include <vespa/config/common/timingvalues.h> namespace config { +class ConnectionFactory; + /** * Class for sending and receiving config requests via FRT. */ class FRTSourceFactory : public SourceFactory { public: - FRTSourceFactory(ConnectionFactory::UP connectionFactory, const TimingValues & timingValues, int traceLevel, const VespaVersion & vespaVersion, const CompressionType & compressionType); - + FRTSourceFactory(const FRTSourceFactory &) = delete; + FRTSourceFactory & operator =(const FRTSourceFactory &) = delete; + FRTSourceFactory(std::unique_ptr<ConnectionFactory> connectionFactory, const TimingValues & timingValues, + int traceLevel, const VespaVersion & vespaVersion, const CompressionType & compressionType); + ~FRTSourceFactory() override; /** * Create source handling config described by key. */ std::unique_ptr<Source> createSource(std::shared_ptr<IConfigHolder> holder, const ConfigKey & key) const override; private: - ConnectionFactory::SP _connectionFactory; + std::shared_ptr<ConnectionFactory> _connectionFactory; FRTConfigRequestFactory _requestFactory; const TimingValues _timingValues; }; diff --git a/config/src/vespa/config/subscription/configsubscriptionset.cpp b/config/src/vespa/config/subscription/configsubscriptionset.cpp index 0b3944acc4e..40dc40d11ef 100644 --- a/config/src/vespa/config/subscription/configsubscriptionset.cpp +++ b/config/src/vespa/config/subscription/configsubscriptionset.cpp @@ -112,7 +112,7 @@ ConfigSubscriptionSet::close() { _state = CLOSED; for (const auto & subscription : _subscriptionList) { - _mgr.unsubscribe(subscription); + _mgr.unsubscribe(*subscription); subscription->close(); } } diff --git a/config/src/vespa/config/subscription/sourcespec.cpp b/config/src/vespa/config/subscription/sourcespec.cpp index 6af981807de..ec70f921179 100644 --- a/config/src/vespa/config/subscription/sourcespec.cpp +++ b/config/src/vespa/config/subscription/sourcespec.cpp @@ -11,6 +11,9 @@ #include <vespa/config/set/configinstancesourcefactory.h> #include <vespa/vespalib/text/stringtokenizer.h> #include <vespa/config/print/asciiconfigwriter.h> +#include <vespa/vespalib/util/size_literals.h> +#include <vespa/fnet/transport.h> +#include <vespa/fastos/thread.h> #include <cassert> namespace config { @@ -25,7 +28,7 @@ RawSpec::RawSpec(const vespalib::string & config) { } -SourceFactory::UP +std::unique_ptr<SourceFactory> RawSpec::createSourceFactory(const TimingValues &) const { return std::make_unique<RawSourceFactory>(_config); @@ -49,7 +52,7 @@ FileSpec::verifyName(const vespalib::string & fileName) } } -SourceFactory::UP +std::unique_ptr<SourceFactory> FileSpec::createSourceFactory(const TimingValues & ) const { return std::make_unique<FileSourceFactory>(*this); @@ -60,7 +63,7 @@ DirSpec::DirSpec(const vespalib::string & dirName) { } -SourceFactory::UP +std::unique_ptr<SourceFactory> DirSpec::createSourceFactory(const TimingValues & ) const { return std::make_unique<DirSourceFactory>(*this); @@ -117,12 +120,15 @@ ServerSpec::ServerSpec(const vespalib::string & hostSpec) initialize(hostSpec); } -SourceFactory::UP +std::unique_ptr<SourceFactory> ServerSpec::createSourceFactory(const TimingValues & timingValues) const { const auto vespaVersion = VespaVersion::getCurrentVersion(); - return std::make_unique<FRTSourceFactory>(std::make_unique<FRTConnectionPool>(*this, timingValues), timingValues, - _traceLevel, vespaVersion, _compressionType); + return std::make_unique<FRTSourceFactory>( + std::make_unique<FRTConnectionPoolWithTransport>(std::make_unique<FastOS_ThreadPool>(64_Ki), + std::make_unique<FNET_Transport>(), + *this, timingValues), + timingValues, _traceLevel, vespaVersion, _compressionType); } @@ -131,7 +137,7 @@ ConfigSet::ConfigSet() { } -SourceFactory::UP +std::unique_ptr<SourceFactory> ConfigSet::createSourceFactory(const TimingValues & ) const { return std::make_unique<ConfigSetSourceFactory>(_builderMap); diff --git a/config/src/vespa/config/subscription/sourcespec.h b/config/src/vespa/config/subscription/sourcespec.h index a57b22ca322..3c23233e3ab 100644 --- a/config/src/vespa/config/subscription/sourcespec.h +++ b/config/src/vespa/config/subscription/sourcespec.h @@ -157,13 +157,6 @@ public: SourceFactorySP createSourceFactory(const TimingValues & timingValues) const override; /** - * Add another host to this source spec, allowing failover. - * - * @param host hostname formatted as tcp/hostname:port - */ - void addHost(const vespalib::string & host) { _hostList.push_back(host); } - - /** * Inspect how many hosts this source refers to. * * @return the number of hosts referred. @@ -193,9 +186,9 @@ public: CompressionType compressionType() const { return _compressionType; } private: void initialize(const vespalib::string & hostSpec); - HostSpecList _hostList; - const int _protocolVersion; - const int _traceLevel; + HostSpecList _hostList; + const int _protocolVersion; + const int _traceLevel; const CompressionType _compressionType; const static int DEFAULT_PROXY_PORT = 19090; }; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java index c414d8dfa9a..9b3b3cef5b5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpErrorResponse.java @@ -20,7 +20,6 @@ import static com.yahoo.jdisc.Response.Status.REQUEST_TIMEOUT; /** * @author Ulf Lilleengen - * @since 5.1 */ public class HttpErrorResponse extends HttpResponse { @@ -45,7 +44,7 @@ public class HttpErrorResponse extends HttpResponse { INVALID_APPLICATION_PACKAGE, METHOD_NOT_ALLOWED, NOT_FOUND, - OUT_OF_CAPACITY, + NODE_ALLOCATION_FAILURE, REQUEST_TIMEOUT, UNKNOWN_VESPA_VERSION, PARENT_HOST_NOT_READY, @@ -66,8 +65,8 @@ public class HttpErrorResponse extends HttpResponse { return new HttpErrorResponse(BAD_REQUEST, ErrorCode.INVALID_APPLICATION_PACKAGE.name(), msg); } - public static HttpErrorResponse outOfCapacity(String msg) { - return new HttpErrorResponse(BAD_REQUEST, ErrorCode.OUT_OF_CAPACITY.name(), msg); + public static HttpErrorResponse nodeAllocationFailure(String msg) { + return new HttpErrorResponse(BAD_REQUEST, ErrorCode.NODE_ALLOCATION_FAILURE.name(), msg); } public static HttpErrorResponse badRequest(String msg) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java index 190005771c7..2700f401a86 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/http/HttpHandler.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.config.server.http; import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.CertificateNotReadyException; -import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.ParentHostUnavailableException; import com.yahoo.config.provision.exception.ActivationConflictException; import com.yahoo.config.provision.exception.LoadBalancerServiceException; @@ -55,8 +55,8 @@ public class HttpHandler extends ThreadedHttpRequestHandler { return HttpErrorResponse.invalidApplicationPackage(getMessage(e, request)); } catch (IllegalArgumentException e) { return HttpErrorResponse.badRequest(getMessage(e, request)); - } catch (OutOfCapacityException e) { - return HttpErrorResponse.outOfCapacity(getMessage(e, request)); + } catch (NodeAllocationException e) { + return HttpErrorResponse.nodeAllocationFailure(getMessage(e, request)); } catch (InternalServerException e) { return HttpErrorResponse.internalServerError(getMessage(e, request)); } catch (UnknownVespaVersionException e) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java index 0fbece2681b..3f655ec66f6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ModelsBuilder.java @@ -13,7 +13,7 @@ import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.DockerImage; -import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.TransientException; import com.yahoo.config.provision.Zone; import com.yahoo.lang.SettableOptional; @@ -123,7 +123,7 @@ public abstract class ModelsBuilder<MODELRESULT extends ModelResult> { buildLatestModelForThisMajor, majorVersion)); buildLatestModelForThisMajor = false; // We have successfully built latest model version, do it only for this major } - catch (OutOfCapacityException | ApplicationLockException | TransientException e) { + catch (NodeAllocationException | ApplicationLockException | TransientException e) { // Don't wrap this exception, and don't try to load other model versions as this is (most likely) // caused by the state of the system, not the model version/application combination throw e; diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java index 0b8b142e3aa..9071b12507e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java @@ -8,7 +8,7 @@ import com.yahoo.config.provision.ApplicationLockException; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.InstanceName; -import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.jdisc.http.HttpRequest; @@ -246,15 +246,15 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { @Test public void test_out_of_capacity_response() throws IOException { long sessionId = applicationRepository.createSession(applicationId(), timeoutBudget, app); - String exceptionMessage = "Out of capacity"; + String exceptionMessage = "Node allocation failure"; FailingSessionPrepareHandler handler = new FailingSessionPrepareHandler(SessionPrepareHandler.testContext(), applicationRepository, configserverConfig, - new OutOfCapacityException(exceptionMessage)); + new NodeAllocationException(exceptionMessage)); HttpResponse response = handler.handle(createTestRequest(pathPrefix, HttpRequest.Method.PUT, Cmd.PREPARED, sessionId)); assertEquals(400, response.getStatus()); Slime data = getData(response); - assertEquals(HttpErrorResponse.ErrorCode.OUT_OF_CAPACITY.name(), data.get().field("error-code").asString()); + assertEquals(HttpErrorResponse.ErrorCode.NODE_ALLOCATION_FAILURE.name(), data.get().field("error-code").asString()); assertEquals(exceptionMessage, data.get().field("message").asString()); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java index f0b681fd9c9..47373413a0d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java @@ -34,7 +34,7 @@ public class ConfigServerException extends RuntimeException { INVALID_APPLICATION_PACKAGE, METHOD_NOT_ALLOWED, NOT_FOUND, - OUT_OF_CAPACITY, + NODE_ALLOCATION_FAILURE, REQUEST_TIMEOUT, UNKNOWN_VESPA_VERSION, PARENT_HOST_NOT_READY, diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/DeploymentFailureMails.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/DeploymentFailureMails.java index 10727f1d4eb..2dcb80d28fb 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/DeploymentFailureMails.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/DeploymentFailureMails.java @@ -20,10 +20,10 @@ public class DeploymentFailureMails { this.registry = registry; } - public Mail outOfCapacity(RunId id, Collection<String> recipients) { - return mail(id, recipients, " due to lack of capacity", - "as the zone does not have enough free capacity to " + - "accomodate the deployment. Please contact the Vespa team to request more!"); + public Mail nodeAllocationFailure(RunId id, Collection<String> recipients) { + return mail(id, recipients, " due to node allocation failure", + "as your node resource request could not be " + + "fulfilled for your tenant. Contact Vespa Cloud support."); } public Mail deploymentFailure(RunId id, Collection<String> recipients) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java index 1354f173633..9ef7cbcebf6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java @@ -18,8 +18,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.outOfCapacity; import static java.util.Comparator.comparing; import static java.util.Comparator.naturalOrder; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java index 4f546081c07..fb4584d4672 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java @@ -744,7 +744,7 @@ public class DeploymentStatus { Versions lastVersions = job.lastCompleted().get().versions(); if (change.platform().isPresent() && ! change.platform().get().equals(lastVersions.targetPlatform())) return Optional.empty(); if (change.application().isPresent() && ! change.application().get().equals(lastVersions.targetApplication())) return Optional.empty(); - if (job.id().type().environment().isTest() && job.isOutOfCapacity()) return Optional.empty(); + if (job.id().type().environment().isTest() && job.isNodeAllocationFailure()) return Optional.empty(); Instant firstFailing = job.firstFailing().get().end().get(); Instant lastCompleted = job.lastCompleted().get().end().get(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index 64f879a3a5c..41db36b136e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -411,7 +411,7 @@ public class DeploymentTrigger { // ---------- Version and job helpers ---------- private Job deploymentJob(Instance instance, Versions versions, JobType jobType, JobStatus jobStatus, Instant availableSince) { - return new Job(instance, versions, jobType, availableSince, jobStatus.isOutOfCapacity(), instance.change().application().isPresent()); + return new Job(instance, versions, jobType, availableSince, jobStatus.isNodeAllocationFailure(), instance.change().application().isPresent()); } // ---------- Data containers ---------- diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index e4987c64b2b..f89e262aac2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -86,7 +86,7 @@ import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Nod import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.outOfCapacity; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.nodeAllocationFailure; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.testFailure; @@ -263,11 +263,11 @@ public class InternalStepRunner implements StepRunner { case PARENT_HOST_NOT_READY: logger.log(e.message()); // Consider splitting these messages in summary and details, on config server. return result; - case OUT_OF_CAPACITY: + case NODE_ALLOCATION_FAILURE: logger.log(e.message()); return controller.system().isCd() && startTime.plus(timeouts.capacity()).isAfter(controller.clock().instant()) ? result - : Optional.of(outOfCapacity); + : Optional.of(nodeAllocationFailure); case INVALID_APPLICATION_PACKAGE: case BAD_REQUEST: logger.log(WARNING, e.getMessage()); @@ -812,8 +812,8 @@ public class InternalStepRunner implements StepRunner { case success: controller.notificationsDb().removeNotification(source, Notification.Type.deployment); return; - case outOfCapacity: - if ( ! run.id().type().environment().isTest()) updater.accept("lack of capacity. Please contact the Vespa team to request more!"); + case nodeAllocationFailure: + if ( ! run.id().type().environment().isTest()) updater.accept("could not allocate the requested capacity to your tenant. Contact Vespa Cloud support."); return; case deploymentFailed: updater.accept("invalid application configuration, or timeout of other deployments of the same application"); @@ -840,8 +840,8 @@ public class InternalStepRunner implements StepRunner { case aborted: case success: return Optional.empty(); - case outOfCapacity: - return run.id().type().isProduction() ? Optional.of(mails.outOfCapacity(run.id(), recipients)) : Optional.empty(); + case nodeAllocationFailure: + return run.id().type().isProduction() ? Optional.of(mails.nodeAllocationFailure(run.id(), recipients)) : Optional.empty(); case deploymentFailed: return Optional.of(mails.deploymentFailure(run.id(), recipients)); case installationFailed: diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java index c54d17c2596..5de07bad859 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java @@ -60,7 +60,7 @@ public class JobList extends AbstractFilteringList<JobStatus, JobList> { } public JobList outOfTestCapacity() { - return matching(job -> job.isOutOfCapacity() && job.id().type().environment().isTest()); + return matching(job -> job.isNodeAllocationFailure() && job.id().type().environment().isTest()); } public JobList running() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java index 80a03f910aa..874b1828f5f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobMetrics.java @@ -16,7 +16,7 @@ import java.util.function.Supplier; public class JobMetrics { public static final String start = "deployment.start"; - public static final String outOfCapacity = "deployment.outOfCapacity"; + public static final String nodeAllocationFailure = "deployment.nodeAllocationFailure"; public static final String endpointCertificateTimeout = "deployment.endpointCertificateTimeout"; public static final String deploymentFailure = "deployment.deploymentFailure"; public static final String convergenceFailure = "deployment.convergenceFailure"; @@ -51,7 +51,7 @@ public class JobMetrics { static String valueOf(RunStatus status) { switch (status) { - case outOfCapacity: return outOfCapacity; + case nodeAllocationFailure: return nodeAllocationFailure; case endpointCertificateTimeout: return endpointCertificateTimeout; case deploymentFailed: return deploymentFailure; case installationFailed: return convergenceFailure; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobStatus.java index cf8081de7ca..aad5d510261 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobStatus.java @@ -67,8 +67,8 @@ public class JobStatus { return lastTriggered.isPresent() && ! lastTriggered.get().hasEnded(); } - public boolean isOutOfCapacity() { - return lastStatus().isPresent() && lastStatus().get() == RunStatus.outOfCapacity; + public boolean isNodeAllocationFailure() { + return lastStatus().isPresent() && lastStatus().get() == RunStatus.nodeAllocationFailure; } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java index 375bdd239d3..0bb4a30425e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java @@ -11,8 +11,8 @@ public enum RunStatus { /** Run is still proceeding normally, i.e., without failures. */ running, - /** Deployment was rejected due to lack of capacity. */ - outOfCapacity, + /** Deployment was rejected due node allocation failure. */ + nodeAllocationFailure, /** Deployment of the real application was rejected. */ deploymentFailed, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java index 5935a0d51fd..c39ee031e27 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/notification/Notification.java @@ -69,7 +69,7 @@ public class Notification { /** Related to contents of application package, e.g. usage of deprecated features/syntax */ applicationPackage, - /** Related to deployment of application, e.g. system test failure, out of capacity, internal errors, etc. */ + /** Related to deployment of application, e.g. system test failure, node allocation failure, internal errors, etc. */ deployment, /** Application cluster is (near) external feed blocked */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index 800d5e2750b..9a301f50b1a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -36,7 +36,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentF import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.endpointCertificateTimeout; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.error; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; -import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.outOfCapacity; +import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.nodeAllocationFailure; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; @@ -360,7 +360,7 @@ class RunSerializer { static String valueOf(RunStatus status) { switch (status) { case running : return "running"; - case outOfCapacity : return "outOfCapacity"; + case nodeAllocationFailure : return "nodeAllocationFailure"; case endpointCertificateTimeout : return "endpointCertificateTimeout"; case deploymentFailed : return "deploymentFailed"; case installationFailed : return "installationFailed"; @@ -377,7 +377,8 @@ class RunSerializer { static RunStatus runStatusOf(String status) { switch (status) { case "running" : return running; - case "outOfCapacity" : return outOfCapacity; + case "outOfCapacity" : return nodeAllocationFailure; // TODO: Remove after March 2022 + case "nodeAllocationFailure" : return nodeAllocationFailure; case "endpointCertificateTimeout" : return endpointCertificateTimeout; case "deploymentFailed" : return deploymentFailed; case "installationFailed" : return installationFailed; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 0da8934f2a6..799907f258e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -237,7 +237,7 @@ class JobControllerApiHandlerHelper { case error: return "error"; case testFailure: return "testFailure"; case endpointCertificateTimeout: return "endpointCertificateTimeout"; - case outOfCapacity: return "outOfCapacity"; + case nodeAllocationFailure: return "nodeAllocationFailure"; case installationFailed: return "installationFailed"; case deploymentFailed: return "deploymentFailed"; case success: return "success"; @@ -246,7 +246,8 @@ class JobControllerApiHandlerHelper { } /** - * @return Response with all job types that have recorded runs for the application _and_ the status for the last run of that type + * Returns response with all job types that have recorded runs for the application + * _and_ the status for the last run of that type */ static HttpResponse overviewResponse(Controller controller, TenantAndApplicationId id, URI baseUriForDeployments) { Application application = controller.applications().requireApplication(id); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index 1e41421de53..83397768264 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -49,7 +49,6 @@ import java.security.cert.X509Certificate; import java.time.Duration; import java.time.Instant; import java.util.HashMap; -import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -299,10 +298,10 @@ public class DeploymentContext { } /** Fail current deployment in given job */ - public DeploymentContext outOfCapacity(JobType type) { + public DeploymentContext nodeAllocationFailure(JobType type) { return failDeployment(type, - new ConfigServerException(ConfigServerException.ErrorCode.OUT_OF_CAPACITY, - "Out of capacity", + new ConfigServerException(ConfigServerException.ErrorCode.NODE_ALLOCATION_FAILURE, + "Node allocation failure", "Failed to deploy application")); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index a4f9ce01e97..6cd4e3e92c3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -790,7 +790,7 @@ public class DeploymentTriggerTest { } @Test - public void requeueOutOfCapacityStagingJob() { + public void requeueNodeAllocationFailureStagingJob() { ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .region("us-east-3") .build(); @@ -826,9 +826,9 @@ public class DeploymentTriggerTest { tester.readyJobsTrigger().maintain(); assertEquals(3, tester.jobs().active().size()); - // Remove the jobs for app1 and app2, and then let app3 fail with outOfCapacity. - // All three jobs are now eligible, but the one for app3 should trigger first as an outOfCapacity-retry. - app3.outOfCapacity(stagingTest); + // Remove the jobs for app1 and app2, and then let app3 fail node allocation. + // All three jobs are now eligible, but the one for app3 should trigger first as a nodeAllocationFailure-retry. + app3.nodeAllocationFailure(stagingTest); app1.abortJob(stagingTest); app2.abortJob(stagingTest); @@ -858,9 +858,9 @@ public class DeploymentTriggerTest { app1.assertRunning(stagingTest); assertEquals(2, tester.jobs().active().size()); - // Let the test jobs start, remove everything except system test for app3, which fails with outOfCapacity again. + // Let the test jobs start, remove everything except system test for app3, which fails node allocation again. tester.triggerJobs(); - app3.outOfCapacity(systemTest); + app3.nodeAllocationFailure(systemTest); app1.abortJob(systemTest); app1.abortJob(stagingTest); app2.abortJob(systemTest); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java index 4a19a9c5c4d..6b380981e15 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunnerTest.java @@ -405,7 +405,7 @@ public class JobRunnerTest { assertEquals(1, metric.getMetric(context::equals, JobMetrics.success).get().intValue()); assertEquals(1, metric.getMetric(context::equals, JobMetrics.convergenceFailure).get().intValue()); assertEquals(1, metric.getMetric(context::equals, JobMetrics.deploymentFailure).get().intValue()); - assertEquals(1, metric.getMetric(context::equals, JobMetrics.outOfCapacity).get().intValue()); + assertEquals(1, metric.getMetric(context::equals, JobMetrics.nodeAllocationFailure).get().intValue()); assertEquals(1, metric.getMetric(context::equals, JobMetrics.endpointCertificateTimeout).get().intValue()); assertEquals(1, metric.getMetric(context::equals, JobMetrics.testFailure).get().intValue()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializerTest.java index 1fec1dbce02..cbb595d2a3b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/NotificationsSerializerTest.java @@ -36,7 +36,7 @@ public class NotificationsSerializerTest { Notification.Type.deployment, Notification.Level.error, NotificationSource.from(new RunId(ApplicationId.from(tenantName.value(), "app1", "instance1"), JobType.systemTest, 12)), - List.of("Failed to deploy: Out of capacity"))); + List.of("Failed to deploy: Node allocation failure"))); Slime serialized = NotificationsSerializer.toSlime(notifications); assertEquals("{\"notifications\":[" + @@ -50,7 +50,7 @@ public class NotificationsSerializerTest { "\"at\":2345000," + "\"type\":\"deployment\"," + "\"level\":\"error\"," + - "\"messages\":[\"Failed to deploy: Out of capacity\"]," + + "\"messages\":[\"Failed to deploy: Node allocation failure\"]," + "\"application\":\"app1\"," + "\"instance\":\"instance1\"," + "\"jobId\":\"system-test\"," + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 9a8fced2288..b4230cb1fc1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -1867,7 +1867,7 @@ public class ApplicationApiTest extends ControllerContainerTest { NotificationSource.from(new RunId(ApplicationId.from(tenantName.value(), "app2", "instance1"), JobType.systemTest, 12)), Notification.Type.deployment, Notification.Level.error, - "Failed to deploy: Out of capacity"); + "Failed to deploy: Node allocation failure"); } private void assertGlobalRouting(DeploymentId deployment, RoutingStatus.Value value, RoutingStatus.Agent agent) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1-app2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1-app2.json index 0e0ca7405ca..277831f2a9f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1-app2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1-app2.json @@ -5,7 +5,7 @@ "level": "error", "type": "deployment", "messages": [ - "Failed to deploy: Out of capacity" + "Failed to deploy: Node allocation failure" ], "application": "app2", "instance": "instance1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1.json index 7d3dd5e672f..33755843486 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/notifications-tenant1.json @@ -14,7 +14,7 @@ "level": "error", "type": "deployment", "messages": [ - "Failed to deploy: Out of capacity" + "Failed to deploy: Node allocation failure" ], "application": "app2", "instance": "instance1", diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java index 1352220166c..11a4d096312 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java @@ -551,7 +551,7 @@ public abstract class ControllerHttpClient { case "aborted": return DeploymentLog.Status.aborted; case "error": return DeploymentLog.Status.error; case "testFailure": return DeploymentLog.Status.testFailure; - case "outOfCapacity": return DeploymentLog.Status.outOfCapacity; + case "nodeAllocationFailure": return DeploymentLog.Status.nodeAllocationFailure; case "installationFailed": return DeploymentLog.Status.installationFailed; case "deploymentFailed": return DeploymentLog.Status.deploymentFailed; case "endpointCertificateTimeout": return DeploymentLog.Status.endpointCertificateTimeout; diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java b/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java index b7097ae8c9c..ea35732a7d6 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentLog.java @@ -117,7 +117,7 @@ public class DeploymentLog { aborted, error, testFailure, - outOfCapacity, + nodeAllocationFailure, installationFailed, deploymentFailed, endpointCertificateTimeout, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java index e6476cd7373..6eaee7b33de 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java @@ -8,7 +8,7 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.jdisc.Metric; import com.yahoo.lang.MutableInteger; import com.yahoo.transaction.Mutex; @@ -121,11 +121,11 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { List<Node> excessHosts; try { excessHosts = provision(nodes); - } catch (OutOfCapacityException | IllegalStateException e) { - log.log(Level.WARNING, "Failed to provision preprovision capacity and/or find excess hosts: " + e.getMessage()); + } catch (NodeAllocationException | IllegalStateException e) { + log.log(Level.WARNING, "Failed to allocate preprovisioned capacity and/or find excess hosts: " + e.getMessage()); return; // avoid removing excess hosts } catch (RuntimeException e) { - log.log(Level.WARNING, "Failed to provision preprovision capacity and/or find excess hosts", e); + log.log(Level.WARNING, "Failed to allocate preprovisioned capacity and/or find excess hosts", e); return; // avoid removing excess hosts } @@ -213,7 +213,7 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { /** * @return the nodes in {@code nodeList} plus all hosts provisioned, plus all preprovision capacity * nodes that were allocated. - * @throws OutOfCapacityException if there were problems provisioning hosts, and in case message + * @throws NodeAllocationException if there were problems provisioning hosts, and in case message * should be sufficient (avoid no stack trace) * @throws IllegalStateException if there was an algorithmic problem, and in case message * should be sufficient (avoid no stack trace). @@ -251,8 +251,8 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { .collect(Collectors.toList()); nodeRepository().nodes().addNodes(hosts, Agent.DynamicProvisioningMaintainer); return hosts; - } catch (OutOfCapacityException | IllegalArgumentException | IllegalStateException e) { - throw new OutOfCapacityException("Failed to provision " + count + " " + nodeResources + ": " + e.getMessage()); + } catch (NodeAllocationException | IllegalArgumentException | IllegalStateException e) { + throw new NodeAllocationException("Failed to provision " + count + " " + nodeResources + ": " + e.getMessage()); } catch (RuntimeException e) { throw new RuntimeException("Failed to provision " + count + " " + nodeResources + ", will retry in " + interval(), e); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java index 8c549c3ca53..e5e5ce27cf1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java @@ -244,9 +244,8 @@ public class Nodes { if ( ! zone.environment().isProduction() || zone.system().isCd()) return deallocate(nodes, Agent.application, "Deactivated by application", transaction.nested()); - NodeList nodeList = NodeList.copyOf(nodes); - NodeList stateless = nodeList.stateless(); - NodeList stateful = nodeList.stateful(); + var stateless = NodeList.copyOf(nodes).stateless(); + var stateful = NodeList.copyOf(nodes).stateful(); List<Node> written = new ArrayList<>(); written.addAll(deallocate(stateless.asList(), Agent.application, "Deactivated by application", transaction.nested())); written.addAll(db.writeTo(Node.State.inactive, stateful.asList(), Agent.application, Optional.empty(), transaction.nested())); @@ -471,7 +470,7 @@ public class Nodes { */ public Node markNodeAvailableForNewAllocation(String hostname, Agent agent, String reason) { Node node = requireNode(hostname); - if (removeOnReadyingOf(node)) { + if (node.flavor().getType() == Flavor.Type.DOCKER_CONTAINER && node.type() == NodeType.tenant) { if (node.state() != Node.State.dirty) illegal("Cannot make " + node + " available for new allocation as it is not in state [dirty]"); return removeRecursively(node, true).get(0); @@ -835,7 +834,7 @@ public class Nodes { private static boolean parkOnDeallocationOf(Node node, Agent agent) { if (node.state() == Node.State.parked) return false; if (agent == Agent.operator) return false; - if (!node.type().isHost() && node.status().wantToDeprovision()) return false; + if (node.type() == NodeType.tenant && node.status().wantToDeprovision()) return false; boolean retirementRequestedByOperator = node.status().wantToRetire() && node.history().event(History.Event.Type.wantToRetire) .map(History.Event::agent) @@ -846,12 +845,6 @@ public class Nodes { retirementRequestedByOperator; } - /** Returns whether node should be deleted when it's moved to ready */ - private static boolean removeOnReadyingOf(Node node) { - if (node.flavor().getType() != Flavor.Type.DOCKER_CONTAINER) return false; - return node.type() == NodeType.tenant || (node.status().wantToRetire() && node.status().wantToDeprovision()); - } - /** The different ways a host can be decommissioned */ private enum DecommissionOperation { deprovision, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java index 5229fbb8f37..ba59ff8e425 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java @@ -37,10 +37,10 @@ public class RetiringOsUpgrader implements OsUpgrader { NodeList allNodes = nodeRepository.nodes().list(); Instant now = nodeRepository.clock().instant(); NodeList candidates = candidates(now, target, allNodes); - candidates.not().matching(host -> deprovisioning(host, allNodes)) + candidates.not().deprovisioning() .byIncreasingOsVersion() - .first() - .ifPresent(node -> deprovision(node, target.version(), now)); + .first(1) + .forEach(node -> deprovision(node, target.version(), now)); } @Override @@ -62,7 +62,7 @@ public class RetiringOsUpgrader implements OsUpgrader { /** Upgrade given host by retiring and deprovisioning it */ private void deprovision(Node host, Version target, Instant now) { - LOG.info("Retiring and deprovisioning " + host + " and its children: On stale OS version " + + LOG.info("Retiring and deprovisioning " + host + ": On stale OS version " + host.status().osVersion().current().map(Version::toFullString).orElse("<unset>") + ", want " + target); nodeRepository.nodes().deprovision(host.hostname(), Agent.RetiringUpgrader, now); @@ -70,10 +70,4 @@ public class RetiringOsUpgrader implements OsUpgrader { nodeRepository.osVersions().writeChange((change) -> change.withRetirementAt(now, host.type())); } - /** Returns whether host and all its children are deprovisioning */ - private static boolean deprovisioning(Node host, NodeList allNodes) { - return host.status().wantToRetire() && host.status().wantToDeprovision() && - allNodes.childrenOf(host).not().deprovisioning().isEmpty(); - } - } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index ae65f367684..9148a2165a5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -5,7 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; @@ -125,8 +125,9 @@ public class GroupPreparer { } if (! allocation.fulfilled() && requestedNodes.canFail()) - throw new OutOfCapacityException((cluster.group().isPresent() ? "Out of capacity on " + cluster.group().get() :"") + - allocation.outOfCapacityDetails()); + throw new NodeAllocationException((cluster.group().isPresent() ? "Node allocation failure on " + + cluster.group().get() : "") + + allocation.allocationFailureDetails()); // Carry out and return allocation nodeRepository.nodes().reserve(allocation.reservableNodes()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index 6c22a26d88a..7836ca3265e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -441,7 +441,7 @@ class NodeAllocation { .collect(Collectors.toList()); } - public String outOfCapacityDetails() { + public String allocationFailureDetails() { List<String> reasons = new ArrayList<>(); if (rejectedDueToExclusivity > 0) reasons.add("host exclusivity constraints"); @@ -453,7 +453,7 @@ class NodeAllocation { reasons.add("insufficient real resources on hosts"); if (reasons.isEmpty()) return ""; - return ": Not enough nodes available due to " + String.join(", ", reasons); + return ": Not enough suitable nodes available due to " + String.join(", ", reasons); } private static Integer parseIndex(String hostname) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java index c98ff31ecb6..838d6751d09 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; @@ -40,10 +40,10 @@ class Preparer { prepareLoadBalancer(application, cluster, requestedNodes); return nodes; } - catch (OutOfCapacityException e) { - throw new OutOfCapacityException("Could not satisfy " + requestedNodes + - ( wantedGroups > 1 ? " (in " + wantedGroups + " groups)" : "") + - " in " + application + " " + cluster + ": " + e.getMessage()); + catch (NodeAllocationException e) { + throw new NodeAllocationException("Could not satisfy " + requestedNodes + + ( wantedGroups > 1 ? " (in " + wantedGroups + " groups)" : "") + + " in " + application + " " + cluster + ": " + e.getMessage()); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java index 49fc3c2b7b2..0a125c5fb95 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java @@ -7,7 +7,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.IP; @@ -59,7 +59,7 @@ public class MockHostProvisioner implements HostProvisioner { Optional<ClusterSpec.Type> clusterType) { Flavor hostFlavor = this.hostFlavor.orElseGet(() -> flavors.stream().filter(f -> compatible(f, resources)) .findFirst() - .orElseThrow(() -> new OutOfCapacityException("No host flavor matches " + resources))); + .orElseThrow(() -> new NodeAllocationException("No host flavor matches " + resources))); List<ProvisionedHost> hosts = new ArrayList<>(); for (int index : provisionIndices) { String hostHostname = hostType == NodeType.host ? "hostname" + index : hostType.name() + index; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java index 7ce26354739..68d75db8a4c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java @@ -486,19 +486,21 @@ public class DynamicProvisioningMaintainerTest { tester.prepareAndActivateInfraApplication(configSrvApp, hostType.childNodeType()); assertEquals("Node moves to inactive", Node.State.inactive, nodeToRemove.get().state()); - // Node is completely removed (done by InactiveExpirer and host-admin in a real system) - Node inactiveConfigServer = nodeToRemove.get(); - int removedIndex = inactiveConfigServer.allocation().get().membership().index(); - tester.nodeRepository().nodes().removeRecursively(inactiveConfigServer, true); - assertEquals(2, tester.nodeRepository().nodes().list().nodeType(hostType.childNodeType()).size()); + // Node is parked (done by InactiveExpirer and host-admin in a real system) + int removedIndex = nodeToRemove.get().allocation().get().membership().index(); + Node parkedConfigServer = tester.nodeRepository().nodes().deallocate(nodeToRemove.get(), Agent.system, getClass().getSimpleName()); + assertSame("Node moves to parked", Node.State.parked, parkedConfigServer.state()); + assertEquals(2, tester.nodeRepository().nodes().list().nodeType(hostType.childNodeType()).state(Node.State.active).size()); - // ExpiredRetirer moves host to inactive after child has moved to parked + // ... same for host tester.nodeRepository().nodes().deallocate(hostToRemove.get(), Agent.system, getClass().getSimpleName()); assertSame("Host moves to parked", Node.State.parked, hostToRemove.get().state()); - // Host is removed + // Host and child is removed dynamicProvisioningTester.maintainer.maintain(); - assertEquals(2, tester.nodeRepository().nodes().list().nodeType(hostType).size()); + allNodes = tester.nodeRepository().nodes().list(); + assertEquals(2, allNodes.nodeType(hostType).size()); + assertEquals(2, allNodes.nodeType(hostType.childNodeType()).size()); // Deployment by the removed host has no effect HostName.setHostNameForTestingOnly("cfg2.example.com"); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java index 5a64d99ea33..26be6d95336 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java @@ -227,41 +227,25 @@ public class OsVersionsTest { @Test public void upgrade_by_retiring_everything_at_once() { - OsVersions versions = new OsVersions(tester.nodeRepository(), true, Integer.MAX_VALUE); - int nodeCount = 3; - ApplicationId configServerApp = ApplicationId.from("hosted-vespa", "zone-config-servers", "default"); - List<Node> configHosts = provisionInfraApplication(nodeCount, infraApplication, NodeType.confighost); - NodeResources resources = new NodeResources(2, 4, 8, 1); - for (int i = 0; i < nodeCount; i++) { - tester.makeReadyChildren(1, i, resources, NodeType.config, configHosts.get(i).hostname(), (index) -> "cfg" + index); - } - tester.prepareAndActivateInfraApplication(configServerApp, NodeType.config); + var versions = new OsVersions(tester.nodeRepository(), true, Integer.MAX_VALUE); + int hostCount = 3; + provisionInfraApplication(hostCount, infraApplication, NodeType.confighost); + Supplier<NodeList> hostNodes = () -> tester.nodeRepository().nodes().list() + .nodeType(NodeType.confighost) + .not().state(Node.State.deprovisioned); // Target is set with zero budget and upgrade started var version1 = Version.fromString("7.1"); versions.setTarget(NodeType.confighost, version1, Duration.ZERO,false); - for (int i = 0; i < nodeCount; i++) { + for (int i = 0; i < hostCount; i++) { versions.resumeUpgradeOf(NodeType.confighost, true); } // All hosts are deprovisioning - NodeList nodes = tester.nodeRepository().nodes().list().not().state(Node.State.deprovisioned); - NodeList configNodes = nodes.nodeType(NodeType.config); - assertEquals(nodeCount, nodes.nodeType(NodeType.confighost).deprovisioning().size()); - assertEquals(nodeCount, configNodes.deprovisioning().size()); - - // One child is inadvertently unretired - tester.patchNode(configNodes.first().get(), (node) -> node.withWantToRetire(false, false, - Agent.system, tester.clock().instant())); - - // Resuming upgrade of host re-retires child - versions.resumeUpgradeOf(NodeType.confighost, true); - nodes = tester.nodeRepository().nodes().list().not().state(Node.State.deprovisioned); - assertEquals(nodeCount, nodes.nodeType(NodeType.config).deprovisioning().size()); - + assertEquals(hostCount, hostNodes.get().deprovisioning().size()); // Nodes complete their upgrade by being reprovisioned - completeReprovisionOf(nodes.nodeType(NodeType.confighost).deprovisioning().asList(), NodeType.confighost); - assertEquals(nodeCount, tester.nodeRepository().nodes().list().nodeType(NodeType.confighost).onOsVersion(version1).size()); + completeReprovisionOf(hostNodes.get().deprovisioning().asList(), NodeType.confighost); + assertEquals(hostCount, hostNodes.get().onOsVersion(version1).size()); } @Test @@ -544,7 +528,7 @@ public class OsVersionsTest { ApplicationId application = node.allocation().get().owner(); tester.nodeRepository().nodes().park(node.hostname(), false, Agent.system, getClass().getSimpleName()); - tester.nodeRepository().nodes().removeRecursively(node, true); + tester.nodeRepository().nodes().removeRecursively(node.hostname()); node = provisionInfraApplication(1, application, nodeType).get(0); } return node.with(node.status().withOsVersion(node.status().osVersion().withCurrent(wantedOsVersion))); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java index 583ccdda656..2cc687e51d3 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicAllocationTest.java @@ -11,7 +11,7 @@ import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.Zone; @@ -220,7 +220,7 @@ public class DynamicAllocationTest { assertEquals(2, hostsWithChildren.size()); } - @Test(expected = OutOfCapacityException.class) + @Test(expected = NodeAllocationException.class) public void multiple_groups_are_on_separate_parent_hosts() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(5, "host-small", NodeType.host, 32); @@ -257,7 +257,7 @@ public class DynamicAllocationTest { try { hosts = tester.prepare(application1, clusterSpec, 4, 1, flavor); fail("Was able to deploy with 4 nodes, should not be able to use spare capacity"); - } catch (OutOfCapacityException ignored) { } + } catch (NodeAllocationException ignored) { } tester.fail(hosts.get(0)); hosts = tester.prepare(application1, clusterSpec, 3, 1, flavor); @@ -283,7 +283,7 @@ public class DynamicAllocationTest { try { tester.prepare(application1, clusterSpec, 4, 1, flavor); fail("Should not be able to deploy 4 nodes on 4 hosts because 1 is suspended"); - } catch (OutOfCapacityException ignored) { } + } catch (NodeAllocationException ignored) { } // Resume the host, the deployment goes through tester.orchestrator().resume(randomHost); @@ -319,7 +319,7 @@ public class DynamicAllocationTest { tester.activate(application1, Set.copyOf(hosts)); } - @Test(expected = OutOfCapacityException.class) + @Test(expected = NodeAllocationException.class) public void allocation_should_fail_when_host_is_not_in_allocatable_state() { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeProvisionedNodes(3, "host-small", NodeType.host, 32).forEach(node -> @@ -377,7 +377,7 @@ public class DynamicAllocationTest { tester.activate(application, hosts); } - private void provisionFastAndSlowThenDeploy(NodeResources.DiskSpeed requestDiskSpeed, boolean expectOutOfCapacity) { + private void provisionFastAndSlowThenDeploy(NodeResources.DiskSpeed requestDiskSpeed, boolean expectNodeAllocationFailure) { ProvisioningTester tester = new ProvisioningTester.Builder().zone(new Zone(Environment.prod, RegionName.from("us-east"))).flavorsConfig(flavorsConfig()).build(); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.fast)), NodeType.host, 10, true); tester.makeReadyNodes(2, new Flavor(new NodeResources(1, 8, 120, 1, NodeResources.DiskSpeed.slow)), NodeType.host, 10, true); @@ -389,12 +389,12 @@ public class DynamicAllocationTest { try { List<HostSpec> hosts = tester.prepare(application, cluster, 4, 1, resources); - if (expectOutOfCapacity) fail("Expected out of capacity"); + if (expectNodeAllocationFailure) fail("Expected node allocation fail"); assertEquals(4, hosts.size()); tester.activate(application, hosts); } - catch (OutOfCapacityException e) { - if ( ! expectOutOfCapacity) throw e; + catch (NodeAllocationException e) { + if ( ! expectNodeAllocationFailure) throw e; } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java index 70f6d3abf6f..a03811778d2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InPlaceResizeProvisionTest.java @@ -7,7 +7,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.flags.InMemoryFlagSource; @@ -133,8 +133,8 @@ public class InPlaceResizeProvisionTest { // Attempt to increase resources of the other app try { new PrepareHelper(tester, app).prepare(container1, 4, 1, largeResources); - fail("Expected to fail due to out of capacity"); - } catch (OutOfCapacityException ignored) { } + fail("Expected to fail node allocation"); + } catch (NodeAllocationException ignored) { } // Add 2 more parent host, now we should be able to do the same deployment that failed earlier // 2 of the nodes will be increased in-place and 2 will be allocated to the new hosts. @@ -211,7 +211,7 @@ public class InPlaceResizeProvisionTest { assertSizeAndResources(listCluster(content1).not().retired(), 8, halvedResources); } - @Test(expected = OutOfCapacityException.class) + @Test(expected = NodeAllocationException.class) public void cannot_inplace_decrease_resources_while_increasing_cluster_size() { addParentHosts(6, mediumResources.with(fast).with(local)); @@ -221,7 +221,7 @@ public class InPlaceResizeProvisionTest { new PrepareHelper(tester, app).prepare(content1, 6, 1, smallResources); } - @Test(expected = OutOfCapacityException.class) + @Test(expected = NodeAllocationException.class) public void cannot_inplace_change_resources_while_decreasing_cluster_size() { addParentHosts(4, largeResources.with(fast).with(local)); @@ -231,7 +231,7 @@ public class InPlaceResizeProvisionTest { new PrepareHelper(tester, app).prepare(container1, 2, 1, smallResources); } - @Test(expected = OutOfCapacityException.class) + @Test(expected = NodeAllocationException.class) public void cannot_inplace_change_resources_while_changing_topology() { addParentHosts(4, largeResources.with(fast).with(local)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index c9b384b95a6..d0eb55469b6 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -14,7 +14,7 @@ import com.yahoo.config.provision.HostFilter; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.ParentHostUnavailableException; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; @@ -333,9 +333,9 @@ public class ProvisioningTest { // redeploy a too large application try { SystemState state2 = prepare(application1, 3, 0, 3, 0, defaultResources, tester); - fail("Expected out of capacity exception"); + fail("Expected node allocation exception"); } - catch (OutOfCapacityException expected) { + catch (NodeAllocationException expected) { } // deploy first state again @@ -626,7 +626,7 @@ public class ProvisioningTest { prepare(application, 2, 2, 3, 3, defaultResources, tester); fail("Expected exception"); } - catch (OutOfCapacityException e) { + catch (NodeAllocationException e) { assertTrue(e.getMessage().startsWith("Could not satisfy request")); } } @@ -653,7 +653,7 @@ public class ProvisioningTest { try { prepare(application, 2, 0, 2, 0, defaultResources, tester); fail("Expected exception"); - } catch (OutOfCapacityException e) { + } catch (NodeAllocationException e) { assertTrue(e.getMessage().startsWith("Could not satisfy request")); } } @@ -826,11 +826,11 @@ public class ProvisioningTest { ApplicationId application = ProvisioningTester.applicationId(); tester.makeReadyHosts(2, defaultResources).activateTenantHosts(); - // Deploy fails with out of capacity + // Node allocation fails try { prepare(application, 2, 0, 2, 0, defaultResources, tester); fail("Expected exception"); - } catch (OutOfCapacityException ignored) {} + } catch (NodeAllocationException ignored) {} assertEquals("Reserved a subset of required nodes", 2, tester.getNodes(application, Node.State.reserved).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java index 9a056541c20..1255cb20cd2 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/VirtualNodeProvisioningTest.java @@ -14,7 +14,7 @@ import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.config.provision.OutOfCapacityException; +import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.ProvisionLock; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; @@ -155,7 +155,7 @@ public class VirtualNodeProvisioningTest { assertEquals(2, nodes.stream().filter(n -> n.allocation().get().membership().retired()).count()); } - @Test(expected = OutOfCapacityException.class) + @Test(expected = NodeAllocationException.class) public void fail_when_too_few_distinct_parent_hosts() { ProvisioningTester tester = new ProvisioningTester.Builder().build(); tester.makeReadyChildren(2, resources1, "parentHost1"); @@ -192,10 +192,10 @@ public class VirtualNodeProvisioningTest { tester.makeReadyChildren(1, resources1, "parentHost1"); tester.makeReadyChildren(2, resources1, "parentHost2"); - OutOfCapacityException expectedException = null; + NodeAllocationException expectedException = null; try { tester.prepare(applicationId, contentClusterSpec, contentNodeCount, groups, resources1); - } catch (OutOfCapacityException e) { + } catch (NodeAllocationException e) { expectedException = e; } assertNotNull(expectedException); @@ -279,7 +279,7 @@ public class VirtualNodeProvisioningTest { ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("myContent")).vespaVersion(wantedVespaVersion).build(), nodeCount, 1, resources2); fail("Expected the allocation to fail due to parent hosts not being active yet"); - } catch (OutOfCapacityException expected) { } + } catch (NodeAllocationException expected) { } // Activate the hosts, thereby allocating the parents tester.activateTenantHosts(); @@ -321,7 +321,7 @@ public class VirtualNodeProvisioningTest { 5, 1, resources); fail("Expected exception"); } - catch (OutOfCapacityException e) { + catch (NodeAllocationException e) { // Success: Not enough nonreserved hosts left } @@ -349,8 +349,8 @@ public class VirtualNodeProvisioningTest { tester.prepare(applicationId, ClusterSpec.request(ClusterSpec.Type.container, ClusterSpec.Id.from("myContent")).vespaVersion(wantedVespaVersion).build(), 6, 1, resources); - fail("Expected to fail due to out of capacity"); - } catch (OutOfCapacityException ignored) { } + fail("Expected to fail node allocation"); + } catch (NodeAllocationException ignored) { } // Same cluster, but content type is now 'content' List<HostSpec> nodes = tester.prepare(applicationId, @@ -447,8 +447,8 @@ public class VirtualNodeProvisioningTest { "Could not satisfy request for 3 nodes with " + "[vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps] " + "in tenant2.app2 container cluster 'myContainer' 6.39: " + - "Out of capacity on group 0: " + - "Not enough nodes available due to host exclusivity constraints", + "Node allocation failure on group 0: " + + "Not enough suitable nodes available due to host exclusivity constraints", e.getMessage()); } @@ -470,11 +470,11 @@ public class VirtualNodeProvisioningTest { 2, 1, resources2.with(NodeResources.StorageType.remote)); } - catch (OutOfCapacityException e) { + catch (NodeAllocationException e) { assertEquals("Could not satisfy request for 2 nodes with " + "[vcpu: 1.0, memory: 4.0 Gb, disk 100.0 Gb, bandwidth: 1.0 Gbps, storage type: remote] " + "in tenant.app1 content cluster 'myContent'" + - " 6.42: Out of capacity on group 0", + " 6.42: Node allocation failure on group 0", e.getMessage()); } } diff --git a/searchlib/src/vespa/searchlib/attribute/attributemanager.cpp b/searchlib/src/vespa/searchlib/attribute/attributemanager.cpp index 5aabad6fa02..4b87435dd2e 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributemanager.cpp +++ b/searchlib/src/vespa/searchlib/attribute/attributemanager.cpp @@ -120,16 +120,6 @@ uint64_t AttributeManager::getMemoryFootprint() const return sum; } -bool AttributeManager::hasReaders() const -{ - for(AttributeMap::const_iterator it(_attributes.begin()), mt(_attributes.end()); it != mt; it++) { - if (it->second->hasReaders()) - return true; - } - - return false; -} - const AttributeManager::VectorHolder * AttributeManager::findAndLoadAttribute(const string & name) const { diff --git a/searchlib/src/vespa/searchlib/attribute/attributemanager.h b/searchlib/src/vespa/searchlib/attribute/attributemanager.h index 7e390d5dd19..daa6c725908 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributemanager.h +++ b/searchlib/src/vespa/searchlib/attribute/attributemanager.h @@ -54,7 +54,6 @@ public: const Snapshot & getSnapshot() const { return _snapShot; } const string & getBaseDir() const { return _baseDir; } void setBaseDir(const string & base); - bool hasReaders() const; uint64_t getMemoryFootprint() const; protected: diff --git a/searchlib/src/vespa/searchlib/attribute/attributevector.h b/searchlib/src/vespa/searchlib/attribute/attributevector.h index 8b42b19cc60..5df8a2aa768 100644 --- a/searchlib/src/vespa/searchlib/attribute/attributevector.h +++ b/searchlib/src/vespa/searchlib/attribute/attributevector.h @@ -623,13 +623,6 @@ public: } /** - * Returns true if we might still have readers. False positives - * are possible if writer hasn't updated first used generation - * after last reader left. - */ - bool hasReaders() const { return _genHandler.hasReaders(); } - - /** * Add reserved initial document with docId 0 and undefined value. */ void addReservedDoc(); diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeployMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeployMojo.java index ea645a9dd2b..6b4fe40d719 100644 --- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeployMojo.java +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/DeployMojo.java @@ -60,7 +60,7 @@ public class DeployMojo extends AbstractVespaDeploymentMojo { case success: return; case error: throw new MojoExecutionException("Unexpected error during deployment; see log for details"); case aborted: throw new MojoFailureException("Deployment was aborted, probably by a newer deployment"); - case outOfCapacity: throw new MojoFailureException("No capacity left in zone; please contact the Vespa team"); + case nodeAllocationFailure: throw new MojoFailureException("Specified node capacity could not be fulfilled for you tenant; contact Vespa Ckoud support"); case deploymentFailed: throw new MojoFailureException("Deployment failed; see log for details"); case installationFailed: throw new MojoFailureException("Installation failed; see Vespa log for details"); case running: throw new MojoFailureException("Deployment not completed"); diff --git a/vespalib/src/tests/util/generationhandler/generationhandler_test.cpp b/vespalib/src/tests/util/generationhandler/generationhandler_test.cpp index 07805a6942c..f269fe729fa 100644 --- a/vespalib/src/tests/util/generationhandler/generationhandler_test.cpp +++ b/vespalib/src/tests/util/generationhandler/generationhandler_test.cpp @@ -97,18 +97,18 @@ Test::requireThatTheFirstUsedGenerationIsCorrect() { GenGuard g1 = gh.takeGuard(); gh.incGeneration(); - EXPECT_EQUAL(true, gh.hasReaders()); + EXPECT_EQUAL(1u, gh.getGenerationRefCount()); EXPECT_EQUAL(1u, gh.getFirstUsedGeneration()); } EXPECT_EQUAL(1u, gh.getFirstUsedGeneration()); gh.updateFirstUsedGeneration(); // Only writer should call this - EXPECT_EQUAL(false, gh.hasReaders()); + EXPECT_EQUAL(0u, gh.getGenerationRefCount()); EXPECT_EQUAL(2u, gh.getFirstUsedGeneration()); { GenGuard g1 = gh.takeGuard(); gh.incGeneration(); gh.incGeneration(); - EXPECT_EQUAL(true, gh.hasReaders()); + EXPECT_EQUAL(1u, gh.getGenerationRefCount()); EXPECT_EQUAL(2u, gh.getFirstUsedGeneration()); { GenGuard g2 = gh.takeGuard(); @@ -117,7 +117,7 @@ Test::requireThatTheFirstUsedGenerationIsCorrect() } EXPECT_EQUAL(2u, gh.getFirstUsedGeneration()); gh.updateFirstUsedGeneration(); // Only writer should call this - EXPECT_EQUAL(false, gh.hasReaders()); + EXPECT_EQUAL(0u, gh.getGenerationRefCount()); EXPECT_EQUAL(4u, gh.getFirstUsedGeneration()); } diff --git a/vespalib/src/tests/util/generationhandler_stress/generation_handler_stress_test.cpp b/vespalib/src/tests/util/generationhandler_stress/generation_handler_stress_test.cpp index 8cea96e0f68..fa2c525b518 100644 --- a/vespalib/src/tests/util/generationhandler_stress/generation_handler_stress_test.cpp +++ b/vespalib/src/tests/util/generationhandler_stress/generation_handler_stress_test.cpp @@ -13,7 +13,7 @@ using vespalib::ThreadStackExecutor; struct WorkContext { - uint64_t _generation; + std::atomic<uint64_t> _generation; WorkContext() noexcept : _generation(0) @@ -84,7 +84,7 @@ Fixture::readWork(const WorkContext &context) for (i = 0; i < cnt && _stopRead.load() == 0; ++i) { auto guard = _generationHandler.takeGuard(); - auto generation = context._generation; + auto generation = context._generation.load(std::memory_order_relaxed); EXPECT_GREATER_EQUAL(generation, guard.getGeneration()); } _doneReadWork += i; @@ -96,7 +96,7 @@ void Fixture::writeWork(uint32_t cnt, WorkContext &context) { for (uint32_t i = 0; i < cnt; ++i) { - context._generation = _generationHandler.getNextGeneration(); + context._generation.store(_generationHandler.getNextGeneration(), std::memory_order_relaxed); _generationHandler.incGeneration(); } _doneWriteWork += cnt; diff --git a/vespalib/src/vespa/vespalib/util/generationhandler.cpp b/vespalib/src/vespa/vespalib/util/generationhandler.cpp index 23e9c9ce8ac..5cee18108d6 100644 --- a/vespalib/src/vespa/vespalib/util/generationhandler.cpp +++ b/vespalib/src/vespa/vespalib/util/generationhandler.cpp @@ -114,7 +114,7 @@ void GenerationHandler::updateFirstUsedGeneration() { for (;;) { - if (_first == _last) + if (_first == _last.load(std::memory_order_relaxed)) break; // No elements can be freed if (!_first->setInvalid()) { break; // First element still in use @@ -122,9 +122,6 @@ GenerationHandler::updateFirstUsedGeneration() GenerationHold *toFree = _first; assert(toFree->_next != nullptr); _first = toFree->_next; - // Must ensure _first is updated before changing next pointer to - // avoid temporarily inconsistent state (breaks hasReaders()) - std::atomic_thread_fence(std::memory_order_release); toFree->_next = _free; _free = toFree; } @@ -141,14 +138,14 @@ GenerationHandler::GenerationHandler() { _last = _first = new GenerationHold; ++_numHolds; - _last->_generation = _generation; - _last->setValid(); + _first->_generation.store(_generation, std::memory_order_relaxed); + _first->setValid(); } GenerationHandler::~GenerationHandler(void) { updateFirstUsedGeneration(); - assert(_first == _last); + assert(_first == _last.load(std::memory_order_relaxed)); while (_free != nullptr) { GenerationHold *toFree = _free; _free = toFree->_next; @@ -162,17 +159,16 @@ GenerationHandler::~GenerationHandler(void) GenerationHandler::Guard GenerationHandler::takeGuard() const { - Guard guard(_last); + Guard guard(_last.load(std::memory_order_acquire)); for (;;) { // Must check valid() after increasing refcount - std::atomic_thread_fence(std::memory_order_acquire); if (guard.valid()) break; // Might still be marked invalid, that's OK /* * Clashed with writer freeing entry. Must abandon current * guard and try again. */ - guard = Guard(_last); + guard = Guard(_last.load(std::memory_order_acquire)); } // Guard has been valid after bumping refCount return guard; @@ -183,13 +179,12 @@ GenerationHandler::incGeneration() { generation_t ngen = getNextGeneration(); - std::atomic_thread_fence(std::memory_order_seq_cst); - if (_last->getRefCount() == 0) { + auto last = _last.load(std::memory_order_relaxed); + if (last->getRefCount() == 0) { // Last generation is unused, morph it to new generation. This is // the typical case when no readers are present. _generation = ngen; - _last->_generation = ngen; - std::atomic_thread_fence(std::memory_order_release); + last->_generation.store(ngen, std::memory_order_relaxed); updateFirstUsedGeneration(); return; } @@ -201,21 +196,12 @@ GenerationHandler::incGeneration() nhold = _free; _free = nhold->_next; } - nhold->_generation = ngen; + nhold->_generation.store(ngen, std::memory_order_relaxed); nhold->_next = nullptr; nhold->setValid(); - - // new hold must be updated before next pointer is updated - std::atomic_thread_fence(std::memory_order_release); - _last->_next = nhold; - - // next pointer must be updated before _last is updated - std::atomic_thread_fence(std::memory_order_release); + last->_next = nhold; _generation = ngen; - _last = nhold; - - // _last must be updated before _first is changed - std::atomic_thread_fence(std::memory_order_release); + _last.store(nhold, std::memory_order_release); updateFirstUsedGeneration(); } @@ -227,7 +213,7 @@ GenerationHandler::getGenerationRefCount(generation_t gen) const if (static_cast<sgeneration_t>(_firstUsedGeneration - gen) > 0) return 0u; for (GenerationHold *hold = _first; hold != nullptr; hold = hold->_next) { - if (hold->_generation == gen) + if (hold->_generation.load(std::memory_order_relaxed) == gen) return hold->getRefCount(); } return 0u; @@ -243,10 +229,4 @@ GenerationHandler::getGenerationRefCount(void) const return ret; } -bool -GenerationHandler::hasReaders(void) const -{ - return (_first != _last) ? true : (_first->getRefCount() > 0); -} - } diff --git a/vespalib/src/vespa/vespalib/util/generationhandler.h b/vespalib/src/vespa/vespalib/util/generationhandler.h index e03a3d24734..0c4b49a2d5b 100644 --- a/vespalib/src/vespa/vespalib/util/generationhandler.h +++ b/vespalib/src/vespa/vespalib/util/generationhandler.h @@ -30,7 +30,7 @@ public: static bool valid(uint32_t refCount) { return (refCount & 1) == 0u; } public: - generation_t _generation; + std::atomic<generation_t> _generation; GenerationHold *_next; // next free element or next newer element. GenerationHold(); @@ -68,13 +68,13 @@ public: bool valid() const { return _hold != nullptr; } - generation_t getGeneration() const { return _hold->_generation; } + generation_t getGeneration() const { return _hold->_generation.load(std::memory_order_relaxed); } }; private: generation_t _generation; generation_t _firstUsedGeneration; - GenerationHold *_last; // Points to "current generation" entry + std::atomic<GenerationHold *> _last; // Points to "current generation" entry GenerationHold *_first; // Points to "firstUsedGeneration" entry GenerationHold *_free; // List of free entries uint32_t _numHolds; // Number of allocated generation hold entries @@ -134,13 +134,6 @@ public: * Should be called by the writer thread. */ uint64_t getGenerationRefCount() const; - - /** - * Returns true if we still have readers. False positives and - * negatives are possible if readers come and go while writer - * updates generations. - */ - bool hasReaders() const; }; } |