diff options
Diffstat (limited to 'config-model')
96 files changed, 1294 insertions, 585 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java index e2c6b788b02..4ef591cda9f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java +++ b/config-model/src/main/java/com/yahoo/config/model/admin/AdminModel.java @@ -84,7 +84,7 @@ public class AdminModel extends ConfigModel { } TreeConfigProducer<AnyConfigProducer> parent = modelContext.getParentProducer(); ModelContext.Properties properties = modelContext.getDeployState().getProperties(); - DomAdminV2Builder domBuilder = new DomAdminV2Builder(modelContext.getApplicationType(), + DomAdminV2Builder domBuilder = new DomAdminV2Builder(modelContext, properties.multitenant(), properties.configServerSpecs()); model.admin = domBuilder.build(modelContext.getDeployState(), parent, adminElement); diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 3c45588a054..c2d6adddeed 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -55,7 +55,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private double feedNiceness = 0.0; private int maxActivationInhibitedOutOfSyncGroups = 0; private List<TenantSecretStore> tenantSecretStores = List.of(); - private String jvmOmitStackTraceInFastThrowOption; private boolean allowDisableMtls = true; private List<X509Certificate> operatorCertificates = List.of(); private double resourceLimitDisk = 0.75; @@ -84,6 +83,9 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private int contentLayerMetadataFeatureLevel = 0; private int persistenceThreadMaxFeedOpBatchSize = 1; private boolean logserverOtelCol = false; + private boolean symmetricPutAndActivateReplicaSelection = false; + private boolean enforceStrictlyIncreasingClusterStateVersions = false; + private boolean launchApplicationAthenzService = false; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -111,7 +113,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public double feedNiceness() { return feedNiceness; } @Override public int maxActivationInhibitedOutOfSyncGroups() { return maxActivationInhibitedOutOfSyncGroups; } @Override public List<TenantSecretStore> tenantSecretStores() { return tenantSecretStores; } - @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return jvmOmitStackTraceInFastThrowOption; } @Override public boolean allowDisableMtls() { return allowDisableMtls; } @Override public List<X509Certificate> operatorCertificates() { return operatorCertificates; } @Override public double resourceLimitDisk() { return resourceLimitDisk; } @@ -142,6 +143,9 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public int contentLayerMetadataFeatureLevel() { return contentLayerMetadataFeatureLevel; } @Override public int persistenceThreadMaxFeedOpBatchSize() { return persistenceThreadMaxFeedOpBatchSize; } @Override public boolean logserverOtelCol() { return logserverOtelCol; } + @Override public boolean symmetricPutAndActivateReplicaSelection() { return symmetricPutAndActivateReplicaSelection; } + @Override public boolean enforceStrictlyIncreasingClusterStateVersions() { return enforceStrictlyIncreasingClusterStateVersions; } + @Override public boolean launchApplicationAthenzService() { return launchApplicationAthenzService; } public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) { this.sharedStringRepoNoReclaim = sharedStringRepoNoReclaim; @@ -276,11 +280,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties setJvmOmitStackTraceInFastThrowOption(String value) { - this.jvmOmitStackTraceInFastThrowOption = value; - return this; - } - public TestProperties allowDisableMtls(boolean value) { this.allowDisableMtls = value; return this; @@ -382,6 +381,21 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setSymmetricPutAndActivateReplicaSelection(boolean symmetricReplicaSelection) { + this.symmetricPutAndActivateReplicaSelection = symmetricReplicaSelection; + return this; + } + + public TestProperties setEnforceStrictlyIncreasingClusterStateVersions(boolean enforce) { + this.enforceStrictlyIncreasingClusterStateVersions = enforce; + return this; + } + + public TestProperties setLaunchApplicationAthenzService(boolean launch) { + this.launchApplicationAthenzService = launch; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index da0fd265724..dcaa7acf823 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -305,8 +305,8 @@ public class InMemoryProvisioner implements HostProvisioner { @Override public int compare(NodeResources a, NodeResources b) { - if (a.memoryGb() > b.memoryGb()) return 1; - if (a.memoryGb() < b.memoryGb()) return -1; + if (a.memoryGiB() > b.memoryGiB()) return 1; + if (a.memoryGiB() < b.memoryGiB()) return -1; if (a.diskGb() > b.diskGb()) return 1; if (a.diskGb() < b.diskGb()) return -1; return Double.compare(a.vcpu(), b.vcpu()); diff --git a/config-model/src/main/java/com/yahoo/schema/RankProfile.java b/config-model/src/main/java/com/yahoo/schema/RankProfile.java index 60674b5487c..ed1a4e98b49 100644 --- a/config-model/src/main/java/com/yahoo/schema/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/RankProfile.java @@ -73,7 +73,8 @@ public class RankProfile implements Cloneable { /** The resolved inherited profiles, or null when not resolved. */ private List<RankProfile> inherited; - private MatchPhaseSettings matchPhaseSettings = null; + private MatchPhaseSettings matchPhase = null; + private DiversitySettings diversity = null; protected Set<RankSetting> rankSettings = new java.util.LinkedHashSet<>(); @@ -106,6 +107,7 @@ public class RankProfile implements Cloneable { /** The drop limit used to drop hits with rank score less than or equal to this value */ private double rankScoreDropLimit = -Double.MAX_VALUE; + private double secondPhaseRankScoreDropLimit = -Double.MAX_VALUE; private Set<ReferenceNode> summaryFeatures; private String inheritedSummaryFeaturesProfileName; @@ -224,7 +226,7 @@ public class RankProfile implements Cloneable { public boolean useSignificanceModel() { if (useSignificanceModel != null) return useSignificanceModel; - return uniquelyInherited(p -> p.useSignificanceModel(), "use-model") + return uniquelyInherited(RankProfile::useSignificanceModel, "use-model") .orElse(false); // Disabled by default } @@ -306,20 +308,28 @@ public class RankProfile implements Cloneable { return false; } - public void setMatchPhaseSettings(MatchPhaseSettings settings) { + public void setMatchPhase(MatchPhaseSettings settings) { settings.checkValid(); - this.matchPhaseSettings = settings; + this.matchPhase = settings; } - public MatchPhaseSettings getMatchPhaseSettings() { - if (matchPhaseSettings != null) return matchPhaseSettings; - return uniquelyInherited(p -> p.getMatchPhaseSettings(), "match phase settings").orElse(null); + public MatchPhaseSettings getMatchPhase() { + if (matchPhase != null) return matchPhase; + return uniquelyInherited(RankProfile::getMatchPhase, "match phase settings").orElse(null); + } + public void setDiversity(DiversitySettings value) { + value.checkValid(); + diversity = value; + } + public DiversitySettings getDiversity() { + if (diversity != null) return diversity; + return uniquelyInherited(RankProfile::getDiversity, "diversity settings").orElse(null); } /** Returns the uniquely determined property, where non-empty is defined as non-null */ private <T> Optional<T> uniquelyInherited(Function<RankProfile, T> propertyRetriever, String propertyDescription) { - return uniquelyInherited(propertyRetriever, p -> p != null, propertyDescription); + return uniquelyInherited(propertyRetriever, Objects::nonNull, propertyDescription); } /** @@ -334,8 +344,8 @@ public class RankProfile implements Cloneable { Predicate<T> nonEmptyValueFilter, String propertyDescription) { Set<T> uniqueProperties = inherited().stream() - .map(p -> propertyRetriever.apply(p)) - .filter(p -> nonEmptyValueFilter.test(p)) + .map(propertyRetriever) + .filter(nonEmptyValueFilter) .collect(Collectors.toSet()); if (uniqueProperties.isEmpty()) return Optional.empty(); if (uniqueProperties.size() == 1) return uniqueProperties.stream().findAny(); @@ -494,7 +504,7 @@ public class RankProfile implements Cloneable { public RankingExpressionFunction getFirstPhase() { if (firstPhaseRanking != null) return firstPhaseRanking; - return uniquelyInherited(p -> p.getFirstPhase(), "first-phase expression").orElse(null); + return uniquelyInherited(RankProfile::getFirstPhase, "first-phase expression").orElse(null); } void setFirstPhaseRanking(RankingExpression rankingExpression) { @@ -521,7 +531,7 @@ public class RankProfile implements Cloneable { public RankingExpressionFunction getSecondPhase() { if (secondPhaseRanking != null) return secondPhaseRanking; - return uniquelyInherited(p -> p.getSecondPhase(), "second-phase expression").orElse(null); + return uniquelyInherited(RankProfile::getSecondPhase, "second-phase expression").orElse(null); } public void setSecondPhaseRanking(String expression) { @@ -541,7 +551,7 @@ public class RankProfile implements Cloneable { public RankingExpressionFunction getGlobalPhase() { if (globalPhaseRanking != null) return globalPhaseRanking; - return uniquelyInherited(p -> p.getGlobalPhase(), "global-phase expression").orElse(null); + return uniquelyInherited(RankProfile::getGlobalPhase, "global-phase expression").orElse(null); } public void setGlobalPhaseRanking(String expression) { @@ -600,7 +610,7 @@ public class RankProfile implements Cloneable { return Collections.unmodifiableSet(combined); } if (summaryFeatures != null) return Collections.unmodifiableSet(summaryFeatures); - return uniquelyInherited(p -> p.getSummaryFeatures(), f -> ! f.isEmpty(), "summary features") + return uniquelyInherited(RankProfile::getSummaryFeatures, f -> ! f.isEmpty(), "summary features") .orElse(Set.of()); } @@ -617,13 +627,13 @@ public class RankProfile implements Cloneable { return Collections.unmodifiableSet(combined); } if (matchFeatures != null) return Collections.unmodifiableSet(matchFeatures); - return uniquelyInherited(p -> p.getMatchFeatures(), f -> ! f.isEmpty(), "match features") + return uniquelyInherited(RankProfile::getMatchFeatures, f -> ! f.isEmpty(), "match features") .orElse(Set.of()); } public Set<ReferenceNode> getHiddenMatchFeatures() { if (hiddenMatchFeatures != null) return Collections.unmodifiableSet(hiddenMatchFeatures); - return uniquelyInherited(p -> p.getHiddenMatchFeatures(), f -> ! f.isEmpty(), "hidden match features") + return uniquelyInherited(RankProfile::getHiddenMatchFeatures, f -> ! f.isEmpty(), "hidden match features") .orElse(Set.of()); } @@ -661,7 +671,7 @@ public class RankProfile implements Cloneable { /** Returns a read-only view of the rank features to use in this profile. This is never null */ public Set<ReferenceNode> getRankFeatures() { if (rankFeatures != null) return Collections.unmodifiableSet(rankFeatures); - return uniquelyInherited(p -> p.getRankFeatures(), f -> ! f.isEmpty(), "summary-features") + return uniquelyInherited(RankProfile::getRankFeatures, f -> ! f.isEmpty(), "summary-features") .orElse(Set.of()); } @@ -692,7 +702,7 @@ public class RankProfile implements Cloneable { if (rankProperties.isEmpty() && inherited().isEmpty()) return Map.of(); if (inherited().isEmpty()) return Collections.unmodifiableMap(rankProperties); - var inheritedProperties = uniquelyInherited(p -> p.getRankPropertyMap(), m -> ! m.isEmpty(), "rank-properties") + var inheritedProperties = uniquelyInherited(RankProfile::getRankPropertyMap, m -> ! m.isEmpty(), "rank-properties") .orElse(Map.of()); if (rankProperties.isEmpty()) return inheritedProperties; @@ -734,21 +744,21 @@ public class RankProfile implements Cloneable { public int getRerankCount() { if (rerankCount >= 0) return rerankCount; - return uniquelyInherited(p -> p.getRerankCount(), c -> c >= 0, "rerank-count").orElse(-1); + return uniquelyInherited(RankProfile::getRerankCount, c -> c >= 0, "rerank-count").orElse(-1); } public void setGlobalPhaseRerankCount(int count) { this.globalPhaseRerankCount = count; } public int getGlobalPhaseRerankCount() { if (globalPhaseRerankCount >= 0) return globalPhaseRerankCount; - return uniquelyInherited(p -> p.getGlobalPhaseRerankCount(), c -> c >= 0, "global-phase rerank-count").orElse(-1); + return uniquelyInherited(RankProfile::getGlobalPhaseRerankCount, c -> c >= 0, "global-phase rerank-count").orElse(-1); } public void setNumThreadsPerSearch(int numThreads) { this.numThreadsPerSearch = numThreads; } public int getNumThreadsPerSearch() { if (numThreadsPerSearch >= 0) return numThreadsPerSearch; - return uniquelyInherited(p -> p.getNumThreadsPerSearch(), n -> n >= 0, "num-threads-per-search") + return uniquelyInherited(RankProfile::getNumThreadsPerSearch, n -> n >= 0, "num-threads-per-search") .orElse(-1); } @@ -756,14 +766,14 @@ public class RankProfile implements Cloneable { public int getMinHitsPerThread() { if (minHitsPerThread >= 0) return minHitsPerThread; - return uniquelyInherited(p -> p.getMinHitsPerThread(), n -> n >= 0, "min-hits-per-search").orElse(-1); + return uniquelyInherited(RankProfile::getMinHitsPerThread, n -> n >= 0, "min-hits-per-search").orElse(-1); } public void setNumSearchPartitions(int numSearchPartitions) { this.numSearchPartitions = numSearchPartitions; } public int getNumSearchPartitions() { if (numSearchPartitions >= 0) return numSearchPartitions; - return uniquelyInherited(p -> p.getNumSearchPartitions(), n -> n >= 0, "num-search-partitions").orElse(-1); + return uniquelyInherited(RankProfile::getNumSearchPartitions, n -> n >= 0, "num-search-partitions").orElse(-1); } public void setTermwiseLimit(double termwiseLimit) { this.termwiseLimit = termwiseLimit; } @@ -773,7 +783,7 @@ public class RankProfile implements Cloneable { public OptionalDouble getTermwiseLimit() { if (termwiseLimit != null) return OptionalDouble.of(termwiseLimit); - return uniquelyInherited(p -> p.getTermwiseLimit(), l -> l.isPresent(), "termwise-limit") + return uniquelyInherited(RankProfile::getTermwiseLimit, OptionalDouble::isPresent, "termwise-limit") .orElse(OptionalDouble.empty()); } @@ -781,21 +791,21 @@ public class RankProfile implements Cloneable { if (postFilterThreshold != null) { return OptionalDouble.of(postFilterThreshold); } - return uniquelyInherited(p -> p.getPostFilterThreshold(), l -> l.isPresent(), "post-filter-threshold").orElse(OptionalDouble.empty()); + return uniquelyInherited(RankProfile::getPostFilterThreshold, OptionalDouble::isPresent, "post-filter-threshold").orElse(OptionalDouble.empty()); } public OptionalDouble getApproximateThreshold() { if (approximateThreshold != null) { return OptionalDouble.of(approximateThreshold); } - return uniquelyInherited(p -> p.getApproximateThreshold(), l -> l.isPresent(), "approximate-threshold").orElse(OptionalDouble.empty()); + return uniquelyInherited(RankProfile::getApproximateThreshold, OptionalDouble::isPresent, "approximate-threshold").orElse(OptionalDouble.empty()); } public OptionalDouble getTargetHitsMaxAdjustmentFactor() { if (targetHitsMaxAdjustmentFactor != null) { return OptionalDouble.of(targetHitsMaxAdjustmentFactor); } - return uniquelyInherited(p -> p.getTargetHitsMaxAdjustmentFactor(), l -> l.isPresent(), "target-hits-max-adjustment-factor").orElse(OptionalDouble.empty()); + return uniquelyInherited(RankProfile::getTargetHitsMaxAdjustmentFactor, OptionalDouble::isPresent, "target-hits-max-adjustment-factor").orElse(OptionalDouble.empty()); } /** Whether we should ignore the default rank features. Set to null to use inherited */ @@ -805,24 +815,34 @@ public class RankProfile implements Cloneable { public Boolean getIgnoreDefaultRankFeatures() { if (ignoreDefaultRankFeatures != null) return ignoreDefaultRankFeatures; - return uniquelyInherited(p -> p.getIgnoreDefaultRankFeatures(), "ignore-default-rank-features").orElse(false); + return uniquelyInherited(RankProfile::getIgnoreDefaultRankFeatures, "ignore-default-rank-features").orElse(false); } public void setKeepRankCount(int rerankArraySize) { this.keepRankCount = rerankArraySize; } public int getKeepRankCount() { if (keepRankCount >= 0) return keepRankCount; - return uniquelyInherited(p -> p.getKeepRankCount(), c -> c >= 0, "keep-rank-count").orElse(-1); + return uniquelyInherited(RankProfile::getKeepRankCount, c -> c >= 0, "keep-rank-count").orElse(-1); } public void setRankScoreDropLimit(double rankScoreDropLimit) { this.rankScoreDropLimit = rankScoreDropLimit; } public double getRankScoreDropLimit() { if (rankScoreDropLimit > -Double.MAX_VALUE) return rankScoreDropLimit; - return uniquelyInherited(p -> p.getRankScoreDropLimit(), c -> c > -Double.MAX_VALUE, "rank.score-drop-limit") + return uniquelyInherited(RankProfile::getRankScoreDropLimit, c -> c > -Double.MAX_VALUE, "rank.score-drop-limit") .orElse(rankScoreDropLimit); } + public void setSecondPhaseRankScoreDropLimit(double limit) { this.secondPhaseRankScoreDropLimit = limit; } + + public double getSecondPhaseRankScoreDropLimit() { + if (secondPhaseRankScoreDropLimit > -Double.MAX_VALUE) { + return secondPhaseRankScoreDropLimit; + } + return uniquelyInherited(RankProfile::getSecondPhaseRankScoreDropLimit, c -> c > -Double.MAX_VALUE, "second-phase rank-score-drop-limit") + .orElse(secondPhaseRankScoreDropLimit); + } + public void addFunction(String name, List<String> arguments, String expression, boolean inline) { try { addFunction(parseRankingExpression(name, arguments, expression), inline); @@ -947,7 +967,7 @@ public class RankProfile implements Cloneable { } private boolean needToUpdateFunctionCache() { - if (inherited().stream().anyMatch(profile -> profile.needToUpdateFunctionCache())) return true; + if (inherited().stream().anyMatch(RankProfile::needToUpdateFunctionCache)) return true; return allFunctionsCached == null; } @@ -955,7 +975,7 @@ public class RankProfile implements Cloneable { /** Returns all filter fields in this profile and any profile it inherits. */ public Set<String> allFilterFields() { - Set<String> inheritedFilterFields = uniquelyInherited(p -> p.allFilterFields(), fields -> ! fields.isEmpty(), + Set<String> inheritedFilterFields = uniquelyInherited(RankProfile::allFilterFields, fields -> ! fields.isEmpty(), "filter fields").orElse(Set.of()); if (inheritedFilterFields.isEmpty()) return Collections.unmodifiableSet(filterFields); @@ -966,7 +986,7 @@ public class RankProfile implements Cloneable { } private ExpressionFunction parseRankingExpression(String name, List<String> arguments, String expression) throws ParseException { - if (expression.trim().length() == 0) + if (expression.trim().isEmpty()) throw new ParseException("Encountered an empty ranking expression in " + name() + ", " + name + "."); try (Reader rankingExpressionReader = openRankingExpressionReader(name, expression.trim())) { @@ -1008,7 +1028,8 @@ public class RankProfile implements Cloneable { try { RankProfile clone = (RankProfile)super.clone(); clone.rankSettings = new LinkedHashSet<>(this.rankSettings); - clone.matchPhaseSettings = this.matchPhaseSettings; // hmm? + clone.matchPhase = this.matchPhase; // hmm? + clone.diversity = this.diversity; clone.summaryFeatures = summaryFeatures != null ? new LinkedHashSet<>(this.summaryFeatures) : null; clone.matchFeatures = matchFeatures != null ? new LinkedHashSet<>(this.matchFeatures) : null; clone.rankFeatures = rankFeatures != null ? new LinkedHashSet<>(this.rankFeatures) : null; @@ -1189,7 +1210,7 @@ public class RankProfile implements Cloneable { private Map<Reference, TensorType> featureTypes() { Map<Reference, TensorType> featureTypes = inputs().values().stream() - .collect(Collectors.toMap(input -> input.name(), + .collect(Collectors.toMap(Input::name, input -> input.type().tensorType())); allFields().forEach(field -> addAttributeFeatureTypes(field, featureTypes)); allImportedFields().forEach(field -> addAttributeFeatureTypes(field, featureTypes)); @@ -1506,15 +1527,9 @@ public class RankProfile implements Cloneable { private boolean ascending = false; private int maxHits = 0; // try to get this many hits before degrading the match phase private double maxFilterCoverage = 0.2; // Max coverage of original corpus that will trigger the filter. - private DiversitySettings diversity = null; private double evaluationPoint = 0.20; private double prePostFilterTippingPoint = 1.0; - public void setDiversity(DiversitySettings value) { - value.checkValid(); - diversity = value; - } - public void setAscending(boolean value) { ascending = value; } public void setAttribute(String value) { attribute = value; } public void setMaxHits(int value) { maxHits = value; } @@ -1526,7 +1541,6 @@ public class RankProfile implements Cloneable { public String getAttribute() { return attribute; } public int getMaxHits() { return maxHits; } public double getMaxFilterCoverage() { return maxFilterCoverage; } - public DiversitySettings getDiversity() { return diversity; } public double getEvaluationPoint() { return evaluationPoint; } public double getPrePostFilterTippingPoint() { return prePostFilterTippingPoint; } @@ -1690,7 +1704,7 @@ public class RankProfile implements Cloneable { } - public static record RankFeatureNormalizer(Reference original, String name, String input, String algo, double kparam) { + public record RankFeatureNormalizer(Reference original, String name, String input, String algo, double kparam) { @Override public String toString() { return "normalizer{name=" + name + ",input=" + input + ",algo=" + algo + ",k=" + kparam + "}"; @@ -1711,7 +1725,7 @@ public class RankProfile implements Cloneable { } } - private List<RankFeatureNormalizer> featureNormalizers = new ArrayList<>(); + private final List<RankFeatureNormalizer> featureNormalizers = new ArrayList<>(); public Map<String, RankFeatureNormalizer> getFeatureNormalizers() { Map<String, RankFeatureNormalizer> all = new LinkedHashMap<>(); diff --git a/config-model/src/main/java/com/yahoo/schema/Schema.java b/config-model/src/main/java/com/yahoo/schema/Schema.java index 3402ba31be9..127d12594b4 100644 --- a/config-model/src/main/java/com/yahoo/schema/Schema.java +++ b/config-model/src/main/java/com/yahoo/schema/Schema.java @@ -721,7 +721,7 @@ public class Schema implements ImmutableSchema { "', but this schema does not exist"); // Require schema and document type inheritance to be consistent to keep things simple - // And require it to be explicit so we have the option to support other possibilities later + // and require it to be explicit, so we have the option to support other possibilities later var parentDocument = owner.schemas().get(inherited.get()).getDocument(); if ( ! getDocument().inheritedTypes().containsKey(new DataTypeName(parentDocument.getName()))) throw new IllegalArgumentException(this + " inherits '" + inherited.get() + diff --git a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java index c1b698df55f..3fb185e333d 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java @@ -106,15 +106,18 @@ public class IndexInfo extends Derived { } private static boolean isPositionField(ImmutableSDField field) { - return GeoPos.isAnyPos(field); + return (field != null) && GeoPos.isAnyPos(field); + } + private static boolean isMultivalueField(ImmutableSDField field) { + return (field != null) && field.getDataType().isMultivalue(); } @Override protected void derive(ImmutableSDField field, Schema schema) { - derive(field, schema, false); + derive(field, schema, null); } - protected void derive(ImmutableSDField field, Schema schema, boolean inPosition) { + protected void derive(ImmutableSDField field, Schema schema, ImmutableSDField parent) { if (field.getDataType().equals(DataType.PREDICATE)) { addIndexCommand(field, CMD_PREDICATE); Index index = field.getIndex(field.getName()); @@ -134,14 +137,13 @@ public class IndexInfo extends Derived { String name = e.getValue(); addIndexAlias(alias, name); } - boolean isPosition = isPositionField(field); if (field.usesStructOrMap()) { for (ImmutableSDField structField : field.getStructFields()) { - derive(structField, schema, isPosition); // Recursion + derive(structField, schema, field); // Recursion } } - if (isPosition) { + if (isPositionField(field)) { addIndexCommand(field.getName(), CMD_DEFAULT_POSITION); } @@ -153,12 +155,12 @@ public class IndexInfo extends Derived { addIndexCommand(field, CMD_LOWERCASE); } - if (field.getDataType().isMultivalue()) { + if (isMultivalueField(field) || isMultivalueField(parent)) { addIndexCommand(field, CMD_MULTIVALUE); } Attribute attribute = field.getAttribute(); - if ((field.doesAttributing() || (attribute != null && !inPosition)) && !field.doesIndexing()) { + if ((field.doesAttributing() || (attribute != null && !isPositionField(parent))) && !field.doesIndexing()) { addIndexCommand(field.getName(), CMD_ATTRIBUTE); if (attribute != null && attribute.isFastSearch()) addIndexCommand(field.getName(), CMD_FAST_SEARCH); diff --git a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java index b057624f055..15e5891a3e3 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java @@ -159,6 +159,7 @@ public class RawRankProfile { private final boolean ignoreDefaultRankFeatures; private final RankProfile.MatchPhaseSettings matchPhaseSettings; + private final RankProfile.DiversitySettings diversitySettings; private final int rerankCount; private final int keepRankCount; private final int numThreadsPerSearch; @@ -169,6 +170,7 @@ public class RawRankProfile { private final OptionalDouble approximateThreshold; private final OptionalDouble targetHitsMaxAdjustmentFactor; private final double rankScoreDropLimit; + private final double secondPhaseRankScoreDropLimit; private final boolean sortBlueprintsByCost; private final boolean alwaysMarkPhraseExpensive; @@ -207,7 +209,8 @@ public class RawRankProfile { rankFeatures = compiled.getRankFeatures(); rerankCount = compiled.getRerankCount(); globalPhaseRerankCount = compiled.getGlobalPhaseRerankCount(); - matchPhaseSettings = compiled.getMatchPhaseSettings(); + matchPhaseSettings = compiled.getMatchPhase(); + diversitySettings = compiled.getDiversity(); numThreadsPerSearch = compiled.getNumThreadsPerSearch(); minHitsPerThread = compiled.getMinHitsPerThread(); numSearchPartitions = compiled.getNumSearchPartitions(); @@ -219,6 +222,7 @@ public class RawRankProfile { targetHitsMaxAdjustmentFactor = compiled.getTargetHitsMaxAdjustmentFactor(); keepRankCount = compiled.getKeepRankCount(); rankScoreDropLimit = compiled.getRankScoreDropLimit(); + secondPhaseRankScoreDropLimit = compiled.getSecondPhaseRankScoreDropLimit(); ignoreDefaultRankFeatures = compiled.getIgnoreDefaultRankFeatures(); rankProperties = new ArrayList<>(compiled.getRankProperties()); @@ -486,13 +490,12 @@ public class RawRankProfile { properties.add(new Pair<>("vespa.matchphase.degradation.maxfiltercoverage", matchPhaseSettings.getMaxFilterCoverage() + "")); properties.add(new Pair<>("vespa.matchphase.degradation.samplepercentage", matchPhaseSettings.getEvaluationPoint() + "")); properties.add(new Pair<>("vespa.matchphase.degradation.postfiltermultiplier", matchPhaseSettings.getPrePostFilterTippingPoint() + "")); - RankProfile.DiversitySettings diversitySettings = matchPhaseSettings.getDiversity(); - if (diversitySettings != null) { - properties.add(new Pair<>("vespa.matchphase.diversity.attribute", diversitySettings.getAttribute())); - properties.add(new Pair<>("vespa.matchphase.diversity.mingroups", String.valueOf(diversitySettings.getMinGroups()))); - properties.add(new Pair<>("vespa.matchphase.diversity.cutoff.factor", String.valueOf(diversitySettings.getCutoffFactor()))); - properties.add(new Pair<>("vespa.matchphase.diversity.cutoff.strategy", String.valueOf(diversitySettings.getCutoffStrategy()))); - } + } + if (diversitySettings != null) { + properties.add(new Pair<>("vespa.matchphase.diversity.attribute", diversitySettings.getAttribute())); + properties.add(new Pair<>("vespa.matchphase.diversity.mingroups", String.valueOf(diversitySettings.getMinGroups()))); + properties.add(new Pair<>("vespa.matchphase.diversity.cutoff.factor", String.valueOf(diversitySettings.getCutoffFactor()))); + properties.add(new Pair<>("vespa.matchphase.diversity.cutoff.strategy", String.valueOf(diversitySettings.getCutoffStrategy()))); } if (rerankCount > -1) { properties.add(new Pair<>("vespa.hitcollector.heapsize", rerankCount + "")); @@ -506,6 +509,9 @@ public class RawRankProfile { if (rankScoreDropLimit > -Double.MAX_VALUE) { properties.add(new Pair<>("vespa.hitcollector.rankscoredroplimit", rankScoreDropLimit + "")); } + if (secondPhaseRankScoreDropLimit > -Double.MAX_VALUE) { + properties.add(new Pair<>("vespa.hitcollector.secondphase.rankscoredroplimit", secondPhaseRankScoreDropLimit + "")); + } if (ignoreDefaultRankFeatures) { properties.add(new Pair<>("vespa.dump.ignoredefaultfeatures", String.valueOf(true))); } diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java index 77a10862f9c..ff78a4a3b60 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java @@ -38,8 +38,8 @@ public class ConvertParsedRanking { for (String name : parsed.getInherited()) profile.inherit(name); - parsed.isStrict().ifPresent(value -> profile.setStrict(value)); - parsed.isUseSignificanceModel().ifPresent(value -> profile.setUseSignificanceModel(value)); + parsed.isStrict().ifPresent(profile::setStrict); + parsed.isUseSignificanceModel().ifPresent(profile::setUseSignificanceModel); for (var constant : parsed.getConstants().values()) profile.add(constant); @@ -58,39 +58,26 @@ public class ConvertParsedRanking { profile.addFunction(name, parameters, expression, inline); } - parsed.getRankScoreDropLimit().ifPresent - (value -> profile.setRankScoreDropLimit(value)); - parsed.getTermwiseLimit().ifPresent - (value -> profile.setTermwiseLimit(value)); - parsed.getPostFilterThreshold().ifPresent - (value -> profile.setPostFilterThreshold(value)); - parsed.getApproximateThreshold().ifPresent - (value -> profile.setApproximateThreshold(value)); - parsed.getTargetHitsMaxAdjustmentFactor().ifPresent - (value -> profile.setTargetHitsMaxAdjustmentFactor(value)); - parsed.getKeepRankCount().ifPresent - (value -> profile.setKeepRankCount(value)); - parsed.getMinHitsPerThread().ifPresent - (value -> profile.setMinHitsPerThread(value)); - parsed.getNumSearchPartitions().ifPresent - (value -> profile.setNumSearchPartitions(value)); - parsed.getNumThreadsPerSearch().ifPresent - (value -> profile.setNumThreadsPerSearch(value)); - parsed.getReRankCount().ifPresent - (value -> profile.setRerankCount(value)); - - parsed.getMatchPhaseSettings().ifPresent - (value -> profile.setMatchPhaseSettings(value)); - - parsed.getFirstPhaseExpression().ifPresent - (value -> profile.setFirstPhaseRanking(value)); - parsed.getSecondPhaseExpression().ifPresent - (value -> profile.setSecondPhaseRanking(value)); - - parsed.getGlobalPhaseExpression().ifPresent - (value -> profile.setGlobalPhaseRanking(value)); - parsed.getGlobalPhaseRerankCount().ifPresent - (value -> profile.setGlobalPhaseRerankCount(value)); + parsed.getRankScoreDropLimit().ifPresent(profile::setRankScoreDropLimit); + parsed.getSecondPhaseRankScoreDropLimit().ifPresent(profile::setSecondPhaseRankScoreDropLimit); + parsed.getTermwiseLimit().ifPresent(profile::setTermwiseLimit); + parsed.getPostFilterThreshold().ifPresent(profile::setPostFilterThreshold); + parsed.getApproximateThreshold().ifPresent(profile::setApproximateThreshold); + parsed.getTargetHitsMaxAdjustmentFactor().ifPresent(profile::setTargetHitsMaxAdjustmentFactor); + parsed.getKeepRankCount().ifPresent(profile::setKeepRankCount); + parsed.getMinHitsPerThread().ifPresent(profile::setMinHitsPerThread); + parsed.getNumSearchPartitions().ifPresent(profile::setNumSearchPartitions); + parsed.getNumThreadsPerSearch().ifPresent(profile::setNumThreadsPerSearch); + parsed.getReRankCount().ifPresent(profile::setRerankCount); + + parsed.getMatchPhase().ifPresent(profile::setMatchPhase); + parsed.getDiversity().ifPresent(profile::setDiversity); + + parsed.getFirstPhaseExpression().ifPresent(profile::setFirstPhaseRanking); + parsed.getSecondPhaseExpression().ifPresent(profile::setSecondPhaseRanking); + + parsed.getGlobalPhaseExpression().ifPresent(profile::setGlobalPhaseRanking); + parsed.getGlobalPhaseRerankCount().ifPresent(profile::setGlobalPhaseRerankCount); for (var value : parsed.getMatchFeatures()) { profile.addMatchFeatures(value); @@ -102,10 +89,8 @@ public class ConvertParsedRanking { profile.addSummaryFeatures(value); } - parsed.getInheritedMatchFeatures().ifPresent - (value -> profile.setInheritedMatchFeatures(value)); - parsed.getInheritedSummaryFeatures().ifPresent - (value -> profile.setInheritedSummaryFeatures(value)); + parsed.getInheritedMatchFeatures().ifPresent(profile::setInheritedMatchFeatures); + parsed.getInheritedSummaryFeatures().ifPresent(profile::setInheritedSummaryFeatures); if (parsed.getIgnoreDefaultRankFeatures()) { profile.setIgnoreDefaultRankFeatures(true); } diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java index 93319e82076..2a117a4af4b 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java @@ -4,6 +4,7 @@ package com.yahoo.schema.parser; import com.yahoo.schema.OnnxModel; import com.yahoo.schema.RankProfile; import com.yahoo.schema.RankProfile.MatchPhaseSettings; +import com.yahoo.schema.RankProfile.DiversitySettings; import com.yahoo.schema.RankProfile.MutateOperation; import com.yahoo.searchlib.rankingexpression.FeatureList; import com.yahoo.searchlib.rankingexpression.Reference; @@ -26,6 +27,7 @@ class ParsedRankProfile extends ParsedBlock { private boolean ignoreDefaultRankFeatures = false; private Double rankScoreDropLimit = null; + private Double secondPhaseRankScoreDropLimit = null; private Double termwiseLimit = null; private Double postFilterThreshold = null; private Double approximateThreshold = null; @@ -38,7 +40,8 @@ class ParsedRankProfile extends ParsedBlock { private Integer numSearchPartitions = null; private Integer numThreadsPerSearch = null; private Integer reRankCount = null; - private MatchPhaseSettings matchPhaseSettings = null; + private MatchPhaseSettings matchPhase = null; + private DiversitySettings diversity = null; private String firstPhaseExpression = null; private String inheritedSummaryFeatures = null; private String inheritedMatchFeatures = null; @@ -64,6 +67,7 @@ class ParsedRankProfile extends ParsedBlock { boolean getIgnoreDefaultRankFeatures() { return this.ignoreDefaultRankFeatures; } Optional<Double> getRankScoreDropLimit() { return Optional.ofNullable(this.rankScoreDropLimit); } + Optional<Double> getSecondPhaseRankScoreDropLimit() { return Optional.ofNullable(this.secondPhaseRankScoreDropLimit); } Optional<Double> getTermwiseLimit() { return Optional.ofNullable(this.termwiseLimit); } Optional<Double> getPostFilterThreshold() { return Optional.ofNullable(this.postFilterThreshold); } Optional<Double> getApproximateThreshold() { return Optional.ofNullable(this.approximateThreshold); } @@ -76,7 +80,8 @@ class ParsedRankProfile extends ParsedBlock { Optional<Integer> getNumSearchPartitions() { return Optional.ofNullable(this.numSearchPartitions); } Optional<Integer> getNumThreadsPerSearch() { return Optional.ofNullable(this.numThreadsPerSearch); } Optional<Integer> getReRankCount() { return Optional.ofNullable(this.reRankCount); } - Optional<MatchPhaseSettings> getMatchPhaseSettings() { return Optional.ofNullable(this.matchPhaseSettings); } + Optional<MatchPhaseSettings> getMatchPhase() { return Optional.ofNullable(this.matchPhase); } + Optional<DiversitySettings> getDiversity() { return Optional.ofNullable(this.diversity); } Optional<String> getFirstPhaseExpression() { return Optional.ofNullable(this.firstPhaseExpression); } Optional<String> getInheritedMatchFeatures() { return Optional.ofNullable(this.inheritedMatchFeatures); } List<ParsedRankFunction> getFunctions() { return List.copyOf(functions.values()); } @@ -171,9 +176,13 @@ class ParsedRankProfile extends ParsedBlock { this.keepRankCount = count; } - void setMatchPhaseSettings(MatchPhaseSettings settings) { - verifyThat(matchPhaseSettings == null, "already has match-phase"); - this.matchPhaseSettings = settings; + void setMatchPhase(MatchPhaseSettings settings) { + verifyThat(matchPhase == null, "already has match-phase"); + this.matchPhase = settings; + } + void setDiversity(DiversitySettings settings) { + verifyThat(diversity == null, "already has diversity"); + this.diversity = settings; } void setMinHitsPerThread(int minHits) { @@ -196,6 +205,11 @@ class ParsedRankProfile extends ParsedBlock { this.rankScoreDropLimit = limit; } + void setSecondPhaseRankScoreDropLimit(double limit) { + verifyThat(secondPhaseRankScoreDropLimit == null, "already has rank-score-drop-limit for second phase"); + this.secondPhaseRankScoreDropLimit = limit; + } + void setRerankCount(int count) { verifyThat(reRankCount == null, "already has rerank-count"); this.reRankCount = count; diff --git a/config-model/src/main/java/com/yahoo/schema/processing/DiversitySettingsValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/DiversitySettingsValidator.java index 5c06ce25184..a8e0f86de8c 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/DiversitySettingsValidator.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/DiversitySettingsValidator.java @@ -23,8 +23,8 @@ public class DiversitySettingsValidator extends Processor { if (documentsOnly) return; for (RankProfile rankProfile : rankProfileRegistry.rankProfilesOf(schema)) { - if (rankProfile.getMatchPhaseSettings() != null && rankProfile.getMatchPhaseSettings().getDiversity() != null) { - validate(rankProfile, rankProfile.getMatchPhaseSettings().getDiversity()); + if (rankProfile.getDiversity() != null) { + validate(rankProfile, rankProfile.getDiversity()); } } } @@ -32,6 +32,9 @@ public class DiversitySettingsValidator extends Processor { String attributeName = settings.getAttribute(); new AttributeValidator(schema.getName(), rankProfile.name(), schema.getAttribute(attributeName), attributeName).validate(); + if ((rankProfile.getMatchPhase() == null) && (rankProfile.getSecondPhaseRanking() == null)) { + throw new IllegalArgumentException("In schema '" + schema.getName() + "', rank-profile '" + rankProfile.name() + "': 'diversity' requires either 'match-phase' or 'second-phase' to be specified."); + } } private static class AttributeValidator extends MatchPhaseSettingsValidator.AttributeValidator { diff --git a/config-model/src/main/java/com/yahoo/schema/processing/MatchPhaseSettingsValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/MatchPhaseSettingsValidator.java index f3a8f7cee18..d29820e0d51 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/MatchPhaseSettingsValidator.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/MatchPhaseSettingsValidator.java @@ -25,7 +25,7 @@ public class MatchPhaseSettingsValidator extends Processor { if (documentsOnly) return; for (RankProfile rankProfile : rankProfileRegistry.rankProfilesOf(schema)) { - RankProfile.MatchPhaseSettings settings = rankProfile.getMatchPhaseSettings(); + RankProfile.MatchPhaseSettings settings = rankProfile.getMatchPhase(); if (settings != null) { validateMatchPhaseSettings(rankProfile, settings); } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/PagedAttributeValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/PagedAttributeValidator.java index 4feb065a90b..5a13267f507 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/PagedAttributeValidator.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/PagedAttributeValidator.java @@ -18,6 +18,9 @@ import java.util.Optional; */ public class PagedAttributeValidator extends Processor { + private static final String disavantagesUrl = + "https://docs.vespa.ai/en/attributes.html#paged-attributes-disadvantages"; + public PagedAttributeValidator(Schema schema, DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, @@ -43,6 +46,10 @@ public class PagedAttributeValidator extends Processor { if (!isSupportedType(attribute)) { fail(schema, field, "The 'paged' attribute setting is not supported for fast-rank tensor and predicate types"); } + if (attribute.getType() == Attribute.Type.TENSOR && attribute.hnswIndexParams().isPresent()) { + warn(schema.getName(), field.getName(), "The 'paged' attribute setting in combination with " + + "HNSW indexing is strongly discouraged, see " + disavantagesUrl + " for details"); + } } private boolean isSupportedType(Attribute attribute) { @@ -58,8 +65,4 @@ public class PagedAttributeValidator extends Processor { return true; } - private boolean isDenseTensorType(TensorType type) { - return type.dimensions().stream().allMatch(d -> d.isIndexed()); - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java index 10de46ae6d8..e48aa0eef7c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java @@ -9,7 +9,8 @@ import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.logging.Level; + +import static java.util.logging.Level.WARNING; /** * A document summary definition - a list of summary fields. @@ -121,13 +122,12 @@ public class DocumentSummary extends FieldView { public void validate(DeployLogger logger) { for (var inheritedName : inherited) { var inheritedSummary = owner.getSummary(inheritedName); - if (inheritedSummary == null) { - // TODO Vespa 9: Throw IllegalArgumentException instead - logger.logApplicationPackage(Level.WARNING, - this + " inherits '" + inheritedName + "' but this" + " is not present in " + owner); - } + // TODO: Throw when no one is doing this anymore + if (inheritedName.equals("default")) + logger.logApplicationPackage(WARNING, this + " inherits '" + inheritedName + "', which makes no sense. Remove this inheritance"); + else if (inheritedSummary == null ) + throw new IllegalArgumentException(this + " inherits '" + inheritedName + "', but this is not present in " + owner); } - } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Host.java b/config-model/src/main/java/com/yahoo/vespa/model/Host.java index a8085919a98..f87f1382ffb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/Host.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/Host.java @@ -16,7 +16,7 @@ import java.util.Objects; public final class Host extends TreeConfigProducer<AnyConfigProducer> implements SentinelConfig.Producer, Comparable<Host> { // Memory needed for auxiliary processes always running on the node (config-proxy, metrics-proxy). - // Keep in sync with node-repository/ClusterModel. + // Keep in sync with node-repository/ClusterModel and startup scripts (go and bash). public static final double memoryOverheadGb = 0.7; private ConfigSentinel configSentinel = null; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java index e18ffea731e..12892cb12ac 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java @@ -1,23 +1,18 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin; -import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.TreeConfigProducer; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.container.Container; /** * Container that should be running on same host as the logserver. Sets up a handler for getting logs from logserver. - * Only in use in hosted Vespa. */ public class LogserverContainer extends Container { public LogserverContainer(TreeConfigProducer<?> parent, DeployState deployState) { super(parent, "" + 0, 0, deployState); - if (deployState.isHosted() && deployState.getProperties().applicationId().instance().isTester()) - useDynamicPorts(); } @Override @@ -30,9 +25,4 @@ public class LogserverContainer extends Container { return ""; } - @Override - public String jvmOmitStackTraceInFastThrowOption(ModelContext.FeatureFlags featureFlags) { - return featureFlags.jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type.admin); - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index e702a29b640..be38298279f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -97,11 +97,6 @@ public class ClusterControllerContainer extends Container implements return ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; } - @Override - public String jvmOmitStackTraceInFastThrowOption(ModelContext.FeatureFlags featureFlags) { - return featureFlags.jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type.admin); - } - private void configureZooKeeperServer(boolean runStandaloneZooKeeper) { if (runStandaloneZooKeeper) ContainerModelBuilder.addReconfigurableZooKeeperServerComponents(this); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index 2ff58438a07..8f262281edc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -10,7 +10,6 @@ import ai.vespa.metricsproxy.rpc.RpcConnector; import ai.vespa.metricsproxy.rpc.RpcConnectorConfig; import ai.vespa.metricsproxy.service.VespaServices; import ai.vespa.metricsproxy.service.VespaServicesConfig; -import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ApplicationId; @@ -78,11 +77,6 @@ public class MetricsProxyContainer extends Container implements } @Override - public String jvmOmitStackTraceInFastThrowOption(ModelContext.FeatureFlags featureFlags) { - return featureFlags.jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type.admin); - } - - @Override public int getWantedPort() { return BASEPORT; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryCollector.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryCollector.java index 03b96b12c03..364048ff261 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryCollector.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryCollector.java @@ -19,11 +19,13 @@ public class OpenTelemetryCollector extends AbstractService implements OpenTelem private final Zone zone; private final ApplicationId applicationId; + private final boolean isHostedVespa; public OpenTelemetryCollector(TreeConfigProducer<?> parent) { super(parent, "otelcol"); this.zone = null; this.applicationId = null; + this.isHostedVespa = false; setProp("clustertype", "admin"); setProp("clustername", "admin"); } @@ -32,6 +34,7 @@ public class OpenTelemetryCollector extends AbstractService implements OpenTelem super(parent, "otelcol"); this.zone = deployState.zone(); this.applicationId = deployState.getProperties().applicationId(); + this.isHostedVespa = deployState.isHosted(); setProp("clustertype", "admin"); setProp("clustername", "admin"); } @@ -54,7 +57,7 @@ public class OpenTelemetryCollector extends AbstractService implements OpenTelem @Override public void getConfig(OpenTelemetryConfig.Builder builder) { - var generator = new OpenTelemetryConfigGenerator(zone, applicationId); + var generator = new OpenTelemetryConfigGenerator(zone, applicationId, isHostedVespa); AnyConfigProducer pp = this; AnyConfigProducer p = pp.getParent(); while (p != null && p != pp) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java index 3f7ca7b46a7..67c45d58a95 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.admin.otel; import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; +import ai.vespa.metricsproxy.metric.model.prometheus.PrometheusUtil; import com.fasterxml.jackson.core.JsonEncoding; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; @@ -11,6 +12,7 @@ import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.model.Service; +import com.yahoo.vespa.model.admin.monitoring.MetricsConsumer; import java.io.ByteArrayOutputStream; import java.nio.charset.StandardCharsets; @@ -18,6 +20,9 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.regex.Pattern; +import java.util.Set; +import java.util.TreeSet; import static com.yahoo.vespa.defaults.Defaults.getDefaults; @@ -33,10 +38,12 @@ public class OpenTelemetryConfigGenerator { private List<StatePortInfo> statePorts = new ArrayList<>(); private final Zone zone; private final ApplicationId applicationId; + private final boolean isHostedVespa; - OpenTelemetryConfigGenerator(Zone zone, ApplicationId applicationId) { + OpenTelemetryConfigGenerator(Zone zone, ApplicationId applicationId, boolean isHostedVespa) { this.zone = zone; this.applicationId = applicationId; + this.isHostedVespa = isHostedVespa; boolean isCd = true; boolean isPublic = true; if (zone != null) { @@ -101,10 +108,29 @@ public class OpenTelemetryConfigGenerator { g.writeStringField(entry.getKey(), entry.getValue()); } } + String ph = findParentHost(statePort.hostName()); + if (isHostedVespa && ph != null) { + g.writeStringField("parentHostname", ph); + } g.writeEndObject(); } g.writeEndObject(); } + // note: this pattern should match entire node name + static private final Pattern expectedNodeName1 = Pattern.compile("[a-z0-9]+-v6-[0-9]+[.].+"); + static private final Pattern expectedNodeName2 = Pattern.compile("[a-z]*[0-9]+[a-z][.].+"); + // matches the part we want to replace with just a dot + static private final Pattern replaceNodeName1 = Pattern.compile("-v6-[0-9]+[.]"); + static private final Pattern replaceNodeName2 = Pattern.compile("[a-z][.]"); + static String findParentHost(String nodeName) { + if (expectedNodeName1.matcher(nodeName).matches()) { + return replaceNodeName1.matcher(nodeName).replaceFirst("."); + } + if (expectedNodeName2.matcher(nodeName).matches()) { + return replaceNodeName2.matcher(nodeName).replaceFirst("."); + } + return null; + } private void addTls(JsonGenerator g) throws java.io.IOException { g.writeFieldName("tls"); g.writeStartObject(); @@ -145,6 +171,31 @@ public class OpenTelemetryConfigGenerator { g.writeFieldName("processors"); g.writeStartObject(); addResourceProcessor(g); + addRenameProcessor(g); + addFilterProcessor(g); + g.writeEndObject(); + } + private void addRenameProcessor(JsonGenerator g) throws java.io.IOException { + g.writeFieldName("metricstransform/rename"); + g.writeStartObject(); + g.writeFieldName("transforms"); + g.writeStartArray(); + var metrics = MetricsConsumer.vespa9.metrics(); + for (var metric : metrics.values()) { + if (! metric.name.equals(metric.outputName)) { + String oldName = PrometheusUtil.sanitize(metric.name); + String newName = PrometheusUtil.sanitize(metric.outputName); + addRenameAction(g, oldName, newName); + } + } + g.writeEndArray(); + g.writeEndObject(); + } + private void addRenameAction(JsonGenerator g, String oldName, String newName) throws java.io.IOException { + g.writeStartObject(); + g.writeStringField("include", oldName); + g.writeStringField("action", "update"); + g.writeStringField("new_name", newName); g.writeEndObject(); } private void addResourceProcessor(JsonGenerator g) throws java.io.IOException { @@ -172,6 +223,35 @@ public class OpenTelemetryConfigGenerator { g.writeStringField("action", "insert"); g.writeEndObject(); } + static private Set<String> wantedMetrics() { + Set<String> result = new TreeSet<>(); + var metrics = MetricsConsumer.vespa9.metrics(); + for (var metric : metrics.values()) { + String oldName = PrometheusUtil.sanitize(metric.name); + String newName = PrometheusUtil.sanitize(metric.outputName); + result.add(oldName); + result.add(newName); + } + return result; + } + private void addFilterProcessor(JsonGenerator g) throws java.io.IOException { + g.writeFieldName("filter/metricset"); + g.writeStartObject(); + g.writeFieldName("metrics"); + g.writeStartObject(); + g.writeFieldName("include"); + g.writeStartObject(); + g.writeStringField("match_type", "strict"); + g.writeFieldName("metric_names"); + g.writeStartArray(); + for (String metricName : wantedMetrics()) { + g.writeString(metricName); + } + g.writeEndArray(); + g.writeEndObject(); + g.writeEndObject(); + g.writeEndObject(); + } private void addServiceBlock(JsonGenerator g) throws java.io.IOException { g.writeFieldName("service"); g.writeStartObject(); @@ -208,12 +288,14 @@ public class OpenTelemetryConfigGenerator { } g.writeFieldName("processors"); g.writeStartArray(); + g.writeString("metricstransform/rename"); + g.writeString("filter/metricset"); g.writeString("resource"); g.writeEndArray(); { g.writeFieldName("exporters"); g.writeStartArray(); - g.writeString("file"); + g.writeString("otlphttp/gw"); g.writeEndArray(); } g.writeEndObject(); // metrics @@ -296,16 +378,18 @@ public class OpenTelemetryConfigGenerator { private Map<String, String> serviceAttributes(Service svc) { Map<String, String> dimvals = new LinkedHashMap<>(); - dimvals.put("instance", svc.getServiceName()); // should maybe be "local_service_name" ? - dimvals.put("instanceType", svc.getServiceType()); // maybe "local_service_type", or remove + // currently unused - can be added if necessary: + // dimvals.put("local_service_name", svc.getServiceName()); + // dimvals.put("local_service_type", svc.getServiceType()); String cName = svc.getServicePropertyString("clustername", null); if (cName != null) { - // what about "clusterid" below, is it always the same? - dimvals.put("clustername", cName); + // overridden by cluster membership below (if available) + dimvals.put(PublicDimensions.INTERNAL_CLUSTER_ID, cName); } String cType = svc.getServicePropertyString("clustertype", null); if (cType != null) { - dimvals.put("clustertype", cType); + // overridden by cluster membership below (if available) + dimvals.put(PublicDimensions.INTERNAL_CLUSTER_TYPE, cType); } var hostResource = svc.getHost(); if (hostResource != null) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java index 9cf5fe84c21..4900b56801c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java @@ -23,21 +23,22 @@ public class JvmHeapSizeValidator implements Validator { context.model().getContainerClusters().forEach((clusterId, appCluster) -> { var mp = appCluster.getMemoryPercentage().orElse(null); if (mp == null) return; - if (mp.availableMemoryGb().isEmpty()) { + if (mp.asAbsoluteGb().isEmpty()) { context.deployState().getDeployLogger().log(Level.FINE, "Host resources unknown or percentage overridden with 'allocated-memory'"); return; } long jvmModelCost = appCluster.onnxModelCostCalculator().aggregatedModelCostInBytes(); if (jvmModelCost > 0) { - double availableMemoryGb = mp.availableMemoryGb().getAsDouble(); + double availableMemoryGb = mp.asAbsoluteGb().getAsDouble(); + int percentageOfTotal = mp.ofContainerTotal().getAsInt(); double modelCostGb = jvmModelCost / (1024D * 1024 * 1024); context.deployState().getDeployLogger().log(Level.FINE, () -> Text.format("JVM: %d%% (limit: %d%%), %.2fGB (limit: %.2fGB), ONNX: %.2fGB", - mp.percentage(), percentLimit, availableMemoryGb, gbLimit, modelCostGb)); - if (mp.percentage() < percentLimit) { + percentageOfTotal, percentLimit, availableMemoryGb, gbLimit, modelCostGb)); + if (percentageOfTotal < percentLimit) { context.illegal(Text.format("Allocated percentage of memory of JVM in cluster '%s' is too low (%d%% < %d%%). " + "Estimated cost of ONNX models is %.2fGB. Either use a node flavor with more memory or use less expensive models. " + "You may override this validation by specifying 'allocated-memory' (https://docs.vespa.ai/en/performance/container-tuning.html#jvm-heap-size).", - clusterId, mp.percentage(), percentLimit, modelCostGb)); + clusterId, percentageOfTotal, percentLimit, modelCostGb)); } if (availableMemoryGb < gbLimit) { context.illegal( diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 7f624032627..f073c0f27c1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -23,6 +23,7 @@ import com.yahoo.vespa.model.application.validation.change.RestartOnDeployForLoc import com.yahoo.vespa.model.application.validation.change.RestartOnDeployForOnnxModelChangesValidator; import com.yahoo.vespa.model.application.validation.change.StartupCommandChangeValidator; import com.yahoo.vespa.model.application.validation.change.StreamingSearchClusterChangeValidator; +import com.yahoo.vespa.model.application.validation.change.VespaRestartAction; import com.yahoo.vespa.model.application.validation.first.RedundancyValidator; import com.yahoo.yolean.Exceptions; @@ -215,6 +216,9 @@ public class Validation { @Override public void require(ConfigChangeAction action) { + if (action instanceof VespaRestartAction && action.getServices().isEmpty()) + throw new IllegalStateException("restart actions must have services specified"); + actions.add(action); action.validationId().ifPresent(id -> invalid(id, action.getMessage())); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java index bfd100f40c9..3722dd84e29 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidator.java @@ -15,7 +15,6 @@ import com.yahoo.vespa.model.utils.internal.ReflectionUtil; import java.util.Arrays; import java.util.List; import java.util.Optional; -import java.util.function.Consumer; import java.util.logging.Level; import java.util.stream.Stream; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/GlobalDocumentChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/GlobalDocumentChangeValidator.java index df8bf0e9b01..f86a3b31182 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/GlobalDocumentChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/GlobalDocumentChangeValidator.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.documentmodel.NewDocumentType; +import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; import com.yahoo.vespa.model.content.cluster.ContentCluster; @@ -38,7 +39,8 @@ public class GlobalDocumentChangeValidator implements ChangeValidator { if ( ! context.deployState().validationOverrides().allows(ValidationId.globalDocumentChange, context.deployState().now())) context.invalid(ValidationId.globalDocumentChange, reason); else if (context.deployState().isHosted()) - context.require(new VespaRestartAction(ClusterSpec.Id.from(clusterName), reason)); + context.require(new VespaRestartAction(ClusterSpec.Id.from(clusterName), reason, + nextCluster.getSearch().getSearchNodes().stream().map(AbstractService::getServiceInfo).toList())); } } }); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidator.java index 0d4776ad00a..ce24d11121c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/NodeResourceChangeValidator.java @@ -36,7 +36,7 @@ public class NodeResourceChangeValidator implements ChangeValidator { } private boolean changeRequiresRestart(NodeResources currentResources, NodeResources nextResources) { - return currentResources.memoryGb() != nextResources.memoryGb(); + return currentResources.memoryGiB() != nextResources.memoryGiB(); } private Optional<NodeResources> resourcesOf(ClusterSpec.Id clusterId, VespaModel model) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java index 42410dc3acf..a1f8a97d47f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java @@ -42,7 +42,7 @@ public class ResourcesReductionValidator implements ChangeValidator { NodeResources currentResources = current.totalResources(); NodeResources nextResources = next.totalResources(); if (nextResources.vcpu() < 0.5 * currentResources.vcpu() || - nextResources.memoryGb() < 0.5 * currentResources.memoryGb() || + nextResources.memoryGiB() < 0.5 * currentResources.memoryGiB() || nextResources.diskGb() < 0.5 * currentResources.diskGb()) context.invalid(ValidationId.resourcesReduction, "Resource reduction in '" + clusterId.value() + "' is too large: " + diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidator.java index ccfc611c3dc..ba1f19ff68c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidator.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; import com.yahoo.vespa.model.container.ApplicationContainerCluster; @@ -33,7 +34,9 @@ public class RestartOnDeployForLocalLLMValidator implements ChangeValidator { // Only restart services if we use a local LLM in both the next and previous generation for (var clusterId : intersect(previousClustersWithLocalLLM, nextClustersWithLocalLLM)) { String message = "Need to restart services in %s due to use of local LLM".formatted(clusterId); - context.require(new VespaRestartAction(clusterId, message)); + context.require(new VespaRestartAction(clusterId, message, + context.model().getContainerClusters().get(clusterId.value()).getContainers() + .stream().map(AbstractService::getServiceInfo).toList())); log.log(INFO, message); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java index 008a3fc5547..f95f0a6c079 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.application.validation.change; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.OnnxModelCost; +import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.Host; import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; import com.yahoo.vespa.model.container.ApplicationContainer; @@ -99,7 +100,7 @@ public class RestartOnDeployForOnnxModelChangesValidator implements ChangeValida log.log(INFO, message); cluster.onnxModelCostCalculator().setRestartOnDeploy(); cluster.onnxModelCostCalculator().store(); - actions.add(new VespaRestartAction(cluster.id(), message)); + actions.add(new VespaRestartAction(cluster.id(), message, cluster.getContainers().stream().map(AbstractService::getServiceInfo).toList())); } private static boolean enoughMemoryToAvoidRestart(ApplicationContainerCluster clusterInCurrentModel, @@ -111,11 +112,11 @@ public class RestartOnDeployForOnnxModelChangesValidator implements ChangeValida double currentModelCostInGb = onnxModelCostInGb(clusterInCurrentModel); double nextModelCostInGb = onnxModelCostInGb(cluster); - double totalMemory = containers.stream().mapToDouble(c -> c.getHostResource().realResources().memoryGb()).min().orElseThrow(); + double totalMemory = containers.stream().mapToDouble(c -> c.getHostResource().realResources().memoryGiB()).min().orElseThrow(); double memoryUsedByModels = currentModelCostInGb + nextModelCostInGb; double availableMemory = Math.max(0, totalMemory - Host.memoryOverheadGb - memoryUsedByModels); - var availableMemoryPercentage = cluster.availableMemoryPercentage(); + var availableMemoryPercentage = cluster.heapSizePercentageOfAvailable(); int memoryPercentage = (int) (availableMemory / totalMemory * availableMemoryPercentage); var prefix = "Validating Onnx models memory usage for %s".formatted(cluster); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java index c0b55d856ab..e36fa5b7373 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/VespaRestartAction.java @@ -16,6 +16,7 @@ public class VespaRestartAction extends VespaConfigChangeAction implements Confi private final boolean ignoreForInternalRedeploy; + /** <strong>This does <em>not</em> trigger restarts; you <em>need</em> the {@code ServiceInfo}!</strong>*/ public VespaRestartAction(ClusterSpec.Id id, String message) { this(id, message, List.of()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java index 692de1769d3..34df7e9d963 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2Builder.java @@ -12,15 +12,18 @@ import com.yahoo.vespa.model.SimpleConfigProducer; import com.yahoo.vespa.model.admin.Admin; import com.yahoo.vespa.model.admin.Configserver; import com.yahoo.vespa.model.admin.Logserver; +import com.yahoo.vespa.model.admin.LogserverContainer; +import com.yahoo.vespa.model.admin.LogserverContainerCluster; import com.yahoo.vespa.model.admin.Slobrok; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerCluster; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainer; import com.yahoo.vespa.model.admin.clustercontroller.ClusterControllerContainerCluster; import com.yahoo.vespa.model.admin.otel.OpenTelemetryCollector; -import com.yahoo.vespa.model.admin.otel.OpenTelemetryConfigGenerator; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder.DomConfigProducerBuilderBase; import com.yahoo.vespa.model.container.Container; +import com.yahoo.vespa.model.container.ContainerModel; import org.w3c.dom.Element; + import java.util.ArrayList; import java.util.List; import java.util.Objects; @@ -35,10 +38,13 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { private static final String ATTRIBUTE_CLUSTER_CONTROLLER_STANDALONE_ZK = "standalone-zookeeper"; - public DomAdminV2Builder(ConfigModelContext.ApplicationType applicationType, + private final ConfigModelContext context; + + public DomAdminV2Builder(ConfigModelContext context, boolean multitenant, List<ConfigServerSpec> configServerSpecs) { - super(applicationType, multitenant, configServerSpecs); + super(context.getApplicationType(), multitenant, configServerSpecs); + this.context = context; } private void addOtelcol(TreeConfigProducer<?> parent, DeployState deployState, HostResource hostResource) { @@ -52,6 +58,7 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { List<Configserver> configservers = parseConfigservers(deployState, admin, adminE); var logserver = parseLogserver(deployState, admin, adminE); admin.setLogserver(logserver); + createContainerOnLogserverHost(deployState, admin, logserver.getHostResource()); if (deployState.featureFlags().logserverOtelCol()) { // for manual testing addOtelcol(admin, deployState, logserver.getHostResource()); @@ -65,6 +72,21 @@ public class DomAdminV2Builder extends DomAdminBuilderBase { addLoggingSpecs(new ModelElement(adminE).child("logging"), admin); } + private void createContainerOnLogserverHost(DeployState deployState, Admin admin, HostResource hostResource) { + LogserverContainerCluster logServerCluster = new LogserverContainerCluster(admin, "logs", deployState); + ContainerModel logserverClusterModel = new ContainerModel(context.withParent(admin).withId(logServerCluster.getSubId())); + logserverClusterModel.setCluster(logServerCluster); + + LogserverContainer container = new LogserverContainer(logServerCluster, deployState); + container.useDynamicPorts(); + container.setHostResource(hostResource); + container.initService(deployState); + logServerCluster.addContainer(container); + admin.addAndInitializeService(deployState, hostResource, container); + admin.setLogserverContainerCluster(logServerCluster); + context.getConfigModelRepoAdder().add(logserverClusterModel); + } + private List<Configserver> parseConfigservers(DeployState deployState, Admin admin, Element adminE) { List<Configserver> configservers; if (multitenant) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java index 347bb504857..c6086327a1c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV4Builder.java @@ -111,6 +111,8 @@ public class DomAdminV4Builder extends DomAdminBuilderBase { logserverClusterModel.setCluster(logServerCluster); LogserverContainer container = new LogserverContainer(logServerCluster, deployState); + if (deployState.getProperties().applicationId().instance().isTester()) + container.useDynamicPorts(); // TODO: read current version in ApplicationRepository, and always use this. container.setHostResource(hostResource); container.initService(deployState); logServerCluster.addContainer(container); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java index d877600db13..11f4c9794aa 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java @@ -137,7 +137,7 @@ public class NodesSpecification { int defaultMinGroups = nodes.from().orElse(1) / groupSize.to().orElse(nodes.from().orElse(1)); int defaultMaxGroups = groupSize.isEmpty() ? 1 : nodes.to().orElse(1) / groupSize.from().orElse(1); - var min = new ClusterResources(nodes.from().orElse(1), groups.from().orElse(defaultMinGroups), nodeResources(nodesElement).getFirst()); + var min = new ClusterResources(nodes.from().orElse(1), groups.from().orElse(defaultMinGroups), nodeResources(nodesElement).getFirst()); var max = new ClusterResources(nodes.to().orElse(1), groups.to().orElse(defaultMaxGroups), nodeResources(nodesElement).getSecond()); return new ResourceConstraints(min, max, groupSize); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index ed7646b3066..014d183071d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -209,22 +209,27 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat if (memoryPercentage != null) return Optional.of(JvmMemoryPercentage.of(memoryPercentage)); if (isHostedVespa()) { - int availableMemoryPercentage = availableMemoryPercentage(); - if (getContainers().isEmpty()) return Optional.of(JvmMemoryPercentage.of(availableMemoryPercentage)); // Node memory is not known - - // Node memory is known so convert available memory percentage to node memory percentage - double totalMemory = getContainers().stream().mapToDouble(c -> c.getHostResource().realResources().memoryGb()).min().orElseThrow(); - double jvmHeapDeductionGb = onnxModelCostCalculator.aggregatedModelCostInBytes() / (1024D * 1024 * 1024); - double availableMemory = Math.max(0, totalMemory - Host.memoryOverheadGb - jvmHeapDeductionGb); - int memoryPercentage = (int) (availableMemory / totalMemory * availableMemoryPercentage); - logger.log(FINE, () -> "cluster id '%s': memoryPercentage=%d, availableMemory=%f, totalMemory=%f, availableMemoryPercentage=%d, jvmHeapDeductionGb=%f" - .formatted(id(), memoryPercentage, availableMemory, totalMemory, availableMemoryPercentage, jvmHeapDeductionGb)); - return Optional.of(JvmMemoryPercentage.of(memoryPercentage, availableMemory)); + int heapSizePercentageOfAvailable = heapSizePercentageOfAvailable(); + if (getContainers().isEmpty()) return Optional.of(JvmMemoryPercentage.of(heapSizePercentageOfAvailable)); // Node memory is not known + + // Node memory is known, so compute heap size as a percentage of available memory (excluding overhead, which the startup scripts also account for) + double totalMemoryGb = getContainers().stream().mapToDouble(c -> c.getHostResource().realResources().memoryGiB()).min().orElseThrow(); + double totalMemoryMinusOverhead = Math.max(0, totalMemoryGb - Host.memoryOverheadGb); + double onnxModelCostGb = onnxModelCostCalculator.aggregatedModelCostInBytes() / (1024D * 1024 * 1024); + double availableMemoryGb = Math.max(0, totalMemoryMinusOverhead - onnxModelCostGb); + int memoryPercentageOfAvailable = (int) (heapSizePercentageOfAvailable * availableMemoryGb / totalMemoryMinusOverhead); + int memoryPercentageOfTotal = (int) (heapSizePercentageOfAvailable * availableMemoryGb / totalMemoryGb); + logger.log(FINE, () -> ("cluster id '%s': memoryPercentageOfAvailable=%d, memoryPercentageOfTotal=%d, " + + "availableMemoryGb=%f, totalMemoryGb=%f, heapSizePercentageOfAvailable=%d, onnxModelCostGb=%f") + .formatted(id(), memoryPercentageOfAvailable, memoryPercentageOfTotal, + availableMemoryGb, totalMemoryGb, heapSizePercentageOfAvailable, onnxModelCostGb)); + return Optional.of(JvmMemoryPercentage.of(memoryPercentageOfAvailable, memoryPercentageOfTotal, + availableMemoryGb * heapSizePercentageOfAvailable * 1e-2)); } return Optional.empty(); } - public int availableMemoryPercentage() { + public int heapSizePercentageOfAvailable() { return getHostClusterId().isPresent() ? heapSizePercentageOfTotalAvailableMemoryWhenCombinedCluster : heapSizePercentageOfAvailableMemory; @@ -310,14 +315,23 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat public void getConfig(QrStartConfig.Builder builder) { super.getConfig(builder); var memoryPct = getMemoryPercentage().orElse(null); - int heapsize = memoryPct != null && memoryPct.availableMemoryGb().isPresent() - ? (int) (memoryPct.availableMemoryGb().getAsDouble() * 1024) : 1536; + int heapsize = truncateTo4SignificantBits(memoryPct != null && memoryPct.asAbsoluteGb().isPresent() + ? (int) (memoryPct.asAbsoluteGb().getAsDouble() * 1024) : 1536); builder.jvm.verbosegc(true) .availableProcessors(0) .compressedClassSpaceSize(0) - .minHeapsize(heapsize) + .minHeapsize(heapsize) // These cause restarts when changed, so we try to keep them stable. .heapsize(heapsize); - if (memoryPct != null) builder.jvm.heapSizeAsPercentageOfPhysicalMemory(memoryPct.percentage()); + if (memoryPct != null) builder.jvm.heapSizeAsPercentageOfPhysicalMemory(memoryPct.ofContainerAvailable()); + } + + static int truncateTo4SignificantBits(int i) { + if (i == Integer.MIN_VALUE) return i; + if (i < 0) return -truncateTo4SignificantBits(-i); + if (i <= 16) return i; + int mask = Integer.highestOneBit(i); + mask += mask - (mask >> 3); + return i & mask; } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index 864cbc8691b..c2368fd29a2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -108,10 +108,6 @@ public abstract class Container extends AbstractService implements addEnvironmentVariable("VESPA_MALLOC_MMAP_THRESHOLD","0x1000000"); // 16M } - public String jvmOmitStackTraceInFastThrowOption(ModelContext.FeatureFlags featureFlags) { - return featureFlags.jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type.container); - } - void setOwner(ContainerCluster<?> owner) { this.owner = owner; } /** True if this container is retired (slated for removal) */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index aeb6c030a49..00ab47e8ddb 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -73,6 +73,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.OptionalDouble; +import java.util.OptionalInt; import java.util.Set; import java.util.TreeSet; @@ -721,10 +722,10 @@ public abstract class ContainerCluster<CONTAINER extends Container> * Returns the percentage of host physical memory this application has specified for nodes in this cluster, * or empty if this is not specified by the application. */ - public record JvmMemoryPercentage(int percentage, OptionalDouble availableMemoryGb) { - static JvmMemoryPercentage of(int percentage) { return new JvmMemoryPercentage(percentage, OptionalDouble.empty()); } - static JvmMemoryPercentage of(int percentage, double availableMemoryGb) { - return new JvmMemoryPercentage(percentage, OptionalDouble.of(availableMemoryGb)); + public record JvmMemoryPercentage(int ofContainerAvailable, OptionalInt ofContainerTotal, OptionalDouble asAbsoluteGb) { // optionalInt pctOfTotal < int pctOfAvailable + static JvmMemoryPercentage of(int percentageOfAvailable) { return new JvmMemoryPercentage(percentageOfAvailable, OptionalInt.empty(), OptionalDouble.empty()); } + static JvmMemoryPercentage of(int percentageOfAvailable, int percentageOfTotal, double absoluteMemoryGb) { + return new JvmMemoryPercentage(percentageOfAvailable, OptionalInt.of(percentageOfTotal), OptionalDouble.of(absoluteMemoryGb)); } } public Optional<JvmMemoryPercentage> getMemoryPercentage() { return Optional.empty(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java index 133195c7039..e797c92ee52 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/ConfigserverCluster.java @@ -87,6 +87,7 @@ public class ConfigserverCluster extends TreeConfigProducer builder.dynamicReconfiguration(options.hostedVespa().orElse(false)); builder.snapshotMethod(options.zooKeeperSnapshotMethod()); + builder.juteMaxBuffer(options.zookeeperJuteMaxBuffer()); } @Override @@ -197,6 +198,7 @@ public class ConfigserverCluster extends TreeConfigProducer builder.server(curatorBuilder); } builder.zookeeperLocalhostAffinity(true); + builder.juteMaxBuffer(options.zookeeperJuteMaxBuffer()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java index ec0fa973d6c..5b5914b3012 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/configserver/option/CloudConfigOptions.java @@ -38,5 +38,6 @@ public interface CloudConfigOptions { Optional<String> athenzDnsSuffix(); Optional<String> ztsUrl(); String zooKeeperSnapshotMethod(); + Integer zookeeperJuteMaxBuffer(); // in bytes } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java index 5f824950ecd..9c9e20062f8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/http/ssl/HostedSslConnectorFactory.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.container.http.ssl; +import ai.vespa.utils.BytesQuantity; import com.yahoo.config.model.api.EndpointCertificateSecrets; import com.yahoo.jdisc.http.ConnectorConfig; import com.yahoo.security.tls.TlsContext; @@ -21,6 +22,8 @@ import java.util.TreeSet; */ public class HostedSslConnectorFactory extends ConnectorFactory { + private record EntityLoggingEntry(String prefix, double sampleRate, BytesQuantity maxEntitySize) {} + private final SslClientAuth clientAuth; private final List<String> tlsCiphersOverride; private final boolean proxyProtocolEnabled; @@ -28,6 +31,7 @@ public class HostedSslConnectorFactory extends ConnectorFactory { private final List<String> remoteAddressHeaders; private final List<String> remotePortHeaders; private final Set<String> knownServerNames; + private final List<EntityLoggingEntry> entityLoggingEntries; public static Builder builder(String name, int listenPort) { return new Builder(name, listenPort); } @@ -40,6 +44,22 @@ public class HostedSslConnectorFactory extends ConnectorFactory { this.remoteAddressHeaders = List.copyOf(builder.remoteAddressHeaders); this.remotePortHeaders = List.copyOf(builder.remotePortHeaders); this.knownServerNames = Collections.unmodifiableSet(new TreeSet<>(builder.knownServerNames)); + this.entityLoggingEntries = builder.requestPrefixForLoggingContent.stream() + .map(prefix -> { + var parts = prefix.split(":"); + if (parts.length != 3) { + throw new IllegalArgumentException("Expected string of format 'prefix:sample-rate:max-entity-size', got '%s'".formatted(prefix)); + } + var pathPrefix = parts[0]; + if (pathPrefix.isBlank()) + throw new IllegalArgumentException("Path prefix must not be blank"); + var sampleRate = Double.parseDouble(parts[1]); + if (sampleRate < 0 || sampleRate > 1) + throw new IllegalArgumentException("Sample rate must be in range [0, 1], got '%s'".formatted(sampleRate)); + var maxEntitySize = BytesQuantity.fromString(parts[2]); + return new EntityLoggingEntry(pathPrefix, sampleRate, maxEntitySize); + }) + .toList(); } private static SslProvider createSslProvider(Builder builder) { @@ -72,8 +92,14 @@ public class HostedSslConnectorFactory extends ConnectorFactory { .idleTimeout(Duration.ofSeconds(30).toSeconds()) .maxConnectionLife(endpointConnectionTtl != null ? endpointConnectionTtl.toSeconds() : 0) .accessLog(new ConnectorConfig.AccessLog.Builder() - .remoteAddressHeaders(remoteAddressHeaders) - .remotePortHeaders(remotePortHeaders)) + .remoteAddressHeaders(remoteAddressHeaders) + .remotePortHeaders(remotePortHeaders) + .content(entityLoggingEntries.stream() + .map(e -> new ConnectorConfig.AccessLog.Content.Builder() + .pathPrefix(e.prefix) + .sampleRate(e.sampleRate) + .maxSize(e.maxEntitySize.toBytes())) + .toList())) .serverName.known(knownServerNames); } @@ -93,6 +119,7 @@ public class HostedSslConnectorFactory extends ConnectorFactory { String tlsCaCertificatesPath; boolean tokenEndpoint; Set<String> knownServerNames = Set.of(); + Set<String> requestPrefixForLoggingContent = Set.of(); private Builder(String name, int port) { this.name = name; this.port = port; } public Builder clientAuth(SslClientAuth auth) { clientAuth = auth; return this; } @@ -106,6 +133,7 @@ public class HostedSslConnectorFactory extends ConnectorFactory { public Builder remoteAddressHeader(String header) { this.remoteAddressHeaders.add(header); return this; } public Builder remotePortHeader(String header) { this.remotePortHeaders.add(header); return this; } public Builder knownServerNames(Set<String> knownServerNames) { this.knownServerNames = Set.copyOf(knownServerNames); return this; } + public Builder requestPrefixForLoggingContent(Collection<String> v) { this.requestPrefixForLoggingContent = Set.copyOf(v); return this; } public HostedSslConnectorFactory build() { return new HostedSslConnectorFactory(this); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index dbe28d48f9e..4995c20b985 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -140,6 +140,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { // Default path to vip status file for container in Hosted Vespa. static final String HOSTED_VESPA_STATUS_FILE = Defaults.getDefaults().underVespaHome("var/vespa/load-balancer/status.html"); + static final String HOSTED_VESPA_TENANT_PARENT_DOMAIN = "vespa.tenant."; + //Path to vip status file for container in Hosted Vespa. Only used if set, else use HOSTED_VESPA_STATUS_FILE private static final String HOSTED_VESPA_STATUS_FILE_SETTING = "VESPA_LB_STATUS_FILE"; @@ -235,6 +237,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { // Must be added after nodes: addDeploymentSpecConfig(cluster, context, deployState.getDeployLogger()); addZooKeeper(cluster, spec); + addAthenzServiceIdentityProvider(cluster, context, deployState.getDeployLogger()); addParameterStoreValidationHandler(cluster, deployState); } @@ -344,6 +347,20 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { cluster.addComponent(cloudSecretStore); } + private void addAthenzServiceIdentityProvider(ApplicationContainerCluster cluster, ConfigModelContext context, DeployLogger deployLogger) { + if ( ! context.getDeployState().isHosted()) return; + if ( ! context.getDeployState().zone().system().isPublic()) return; // Non-public is handled by deployment spec config. + if ( ! context.properties().launchApplicationAthenzService()) return; + addIdentityProvider(cluster, + context.getDeployState().getProperties().configServerSpecs(), + context.getDeployState().getProperties().loadBalancerName(), + context.getDeployState().getProperties().ztsUrl(), + context.getDeployState().getProperties().athenzDnsSuffix(), + context.getDeployState().zone(), + AthenzDomain.from(HOSTED_VESPA_TENANT_PARENT_DOMAIN + context.properties().applicationId().tenant().value()), + AthenzService.from(context.properties().applicationId().application().value())); + } + private void addDeploymentSpecConfig(ApplicationContainerCluster cluster, ConfigModelContext context, DeployLogger deployLogger) { if ( ! context.getDeployState().isHosted()) return; DeploymentSpec deploymentSpec = app.getDeploymentSpec(); @@ -607,7 +624,8 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { var builder = HostedSslConnectorFactory.builder(serverName, getMtlsDataplanePort(state)) .proxyProtocol(state.zone().cloud().useProxyProtocol()) .tlsCiphersOverride(state.getProperties().tlsCiphersOverride()) - .endpointConnectionTtl(state.getProperties().endpointConnectionTtl()); + .endpointConnectionTtl(state.getProperties().endpointConnectionTtl()) + .requestPrefixForLoggingContent(state.getProperties().requestPrefixForLoggingContent()); var endpointCert = state.endpointCertificateSecrets().orElse(null); if (endpointCert != null) { builder.endpointCertificate(endpointCert); @@ -670,6 +688,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { .remotePortHeader("X-Forwarded-Port") .clientAuth(SslClientAuth.NEED) .knownServerNames(tokenEndpoints) + .requestPrefixForLoggingContent(state.getProperties().requestPrefixForLoggingContent()) .build(); server.addConnector(connector); @@ -815,8 +834,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { !container.getHostResource().realResources().gpuResources().isZero()); onnxModel.setGpuDevice(gpuDevice, hasGpu); } - cluster.onnxModelCostCalculator().registerModel(context.getApplicationPackage().getFile(onnxModel.getFilePath()), onnxModel.onnxModelOptions()); } + for (OnnxModel onnxModel : models.asMap().values()) + cluster.onnxModelCostCalculator().registerModel(context.getApplicationPackage().getFile(onnxModel.getFilePath()), onnxModel.onnxModelOptions()); cluster.setModelEvaluation(new ContainerModelEvaluation(cluster, profiles, models)); } @@ -986,8 +1006,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { AbstractService.distributeCpuSocketAffinity(nodes); cluster.addContainers(nodes); } - // Must be done after setting Jvm options from services.xml (#extractJvmOptions()), otherwise those options will not be set - cluster.getContainers().forEach(container -> container.appendJvmOptions(container.jvmOmitStackTraceInFastThrowOption(context.featureFlags()))); } private ZoneEndpoint zoneEndpoint(ConfigModelContext context, ClusterSpec.Id cluster) { @@ -1039,7 +1057,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } catch (NumberFormatException e) { throw new IllegalArgumentException("The memory percentage given for nodes in " + cluster + - " must be an integer percentage ending by the '%' sign", e); + " must be given as an integer followe by '%'", e); } } @@ -1074,9 +1092,21 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return List.of(node); } + private static void requireFixedSizeSingularNodeIfTester(ConfigModelContext context, NodesSpecification nodes) { + if ( ! context.properties().hostedVespa() || ! context.properties().applicationId().instance().isTester()) + return; + + if ( ! nodes.maxResources().equals(nodes.minResources())) + throw new IllegalArgumentException("tester resources must be absolute, but min and max resources differ: " + nodes); + + if (nodes.maxResources().nodes() > 1) + throw new IllegalArgumentException("tester cannot run on more than 1 node, but " + nodes.maxResources().nodes() + " nodes were specified"); + } + private List<ApplicationContainer> createNodesFromNodeCount(ApplicationContainerCluster cluster, Element containerElement, Element nodesElement, ConfigModelContext context) { try { var nodesSpecification = NodesSpecification.from(new ModelElement(nodesElement), context); + requireFixedSizeSingularNodeIfTester(context, nodesSpecification); var clusterId = ClusterSpec.Id.from(cluster.name()); Map<HostResource, ClusterMembership> hosts = nodesSpecification.provision(cluster.getRoot().hostSystem(), ClusterSpec.Type.container, @@ -1282,37 +1312,36 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } } - private void addIdentityProvider(ApplicationContainerCluster cluster, - List<ConfigServerSpec> configServerSpecs, - HostName loadBalancerName, - URI ztsUrl, - String athenzDnsSuffix, - Zone zone, - DeploymentSpec spec) { - spec.athenzDomain() - .ifPresent(domain -> { - AthenzService service = spec.athenzService(app.getApplicationId().instance(), zone.environment(), zone.region()) - .orElseThrow(() -> new IllegalArgumentException("Missing Athenz service configuration in instance '" + - app.getApplicationId().instance() + "'")); - String zoneDnsSuffix = zone.environment().value() + "-" + zone.region().value() + "." + athenzDnsSuffix; - IdentityProvider identityProvider = new IdentityProvider(domain, - service, - getLoadBalancerName(loadBalancerName, configServerSpecs), - ztsUrl, - zoneDnsSuffix, - zone); - - // Replace AthenzIdentityProviderProvider - cluster.removeComponent(ComponentId.fromString("com.yahoo.container.jdisc.AthenzIdentityProviderProvider")); - cluster.addComponent(identityProvider); - - var serviceIdentityProviderProvider = "com.yahoo.vespa.athenz.identityprovider.client.ServiceIdentityProviderProvider"; - cluster.addComponent(new SimpleComponent(new ComponentModel(serviceIdentityProviderProvider, serviceIdentityProviderProvider, "vespa-athenz"))); - - cluster.getContainers().forEach(container -> { - container.setProp("identity.domain", domain.value()); - container.setProp("identity.service", service.value()); - }); + private void addIdentityProvider(ApplicationContainerCluster cluster, List<ConfigServerSpec> configServerSpecs, HostName loadBalancerName, + URI ztsUrl, String athenzDnsSuffix, Zone zone, DeploymentSpec spec) { + spec.athenzDomain().ifPresent(domain -> { + AthenzService service = spec.athenzService(app.getApplicationId().instance(), zone.environment(), zone.region()) + .orElseThrow(() -> new IllegalArgumentException("Missing Athenz service configuration in instance '" + + app.getApplicationId().instance() + "'")); + addIdentityProvider(cluster, configServerSpecs, loadBalancerName, ztsUrl, athenzDnsSuffix, zone, domain, service); + }); + } + + private void addIdentityProvider(ApplicationContainerCluster cluster, List<ConfigServerSpec> configServerSpecs, HostName loadBalancerName, + URI ztsUrl, String athenzDnsSuffix, Zone zone, AthenzDomain domain, AthenzService service) { + String zoneDnsSuffix = zone.environment().value() + "-" + zone.region().value() + "." + athenzDnsSuffix; + IdentityProvider identityProvider = new IdentityProvider(domain, + service, + getLoadBalancerName(loadBalancerName, configServerSpecs), + ztsUrl, + zoneDnsSuffix, + zone); + + // Replace AthenzIdentityProviderProvider + cluster.removeComponent(ComponentId.fromString("com.yahoo.container.jdisc.AthenzIdentityProviderProvider")); + cluster.addComponent(identityProvider); + + var serviceIdentityProviderProvider = "com.yahoo.vespa.athenz.identityprovider.client.ServiceIdentityProviderProvider"; + cluster.addComponent(new SimpleComponent(new ComponentModel(serviceIdentityProviderProvider, serviceIdentityProviderProvider, "vespa-athenz"))); + + cluster.getContainers().forEach(container -> { + container.setProp("identity.domain", domain.value()); + container.setProp("identity.service", service.value()); }); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index fbac9e9d710..994837a2dbe 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -21,7 +21,6 @@ import com.yahoo.vespa.model.search.NodeSpec; import com.yahoo.vespa.model.search.SchemaDefinitionXMLHandler; import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.vespa.model.search.SearchNode; -import com.yahoo.vespa.model.search.TransactionLogServer; import com.yahoo.vespa.model.search.Tuning; import org.w3c.dom.Element; @@ -237,24 +236,17 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> NodeSpec spec = getNextSearchNodeSpec(parentGroup); SearchNode searchNode; - TransactionLogServer tls; if (element == null) { searchNode = SearchNode.create(parent, "" + node.getDistributionKey(), node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), - fractionOfMemoryReserved, deployState.featureFlags()); + fractionOfMemoryReserved, deployState.featureFlags(), syncTransactionLog); searchNode.setHostResource(node.getHostResource()); searchNode.initService(deployState); - - tls = new TransactionLogServer(searchNode, clusterName, syncTransactionLog); - tls.setHostResource(searchNode.getHostResource()); - tls.initService(deployState); } else { - searchNode = new SearchNode.Builder(""+node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, - tuning, resourceLimits, fractionOfMemoryReserved) + searchNode = new SearchNode.Builder("" + node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, + tuning, resourceLimits, fractionOfMemoryReserved, syncTransactionLog) .build(deployState, parent, element.getXml()); - tls = new TransactionLogServer.Builder(clusterName, syncTransactionLog).build(deployState, searchNode, element.getXml()); } - searchNode.setTls(tls); if (searchCluster != null) { searchCluster.addSearcher(searchNode); } else { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java index c182f0e0507..f218cedb74e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java @@ -35,6 +35,9 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen private final boolean hasIndexedDocumentType; private final int maxActivationInhibitedOutOfSyncGroups; private final int contentLayerMetadataFeatureLevel; + private final boolean symmetricPutAndActivateReplicaSelection; + private final boolean enforceStrictlyIncreasingClusterStateVersions; + public static class Builder extends VespaDomBuilder.DomConfigProducerBuilderBase<DistributorCluster> { ContentCluster parent; @@ -97,19 +100,25 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen var featureFlags = deployState.getProperties().featureFlags(); int maxInhibitedGroups = featureFlags.maxActivationInhibitedOutOfSyncGroups(); int contentLayerMetadataFeatureLevel = featureFlags.contentLayerMetadataFeatureLevel(); + boolean symmetricPutAndActivateReplicaSelection = featureFlags.symmetricPutAndActivateReplicaSelection(); + boolean enforceStrictlyIncreasingClusterStateVersions = featureFlags.enforceStrictlyIncreasingClusterStateVersions(); return new DistributorCluster(parent, new BucketSplitting.Builder().build(new ModelElement(producerSpec)), gc, hasIndexedDocumentType, maxInhibitedGroups, - contentLayerMetadataFeatureLevel); + contentLayerMetadataFeatureLevel, + symmetricPutAndActivateReplicaSelection, + enforceStrictlyIncreasingClusterStateVersions); } } private DistributorCluster(ContentCluster parent, BucketSplitting bucketSplitting, GcOptions gc, boolean hasIndexedDocumentType, int maxActivationInhibitedOutOfSyncGroups, - int contentLayerMetadataFeatureLevel) + int contentLayerMetadataFeatureLevel, + boolean symmetricPutAndActivateReplicaSelection, + boolean enforceStrictlyIncreasingClusterStateVersions) { super(parent, "distributor"); this.parent = parent; @@ -118,6 +127,8 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen this.hasIndexedDocumentType = hasIndexedDocumentType; this.maxActivationInhibitedOutOfSyncGroups = maxActivationInhibitedOutOfSyncGroups; this.contentLayerMetadataFeatureLevel = contentLayerMetadataFeatureLevel; + this.symmetricPutAndActivateReplicaSelection = symmetricPutAndActivateReplicaSelection; + this.enforceStrictlyIncreasingClusterStateVersions = enforceStrictlyIncreasingClusterStateVersions; } @Override @@ -132,6 +143,7 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen if (contentLayerMetadataFeatureLevel > 0) { builder.enable_operation_cancellation(true); } + builder.symmetric_put_and_activate_replica_selection(symmetricPutAndActivateReplicaSelection); bucketSplitting.getConfig(builder); } @@ -152,6 +164,7 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen builder.root_folder(""); builder.cluster_name(parent.getName()); builder.is_distributor(true); + builder.require_strictly_increasing_cluster_state_versions(enforceStrictlyIncreasingClusterStateVersions); } public String getClusterName() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index bac86e37e8f..d37e5d5382a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -46,6 +46,7 @@ import com.yahoo.vespa.model.content.IndexedHierarchicDistributionValidator; import com.yahoo.vespa.model.content.Redundancy; import com.yahoo.vespa.model.content.ReservedDocumentTypeNameValidator; import com.yahoo.vespa.model.content.StorageGroup; +import com.yahoo.vespa.model.content.StorageNode; import com.yahoo.vespa.model.content.engines.PersistenceEngine; import com.yahoo.vespa.model.content.engines.ProtonEngine; import com.yahoo.vespa.model.content.storagecluster.StorageCluster; @@ -137,6 +138,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem c.rootGroup = new StorageGroup.Builder(contentElement, context).buildRootGroup(deployState, c, c.search.isStreaming()); c.clusterControllerConfig = createClusterControllerConfig(contentElement, deployState, c, resourceLimits); validateThatGroupSiblingsAreUnique(c.clusterId, c.rootGroup); + warnIfDistributionKeyRangeIsSuboptimal(c.clusterId, c.rootGroup, deployState); c.search.handleRedundancy(c.redundancy); setupSearchCluster(c.search, contentElement, deployState.getDeployLogger()); @@ -247,7 +249,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem for (ContainerModel containerModel : containers) { Optional<String> hostClusterId = containerModel.getCluster().getHostClusterId(); if (hostClusterId.isPresent() && hostClusterId.get().equals(clusterId) && containerModel.getCluster().getMemoryPercentage().isPresent()) { - return containerModel.getCluster().getMemoryPercentage().get().percentage() * 0.01; + return containerModel.getCluster().getMemoryPercentage().get().ofContainerAvailable() * 0.01; } } return 0.0; @@ -275,6 +277,38 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem } } + private static class HighestDistributionKeyAggregator { + public int nodeCount = 0; + public int highestNodeDistributionKey = 0; + + void aggregateNodeStats(StorageGroup group) { + for (StorageNode n : group.getNodes()) { + nodeCount++; + highestNodeDistributionKey = Math.max(highestNodeDistributionKey, n.getDistributionKey()); + } + for (StorageGroup g : group.getSubgroups()) { + aggregateNodeStats(g); + } + } + } + + private void warnIfDistributionKeyRangeIsSuboptimal(String clusterId, StorageGroup rootGroup, DeployState deployState) { + if (rootGroup == null) { + return; // Unit testing case + } + var aggr = new HighestDistributionKeyAggregator(); + aggr.aggregateNodeStats(rootGroup); + int warnThreshold = 100; // ... Not scientifically chosen + if ((aggr.highestNodeDistributionKey - aggr.nodeCount) >= warnThreshold) { + deployState.getDeployLogger().logApplicationPackage(WARNING, + ("Content cluster '%s' has %d node(s), but the highest distribution key is %d. " + + "Having much higher distribution keys than the number of nodes is not recommended, " + + "as it may negatively affect performance. " + + "See https://docs.vespa.ai/en/reference/services-content.html#node") + .formatted(clusterId, aggr.nodeCount, aggr.highestNodeDistributionKey)); + } + } + private void addClusterControllers(ConfigModelContext context, ModelElement contentElement, ContentCluster contentCluster, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java index 701da93a329..8bde038b160 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java @@ -31,13 +31,15 @@ public class StorageCluster extends TreeConfigProducer<StorageNode> protected StorageCluster doBuild(DeployState deployState, TreeConfigProducer<AnyConfigProducer> ancestor, Element producerSpec) { final ModelElement clusterElem = new ModelElement(producerSpec); final ContentCluster cluster = (ContentCluster)ancestor; + var featureFlags = deployState.getProperties().featureFlags(); return new StorageCluster(ancestor, ContentCluster.getClusterId(clusterElem), new FileStorProducer.Builder().build(deployState.getProperties(), cluster, clusterElem), new StorServerProducer.Builder().build(deployState.getProperties(), clusterElem), new StorVisitorProducer.Builder().build(clusterElem), - new PersistenceProducer.Builder().build(clusterElem)); + new PersistenceProducer.Builder().build(clusterElem), + featureFlags.enforceStrictlyIncreasingClusterStateVersions()); } } @@ -46,19 +48,22 @@ public class StorageCluster extends TreeConfigProducer<StorageNode> private final StorServerProducer storServerProducer; private final StorVisitorProducer storVisitorProducer; private final PersistenceProducer persistenceProducer; + private final boolean enforceStrictlyIncreasingClusterStateVersions; StorageCluster(TreeConfigProducer<?> parent, String clusterName, FileStorProducer fileStorProducer, StorServerProducer storServerProducer, StorVisitorProducer storVisitorProducer, - PersistenceProducer persistenceProducer) { + PersistenceProducer persistenceProducer, + boolean enforceStrictlyIncreasingClusterStateVersions) { super(parent, "storage"); this.clusterName = clusterName; this.fileStorProducer = fileStorProducer; this.storServerProducer = storServerProducer; this.storVisitorProducer = storVisitorProducer; this.persistenceProducer = persistenceProducer; + this.enforceStrictlyIncreasingClusterStateVersions = enforceStrictlyIncreasingClusterStateVersions; } @Override @@ -85,6 +90,7 @@ public class StorageCluster extends TreeConfigProducer<StorageNode> @Override public void getConfig(StorServerConfig.Builder builder) { storServerProducer.getConfig(builder); + builder.require_strictly_increasing_cluster_state_versions(enforceStrictlyIncreasingClusterStateVersions); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java index 2616dd8a93c..d60e6ac2df6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.Host; -import com.yahoo.vespa.model.content.Redundancy; import static java.lang.Long.min; import static java.lang.Long.max; @@ -21,8 +20,9 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { private final static double MEMORY_GAIN_AS_FRACTION_OF_MEMORY = 0.08; private final static double MIN_MEMORY_PER_FLUSH_THREAD_GB = 11.0; private final static double TLS_SIZE_FRACTION = 0.02; - final static long MB = 1024 * 1024; - public final static long GB = MB * 1024; + final static long MiB = 1024 * 1024; + public final static long GiB = MiB * 1024; + public final static long GB = 1_000_000_000; private final NodeResources resources; private final int threadsPerSearch; private final double fractionOfMemoryReserved; @@ -50,14 +50,14 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { } private void tuneSummaryCache(ProtonConfig.Summary.Cache.Builder builder) { - long memoryLimitBytes = (long) ((usableMemoryGb() * SUMMARY_CACHE_SIZE_AS_FRACTION_OF_MEMORY) * GB); + long memoryLimitBytes = (long) ((usableMemoryGb() * SUMMARY_CACHE_SIZE_AS_FRACTION_OF_MEMORY) * GiB); builder.maxbytes(memoryLimitBytes); } private void setHwInfo(ProtonConfig.Builder builder) { builder.hwinfo.disk.shared(true); builder.hwinfo.cpu.cores((int)resources.vcpu()); - builder.hwinfo.memory.size((long)(usableMemoryGb() * GB)); + builder.hwinfo.memory.size((long)(usableMemoryGb() * GiB)); builder.hwinfo.disk.size((long)(resources.diskGb() * GB)); } @@ -68,12 +68,12 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { } private void tuneDocumentStoreMaxFileSize(ProtonConfig.Summary.Log.Builder builder) { - long fileSizeBytes = (long) Math.max(256*MB, usableMemoryGb()*GB*SUMMARY_FILE_SIZE_AS_FRACTION_OF_MEMORY); + long fileSizeBytes = (long) Math.max(256* MiB, usableMemoryGb()* GiB *SUMMARY_FILE_SIZE_AS_FRACTION_OF_MEMORY); builder.maxfilesize(fileSizeBytes); } private void tuneFlushStrategyMemoryLimits(ProtonConfig.Flush.Memory.Builder builder) { - long memoryLimitBytes = (long) ((usableMemoryGb() * MEMORY_GAIN_AS_FRACTION_OF_MEMORY) * GB); + long memoryLimitBytes = (long) ((usableMemoryGb() * MEMORY_GAIN_AS_FRACTION_OF_MEMORY) * GiB); builder.maxmemory(memoryLimitBytes); builder.each.maxmemory(memoryLimitBytes); } @@ -89,7 +89,7 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { private void tuneFlushStrategyTlsSize(ProtonConfig.Flush.Memory.Builder builder) { long tlsSizeBytes = (long) ((resources.diskGb() * TLS_SIZE_FRACTION) * GB); - tlsSizeBytes = max(2*GB, min(tlsSizeBytes, 100 * GB)); + tlsSizeBytes = max(2* GB, min(tlsSizeBytes, 100 * GB)); builder.maxtlssize(tlsSizeBytes); } @@ -114,7 +114,7 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { /** Returns the memory we can expect will be available for the content node processes */ private double usableMemoryGb() { - double usableMemoryGb = resources.memoryGb() - Host.memoryOverheadGb; + double usableMemoryGb = resources.memoryGiB() - Host.memoryOverheadGb; return usableMemoryGb * (1 - fractionOfMemoryReserved); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index 61933c10504..08743290ae3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -55,17 +55,18 @@ public class SearchNode extends AbstractService implements private static final int UNUSED_2 = 2; private static final int UNUSED_3 = 3; private static final int HEALTH_PORT = 4; + private static final int TLS_PORT = 5; private final boolean isHostedVespa; private final boolean flushOnShutdown; private final NodeSpec nodeSpec; private final int distributionKey; private final String clusterName; - private TransactionLogServer tls; private final AbstractService serviceLayerService; private final Tuning tuning; private final ResourceLimits resourceLimits; private final double fractionOfMemoryReserved; + private final Boolean syncTransactionLog; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilderBase<SearchNode> { @@ -77,10 +78,11 @@ public class SearchNode extends AbstractService implements private final Tuning tuning; private final ResourceLimits resourceLimits; private final double fractionOfMemoryReserved; + private final Boolean syncTransactionLog; public Builder(String name, NodeSpec nodeSpec, String clusterName, ContentNode node, boolean flushOnShutdown, Tuning tuning, ResourceLimits resourceLimits, - double fractionOfMemoryReserved) { + double fractionOfMemoryReserved, Boolean syncTransactionLog) { this.name = name; this.nodeSpec = nodeSpec; this.clusterName = clusterName; @@ -89,6 +91,7 @@ public class SearchNode extends AbstractService implements this.tuning = tuning; this.resourceLimits = resourceLimits; this.fractionOfMemoryReserved = fractionOfMemoryReserved; + this.syncTransactionLog = syncTransactionLog; } @Override @@ -96,7 +99,7 @@ public class SearchNode extends AbstractService implements Element producerSpec) { return SearchNode.create(ancestor, name, contentNode.getDistributionKey(), nodeSpec, clusterName, contentNode, flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), - fractionOfMemoryReserved, deployState.featureFlags()); + fractionOfMemoryReserved, deployState.featureFlags(), syncTransactionLog); } } @@ -105,9 +108,9 @@ public class SearchNode extends AbstractService implements String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, Tuning tuning, ResourceLimits resourceLimits, boolean isHostedVespa, double fractionOfMemoryReserved, - ModelContext.FeatureFlags featureFlags) { + ModelContext.FeatureFlags featureFlags, Boolean syncTransactionLog) { SearchNode node = new SearchNode(parent, name, distributionKey, nodeSpec, clusterName, serviceLayerService, flushOnShutdown, - tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved); + tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved, syncTransactionLog); if (featureFlags.loadCodeAsHugePages()) { node.addEnvironmentVariable("VESPA_LOAD_CODE_AS_HUGEPAGES", true); } @@ -120,7 +123,7 @@ public class SearchNode extends AbstractService implements private SearchNode(TreeConfigProducer<?> parent, String name, int distributionKey, NodeSpec nodeSpec, String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, Tuning tuning, ResourceLimits resourceLimits, boolean isHostedVespa, - double fractionOfMemoryReserved) { + double fractionOfMemoryReserved, Boolean syncTransactionLog) { super(parent, name); this.distributionKey = distributionKey; this.serviceLayerService = serviceLayerService; @@ -134,9 +137,11 @@ public class SearchNode extends AbstractService implements portsMeta.on(UNUSED_2).tag("unused"); portsMeta.on(UNUSED_3).tag("unused"); portsMeta.on(HEALTH_PORT).tag("http").tag("json").tag("health").tag("state"); + portsMeta.on(TLS_PORT).tag("tls"); // Properties are set in DomSearchBuilder this.tuning = tuning; this.resourceLimits = resourceLimits; + this.syncTransactionLog = syncTransactionLog; setPropertiesElastic(clusterName, distributionKey); addEnvironmentVariable("OMP_NUM_THREADS", 1); } @@ -172,6 +177,7 @@ public class SearchNode extends AbstractService implements from.allocatePort("unused/2"); from.allocatePort("unused/3"); from.allocatePort("health"); + from.allocatePort("tls"); } /** @@ -181,7 +187,7 @@ public class SearchNode extends AbstractService implements */ @Override public int getPortCount() { - return 5; + return 6; } /** @@ -198,6 +204,8 @@ public class SearchNode extends AbstractService implements return getHttpPort(); } + int getTlsPort() { return getRelativePort(TLS_PORT); } + @Override public String getServiceType() { return "searchnode"; @@ -219,7 +227,10 @@ public class SearchNode extends AbstractService implements builder.usefsync(false); } } - tls.getConfig(builder); + builder.listenport(getTlsPort()) + .basedir(getTlsDir()); + if (syncTransactionLog != null) + builder.usefsync(syncTransactionLog); } @Override @@ -227,14 +238,6 @@ public class SearchNode extends AbstractService implements return getHostName(); } - private TransactionLogServer getTransactionLogServer() { - return tls; - } - - public void setTls(TransactionLogServer tls) { - this.tls = tls; - } - public AbstractService getServiceLayerService() { return serviceLayerService; } @@ -260,7 +263,7 @@ public class SearchNode extends AbstractService implements httpport(getHttpPort()). clustername(getClusterName()). basedir(getBaseDir()). - tlsspec("tcp/" + getHost().getHostname() + ":" + getTransactionLogServer().getTlsPort()). + tlsspec("tcp/" + getHost().getHostname() + ":" + getTlsPort()). tlsconfigid(getConfigId()). slobrokconfigid(getClusterConfigId()). routingconfigid(getClusterConfigId()). @@ -305,6 +308,8 @@ public class SearchNode extends AbstractService implements new MetricsmanagerConfig.Consumer.Builder().name("log").tags("logdefault")); } + private String getTlsDir() { return "tls";} + @Override public Optional<String> getPreShutdownCommand() { if (flushOnShutdown) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java b/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java deleted file mode 100644 index 5617fd15cbc..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/TransactionLogServer.java +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.search; - -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.searchlib.TranslogserverConfig; -import com.yahoo.config.model.producer.AnyConfigProducer; -import com.yahoo.config.model.producer.TreeConfigProducer; -import com.yahoo.vespa.model.AbstractService; -import com.yahoo.vespa.model.PortAllocBridge; -import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; -import org.w3c.dom.Element; - -/** - * @author hmusum - */ -public class TransactionLogServer extends AbstractService { - - private final Boolean useFsync; - - public TransactionLogServer(TreeConfigProducer<?> searchNode, String clusterName, Boolean useFsync) { - super(searchNode, "transactionlogserver"); - portsMeta.on(0).tag("tls"); - this.useFsync = useFsync; - setProp("clustername", clusterName); - setProp("clustertype", "search"); - } - - public static class Builder extends VespaDomBuilder.DomConfigProducerBuilderBase<TransactionLogServer> { - - private final String clusterName; - private final Boolean useFsync; - public Builder(String clusterName, Boolean useFsync) { - this.clusterName = clusterName; - this.useFsync = useFsync; - } - - @Override - protected TransactionLogServer doBuild(DeployState deployState, TreeConfigProducer<AnyConfigProducer> ancestor, Element producerSpec) { - return new TransactionLogServer(ancestor, clusterName, useFsync); - } - - } - - public int getPortCount() { - return 1; - } - - @Override - public void allocatePorts(int start, PortAllocBridge from) { - // NB: ignore "start" - from.allocatePort("tls"); - } - - /** - * Returns the port used by the TLS. - * - * @return The port. - */ - int getTlsPort() { - return getRelativePort(0); - } - - /** - * Returns the directory used by the TLS. - * - * @return The directory. - */ - private String getTlsDir() { - return "tls"; - } - - public void getConfig(TranslogserverConfig.Builder builder) { - builder.listenport(getTlsPort()) - .basedir(getTlsDir()); - if (useFsync != null) { - builder.usefsync(useFsync); - } - } - -} diff --git a/config-model/src/main/javacc/SchemaParser.jj b/config-model/src/main/javacc/SchemaParser.jj index 1365c133932..c9eff88764f 100644 --- a/config-model/src/main/javacc/SchemaParser.jj +++ b/config-model/src/main/javacc/SchemaParser.jj @@ -1756,6 +1756,7 @@ void rankProfileItem(ParsedSchema schema, ParsedRankProfile profile) : { } | fieldRankFilter(profile) | firstPhase(profile) | matchPhase(profile) + | diversity(profile) | function(profile) | mutate(profile) | ignoreRankFeatures(profile) @@ -1875,14 +1876,14 @@ void matchPhase(ParsedRankProfile profile) : MatchPhaseSettings settings = new MatchPhaseSettings(); } { - <MATCH_PHASE> lbrace() (matchPhaseItem(settings) (<NL>)*)* <RBRACE> + <MATCH_PHASE> lbrace() (matchPhaseItem(profile, settings) (<NL>)*)* <RBRACE> { settings.checkValid(); - profile.setMatchPhaseSettings(settings); + profile.setMatchPhase(settings); } } -void matchPhaseItem(MatchPhaseSettings settings) : +void matchPhaseItem(ParsedRankProfile profile, MatchPhaseSettings settings) : { String str; int num; @@ -1891,7 +1892,7 @@ void matchPhaseItem(MatchPhaseSettings settings) : } { ( <ATTRIBUTE> <COLON> str = identifier() { settings.setAttribute(str); } - | diversity(settings) + | diversityDeprecated(profile) | <ORDER> <COLON> ( <ASCENDING> { settings.setAscending(true); } | <DESCENDING> { settings.setAscending(false); } ) | <MAX_HITS> <COLON> num = integer() { settings.setMaxHits(num); } @@ -1906,7 +1907,7 @@ void matchPhaseItem(MatchPhaseSettings settings) : * * @param profile The rank profile to modify. */ -void diversity(MatchPhaseSettings profile) : +void diversity(ParsedRankProfile profile) : { DiversitySettings settings = new DiversitySettings(); } @@ -1917,6 +1918,18 @@ void diversity(MatchPhaseSettings profile) : } } +void diversityDeprecated(ParsedRankProfile profile) : +{ + DiversitySettings settings = new DiversitySettings(); +} +{ + <DIVERSITY> lbrace() (diversityItem(settings) (<NL>)*)* <RBRACE> + { + profile.setDiversity(settings); + deployLogger.logApplicationPackage(Level.WARNING, "'diversity is deprecated inside 'match-phase'. Specify it at 'rank-profile' level."); + } +} + void diversityItem(DiversitySettings settings) : { String str; @@ -1981,10 +1994,12 @@ void secondPhaseItem(ParsedRankProfile profile) : { String expression; int rerankCount; + double dropLimit; } { ( expression = expression() { profile.setSecondPhaseRanking(expression); } | (<RERANK_COUNT> <COLON> rerankCount = integer()) { profile.setRerankCount(rerankCount); } + | (<RANK_SCORE_DROP_LIMIT> <COLON> dropLimit = floatValue()) { profile.setSecondPhaseRankScoreDropLimit(dropLimit); } ) } diff --git a/config-model/src/main/resources/schema/deployment.rnc b/config-model/src/main/resources/schema/deployment.rnc index 3491d868f20..f79fc614a53 100644 --- a/config-model/src/main/resources/schema/deployment.rnc +++ b/config-model/src/main/resources/schema/deployment.rnc @@ -2,6 +2,8 @@ # RELAX NG Compact Syntax # Vespa Deployment file +include "common.rnc" + start = element deployment { attribute version { "1.0" } & attribute major-version { text }? & @@ -100,7 +102,7 @@ Test = element test { attribute tester-flavor { xsd:string }? & attribute cloud-account { xsd:string }? & attribute empty-host-ttl { xsd:string }? & - text + Tester? } Staging = element staging { @@ -108,7 +110,7 @@ Staging = element staging { attribute tester-flavor { xsd:string }? & attribute cloud-account { xsd:string }? & attribute empty-host-ttl { xsd:string }? & - text + Tester? } Dev = element dev { @@ -129,7 +131,8 @@ Prod = element prod { Region* & Delay* & ProdTest* & - ParallelSteps* + ParallelSteps* & + Tester? } ProdTest = element test { @@ -197,3 +200,7 @@ MemberRegion = element region { attribute fraction { xsd:double }? & text } + +Tester = element tester { + Nodes? +}
\ No newline at end of file diff --git a/config-model/src/test/cfg/application/ml_serving/services.xml b/config-model/src/test/cfg/application/ml_serving/services.xml index 3a5a4438c78..b1271b1297f 100644 --- a/config-model/src/test/cfg/application/ml_serving/services.xml +++ b/config-model/src/test/cfg/application/ml_serving/services.xml @@ -3,7 +3,13 @@ <services version="1.0"> <container version="1.0"> - <model-evaluation/> + <model-evaluation> + <onnx> + <models> + <model name="sqrt" /> <!-- list one of the models --> + </models> + </onnx> + </model-evaluation> <nodes> <node hostalias="node1" /> </nodes> diff --git a/config-model/src/test/derived/array_of_struct_attribute/index-info.cfg b/config-model/src/test/derived/array_of_struct_attribute/index-info.cfg index 0927045528d..a4fbe6f6c39 100644 --- a/config-model/src/test/derived/array_of_struct_attribute/index-info.cfg +++ b/config-model/src/test/derived/array_of_struct_attribute/index-info.cfg @@ -6,6 +6,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "elem_array.name" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "elem_array.name" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "elem_array.name" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "elem_array.name" indexinfo[].command[].command "fast-search" @@ -16,6 +18,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "elem_array.name" indexinfo[].command[].command "word" indexinfo[].command[].indexname "elem_array.weight" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "elem_array.weight" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "elem_array.weight" indexinfo[].command[].command "numerical" diff --git a/config-model/src/test/derived/exactmatch/index-info.cfg b/config-model/src/test/derived/exactmatch/index-info.cfg index 4a9ba85ad38..73a80d3510e 100644 --- a/config-model/src/test/derived/exactmatch/index-info.cfg +++ b/config-model/src/test/derived/exactmatch/index-info.cfg @@ -22,6 +22,8 @@ indexinfo[].command[].command "exact *!A!!*" indexinfo[].command[].indexname "string_map.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "string_map.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "string_map.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "string_map.key" indexinfo[].command[].command "string" @@ -30,6 +32,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "string_map.key" indexinfo[].command[].command "exact *!B!!*" indexinfo[].command[].indexname "string_map.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "string_map.value" indexinfo[].command[].command "string" indexinfo[].command[].indexname "string_map.value" indexinfo[].command[].command "type string" @@ -38,6 +42,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "string_map" indexinfo[].command[].command "type Map<string,string>" indexinfo[].command[].indexname "elem_map.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "elem_map.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "elem_map.key" indexinfo[].command[].command "type string" @@ -58,6 +64,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "elem_map.value.weight" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "elem_map.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "elem_map.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "elem_map" indexinfo[].command[].command "multivalue" @@ -66,6 +74,8 @@ indexinfo[].command[].command "type Map<string,elem>" indexinfo[].command[].indexname "elem_array.name" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "elem_array.name" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "elem_array.name" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "elem_array.name" indexinfo[].command[].command "string" @@ -74,6 +84,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "elem_array.name" indexinfo[].command[].command "exact @@" indexinfo[].command[].indexname "elem_array.weight" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "elem_array.weight" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "elem_array.weight" indexinfo[].command[].command "integer" @@ -84,6 +96,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "elem_array" indexinfo[].command[].command "type Array<elem>" indexinfo[].command[].indexname "another_map.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "another_map.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "another_map.key" indexinfo[].command[].command "type string" @@ -104,6 +118,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "another_map.value.weight" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "another_map.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "another_map.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "another_map" indexinfo[].command[].command "multivalue" diff --git a/config-model/src/test/derived/imported_struct_fields/index-info.cfg b/config-model/src/test/derived/imported_struct_fields/index-info.cfg index 2b8a6fc344d..7c808c932b2 100644 --- a/config-model/src/test/derived/imported_struct_fields/index-info.cfg +++ b/config-model/src/test/derived/imported_struct_fields/index-info.cfg @@ -12,6 +12,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "my_elem_array.name" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "my_elem_array.name" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "my_elem_array.name" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "my_elem_array.name" indexinfo[].command[].command "fast-search" @@ -22,6 +24,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "my_elem_array.name" indexinfo[].command[].command "word" indexinfo[].command[].indexname "my_elem_array.weight" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "my_elem_array.weight" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "my_elem_array.weight" indexinfo[].command[].command "numerical" @@ -54,10 +58,14 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "my_elem_map.value.weight" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "my_elem_map.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "my_elem_map.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "my_elem_map.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "my_elem_map.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "my_elem_map.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "my_elem_map.key" indexinfo[].command[].command "fast-search" @@ -74,6 +82,8 @@ indexinfo[].command[].command "type Map<string,elem>" indexinfo[].command[].indexname "my_str_int_map.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "my_str_int_map.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "my_str_int_map.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "my_str_int_map.key" indexinfo[].command[].command "fast-search" @@ -84,6 +94,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "my_str_int_map.key" indexinfo[].command[].command "word" indexinfo[].command[].indexname "my_str_int_map.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "my_str_int_map.value" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "my_str_int_map.value" indexinfo[].command[].command "numerical" diff --git a/config-model/src/test/derived/indexschema/index-info.cfg b/config-model/src/test/derived/indexschema/index-info.cfg index 8c2349e37ea..e764fea8d1f 100644 --- a/config-model/src/test/derived/indexschema/index-info.cfg +++ b/config-model/src/test/derived/indexschema/index-info.cfg @@ -260,6 +260,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "f10.text" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "f10.text" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "f10.text" indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "f10.text" indexinfo[].command[].command "normalize" @@ -270,6 +272,8 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "f10.text" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "f10.name" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "f10.name" indexinfo[].command[].command "string" indexinfo[].command[].indexname "f10.name" indexinfo[].command[].command "type string" diff --git a/config-model/src/test/derived/map_attribute/index-info.cfg b/config-model/src/test/derived/map_attribute/index-info.cfg index 64a51db1edd..be9612ba293 100644 --- a/config-model/src/test/derived/map_attribute/index-info.cfg +++ b/config-model/src/test/derived/map_attribute/index-info.cfg @@ -6,6 +6,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "str_map.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "str_map.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "str_map.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "str_map.key" indexinfo[].command[].command "fast-search" @@ -18,6 +20,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "str_map.value" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "str_map.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "str_map.value" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "str_map.value" indexinfo[].command[].command "string" @@ -30,6 +34,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "str_map" indexinfo[].command[].command "type Map<string,string>" indexinfo[].command[].indexname "int_map.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "int_map.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "int_map.key" indexinfo[].command[].command "numerical" @@ -38,6 +44,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "int_map.key" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "int_map.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "int_map.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "int_map.value" indexinfo[].command[].command "integer" diff --git a/config-model/src/test/derived/map_of_struct_attribute/index-info.cfg b/config-model/src/test/derived/map_of_struct_attribute/index-info.cfg index b649fa2aa8a..5f414f14420 100644 --- a/config-model/src/test/derived/map_of_struct_attribute/index-info.cfg +++ b/config-model/src/test/derived/map_of_struct_attribute/index-info.cfg @@ -6,6 +6,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "str_elem_map.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "str_elem_map.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "str_elem_map.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "str_elem_map.key" indexinfo[].command[].command "fast-search" @@ -34,12 +36,16 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "str_elem_map.value.weight" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "str_elem_map.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "str_elem_map.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "str_elem_map" indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "str_elem_map" indexinfo[].command[].command "type Map<string,elem>" indexinfo[].command[].indexname "int_elem_map.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "int_elem_map.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "int_elem_map.key" indexinfo[].command[].command "numerical" @@ -66,6 +72,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "int_elem_map.value.weight" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "int_elem_map.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "int_elem_map.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "int_elem_map" indexinfo[].command[].command "multivalue" diff --git a/config-model/src/test/derived/matchsettings_map_after/index-info.cfg b/config-model/src/test/derived/matchsettings_map_after/index-info.cfg index e91784db387..f96bc5ad99f 100644 --- a/config-model/src/test/derived/matchsettings_map_after/index-info.cfg +++ b/config-model/src/test/derived/matchsettings_map_after/index-info.cfg @@ -4,6 +4,8 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "sddocname" indexinfo[].command[].command "word" indexinfo[].command[].indexname "mse4.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse4.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mse4.key" indexinfo[].command[].command "type string" @@ -24,6 +26,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mse4.value.sf2i" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "mse4.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse4.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "mse4" indexinfo[].command[].command "multivalue" diff --git a/config-model/src/test/derived/matchsettings_map_def/index-info.cfg b/config-model/src/test/derived/matchsettings_map_def/index-info.cfg index 1304354a722..22214e83698 100644 --- a/config-model/src/test/derived/matchsettings_map_def/index-info.cfg +++ b/config-model/src/test/derived/matchsettings_map_def/index-info.cfg @@ -6,6 +6,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "mss3.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "mss3.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mss3.key" indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "mss3.key" indexinfo[].command[].command "normalize" @@ -18,6 +20,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "mss3.value" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "mss3.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mss3.value" indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "mss3.value" indexinfo[].command[].command "normalize" @@ -36,6 +40,8 @@ indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "mss3" indexinfo[].command[].command "type Map<string,string>" indexinfo[].command[].indexname "mse4.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse4.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mse4.key" indexinfo[].command[].command "type string" @@ -52,6 +58,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mse4.value.sf2i" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "mse4.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse4.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "mse4" indexinfo[].command[].command "multivalue" @@ -60,6 +68,8 @@ indexinfo[].command[].command "type Map<string,elem>" indexinfo[].command[].indexname "mse5.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "mse5.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse5.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "mse5.key" indexinfo[].command[].command "fast-search" @@ -88,6 +98,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mse5.value.sf2i" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "mse5.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse5.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "mse5" indexinfo[].command[].command "multivalue" diff --git a/config-model/src/test/derived/matchsettings_map_in_struct/index-info.cfg b/config-model/src/test/derived/matchsettings_map_in_struct/index-info.cfg index 75273abefa1..c0d16de6393 100644 --- a/config-model/src/test/derived/matchsettings_map_in_struct/index-info.cfg +++ b/config-model/src/test/derived/matchsettings_map_in_struct/index-info.cfg @@ -16,6 +16,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "stuff.cf5e1.sf2m.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stuff.cf5e1.sf2m.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stuff.cf5e1.sf2m.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "stuff.cf5e1.sf2m.key" indexinfo[].command[].command "string" @@ -26,6 +28,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "stuff.cf5e1.sf2m.value" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stuff.cf5e1.sf2m.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stuff.cf5e1.sf2m.value" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "stuff.cf5e1.sf2m.value" indexinfo[].command[].command "string" @@ -50,6 +54,8 @@ indexinfo[].command[].command "exact @elem@" indexinfo[].command[].indexname "stuff.cf5e1.sf4m.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stuff.cf5e1.sf4m.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stuff.cf5e1.sf4m.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "stuff.cf5e1.sf4m.key" indexinfo[].command[].command "string" @@ -60,6 +66,8 @@ indexinfo[].command[].command "exact @elem@" indexinfo[].command[].indexname "stuff.cf5e1.sf4m.value" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stuff.cf5e1.sf4m.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stuff.cf5e1.sf4m.value" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "stuff.cf5e1.sf4m.value" indexinfo[].command[].command "string" @@ -88,6 +96,8 @@ indexinfo[].command[].command "exact @combi@" indexinfo[].command[].indexname "stuff.cf6e2.sf2m.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stuff.cf6e2.sf2m.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stuff.cf6e2.sf2m.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "stuff.cf6e2.sf2m.key" indexinfo[].command[].command "string" @@ -98,6 +108,8 @@ indexinfo[].command[].command "exact @combi@" indexinfo[].command[].indexname "stuff.cf6e2.sf2m.value" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stuff.cf6e2.sf2m.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stuff.cf6e2.sf2m.value" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "stuff.cf6e2.sf2m.value" indexinfo[].command[].command "string" @@ -124,6 +136,8 @@ indexinfo[].command[].command "exact @elem@" indexinfo[].command[].indexname "stuff.cf6e2.sf4m.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stuff.cf6e2.sf4m.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stuff.cf6e2.sf4m.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "stuff.cf6e2.sf4m.key" indexinfo[].command[].command "string" @@ -134,6 +148,8 @@ indexinfo[].command[].command "exact @elem@" indexinfo[].command[].indexname "stuff.cf6e2.sf4m.value" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stuff.cf6e2.sf4m.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stuff.cf6e2.sf4m.value" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "stuff.cf6e2.sf4m.value" indexinfo[].command[].command "string" diff --git a/config-model/src/test/derived/matchsettings_map_wfs/index-info.cfg b/config-model/src/test/derived/matchsettings_map_wfs/index-info.cfg index 2ef04a85a4f..51c13848a05 100644 --- a/config-model/src/test/derived/matchsettings_map_wfs/index-info.cfg +++ b/config-model/src/test/derived/matchsettings_map_wfs/index-info.cfg @@ -6,6 +6,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "mss3.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "mss3.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mss3.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mss3.key" indexinfo[].command[].command "type string" @@ -14,6 +16,8 @@ indexinfo[].command[].command "exact @mss3_key@" indexinfo[].command[].indexname "mss3.value" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "mss3.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mss3.value" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mss3.value" indexinfo[].command[].command "type string" @@ -28,6 +32,8 @@ indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "mss3" indexinfo[].command[].command "type Map<string,string>" indexinfo[].command[].indexname "mse4.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse4.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mse4.key" indexinfo[].command[].command "type string" @@ -48,6 +54,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mse4.value.sf2i" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "mse4.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse4.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "mse4" indexinfo[].command[].command "multivalue" @@ -56,6 +64,8 @@ indexinfo[].command[].command "type Map<string,elem>" indexinfo[].command[].indexname "mse5.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "mse5.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse5.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "mse5.key" indexinfo[].command[].command "fast-search" @@ -84,6 +94,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mse5.value.sf2i" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "mse5.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse5.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "mse5" indexinfo[].command[].command "multivalue" diff --git a/config-model/src/test/derived/matchsettings_map_wss/index-info.cfg b/config-model/src/test/derived/matchsettings_map_wss/index-info.cfg index cf6a2fc5992..c4cf9cd2f1e 100644 --- a/config-model/src/test/derived/matchsettings_map_wss/index-info.cfg +++ b/config-model/src/test/derived/matchsettings_map_wss/index-info.cfg @@ -6,6 +6,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "mss3.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "mss3.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mss3.key" indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "mss3.key" indexinfo[].command[].command "normalize" @@ -18,6 +20,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "mss3.value" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "mss3.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mss3.value" indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "mss3.value" indexinfo[].command[].command "normalize" @@ -36,6 +40,8 @@ indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "mss3" indexinfo[].command[].command "type Map<string,string>" indexinfo[].command[].indexname "mse4.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse4.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mse4.key" indexinfo[].command[].command "type string" @@ -54,6 +60,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mse4.value.sf2i" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "mse4.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse4.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "mse4" indexinfo[].command[].command "multivalue" @@ -62,6 +70,8 @@ indexinfo[].command[].command "type Map<string,elem>" indexinfo[].command[].indexname "mse5.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "mse5.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse5.key" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "mse5.key" indexinfo[].command[].command "fast-search" @@ -90,6 +100,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mse5.value.sf2i" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "mse5.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mse5.value" indexinfo[].command[].command "type elem" indexinfo[].command[].indexname "mse5" indexinfo[].command[].command "multivalue" diff --git a/config-model/src/test/derived/position_array/index-info.cfg b/config-model/src/test/derived/position_array/index-info.cfg index 5484964149e..65587820bc1 100644 --- a/config-model/src/test/derived/position_array/index-info.cfg +++ b/config-model/src/test/derived/position_array/index-info.cfg @@ -4,12 +4,16 @@ indexinfo[].command[].command "index" indexinfo[].command[].indexname "sddocname" indexinfo[].command[].command "word" indexinfo[].command[].indexname "pos.x" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "pos.x" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "pos.x" indexinfo[].command[].command "integer" indexinfo[].command[].indexname "pos.x" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "pos.y" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "pos.y" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "pos.y" indexinfo[].command[].command "integer" diff --git a/config-model/src/test/derived/structandfieldset/index-info.cfg b/config-model/src/test/derived/structandfieldset/index-info.cfg index 87a95f5a908..0a423530cfa 100644 --- a/config-model/src/test/derived/structandfieldset/index-info.cfg +++ b/config-model/src/test/derived/structandfieldset/index-info.cfg @@ -16,6 +16,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "people.first_name" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "people.first_name" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "people.first_name" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "people.first_name" indexinfo[].command[].command "string" @@ -26,6 +28,8 @@ indexinfo[].command[].command "word" indexinfo[].command[].indexname "people.last_name" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "people.last_name" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "people.last_name" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "people.last_name" indexinfo[].command[].command "string" diff --git a/config-model/src/test/derived/structanyorder/index-info.cfg b/config-model/src/test/derived/structanyorder/index-info.cfg index 6ea63818572..43dc6312d3e 100644 --- a/config-model/src/test/derived/structanyorder/index-info.cfg +++ b/config-model/src/test/derived/structanyorder/index-info.cfg @@ -180,10 +180,14 @@ indexinfo[].command[].command "type foo" indexinfo[].command[].indexname "structfield" indexinfo[].command[].command "type sct" indexinfo[].command[].indexname "structarrayfield.s1" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "structarrayfield.s1" indexinfo[].command[].command "string" indexinfo[].command[].indexname "structarrayfield.s1" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "structarrayfield.s2" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "structarrayfield.s2" indexinfo[].command[].command "string" indexinfo[].command[].indexname "structarrayfield.s2" indexinfo[].command[].command "type string" @@ -344,6 +348,8 @@ indexinfo[].command[].command "type int" indexinfo[].command[].indexname "structarrayfield.s3.s4" indexinfo[].command[].command "type foo" indexinfo[].command[].indexname "structarrayfield.s3" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "structarrayfield.s3" indexinfo[].command[].command "type sct" indexinfo[].command[].indexname "structarrayfield.s4.s1" indexinfo[].command[].command "numerical" @@ -352,6 +358,8 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "structarrayfield.s4.s1" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "structarrayfield.s4" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "structarrayfield.s4" indexinfo[].command[].command "type foo" indexinfo[].command[].indexname "structarrayfield" indexinfo[].command[].command "multivalue" diff --git a/config-model/src/test/derived/types/index-info.cfg b/config-model/src/test/derived/types/index-info.cfg index cc49e006f98..6b39e4d1924 100644 --- a/config-model/src/test/derived/types/index-info.cfg +++ b/config-model/src/test/derived/types/index-info.cfg @@ -96,10 +96,14 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "structfield" indexinfo[].command[].command "type sct" indexinfo[].command[].indexname "structarrayfield.s1" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "structarrayfield.s1" indexinfo[].command[].command "string" indexinfo[].command[].indexname "structarrayfield.s1" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "structarrayfield.s2" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "structarrayfield.s2" indexinfo[].command[].command "string" indexinfo[].command[].indexname "structarrayfield.s2" indexinfo[].command[].command "type string" @@ -110,6 +114,8 @@ indexinfo[].command[].command "type Array<sct>" indexinfo[].command[].indexname "stringmapfield.key" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stringmapfield.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stringmapfield.key" indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "stringmapfield.key" indexinfo[].command[].command "normalize" @@ -122,6 +128,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "stringmapfield.value" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "stringmapfield.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "stringmapfield.value" indexinfo[].command[].command "stem:BEST" indexinfo[].command[].indexname "stringmapfield.value" indexinfo[].command[].command "normalize" @@ -140,10 +148,14 @@ indexinfo[].command[].command "plain-tokens" indexinfo[].command[].indexname "stringmapfield" indexinfo[].command[].command "type Map<string,string>" indexinfo[].command[].indexname "intmapfield.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "intmapfield.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "intmapfield.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "intmapfield.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "intmapfield.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "intmapfield.value" indexinfo[].command[].command "integer" @@ -154,10 +166,14 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "intmapfield" indexinfo[].command[].command "type Map<string,int>" indexinfo[].command[].indexname "floatmapfield.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "floatmapfield.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "floatmapfield.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "floatmapfield.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "floatmapfield.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "floatmapfield.value" indexinfo[].command[].command "type float" @@ -166,12 +182,16 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "floatmapfield" indexinfo[].command[].command "type Map<string,float>" indexinfo[].command[].indexname "longmapfield.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "longmapfield.key" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "longmapfield.key" indexinfo[].command[].command "integer" indexinfo[].command[].indexname "longmapfield.key" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "longmapfield.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "longmapfield.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "longmapfield.value" indexinfo[].command[].command "integer" @@ -182,12 +202,16 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "longmapfield" indexinfo[].command[].command "type Map<int,long>" indexinfo[].command[].indexname "doublemapfield.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "doublemapfield.key" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "doublemapfield.key" indexinfo[].command[].command "integer" indexinfo[].command[].indexname "doublemapfield.key" indexinfo[].command[].command "type int" indexinfo[].command[].indexname "doublemapfield.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "doublemapfield.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "doublemapfield.value" indexinfo[].command[].command "type double" @@ -196,6 +220,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "doublemapfield" indexinfo[].command[].command "type Map<int,double>" indexinfo[].command[].indexname "arraymapfield.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "arraymapfield.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "arraymapfield.key" indexinfo[].command[].command "type string" @@ -232,10 +258,14 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mystructfield.bytearr" indexinfo[].command[].command "type Array<byte>" indexinfo[].command[].indexname "mystructfield.mymap.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructfield.mymap.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mystructfield.mymap.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "mystructfield.mymap.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructfield.mymap.value" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mystructfield.mymap.value" indexinfo[].command[].command "type string" @@ -254,6 +284,8 @@ indexinfo[].command[].command "type string" indexinfo[].command[].indexname "mystructfield" indexinfo[].command[].command "type mystruct" indexinfo[].command[].indexname "mystructmap.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructmap.key" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "mystructmap.key" indexinfo[].command[].command "integer" @@ -268,10 +300,14 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mystructmap.value.bytearr" indexinfo[].command[].command "type Array<byte>" indexinfo[].command[].indexname "mystructmap.value.mymap.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructmap.value.mymap.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mystructmap.value.mymap.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "mystructmap.value.mymap.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructmap.value.mymap.value" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mystructmap.value.mymap.value" indexinfo[].command[].command "type string" @@ -288,6 +324,8 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "mystructmap.value.structfield" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "mystructmap.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructmap.value" indexinfo[].command[].command "type mystruct" indexinfo[].command[].indexname "mystructmap" indexinfo[].command[].command "multivalue" @@ -302,10 +340,14 @@ indexinfo[].command[].command "integer" indexinfo[].command[].indexname "mystructarr.bytearr" indexinfo[].command[].command "type Array<byte>" indexinfo[].command[].indexname "mystructarr.mymap.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructarr.mymap.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mystructarr.mymap.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "mystructarr.mymap.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructarr.mymap.value" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mystructarr.mymap.value" indexinfo[].command[].command "type string" @@ -314,10 +356,14 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "mystructarr.mymap" indexinfo[].command[].command "type Map<string,string>" indexinfo[].command[].indexname "mystructarr.title" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructarr.title" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mystructarr.title" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "mystructarr.structfield" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "mystructarr.structfield" indexinfo[].command[].command "string" indexinfo[].command[].indexname "mystructarr.structfield" indexinfo[].command[].command "type string" @@ -326,6 +372,8 @@ indexinfo[].command[].command "multivalue" indexinfo[].command[].indexname "mystructarr" indexinfo[].command[].command "type Array<mystruct>" indexinfo[].command[].indexname "Folders.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.key" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.key" indexinfo[].command[].command "integer" @@ -342,10 +390,14 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.Name" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.FlagsCounter.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.FlagsCounter.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.FlagsCounter.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.FlagsCounter.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.FlagsCounter.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.value.FlagsCounter.value" indexinfo[].command[].command "integer" @@ -366,10 +418,14 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.Name" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.FlagsCounter.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.FlagsCounter.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.value.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "integer" @@ -390,10 +446,14 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.Name" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.FlagsCounter.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.FlagsCounter.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "integer" @@ -414,10 +474,14 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.Name" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "integer" @@ -438,10 +502,14 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.Name" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "integer" @@ -462,10 +530,14 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.Name" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "integer" @@ -486,10 +558,14 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.Name" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "integer" @@ -510,10 +586,14 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.Name" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "integer" @@ -534,10 +614,14 @@ indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.Name" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.key" indexinfo[].command[].command "type string" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "numerical" indexinfo[].command[].indexname "Folders.value.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.anotherfolder.FlagsCounter.value" indexinfo[].command[].command "integer" @@ -566,6 +650,8 @@ indexinfo[].command[].command "type folder" indexinfo[].command[].indexname "Folders.value.anotherfolder" indexinfo[].command[].command "type folder" indexinfo[].command[].indexname "Folders.value" +indexinfo[].command[].command "multivalue" +indexinfo[].command[].indexname "Folders.value" indexinfo[].command[].command "type folder" indexinfo[].command[].indexname "Folders" indexinfo[].command[].command "multivalue" diff --git a/config-model/src/test/java/com/yahoo/config/model/deploy/SystemModelTestCase.java b/config-model/src/test/java/com/yahoo/config/model/deploy/SystemModelTestCase.java index f9f74546a00..8a184e414d7 100644 --- a/config-model/src/test/java/com/yahoo/config/model/deploy/SystemModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/config/model/deploy/SystemModelTestCase.java @@ -62,7 +62,7 @@ public class SystemModelTestCase { VespaModel vespaModel = getVespaModelDoNotValidateXml(TESTDIR + "simpleconfig/"); assertNotNull(vespaModel); - assertEquals(4, vespaModel.configModelRepo().asMap().size(), "There are two instances of the simple model + Routing and AdminModel (set up implicitly)"); + assertEquals(5, vespaModel.configModelRepo().asMap().size(), "There are two instances of the simple model + Routing, Logs and AdminModel (set up implicitly)"); assertNotNull(vespaModel.configModelRepo().asMap().get("simple"), "One gets the default name as there is no explicit id"); assertNotNull(vespaModel.configModelRepo().asMap().get("second"), "The other gets the explicit id as name"); @@ -116,9 +116,9 @@ public class SystemModelTestCase { assertEquals(host1, host2); assertEquals(host2, host3); - // all three host aliases are for the same host, so the number of services should be 3 + 8 - // (3 simpleservices and logd, configproxy, config sentinel, admin server config server, slobrok, logserver and metricsproxy) - assertEquals(10, host1.getServices().size()); + // all three host aliases are for the same host, so the number of services should be 3 + 9 + // (3 simpleservices and logd, configproxy, config sentinel, admin server config server, slobrok, logserver, logserver-container and metricsproxy) + assertEquals(11, host1.getServices().size()); assertNotNull(host1.getService("simpleservice")); assertNotNull(host1.getService("simpleservice2")); @@ -145,7 +145,7 @@ public class SystemModelTestCase { assertNotNull(vespaModel); ApplicationConfigProducerRoot root = vespaModel.getVespa(); - assertEquals(5, vespaModel.configModelRepo().asMap().size()); + assertEquals(6, vespaModel.configModelRepo().asMap().size()); assertTrue(vespaModel.configModelRepo().asMap().containsKey("simple")); assertTrue(vespaModel.configModelRepo().asMap().containsKey("api")); assertTrue(root.getConfigIds().contains("simple/simpleservice.0")); @@ -155,6 +155,8 @@ public class SystemModelTestCase { // Verify that configModelRegistry iterates in dependency order Iterator<ConfigModel> i = vespaModel.configModelRepo().iterator(); ConfigModel plugin = i.next(); + assertEquals("logs", plugin.getId()); + plugin = i.next(); assertEquals("admin", plugin.getId()); plugin = i.next(); assertEquals("simple", plugin.getId()); diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 5ead9812b56..737ac4c248c 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -60,7 +60,7 @@ import static com.yahoo.config.provision.NodeResources.DiskSpeed; import static com.yahoo.config.provision.NodeResources.StorageType; import static com.yahoo.vespa.defaults.Defaults.getDefaults; import static com.yahoo.vespa.model.Host.memoryOverheadGb; -import static com.yahoo.vespa.model.search.NodeResourcesTuning.GB; +import static com.yahoo.vespa.model.search.NodeResourcesTuning.GiB; import static com.yahoo.vespa.model.test.utils.ApplicationPackageUtils.generateSchemas; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -151,7 +151,7 @@ public class ModelProvisioningTest { assertEquals("-Xlog:gc", mydisc2.getContainers().get(1).getJvmOptions()); assertEquals("lib/blablamalloc.so", mydisc2.getContainers().get(0).getPreLoad()); assertEquals("lib/blablamalloc.so", mydisc2.getContainers().get(1).getPreLoad()); - assertEquals(45, mydisc2.getMemoryPercentage().get().percentage()); + assertEquals(45, mydisc2.getMemoryPercentage().get().ofContainerAvailable()); assertEquals(Optional.of("-XX:+UseParNewGC"), mydisc2.getJvmGCOptions()); QrStartConfig.Builder qrStartBuilder = new QrStartConfig.Builder(); mydisc2.getConfig(qrStartBuilder); @@ -234,8 +234,8 @@ public class ModelProvisioningTest { assertEquals(2, model.getContentClusters().get("content1").getRootGroup().getNodes().size(), "Nodes in content1"); assertEquals(1, model.getContainerClusters().get("container1").getContainers().size(), "Nodes in container1"); assertEquals(2, model.getContentClusters().get("content").getRootGroup().getNodes().size(), "Nodes in cluster without ID"); - assertEquals(65, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size for container1"); - assertEquals(84, physicalMemoryPercentage(model.getContainerClusters().get("container2")), "Heap size for container2"); + assertEquals(85, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size for container1"); + assertEquals(85, physicalMemoryPercentage(model.getContainerClusters().get("container2")), "Heap size for container2"); assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Type.content, model); assertProvisioned(1, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model); assertProvisioned(2, ClusterSpec.Id.from("content"), ClusterSpec.Type.content, model); @@ -287,8 +287,8 @@ public class ModelProvisioningTest { VespaModel model = tester.createModel(xmlWithNodes, true, deployStateWithClusterEndpoints("container1").deployLogger(logger)); assertEquals(2, model.getContentClusters().get("content1").getRootGroup().getNodes().size(), "Nodes in content1"); assertEquals(2, model.getContainerClusters().get("container1").getContainers().size(), "Nodes in container1"); - assertEquals(18, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size is lowered with combined clusters"); - assertEquals(2025077080L, protonMemorySize(model.getContentClusters().get("content1")), "Memory for proton is lowered to account for the jvm heap"); + assertEquals(24, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size is lowered with combined clusters"); + assertEquals(1876900708, protonMemorySize(model.getContentClusters().get("content1")), "Memory for proton is lowered to account for the jvm heap"); assertProvisioned(0, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model); assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Id.from("container1"), ClusterSpec.Type.combined, model); var msgs = logger.msgs().stream().filter(m -> m.level().equals(Level.WARNING)).toList(); @@ -356,7 +356,7 @@ public class ModelProvisioningTest { VespaModel model = tester.createModel(xmlWithNodes, true, deployStateWithClusterEndpoints("container1")); assertEquals(2, model.getContentClusters().get("content1").getRootGroup().getNodes().size(), "Nodes in content1"); assertEquals(2, model.getContainerClusters().get("container1").getContainers().size(), "Nodes in container1"); - assertEquals(65, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size is normal"); + assertEquals(85, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size is normal"); assertEquals((long) ((3 - memoryOverheadGb) * (Math.pow(1024, 3))), protonMemorySize(model.getContentClusters().get("content1")), "Memory for proton is normal"); } @@ -2580,7 +2580,7 @@ public class ModelProvisioningTest { ProtonConfig cfg = getProtonConfig(model, cluster.getSearchNodes().get(0).getConfigId()); assertEquals(2000, cfg.flush().memory().maxtlssize()); // from config override assertEquals(1000, cfg.flush().memory().maxmemory()); // from explicit tuning - assertEquals((long) ((128 - memoryOverheadGb) * GB * 0.08), cfg.flush().memory().each().maxmemory()); // from default node flavor tuning + assertEquals((long) ((128 - memoryOverheadGb) * GiB * 0.08), cfg.flush().memory().each().maxmemory()); // from default node flavor tuning } private static ProtonConfig getProtonConfig(VespaModel model, String configId) { diff --git a/config-model/src/test/java/com/yahoo/schema/DiversityTestCase.java b/config-model/src/test/java/com/yahoo/schema/DiversityTestCase.java index df71e30c7d9..d9355e2df7d 100644 --- a/config-model/src/test/java/com/yahoo/schema/DiversityTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/DiversityTestCase.java @@ -3,49 +3,58 @@ package com.yahoo.schema; import com.yahoo.search.query.ranking.Diversity; import com.yahoo.schema.parser.ParseException; +import com.yahoo.vespa.model.test.utils.DeployLoggerStub; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; /** * @author baldersheim */ public class DiversityTestCase { - @Test - void testDiversity() throws ParseException { + private static void verifyDiversity(DeployLoggerStub logger, boolean atRankProfile, boolean atMatchPhase) throws ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); + ApplicationBuilder builder = new ApplicationBuilder(logger, rankProfileRegistry); + String diversitySpec = """ + diversity { + attribute: b + min-groups: 74 + cutoff-factor: 17.3 + cutoff-strategy: strict + } + """; builder.addSchema( - "search test {\n" + - " document test { \n" + - " field a type int { \n" + - " indexing: attribute \n" + - " attribute: fast-search\n" + - " }\n" + - " field b type int {\n" + - " indexing: attribute \n" + - " }\n" + - " }\n" + - " \n" + - " rank-profile parent {\n" + - " match-phase {\n" + - " diversity {\n" + - " attribute: b\n" + - " min-groups: 74\n" + - " cutoff-factor: 17.3\n" + - " cutoff-strategy: strict" + - " }\n" + - " attribute: a\n" + - " max-hits: 120\n" + - " max-filter-coverage: 0.065" + - " }\n" + - " }\n" + - "}\n"); + """ + search test { + document test { + field a type int { + indexing: attribute + attribute: fast-search + } + field b type int { + indexing: attribute + } + } + rank-profile parent { + match-phase {""" + + (atMatchPhase ? diversitySpec : "") + + """ + attribute: a + max-hits: 120 + max-filter-coverage: 0.065 + }""" + + (atRankProfile ? diversitySpec : "") + + """ + } + } + """); builder.build(true); Schema s = builder.getSchema(); - RankProfile.MatchPhaseSettings matchPhase = rankProfileRegistry.get(s, "parent").getMatchPhaseSettings(); - RankProfile.DiversitySettings diversity = matchPhase.getDiversity(); + RankProfile parent = rankProfileRegistry.get(s, "parent"); + RankProfile.MatchPhaseSettings matchPhase = parent.getMatchPhase(); + RankProfile.DiversitySettings diversity = parent.getDiversity(); assertEquals("b", diversity.getAttribute()); assertEquals(74, diversity.getMinGroups()); assertEquals(17.3, diversity.getCutoffFactor(), 1e-16); @@ -54,6 +63,44 @@ public class DiversityTestCase { assertEquals("a", matchPhase.getAttribute()); assertEquals(0.065, matchPhase.getMaxFilterCoverage(), 1e-16); } + @Test + void testDiversity() throws ParseException { + DeployLoggerStub logger = new DeployLoggerStub(); + verifyDiversity(logger, true, false); + assertTrue(logger.entries.isEmpty()); + verifyDiversity(logger, false, true); + assertEquals(1, logger.entries.size()); + assertEquals("'diversity is deprecated inside 'match-phase'. Specify it at 'rank-profile' level.", logger.entries.get(0).message); + try { + verifyDiversity(logger, true, true); + fail("Should throw."); + } catch (Exception e) { + assertEquals("rank-profile 'parent' error: already has diversity", e.getMessage()); + } + } + + @Test + void requireMatchPhaseOrSecondPhase() throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(new RankProfileRegistry()); + builder.addSchema(""" + search test { + document test { + field b type int { indexing: attribute } + } + rank-profile parent { + diversity { + attribute: b + min-groups: 74 + } + } + }"""); + try { + builder.build(true); + fail("Should throw."); + } catch (IllegalArgumentException e) { + assertEquals("In schema 'test', rank-profile 'parent': 'diversity' requires either 'match-phase' or 'second-phase' to be specified.", e.getMessage()); + } + } private static String getMessagePrefix() { return "In search definition 'test', rank-profile 'parent': diversity attribute 'b' "; @@ -82,30 +129,29 @@ public class DiversityTestCase { assertEquals(getMessagePrefix() + "must be single value numeric, or enumerated attribute, but it is 'Array<int>'", e.getMessage()); } } - private ApplicationBuilder getSearchBuilder(String diversity) throws ParseException { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - ApplicationBuilder builder = new ApplicationBuilder(rankProfileRegistry); - builder.addSchema( - "search test {\n" + - " document test { \n" + - " field a type int { \n" + - " indexing: attribute \n" + - " attribute: fast-search\n" + - " }\n" + - diversity + - " }\n" + - " \n" + - " rank-profile parent {\n" + - " match-phase {\n" + - " diversity {\n" + - " attribute: b\n" + - " min-groups: 74\n" + - " }\n" + - " attribute: a\n" + - " max-hits: 120\n" + - " }\n" + - " }\n" + - "}\n"); + private ApplicationBuilder getSearchBuilder(String diversityField) throws ParseException { + ApplicationBuilder builder = new ApplicationBuilder(new RankProfileRegistry()); + builder.addSchema(""" + search test { + document test { + field a type int { + indexing: attribute + attribute: fast-search + }""" + + diversityField + + """ + } + rank-profile parent { + match-phase { + attribute: a + max-hits: 120 + } + diversity { + attribute: b + min-groups: 74 + } + } + }"""); return builder; } } diff --git a/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java b/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java index 59887bfd57e..564bfd4a990 100644 --- a/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java @@ -530,4 +530,28 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { "}"); } + @Test + public void secondPhaseRankScoreDropLimitIsAddedToRankProperties() throws ParseException { + RankProfileRegistry registry = new RankProfileRegistry(); + ApplicationBuilder builder = new ApplicationBuilder(registry); + String input = """ + schema test { + document test { + } + rank-profile test inherits default { + second-phase { + rank-score-drop-limit: 17.0 + } + } + } + """; + builder.addSchema(input); + builder.build(true); + Schema schema = builder.getSchema(); + + assertEquals(3, registry.all().size()); + RawRankProfile rawProfile = createRawRankProfile(registry.get(schema, "test"), schema); + assertEquals("17.0", findProperty(rawProfile.configProperties(), "vespa.hitcollector.secondphase.rankscoredroplimit").get()); + } + } diff --git a/config-model/src/test/java/com/yahoo/schema/SchemaImporterTestCase.java b/config-model/src/test/java/com/yahoo/schema/SchemaImporterTestCase.java index 6252fe62a1d..47d3c64d351 100644 --- a/config-model/src/test/java/com/yahoo/schema/SchemaImporterTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/SchemaImporterTestCase.java @@ -28,7 +28,6 @@ import static org.junit.jupiter.api.Assertions.*; public class SchemaImporterTestCase extends AbstractSchemaTestCase { @Test - @SuppressWarnings("deprecation") void testSimpleImporting() throws IOException, ParseException { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); ApplicationBuilder sb = new ApplicationBuilder(rankProfileRegistry, new QueryProfileRegistry()); diff --git a/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java index 8ffbab84fd7..539ec91ee5f 100644 --- a/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java @@ -6,16 +6,13 @@ import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.model.test.utils.DeployLoggerStub; import com.yahoo.vespa.objects.FieldBase; import com.yahoo.yolean.Exceptions; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import static com.yahoo.config.model.test.TestUtil.joinLines; import java.util.Collection; import java.util.List; -import java.util.Optional; import java.util.logging.Level; -import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.*; @@ -233,16 +230,12 @@ public class SummaryTestCase { " document-summary test_summary inherits nonesuch {" + " }" + "}"); - DeployLoggerStub logger = new DeployLoggerStub(); - ApplicationBuilder.createFromStrings(logger, schema); - assertEquals("document-summary 'test_summary' inherits 'nonesuch' but this is not present in schema 'test'", - logger.entries.get(0).message); - // fail("Expected failure"); + ApplicationBuilder.createFromString(schema); + fail("Expected failure"); } catch (IllegalArgumentException e) { - fail(); - // assertEquals("document-summary 'test_summary' inherits nonesuch but this is not present in schema 'test'", - // e.getMessage()); + assertEquals("document-summary 'test_summary' inherits 'nonesuch', but this is not present in schema 'test'", + e.getMessage()); } } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/PagedAttributeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/PagedAttributeValidatorTestCase.java index 409e3b5df22..3f7c87ae37e 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/PagedAttributeValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/PagedAttributeValidatorTestCase.java @@ -2,6 +2,7 @@ package com.yahoo.schema.processing; import com.yahoo.config.model.application.provider.BaseDeployLogger; +import com.yahoo.schema.derived.TestableDeployLogger; import com.yahoo.schema.parser.ParseException; import org.junit.jupiter.api.Test; @@ -90,6 +91,31 @@ public class PagedAttributeValidatorTestCase { assertPagedSupported("reference<parent>", Optional.of(getSd("parent", "int", false))); } + @Test + void hnsw_index_triggers_warning_with_paged_setting() throws ParseException { + var logger = new TestableDeployLogger(); + var sd = """ + schema test { + document test { + field pos type tensor(x[2]) { + indexing: attribute | index + attribute: paged + index { + hnsw { + } + } + } + } + } + """; + var schema = createFromString(sd, logger); + assertEquals(1, logger.warnings.size()); + assertEquals("For schema 'test', field 'pos': " + + "The 'paged' attribute setting in combination with HNSW indexing is strongly discouraged, see " + + "https://docs.vespa.ai/en/attributes.html#paged-attributes-disadvantages for details", + logger.warnings.get(0)); + } + private void assertPagedSettingNotSupported(String fieldType) throws ParseException { assertPagedSettingNotSupported(fieldType, false); } diff --git a/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java index 9eca2106c5e..5ea097f13fb 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java @@ -9,6 +9,7 @@ import org.junit.jupiter.api.Test; import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; public class SummaryConsistencyTestCase { @@ -42,4 +43,24 @@ public class SummaryConsistencyTestCase { Schema schema = ApplicationBuilder.createFromString(sd).getSchema(); assertEquals(SummaryTransform.ATTRIBUTECOMBINER, schema.getSummaryField("elem_array_unfiltered").getTransform()); } + + @Test + void testDocumentSummaryWithInheritanceOfNonExistingSummary() { + String schemaString = """ + schema foo { + document foo { + field foo type string { + indexing: summary + } + } + document-summary foo_summary inherits non-existent { + summary foo { + source: foo + } + } + } + """; + assertThrows(IllegalArgumentException.class, () -> ApplicationBuilder.createFromString(schemaString).getSchema()); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java index adf570c4664..96c6b1f99eb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/AdminTestCase.java @@ -50,8 +50,8 @@ public class AdminTestCase { void testAdmin20() { VespaModel vespaModel = getVespaModel(TESTDIR + "adminconfig20"); - // Verify that the admin plugin has been loaded (always loads routing). - assertEquals(2, vespaModel.configModelRepo().asMap().size()); + // Verify that the admin plugin has been loaded (always loads 'routing' and 'logs') + assertEquals(3, vespaModel.configModelRepo().asMap().size()); ApplicationConfigProducerRoot root = vespaModel.getVespa(); assertNotNull(root); @@ -81,10 +81,10 @@ public class AdminTestCase { StateserverConfig.Builder ssb = new StateserverConfig.Builder(); vespaModel.getConfig(ssb, "admin/slobrok.0"); - assertEquals(19100, new StateserverConfig(ssb).httpport()); + assertEquals(19103, new StateserverConfig(ssb).httpport()); vespaModel.getConfig(ssb, "admin/slobrok.1"); - assertEquals(19102, new StateserverConfig(ssb).httpport()); + assertEquals(19105, new StateserverConfig(ssb).httpport()); LogdConfig.Builder lb = new LogdConfig.Builder(); vespaModel.getConfig(lb, "admin/slobrok.0"); @@ -98,12 +98,13 @@ public class AdminTestCase { SentinelConfig.Builder b = new SentinelConfig.Builder(); vespaModel.getConfig(b, localhostConfigId); SentinelConfig sentinelConfig = new SentinelConfig(b); - assertEquals(5, sentinelConfig.service().size()); + assertEquals(6, sentinelConfig.service().size()); assertEquals("logserver", sentinelConfig.service(0).name()); - assertEquals("slobrok", sentinelConfig.service(1).name()); - assertEquals("slobrok2", sentinelConfig.service(2).name()); - assertEquals(METRICS_PROXY_CONTAINER.serviceName, sentinelConfig.service(3).name()); - assertEquals("logd", sentinelConfig.service(4).name()); + assertEquals("logserver-container", sentinelConfig.service(1).name()); + assertEquals("slobrok", sentinelConfig.service(2).name()); + assertEquals("slobrok2", sentinelConfig.service(3).name()); + assertEquals(METRICS_PROXY_CONTAINER.serviceName, sentinelConfig.service(4).name()); + assertEquals("logd", sentinelConfig.service(5).name()); } /** @@ -114,8 +115,8 @@ public class AdminTestCase { void testOnlyAdminserver() { VespaModel vespaModel = getVespaModel(TESTDIR + "simpleadminconfig20"); - // Verify that the admin plugin has been loaded (always loads routing). - assertEquals(2, vespaModel.configModelRepo().asMap().size()); + // Verify that the admin plugin has been loaded (always loads 'routing' and 'logs') + assertEquals(3, vespaModel.configModelRepo().asMap().size()); ApplicationConfigProducerRoot root = vespaModel.getVespa(); assertNotNull(root); @@ -134,11 +135,12 @@ public class AdminTestCase { SentinelConfig.Builder b = new SentinelConfig.Builder(); vespaModel.getConfig(b, localhostConfigId); SentinelConfig sentinelConfig = new SentinelConfig(b); - assertEquals(4, sentinelConfig.service().size()); + assertEquals(5, sentinelConfig.service().size()); assertEquals("logserver", sentinelConfig.service(0).name()); - assertEquals("slobrok", sentinelConfig.service(1).name()); - assertEquals(METRICS_PROXY_CONTAINER.serviceName, sentinelConfig.service(2).name()); - assertEquals("logd", sentinelConfig.service(3).name()); + assertEquals("logserver-container", sentinelConfig.service(1).name()); + assertEquals("slobrok", sentinelConfig.service(2).name()); + assertEquals(METRICS_PROXY_CONTAINER.serviceName, sentinelConfig.service(3).name()); + assertEquals("logd", sentinelConfig.service(4).name()); assertEquals(-1, sentinelConfig.service(0).affinity().cpuSocket()); assertTrue(sentinelConfig.service(0).preShutdownCommand().isEmpty()); @@ -175,8 +177,8 @@ public class AdminTestCase { void testMultipleConfigServers() { VespaModel vespaModel = getVespaModel(TESTDIR + "multipleconfigservers"); - // Verify that the admin plugin has been loaded (always loads routing). - assertEquals(2, vespaModel.configModelRepo().asMap().size()); + // Verify that the admin plugin has been loaded (always loads 'routing' and 'logs') + assertEquals(3, vespaModel.configModelRepo().asMap().size()); ApplicationConfigProducerRoot root = vespaModel.getVespa(); assertNotNull(root); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java index ac431f081ed..0acacc340eb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainerTest.java @@ -150,7 +150,7 @@ public class MetricsProxyContainerTest { @Test void vespa_services_config_has_all_services() { VespaServicesConfig vespaServicesConfig = getVespaServicesConfig(hostedServicesWithContent()); - assertEquals(10, vespaServicesConfig.service().size()); + assertEquals(11, vespaServicesConfig.service().size()); for (var service : vespaServicesConfig.service()) { if (service.configId().equals("admin/cluster-controllers/0")) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGeneratorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGeneratorTest.java index c24fcb27dc9..efe9cfe5060 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGeneratorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGeneratorTest.java @@ -31,7 +31,7 @@ public class OpenTelemetryConfigGeneratorTest { void testBuildsYaml() { var mockZone = new Zone(SystemName.PublicCd, Environment.prod, RegionName.from("mock")); var app = ApplicationId.from("mytenant", "myapp", "myinstance"); - var generator = new OpenTelemetryConfigGenerator(mockZone, app); + var generator = new OpenTelemetryConfigGenerator(mockZone, app, true); var root = new MockRoot(); var mockHost = new Host(root, "localhost2.local"); @@ -50,12 +50,15 @@ public class OpenTelemetryConfigGeneratorTest { var mockSvc2 = new MockService(root, "searchnode"); mockSvc2.setProp("clustername", "mycluster"); mockSvc2.setProp("clustertype", "mockup"); - var mockPort2 = new StatePortInfo("other.host.local", 19102, mockSvc2); + var mockPort2 = new StatePortInfo("other123x.host.local", 19102, mockSvc2); generator.addStatePorts(List.of(mockPort1, mockPort2)); String yaml = generator.generate(); // System.err.println(">>>\n" + yaml + "\n<<<"); assertTrue(yaml.contains("sentinel")); + String want = """ + "parentHostname":"other123.host.local"""; + assertTrue(yaml.contains(want)); } static class MockService extends AbstractService { @@ -70,4 +73,22 @@ public class OpenTelemetryConfigGeneratorTest { @Override public void allocatePorts(int start, PortAllocBridge from) { } } + @Test + void testFindParentHost() { + String result; + result = OpenTelemetryConfigGenerator.findParentHost("n1234c.foo.bar.some.cloud"); + assertEquals("n1234.foo.bar.some.cloud", result); + result = OpenTelemetryConfigGenerator.findParentHost("n1234-v6-7.foo.bar.some.cloud"); + assertEquals("n1234.foo.bar.some.cloud", result); + result = OpenTelemetryConfigGenerator.findParentHost("2000a.foo.bar.some.cloud"); + assertEquals("2000.foo.bar.some.cloud", result); + result = OpenTelemetryConfigGenerator.findParentHost("2000-v6-10.foo.bar.some.cloud"); + assertEquals("2000.foo.bar.some.cloud", result); + result = OpenTelemetryConfigGenerator.findParentHost("foobar.some.cloud"); + assertNull(result); + result = OpenTelemetryConfigGenerator.findParentHost("foo123bar.some.cloud"); + assertNull(result); + result = OpenTelemetryConfigGenerator.findParentHost("foo123.some.cloud"); + assertNull(result); + } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java index 45125c8eb68..340968f89d1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java @@ -45,10 +45,10 @@ class JvmHeapSizeValidatorTest { @Test void fails_on_too_low_heap_size() throws IOException, SAXException { - var deployState = createDeployState(2.2, 1024L * 1024 * 1024); + var deployState = createDeployState(2.3, 1024L * 1024 * 1024); var model = new VespaModel(new NullConfigModelRegistry(), deployState); ValidationTester.expect(new JvmHeapSizeValidator(), model, deployState, - "Allocated memory to JVM in cluster 'container' is too low (0.50GB < 0.60GB). Estimated cost of ONNX models is 1.00GB."); + "Allocated memory to JVM in cluster 'container' is too low (0.51GB < 0.60GB). Estimated cost of ONNX models is 1.00GB."); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java index f68a1da7dfb..0f7113748dd 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java @@ -57,10 +57,11 @@ public class ConfigValueChangeValidatorTest { createVespaModel(createQrStartConfigSegment(true, 2096)), createVespaModel(createQrStartConfigSegment(false, 2096)) ); - assertEquals(3, changes.size()); + assertEquals(4, changes.size()); assertComponentsEquals(changes, "default/container.0", 0); - assertComponentsEquals(changes, "admin/cluster-controllers/0", 1); - assertComponentsEquals(changes, "admin/metrics/localhost", 2); + assertComponentsEquals(changes, "admin/logs/0", 1); + assertComponentsEquals(changes, "admin/cluster-controllers/0", 2); + assertComponentsEquals(changes, "admin/metrics/localhost", 3); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java index b7e612127c1..36839e72e10 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidatorTest.java @@ -209,7 +209,7 @@ public class ResourcesReductionValidatorTest { String resourcesStr = resources == null ? "" : String.format(" <resources vcpu='%.0f' memory='%.0fG' disk='%.0fG'/>", - resources.vcpu(), resources.memoryGb(), resources.diskGb()); + resources.vcpu(), resources.memoryGiB(), resources.diskGb()); return "<services version='1.0'>" + " <container id='default' version='1.0'>" + " <nodes count='" + nodes + "'>" + @@ -223,7 +223,7 @@ public class ResourcesReductionValidatorTest { String resourcesStr = resources == null ? "" : String.format(" <resources vcpu='%.0f' memory='%.0fG' disk='%.0fG'/>", - resources.vcpu(), resources.memoryGb(), resources.diskGb()); + resources.vcpu(), resources.memoryGiB(), resources.diskGb()); return "<services version='1.0'>" + " <content id='default' version='1.0'>" + " <redundancy>1</redundancy>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java index 4c0786ea879..484a938476d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java @@ -33,7 +33,7 @@ public class RestartOnDeployForOnnxModelChangesValidatorTest { // Must be so large that changing model set or options requires restart (due to using more memory than available), // but not so large that deployment will not work at all with one model - private static final long defaultCost = 723456789; + private static final long defaultCost = 635241309; private static final long defaultHash = 0; @@ -47,8 +47,8 @@ public class RestartOnDeployForOnnxModelChangesValidatorTest { @Test void validate_changed_estimated_cost() { - VespaModel current = createModel(onnxModelCost(70000000, defaultHash)); - VespaModel next = createModel(onnxModelCost(723456789, defaultHash)); + VespaModel current = createModel(onnxModelCost(defaultCost, defaultHash)); + VespaModel next = createModel(onnxModelCost(19 * defaultCost / 20, defaultHash)); List<ConfigChangeAction> result = validateModel(current, next); assertEquals(1, result.size()); assertTrue(result.get(0).validationId().isEmpty()); @@ -58,8 +58,8 @@ public class RestartOnDeployForOnnxModelChangesValidatorTest { @Test void validate_changed_estimated_cost_non_hosted() { boolean hosted = false; - VespaModel current = createModel(onnxModelCost(70000000, defaultHash), hosted); - VespaModel next = createModel(onnxModelCost(723456789, defaultHash), hosted); + VespaModel current = createModel(onnxModelCost(defaultCost, defaultHash), hosted); + VespaModel next = createModel(onnxModelCost(19 * defaultCost / 20, defaultHash), hosted); List<ConfigChangeAction> result = validateModel(current, next, hosted); assertEquals(0, result.size()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 924419daeae..052d32269ef 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -157,7 +157,7 @@ public class ContentBuilderTest extends DomBuilderTest { assertEquals("clu/storage/0", c.getRootGroup().getNodes().get(0).getConfigId()); // Due to reuse. assertEquals(1, c.getRoot().hostSystem().getHosts().size()); HostResource h = c.getRoot().hostSystem().getHost("mockhost"); - String [] expectedServices = {"configserver", "logserver", "logd", "container-clustercontroller", "metricsproxy-container", "slobrok", "configproxy", "config-sentinel", "container", "storagenode", "searchnode", "distributor", "transactionlogserver"}; + String [] expectedServices = {"configserver", "logserver", "logserver-container", "logd", "container-clustercontroller", "metricsproxy-container", "slobrok", "configproxy", "config-sentinel", "container", "storagenode", "searchnode", "distributor"}; assertServices(h, expectedServices); assertEquals("clu/storage/0", h.getService("storagenode").getConfigId()); assertEquals("clu/search/cluster.clu/0", h.getService("searchnode").getConfigId()); @@ -205,7 +205,7 @@ public class ContentBuilderTest extends DomBuilderTest { HostResource h = cluster.getRoot().hostSystem().getHost("mockhost"); String [] expectedServices = { "logd", "configproxy", "config-sentinel", "configserver", "container", "logserver", - "slobrok", "storagenode", "distributor", "searchnode", "transactionlogserver", + "logserver-container", "slobrok", "storagenode", "distributor", "searchnode", CLUSTERCONTROLLER_CONTAINER.serviceName, METRICS_PROXY_CONTAINER.serviceName }; assertServices(h, expectedServices); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java index 0e616661191..feeff452a91 100755 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomAdminV2BuilderTest.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.model.admin.monitoring.Monitoring; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.w3c.dom.Element; + import java.util.ArrayList; import java.util.List; @@ -196,14 +197,21 @@ public class DomAdminV2BuilderTest extends DomBuilderTest { assertEquals(1, admin.getConfigservers().size()); } + @Test + void containerOnLogserver() { + Admin admin = buildAdmin(servicesAdminNoAdminServerOrConfigServer()); + assertTrue(admin.getLogServerContainerCluster().isPresent()); + assertEquals("logs", admin.getLogServerContainerCluster().get().getName()); + } + private Admin buildAdmin(Element xml) { return buildAdmin(xml, false, new ArrayList<>()); } private Admin buildAdmin(Element xml, boolean multitenant, List<ConfigServerSpec> configServerSpecs) { DeployState deployState = DeployState.createTestState(); - final DomAdminV2Builder domAdminBuilder = - new DomAdminV2Builder(ConfigModelContext.ApplicationType.DEFAULT, multitenant, configServerSpecs); + ConfigModelContext context = ConfigModelContext.create(deployState, null, (m) -> {}, root, "foo"); + DomAdminV2Builder domAdminBuilder = new DomAdminV2Builder(context, multitenant, configServerSpecs); Admin admin = domAdminBuilder.build(deployState, root, xml); admin.addPerHostServices(root.hostSystem().getHosts(), deployState); return admin; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecificationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecificationTest.java index 344471cada0..e0193d10e23 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecificationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecificationTest.java @@ -42,8 +42,8 @@ public class NodesSpecificationTest { assertEquals(2, spec.minResources().nodeResources().vcpu(), 1e-9); assertEquals(2, spec.maxResources().nodeResources().vcpu(), 1e-9); - assertEquals(3e15, spec.minResources().nodeResources().memoryGb(), 1e-9); - assertEquals(3e15, spec.maxResources().nodeResources().memoryGb(), 1e-9); + assertEquals(3e15, spec.minResources().nodeResources().memoryGiB(), 1e-9); + assertEquals(3e15, spec.maxResources().nodeResources().memoryGiB(), 1e-9); assertEquals(4e3, spec.minResources().nodeResources().diskGb(), 1e-9); assertEquals(4e3, spec.maxResources().nodeResources().diskGb(), 1e-9); @@ -54,8 +54,8 @@ public class NodesSpecificationTest { assertEquals(1 << 30, spec.minResources().nodeResources().gpuResources().count()); assertEquals(1 << 30, spec.maxResources().nodeResources().gpuResources().count()); - assertEquals(3e-9, spec.minResources().nodeResources().gpuResources().memoryGb(), 1e-12); - assertEquals(3e-9, spec.maxResources().nodeResources().gpuResources().memoryGb(), 1e-12); + assertEquals(3e-9, spec.minResources().nodeResources().gpuResources().memoryGiB(), 1e-12); + assertEquals(3e-9, spec.maxResources().nodeResources().gpuResources().memoryGiB(), 1e-12); assertEquals(DiskSpeed.fast, spec.minResources().nodeResources().diskSpeed()); assertEquals(DiskSpeed.fast, spec.maxResources().nodeResources().diskSpeed()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 9cf34b55ebf..93362e144a6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -49,6 +49,7 @@ import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.zone; import static com.yahoo.config.provision.SystemName.main; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -498,4 +499,28 @@ public class ContainerClusterTest { (extra.isEmpty() ? "" : " " + extra); } + @Test + void testTruncationTo4Bits() { + assertEquals(0, ApplicationContainerCluster.truncateTo4SignificantBits(0)); + assertEquals(1, ApplicationContainerCluster.truncateTo4SignificantBits(1)); + assertEquals(15, ApplicationContainerCluster.truncateTo4SignificantBits(15)); + assertEquals(16, ApplicationContainerCluster.truncateTo4SignificantBits(16)); + assertEquals(16, ApplicationContainerCluster.truncateTo4SignificantBits(17)); + assertEquals(18, ApplicationContainerCluster.truncateTo4SignificantBits(18)); + assertEquals(18, ApplicationContainerCluster.truncateTo4SignificantBits(19)); + assertEquals(30, ApplicationContainerCluster.truncateTo4SignificantBits(30)); + assertEquals(30, ApplicationContainerCluster.truncateTo4SignificantBits(31)); + assertEquals(32, ApplicationContainerCluster.truncateTo4SignificantBits(32)); + assertEquals(32, ApplicationContainerCluster.truncateTo4SignificantBits(33)); + assertEquals(32, ApplicationContainerCluster.truncateTo4SignificantBits(34)); + assertEquals(32, ApplicationContainerCluster.truncateTo4SignificantBits(35)); + assertEquals(36, ApplicationContainerCluster.truncateTo4SignificantBits(36)); + assertEquals(0x78000000, ApplicationContainerCluster.truncateTo4SignificantBits(0x78000000)); + assertEquals(0x78000000, ApplicationContainerCluster.truncateTo4SignificantBits(0x7fffffff)); + assertEquals(0x80000000, ApplicationContainerCluster.truncateTo4SignificantBits(0x80000000)); + assertEquals(0b10001000000000000000000000000000, ApplicationContainerCluster.truncateTo4SignificantBits(0b10000000000000000000000000000001)); + assertEquals(0b11000100000000000000000000000000, ApplicationContainerCluster.truncateTo4SignificantBits(0b11000000000000000000000000000001)); + assertEquals(0b11111111111111111111111111100010, ApplicationContainerCluster.truncateTo4SignificantBits(0b11111111111111111111111111100001)); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java index 899e92d1111..6d8a8b1b9a0 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/configserver/TestOptions.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.model.container.configserver; import com.yahoo.vespa.model.container.configserver.option.CloudConfigOptions; -import java.util.Objects; import java.util.Optional; /** @@ -18,7 +17,7 @@ public class TestOptions implements CloudConfigOptions { private Optional<String> region = Optional.empty(); private Optional<Boolean> useVespaVersionInRequest = Optional.empty(); private Optional<Boolean> hostedVespa = Optional.empty(); - private String zooKeeperSnapshotMethod = "gz"; + private static final String zooKeeperSnapshotMethod = "gz"; @Override public Optional<Integer> rpcPort() { @@ -111,6 +110,9 @@ public class TestOptions implements CloudConfigOptions { @Override public String zooKeeperSnapshotMethod() { return zooKeeperSnapshotMethod; } + @Override + public Integer zookeeperJuteMaxBuffer() { return 12345; } + public TestOptions configServers(ConfigServer[] configServers) { this.configServers = configServers; return this; @@ -136,10 +138,4 @@ public class TestOptions implements CloudConfigOptions { return this; } - public TestOptions zooKeeperSnapshotMethod(String snapshotMethod) { - Objects.requireNonNull(snapshotMethod); - this.zooKeeperSnapshotMethod = snapshotMethod; - return this; - } - } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java index ea43f5c8124..b782366655f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java @@ -142,7 +142,7 @@ public class SchemaChainsTest extends SchemaChainsTestBase { assertTrue(chain.phases().isEmpty()); assertEquals(1, chain.inherits().size()); assertEquals("native", chain.inherits(0)); - assertEquals(10, chain.components().size()); + assertEquals(11, chain.components().size()); assertEquals("com.yahoo.prelude.querytransform.PhrasingSearcher@vespa", chain.components(0)); assertEquals("com.yahoo.prelude.searcher.FieldCollapsingSearcher@vespa", chain.components(1)); assertEquals("com.yahoo.search.yql.MinimalQueryInserter@vespa", chain.components(2)); @@ -153,13 +153,14 @@ public class SchemaChainsTest extends SchemaChainsTestBase { assertEquals("com.yahoo.prelude.semantics.SemanticSearcher@vespa", chain.components(7)); assertEquals("com.yahoo.search.grouping.GroupingQueryParser@vespa", chain.components(8)); assertEquals("com.yahoo.search.querytransform.WeakAndReplacementSearcher@vespa", chain.components(9)); + assertEquals("com.yahoo.search.searchers.OpportunisticWeakAndSearcher@vespa", chain.components(10)); assertTrue(chain.excludes().isEmpty()); assertEquals(ChainsConfig.Chains.Type.SEARCH, chain.type()); } @Test public void require_all_default_chains_are_correct() { - assertEquals(63, chainsConfig.components().size()); + assertEquals(64, chainsConfig.components().size()); assertEquals(10, chainsConfig.chains().size()); validateVespaPhasesChain(findChain("vespaPhases")); validateNativeChain(findChain("native")); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 0e8ce4748b4..60ea37cad3f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -17,6 +17,7 @@ import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.provision.SingleNodeProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.MockRoot; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; @@ -49,6 +50,7 @@ import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; import com.yahoo.vespa.model.test.VespaModelTester; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; +import com.yahoo.yolean.Exceptions; import org.junit.jupiter.api.Test; import org.w3c.dom.Element; import org.xml.sax.SAXException; @@ -741,6 +743,29 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test + void testerContainer() { + createModelWithTesterNodes("<nodes count='1' docker-image='foo/bar/baz'><resources vcpu='0.1' memory='1Gb' disk='1Gb'/></nodes>"); + + assertEquals("In container cluster 'default': tester cannot run on more than 1 node, but 2 nodes were specified", + Exceptions.toMessageString(assertThrows(IllegalArgumentException.class, + () -> createModelWithTesterNodes("<nodes count='2'/>")))); + + assertEquals("In container cluster 'default': tester resources must be absolute, but min and max resources differ: specification of dedicated " + + "min 1 nodes with [vcpu: 0.0, memory: 1.0 Gb, disk: 0.0 Gb, bandwidth: 0.3 Gbps, architecture: any] " + + "max 1 nodes with [vcpu: 0.0, memory: 2.0 Gb, disk: 0.0 Gb, bandwidth: 0.3 Gbps, architecture: any]", + Exceptions.toMessageString(assertThrows(IllegalArgumentException.class, + () -> createModelWithTesterNodes("<nodes><resources memory='[1Gb, 2Gb]'/></nodes>")))); + } + + void createModelWithTesterNodes(String testerNodesXml) { + String containerXml = "<container id='default' version='1.0'>%s</container>".formatted(testerNodesXml); + VespaModelTester tester = new VespaModelTester(); + tester.setApplicationId("t", "a", "i-t"); + tester.addHosts(3); + tester.createModel(containerXml, true); + } + + @Test void cluster_with_zookeeper() { Function<Integer, String> servicesXml = (nodeCount) -> "<container version='1.0' id='default'>" + "<nodes count='" + nodeCount + "'/>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java index 64afd444989..46097da434e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.model.container.xml; -import com.yahoo.cloud.config.SentinelConfig; import com.yahoo.collections.Pair; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.NullConfigModelRegistry; @@ -206,29 +205,6 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { assertEquals("-XX:+UseParNewGC", qrStartConfig.jvm().gcopts()); } - @Test - void verify_jvm_option_and_value_from_feature_flag_are_both_included() throws IOException, SAXException { - String servicesXml = """ - <container version='1.0'> - <search/> - <nodes> - <jvm options="-Xms1024m -Xmx2048m" /> - <node hostalias="node1" /> - </nodes> - </container> - """; - ApplicationPackage applicationPackage = new MockApplicationPackage.Builder().withServices(servicesXml).build(); - // Need to create VespaModel to make deploy properties have effect - VespaModel model = new VespaModel(new NullConfigModelRegistry(), new DeployState.Builder() - .applicationPackage(applicationPackage) - .properties(new TestProperties().setJvmOmitStackTraceInFastThrowOption("-XX:-OmitStackTraceInFastThrow")) - .build()); - SentinelConfig.Builder builder = new SentinelConfig.Builder(); - model.getConfig(builder, "hosts/localhost"); - SentinelConfig config = builder.build(); - assertEquals("PRELOAD=/opt/vespa/lib64/vespa/malloc/libvespamalloc.so exec ${VESPA_HOME}/libexec/vespa/vespa-wrapper vespa-start-container-daemon -Xms1024m -Xmx2048m -XX:-OmitStackTraceInFastThrow ", config.service().get(0).command()); - } - private void verifyLoggingOfJvmGcOptions(boolean isHosted, String override, String... invalidOptions) throws IOException, SAXException { verifyLogMessage(isHosted, "gc-options", override, invalidOptions); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 786caa4b317..a890f35caa8 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.ModelContext; @@ -48,6 +49,8 @@ import java.util.Map; import java.util.Optional; import java.util.OptionalInt; import java.util.Set; +import java.util.function.Consumer; +import java.util.logging.Level; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -941,8 +944,19 @@ public class ContentClusterTest extends ContentBaseTest { } private static ContentCluster createOneNodeCluster(String clusterXml, TestProperties props, Optional<Flavor> flavor) throws Exception { + return createOneNodeCluster(clusterXml, props, flavor, new StringBuffer()); + } + + private static ContentCluster createOneNodeCluster(String clusterXml, TestProperties props, + Optional<Flavor> flavor, StringBuffer deployWarningsBuffer) throws Exception { + DeployLogger logger = (level, message) -> { + if (level == Level.WARNING) { // only care about warnings + deployWarningsBuffer.append("%s\n".formatted(message)); + } + }; DeployState.Builder deployStateBuilder = new DeployState.Builder() - .properties(props); + .properties(props) + .deployLogger(logger); MockRoot root = flavor.isPresent() ? ContentClusterUtils.createMockRoot(new SingleNodeProvisioner(flavor.get()), List.of(), deployStateBuilder) : @@ -1264,7 +1278,7 @@ public class ContentClusterTest extends ContentBaseTest { void verify_max_tls_size() throws Exception { var flavor = new Flavor(new FlavorsConfig.Flavor(new FlavorsConfig.Flavor.Builder().name("test").minDiskAvailableGb(100))); assertEquals(21474836480L, resolveMaxTLSSize(Optional.empty())); - assertEquals(2147483648L, resolveMaxTLSSize(Optional.of(flavor))); + assertEquals(2_000_000_000, resolveMaxTLSSize(Optional.of(flavor))); } void assertZookeeperServerImplementation(String expectedClassName, @@ -1471,13 +1485,18 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(expectedGroupsAllowedDown, config.max_number_of_groups_allowed_to_be_down()); } - private boolean resolveDistributorOperationCancellationConfig(Integer featureLevel) throws Exception { + private StorDistributormanagerConfig resolveDistributorConfig(Consumer<TestProperties> propertyMutator) throws Exception { var properties = new TestProperties(); - if (featureLevel != null) { - properties.setContentLayerMetadataFeatureLevel(featureLevel); - } - var cfg = resolveStorDistributormanagerConfig(properties); - return cfg.enable_operation_cancellation(); + propertyMutator.accept(properties); + return resolveStorDistributormanagerConfig(properties); + } + + private boolean resolveDistributorOperationCancellationConfig(Integer featureLevel) throws Exception { + return resolveDistributorConfig((props) -> { + if (featureLevel != null) { + props.setContentLayerMetadataFeatureLevel(featureLevel); + } + }).enable_operation_cancellation(); } @Test @@ -1488,6 +1507,21 @@ public class ContentClusterTest extends ContentBaseTest { assertTrue(resolveDistributorOperationCancellationConfig(2)); } + private boolean resolveDistributorSymmetricReplicaSelectionConfig(Boolean flagValue) throws Exception { + return resolveDistributorConfig((props) -> { + if (flagValue != null) { + props.setSymmetricPutAndActivateReplicaSelection(flagValue); + } + }).symmetric_put_and_activate_replica_selection(); + } + + @Test + void distributor_symmetric_replica_selection_config_controlled_by_properties() throws Exception { + assertFalse(resolveDistributorSymmetricReplicaSelectionConfig(null)); // defaults to false + assertFalse(resolveDistributorSymmetricReplicaSelectionConfig(false)); + assertTrue(resolveDistributorSymmetricReplicaSelectionConfig(true)); + } + @Test void node_distribution_key_outside_legal_range_is_disallowed() { // Only [0, UINT16_MAX - 1] is a valid range. UINT16_MAX is a special content layer-internal @@ -1506,6 +1540,68 @@ public class ContentClusterTest extends ContentBaseTest { } } + private String createClusterAndGetDeploymentWarnings(String xml) { + var warningBuf = new StringBuffer(); + try { + createOneNodeCluster(xml, new TestProperties(), Optional.empty(), warningBuf); + } catch (Exception e) { + throw new RuntimeException(e); + } + return warningBuf.toString(); + }; + + private static String clusterXmlWithNodeDistributionKey(int key) { + return "<content version=\"1.0\" id=\"mockcluster\">" + + " <redundancy>1</redundancy>" + + " <documents/>" + + " <group>" + + " <node distribution-key=\"%d\" hostalias=\"mockhost\"/>".formatted(key) + + " </group>" + + "</content>"; + } + + @Test + void distribution_key_much_higher_than_node_count_logs_deployment_warning() { + // "much higher" is somewhat arbitrary, but needs to be kept in track with threshold in `ContentCluster`. + String warnings = createClusterAndGetDeploymentWarnings(clusterXmlWithNodeDistributionKey(101)); + assertEquals(warnings, "Content cluster 'mockcluster' has 1 node(s), but the highest distribution " + + "key is 101. Having much higher distribution keys than the number of nodes " + + "is not recommended, as it may negatively affect performance. " + + "See https://docs.vespa.ai/en/reference/services-content.html#node\n"); + } + + @Test + void distribution_key_not_much_higher_than_node_count_does_not_log_deployment_warning() { + String warnings = createClusterAndGetDeploymentWarnings(clusterXmlWithNodeDistributionKey(100)); + assertEquals(warnings, ""); + } + + private void checkStrictlyIncreasingClusterStateVersionConfig(Boolean flagValue, boolean expected) throws Exception { + var props = new TestProperties(); + if (flagValue != null) { + props.setEnforceStrictlyIncreasingClusterStateVersions(flagValue); + } + var cc = createOneNodeCluster(props); + + // stor-server config should be the same for both distributors and storage nodes + var builder = new StorServerConfig.Builder(); + cc.getStorageCluster().getConfig(builder); + var cfg = builder.build(); + assertEquals(expected, cfg.require_strictly_increasing_cluster_state_versions()); + + builder = new StorServerConfig.Builder(); + cc.getDistributorNodes().getConfig(builder); + cfg = builder.build(); + assertEquals(expected, cfg.require_strictly_increasing_cluster_state_versions()); + } + + @Test + void strictly_increasing_cluster_state_versions_config_controlled_by_feature_flag() throws Exception { + checkStrictlyIncreasingClusterStateVersionConfig(null, false); // TODO change default + checkStrictlyIncreasingClusterStateVersionConfig(false, false); + checkStrictlyIncreasingClusterStateVersionConfig(true, true); + } + private String servicesWithGroups(int groupCount, double minGroupUpRatio) { String services = String.format("<?xml version='1.0' encoding='UTF-8' ?>" + "<services version='1.0'>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java index e9e96d8b0cf..920b2f7a868 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/IndexedTest.java @@ -131,7 +131,7 @@ public class IndexedTest extends ContentBaseTest { // String [] expectedServices = {"logserver", "configserver", "adminserver", "slobrok", // "logd", "configproxy","config-sentinel", // "container", "fleetcontroller", - // "storagenode", "searchnode", "distributor", "transactionlogserver"}; + // "storagenode", "searchnode", "distributor"}; // DomContentBuilderTest.assertServices(h, expectedServices); Routing routing = model.getRouting(); assertNotNull(routing); @@ -159,8 +159,7 @@ public class IndexedTest extends ContentBaseTest { // HostResource h = model.getHostSystem().getHosts().get(0); // String [] expectedServices = {"logserver", "configserver", "adminserver", "slobrok", // "logd", "configproxy","config-sentinel", - // "container", "storagenode", "searchnode", "distributor", - // "transactionlogserver"}; + // "container", "storagenode", "searchnode", "distributor"}; // DomContentBuilderTest.assertServices(h, expectedServices); ContentCluster s = model.getContentClusters().get("test"); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java index 8b83c941631..2a81f2da286 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/cluster/ClusterTest.java @@ -111,9 +111,9 @@ public class ClusterTest { assertEquals(1, nodesConfig.node(1).key()); assertEquals(2, nodesConfig.node(2).key()); - assertEquals(19106, nodesConfig.node(0).port()); - assertEquals(19118, nodesConfig.node(1).port()); - assertEquals(19130, nodesConfig.node(2).port()); + assertEquals(19109, nodesConfig.node(0).port()); + assertEquals(19121, nodesConfig.node(1).port()); + assertEquals(19133, nodesConfig.node(2).port()); assertEquals(0, nodesConfig.node(0).group()); assertEquals(0, nodesConfig.node(1).group()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java index 4fd61f59ed7..6114cc7d8bf 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java @@ -8,9 +8,14 @@ import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter; import ai.vespa.rankingexpression.importer.vespa.VespaImporter; import ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter; import com.yahoo.config.FileReference; +import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.model.ApplicationPackageTester; import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ContainerEndpoint; +import com.yahoo.config.model.api.OnnxModelCost; +import com.yahoo.config.model.api.OnnxModelCost.Calculator; +import com.yahoo.config.model.api.OnnxModelCost.ModelInfo; +import com.yahoo.config.model.api.OnnxModelOptions; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.io.IOUtils; @@ -22,7 +27,10 @@ import org.xml.sax.SAXException; import java.io.IOException; import java.io.UncheckedIOException; +import java.net.URI; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; @@ -47,6 +55,7 @@ public class ImportedModelTester { private final String modelName; private final Path applicationDir; private final DeployState deployState; + public final Calculator calculator = new MockCalculator(); public ImportedModelTester(String modelName, Path applicationDir) { this(modelName, applicationDir, new DeployState.Builder()); @@ -58,6 +67,7 @@ public class ImportedModelTester { deployState = deployStateBuilder.applicationPackage(ApplicationPackageTester.create(applicationDir.toString()).app()) .endpoints(Set.of(new ContainerEndpoint("container", ApplicationClusterEndpoint.Scope.zone, List.of("default.example.com")))) .modelImporters(importers) + .onnxModelCost((pkg, app, cluster) -> calculator) .build(); } @@ -98,4 +108,19 @@ public class ImportedModelTester { } } + public static class MockCalculator implements OnnxModelCost.Calculator { + private final Map<String, ModelInfo> models = new HashMap<>(); + @Override public long aggregatedModelCostInBytes() { return models.size(); } + @Override public void registerModel(ApplicationFile path, OnnxModelOptions onnxModelOptions) { + models.put(path.toString(), new ModelInfo(path.toString(), 1, 1, onnxModelOptions)); + } + @Override public void registerModel(URI uri, OnnxModelOptions onnxModelOptions) { + models.put(uri.toString(), new ModelInfo(uri.toString(), 1, 1, onnxModelOptions)); + } + @Override public Map<String, ModelInfo> models() { return Map.copyOf(models); } + @Override public void setRestartOnDeploy() { } + @Override public boolean restartOnDeploy() { return false; } + @Override public void store() { } + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java index cc33c8561fc..bced4c546c6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java @@ -55,6 +55,7 @@ public class ModelEvaluationTest { RankProfilesConfig config = new RankProfilesConfig(b); assertEquals(0, config.rankprofile().size()); + assertEquals(0, tester.calculator.aggregatedModelCostInBytes()); } finally { IOUtils.recursiveDeleteDir(appDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); @@ -69,6 +70,7 @@ public class ModelEvaluationTest { try { ImportedModelTester tester = new ImportedModelTester("ml_serving", appDir); assertHasMlModels(tester.createVespaModel(), appDir); + assertEquals(3, tester.calculator.aggregatedModelCostInBytes()); // At this point the expression is stored - copy application to another location which do not have a models dir storedAppDir.toFile().mkdirs(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java index fbc4f6768a2..998437b64f5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java @@ -9,7 +9,8 @@ import org.junit.jupiter.api.Test; import static com.yahoo.vespa.model.Host.memoryOverheadGb; import static org.junit.jupiter.api.Assertions.assertEquals; -import static com.yahoo.vespa.model.search.NodeResourcesTuning.MB; +import static com.yahoo.vespa.model.search.NodeResourcesTuning.MiB; +import static com.yahoo.vespa.model.search.NodeResourcesTuning.GiB; import static com.yahoo.vespa.model.search.NodeResourcesTuning.GB; /** @@ -28,7 +29,7 @@ public class NodeResourcesTuningTest { @Test void require_that_hwinfo_memory_size_is_set() { - assertEquals(24 * GB, configFromMemorySetting(24 + memoryOverheadGb, 0).hwinfo().memory().size()); + assertEquals(24 * GiB, configFromMemorySetting(24 + memoryOverheadGb, 0).hwinfo().memory().size()); assertEquals(1.9585050869E10, configFromMemorySetting(24 + memoryOverheadGb, ApplicationContainerCluster.heapSizePercentageOfTotalAvailableMemoryWhenCombinedCluster * 0.01).hwinfo().memory().size(), 1000); } @@ -84,26 +85,26 @@ public class NodeResourcesTuningTest { @Test void require_that_document_store_maxfilesize_is_set_based_on_available_memory() { - assertDocumentStoreMaxFileSize(256 * MB, 4); - assertDocumentStoreMaxFileSize(256 * MB, 6); - assertDocumentStoreMaxFileSize(256 * MB, 8); - assertDocumentStoreMaxFileSize(256 * MB, 12); - assertDocumentStoreMaxFileSize((long) (16 * GB * 0.02), 16); - assertDocumentStoreMaxFileSize((long) (24 * GB * 0.02), 24); - assertDocumentStoreMaxFileSize((long) (32 * GB * 0.02), 32); - assertDocumentStoreMaxFileSize((long) (48 * GB * 0.02), 48); - assertDocumentStoreMaxFileSize((long) (64 * GB * 0.02), 64); - assertDocumentStoreMaxFileSize((long) (128 * GB * 0.02), 128); - assertDocumentStoreMaxFileSize((long) (256 * GB * 0.02), 256); - assertDocumentStoreMaxFileSize((long) (512 * GB * 0.02), 512); + assertDocumentStoreMaxFileSize(256 * MiB, 4); + assertDocumentStoreMaxFileSize(256 * MiB, 6); + assertDocumentStoreMaxFileSize(256 * MiB, 8); + assertDocumentStoreMaxFileSize(256 * MiB, 12); + assertDocumentStoreMaxFileSize((long) (16 * GiB * 0.02), 16); + assertDocumentStoreMaxFileSize((long) (24 * GiB * 0.02), 24); + assertDocumentStoreMaxFileSize((long) (32 * GiB * 0.02), 32); + assertDocumentStoreMaxFileSize((long) (48 * GiB * 0.02), 48); + assertDocumentStoreMaxFileSize((long) (64 * GiB * 0.02), 64); + assertDocumentStoreMaxFileSize((long) (128 * GiB * 0.02), 128); + assertDocumentStoreMaxFileSize((long) (256 * GiB * 0.02), 256); + assertDocumentStoreMaxFileSize((long) (512 * GiB * 0.02), 512); } @Test void require_that_flush_strategy_memory_limits_are_set_based_on_available_memory() { - assertFlushStrategyMemory((long) (4 * GB * DEFAULT_MEMORY_GAIN), 4); - assertFlushStrategyMemory((long) (8 * GB * DEFAULT_MEMORY_GAIN), 8); - assertFlushStrategyMemory((long) (24 * GB * DEFAULT_MEMORY_GAIN), 24); - assertFlushStrategyMemory((long) (64 * GB * DEFAULT_MEMORY_GAIN), 64); + assertFlushStrategyMemory((long) (4 * GiB * DEFAULT_MEMORY_GAIN), 4); + assertFlushStrategyMemory((long) (8 * GiB * DEFAULT_MEMORY_GAIN), 8); + assertFlushStrategyMemory((long) (24 * GiB * DEFAULT_MEMORY_GAIN), 24); + assertFlushStrategyMemory((long) (64 * GiB * DEFAULT_MEMORY_GAIN), 64); } @Test @@ -129,8 +130,8 @@ public class NodeResourcesTuningTest { @Test void require_that_summary_cache_max_bytes_is_set_based_on_memory() { - assertEquals(1 * GB / 25, configFromMemorySetting(1 + memoryOverheadGb, 0).summary().cache().maxbytes()); - assertEquals(256 * GB / 25, configFromMemorySetting(256 + memoryOverheadGb, 0).summary().cache().maxbytes()); + assertEquals(1 * GiB / 25, configFromMemorySetting(1 + memoryOverheadGb, 0).summary().cache().maxbytes()); + assertEquals(256 * GiB / 25, configFromMemorySetting(256 + memoryOverheadGb, 0).summary().cache().maxbytes()); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java index 672bba83e87..7a9c6aaa99c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java @@ -101,18 +101,18 @@ public class SchemaInfoTestCase { void testFieldWithAttributeExporting() { var schemaInfoTester = new SchemaInfoTester(); var schema = schemaInfoTester.createSchema("test"); - var field = (SDField)schema.getDocument().addField(new SDField("f1", DataType.STRING)); + var field = (SDField)schema.getDocument().addField(new SDField("f1", DataType.INT)); var attribute = field.addAttribute(new Attribute(schema.getName(), field.getName(), "f1Attribute", field.getDataType())); attribute.getAliases().add("a1"); attribute.getAliases().add("a2"); assertEquals(""" schema[0].name "test" schema[0].field[0].name "f1" - schema[0].field[0].type "string" + schema[0].field[0].type "int" schema[0].field[0].attribute false schema[0].field[0].index false schema[0].field[1].name "f1Attribute" - schema[0].field[1].type "string" + schema[0].field[1].type "int" schema[0].field[1].alias[0] "a1" schema[0].field[1].alias[1] "a2" schema[0].field[1].attribute true diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java index bc981c3de7c..499df4e5669 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java @@ -12,7 +12,6 @@ import com.yahoo.vespa.model.Host; import com.yahoo.vespa.model.HostResource; import com.yahoo.vespa.model.search.NodeSpec; import com.yahoo.vespa.model.search.SearchNode; -import com.yahoo.vespa.model.search.TransactionLogServer; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.*; @@ -31,13 +30,8 @@ public class SearchNodeTest { assertEquals(expected, cfg.basedir()); } - private void prepare(MockRoot root, SearchNode node, Boolean useFsync) { + private void prepare(MockRoot root, SearchNode node) { Host host = new Host(root, "mockhost"); - TransactionLogServer tls = new TransactionLogServer(root, "mycluster", useFsync); - tls.setHostResource(new HostResource(host)); - tls.setBasePort(100); - tls.initService(root.getDeployState()); - node.setTls(tls); node.setHostResource(new HostResource(host)); node.setBasePort(200); node.initService(root.getDeployState()); @@ -45,13 +39,15 @@ public class SearchNodeTest { } private static SearchNode createSearchNode(MockRoot root, String name, int distributionKey, NodeSpec nodeSpec, - boolean flushOnShutDown, boolean isHosted, ModelContext.FeatureFlags featureFlags) { + boolean flushOnShutDown, boolean isHosted, + ModelContext.FeatureFlags featureFlags, + Boolean syncTransactionLog) { return SearchNode.create(root, name, distributionKey, nodeSpec, "mycluster", null, flushOnShutDown, - null, null, isHosted, 0.0, featureFlags); + null, null, isHosted, 0.0, featureFlags, syncTransactionLog); } - private static SearchNode createSearchNode(MockRoot root) { - return createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), true, true, new TestProperties()); + private static SearchNode createSearchNode(MockRoot root, Boolean syncTransactionLog) { + return createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), true, true, new TestProperties(), syncTransactionLog); } @Test @@ -64,15 +60,17 @@ public class SearchNodeTest { @Test void requireThatBasedirIsCorrectForElasticMode() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted(), new TestProperties()); - prepare(root, node, true); + SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, + root.getDeployState().isHosted(), new TestProperties(), true); + prepare(root, node); assertBaseDir(Defaults.getDefaults().underVespaHome("var/db/vespa/search/cluster.mycluster/n3"), node); } @Test void requireThatPreShutdownCommandIsEmptyWhenNotActivated() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted(), new TestProperties()); + SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, + root.getDeployState().isHosted(), new TestProperties(), true); node.setHostResource(new HostResource(new Host(node, "mynbode"))); node.initService(root.getDeployState()); assertFalse(node.getPreShutdownCommand().isPresent()); @@ -81,7 +79,8 @@ public class SearchNodeTest { @Test void requireThatPreShutdownCommandUsesPrepareRestartWhenActivated() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, root.getDeployState().isHosted(), new TestProperties()); + SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, + root.getDeployState().isHosted(), new TestProperties(), true); node.setHostResource(new HostResource(new Host(node, "mynbode2"))); node.initService(root.getDeployState()); assertTrue(node.getPreShutdownCommand().isPresent()); @@ -90,7 +89,8 @@ public class SearchNodeTest { private void verifyCodePlacement(boolean hugePages) { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, false, new TestProperties().loadCodeAsHugePages(hugePages)); + SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, false, + new TestProperties().loadCodeAsHugePages(hugePages), true); node.setHostResource(new HostResource(new Host(node, "mynbode2"))); node.initService(root.getDeployState()); assertEquals(hugePages, node.getEnvVars().get("VESPA_LOAD_CODE_AS_HUGEPAGES") != null); @@ -104,7 +104,8 @@ public class SearchNodeTest { private void verifySharedStringRepoReclaim(boolean sharedStringRepoNoReclaim) { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, false, new TestProperties().sharedStringRepoNoReclaim(sharedStringRepoNoReclaim)); + SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, false, + new TestProperties().sharedStringRepoNoReclaim(sharedStringRepoNoReclaim), true); node.setHostResource(new HostResource(new Host(node, "mynbode2"))); node.initService(root.getDeployState()); assertEquals(sharedStringRepoNoReclaim, node.getEnvVars().get("VESPA_SHARED_STRING_REPO_NO_RECLAIM") != null); @@ -120,10 +121,10 @@ public class SearchNodeTest { return new MockRoot("", new DeployState.Builder().properties(properties).build()); } - private TranslogserverConfig getTlsConfig(ModelContext.Properties properties, Boolean useFsync) { + private TranslogserverConfig getTlsConfig(ModelContext.Properties properties, Boolean syncTransactionLog) { MockRoot root = createRoot(properties); - SearchNode node = createSearchNode(root); - prepare(root, node, useFsync); + SearchNode node = createSearchNode(root, syncTransactionLog); + prepare(root, node); TranslogserverConfig.Builder tlsBuilder = new TranslogserverConfig.Builder(); node.getConfig(tlsBuilder); return tlsBuilder.build(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java index c7d9ae6b818..2fda76d0d82 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTestCase.java @@ -20,9 +20,6 @@ import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.TestDriver; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.Zone; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.messagebus.MessagebusConfig; import com.yahoo.net.HostName; @@ -37,6 +34,7 @@ import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.xml.sax.SAXException; + import java.io.File; import java.io.IOException; import java.io.StringReader; @@ -265,9 +263,9 @@ public class VespaModelTestCase { assertEquals(1, admin.getConfigservers().size()); Set<HostInfo> hosts = model.getHosts(); assertEquals(1, hosts.size()); - //logd, config proxy, sentinel, config server, slobrok, log server + // logd, config proxy, sentinel, config server, slobrok, logserver, logserver container HostInfo host = hosts.iterator().next(); - assertEquals(7, host.getServices().size()); + assertEquals(8, host.getServices().size()); new LogdConfig((LogdConfig.Builder) model.getConfig(new LogdConfig.Builder(), "admin/model")); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java index 7037480f853..887db9c9cc9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/VespaModelTester.java @@ -88,7 +88,7 @@ public class VespaModelTester { // (for e.g cluster controllers and slobrok nodes) String hostname = String.format("%s-%02d", "node" + "-" + Math.round(resources.vcpu()) + - "-" + Math.round(resources.memoryGb()) + + "-" + Math.round(resources.memoryGiB()) + "-" + Math.round(resources.diskGb()), count - i); hosts.add(new Host(hostname, List.of(), flavor)); |