diff options
Diffstat (limited to 'config-model/src')
119 files changed, 1057 insertions, 284 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java index f324ceef5ab..2461fc64172 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java +++ b/config-model/src/main/java/com/yahoo/config/model/ApplicationConfigProducerRoot.java @@ -233,10 +233,7 @@ public class ApplicationConfigProducerRoot extends TreeConfigProducer<AnyConfigP } } - // add cluster type? - // add cluster name? - public record StatePortInfo(String hostName, int portNumber, - String serviceName, String serviceType) + public record StatePortInfo(String hostName, int portNumber, Service service) {} public List<StatePortInfo> getStatePorts() { @@ -244,8 +241,6 @@ public class ApplicationConfigProducerRoot extends TreeConfigProducer<AnyConfigP for (HostResource modelHost : hostSystem().getHosts()) { String hostName = modelHost.getHostname(); for (Service modelService : modelHost.getServices()) { - String serviceName = modelService.getServiceName(); - String serviceType = modelService.getServiceType(); PortsMeta portsMeta = modelService.getPortsMeta(); for (int i = 0; i < portsMeta.getNumPorts(); i++) { int portNumber = modelService.getRelativePort(i); @@ -256,7 +251,7 @@ public class ApplicationConfigProducerRoot extends TreeConfigProducer<AnyConfigP if (tag.equals("http")) isHttp = true; } if (hasState && isHttp) { - result.add(new StatePortInfo(hostName, portNumber, serviceName, serviceType)); + result.add(new StatePortInfo(hostName, portNumber, modelService)); } } } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 3c45588a054..970597685da 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -55,7 +55,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private double feedNiceness = 0.0; private int maxActivationInhibitedOutOfSyncGroups = 0; private List<TenantSecretStore> tenantSecretStores = List.of(); - private String jvmOmitStackTraceInFastThrowOption; private boolean allowDisableMtls = true; private List<X509Certificate> operatorCertificates = List.of(); private double resourceLimitDisk = 0.75; @@ -84,6 +83,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private int contentLayerMetadataFeatureLevel = 0; private int persistenceThreadMaxFeedOpBatchSize = 1; private boolean logserverOtelCol = false; + private boolean symmetricPutAndActivateReplicaSelection = false; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -111,7 +111,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public double feedNiceness() { return feedNiceness; } @Override public int maxActivationInhibitedOutOfSyncGroups() { return maxActivationInhibitedOutOfSyncGroups; } @Override public List<TenantSecretStore> tenantSecretStores() { return tenantSecretStores; } - @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) { return jvmOmitStackTraceInFastThrowOption; } @Override public boolean allowDisableMtls() { return allowDisableMtls; } @Override public List<X509Certificate> operatorCertificates() { return operatorCertificates; } @Override public double resourceLimitDisk() { return resourceLimitDisk; } @@ -142,6 +141,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public int contentLayerMetadataFeatureLevel() { return contentLayerMetadataFeatureLevel; } @Override public int persistenceThreadMaxFeedOpBatchSize() { return persistenceThreadMaxFeedOpBatchSize; } @Override public boolean logserverOtelCol() { return logserverOtelCol; } + @Override public boolean symmetricPutAndActivateReplicaSelection() { return symmetricPutAndActivateReplicaSelection; } public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) { this.sharedStringRepoNoReclaim = sharedStringRepoNoReclaim; @@ -276,11 +276,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties setJvmOmitStackTraceInFastThrowOption(String value) { - this.jvmOmitStackTraceInFastThrowOption = value; - return this; - } - public TestProperties allowDisableMtls(boolean value) { this.allowDisableMtls = value; return this; @@ -382,6 +377,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setSymmetricPutAndActivateReplicaSelection(boolean symmetricReplicaSelection) { + this.symmetricPutAndActivateReplicaSelection = symmetricReplicaSelection; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java index befe57a97e4..da0fd265724 100644 --- a/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java +++ b/config-model/src/main/java/com/yahoo/config/model/provision/InMemoryProvisioner.java @@ -158,7 +158,7 @@ public class InMemoryProvisioner implements HostProvisioner { @Override public List<HostSpec> prepare(ClusterSpec cluster, Capacity requested, ProvisionLogger logger) { - provisioned.add(cluster.id(), requested); + provisioned.add(cluster, requested); clusters.add(cluster); if (environment == Environment.dev && ! requested.isRequired()) { requested = requested.withLimits(requested.minResources().withNodes(1), diff --git a/config-model/src/main/java/com/yahoo/schema/RankProfile.java b/config-model/src/main/java/com/yahoo/schema/RankProfile.java index 82ed45028b3..60674b5487c 100644 --- a/config-model/src/main/java/com/yahoo/schema/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/RankProfile.java @@ -141,6 +141,8 @@ public class RankProfile implements Cloneable { private Boolean strict; + private Boolean useSignificanceModel; + private final ApplicationPackage applicationPackage; private final DeployLogger deployLogger; @@ -216,6 +218,16 @@ public class RankProfile implements Cloneable { this.strict = strict; } + public void setUseSignificanceModel(Boolean useSignificanceModel) { + this.useSignificanceModel = useSignificanceModel; + } + + public boolean useSignificanceModel() { + if (useSignificanceModel != null) return useSignificanceModel; + return uniquelyInherited(p -> p.useSignificanceModel(), "use-model") + .orElse(false); // Disabled by default + } + /** * Adds a profile to those inherited by this. * The profile must belong to this schema (directly or by inheritance). diff --git a/config-model/src/main/java/com/yahoo/schema/Schema.java b/config-model/src/main/java/com/yahoo/schema/Schema.java index 3402ba31be9..127d12594b4 100644 --- a/config-model/src/main/java/com/yahoo/schema/Schema.java +++ b/config-model/src/main/java/com/yahoo/schema/Schema.java @@ -721,7 +721,7 @@ public class Schema implements ImmutableSchema { "', but this schema does not exist"); // Require schema and document type inheritance to be consistent to keep things simple - // And require it to be explicit so we have the option to support other possibilities later + // and require it to be explicit, so we have the option to support other possibilities later var parentDocument = owner.schemas().get(inherited.get()).getDocument(); if ( ! getDocument().inheritedTypes().containsKey(new DataTypeName(parentDocument.getName()))) throw new IllegalArgumentException(this + " inherits '" + inherited.get() + diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java index f996b2624db..b91404be2dd 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java @@ -183,10 +183,12 @@ public final class SchemaInfo extends Derived { private void addRankProfilesConfig(SchemaInfoConfig.Schema.Builder schemaBuilder) { for (RankProfileInfo rankProfile : rankProfiles().values()) { - var rankProfileConfig = new SchemaInfoConfig.Schema.Rankprofile.Builder(); - rankProfileConfig.name(rankProfile.name()); - rankProfileConfig.hasSummaryFeatures(rankProfile.hasSummaryFeatures()); - rankProfileConfig.hasRankFeatures(rankProfile.hasRankFeatures()); + var rankProfileConfig = new SchemaInfoConfig.Schema.Rankprofile.Builder() + .name(rankProfile.name()) + .hasSummaryFeatures(rankProfile.hasSummaryFeatures()) + .hasRankFeatures(rankProfile.hasRankFeatures()) + .significance(new SchemaInfoConfig.Schema.Rankprofile.Significance.Builder() + .useModel(rankProfile.useSignificanceModel())); for (var input : rankProfile.inputs().entrySet()) { var inputConfig = new SchemaInfoConfig.Schema.Rankprofile.Input.Builder(); inputConfig.name(input.getKey().toString()); @@ -226,6 +228,7 @@ public final class SchemaInfo extends Derived { private final String name; private final boolean hasSummaryFeatures; private final boolean hasRankFeatures; + private final boolean useSignificanceModel; private final Map<Reference, RankProfile.Input> inputs; public RankProfileInfo(RankProfile profile) { @@ -233,11 +236,13 @@ public final class SchemaInfo extends Derived { this.hasSummaryFeatures = ! profile.getSummaryFeatures().isEmpty(); this.hasRankFeatures = ! profile.getRankFeatures().isEmpty(); this.inputs = profile.inputs(); + useSignificanceModel = profile.useSignificanceModel(); } public String name() { return name; } public boolean hasSummaryFeatures() { return hasSummaryFeatures; } public boolean hasRankFeatures() { return hasRankFeatures; } + public boolean useSignificanceModel() { return useSignificanceModel; } public Map<Reference, RankProfile.Input> inputs() { return inputs; } } diff --git a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java index 398897c6b78..73495c066b9 100644 --- a/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java @@ -11,7 +11,6 @@ import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; -import java.io.IOException; import java.util.Collections; import java.util.Map; import java.util.logging.Level; diff --git a/config-model/src/main/java/com/yahoo/schema/document/Matching.java b/config-model/src/main/java/com/yahoo/schema/document/Matching.java index 9d68553fa80..33256fa8586 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/Matching.java +++ b/config-model/src/main/java/com/yahoo/schema/document/Matching.java @@ -33,6 +33,8 @@ public class Matching implements Cloneable, Serializable { private Integer maxLength; /** Maximum number of occurrences for each term */ private Integer maxTermOccurrences; + /** Maximum number of characters in a token. */ + private Integer maxTokenLength; private String exactMatchTerminator = null; @@ -61,6 +63,8 @@ public class Matching implements Cloneable, Serializable { public Matching maxLength(int maxLength) { this.maxLength = maxLength; return this; } public Integer maxTermOccurrences() { return maxTermOccurrences; } public Matching maxTermOccurrences(int maxTermOccurrences) { this.maxTermOccurrences = maxTermOccurrences; return this; } + public Integer maxTokenLength() { return maxTokenLength; } + public Matching maxTokenLength(int maxTokenLength) { this.maxTokenLength = maxTokenLength; return this; } public boolean isTypeUserSet() { return typeUserSet; } public MatchAlgorithm getAlgorithm() { return algorithm; } diff --git a/config-model/src/main/java/com/yahoo/schema/document/SDField.java b/config-model/src/main/java/com/yahoo/schema/document/SDField.java index f165141b16e..2483fa47667 100644 --- a/config-model/src/main/java/com/yahoo/schema/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/schema/document/SDField.java @@ -46,7 +46,7 @@ import java.util.TreeMap; * * @author bratseth */ -public class SDField extends Field implements TypedKey, ImmutableSDField { +public class SDField extends Field implements ImmutableSDField { /** Use this field for modifying index-structure, even if it doesn't have any indexing code */ private boolean indexStructureField = false; @@ -315,7 +315,7 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { supplyStructField.accept(field.getName(), field.getDataType()); } } - if ((subType == null) && (structFields.size() > 0)) { + if ((subType == null) && (!structFields.isEmpty())) { throw new IllegalArgumentException("Cannot find matching (repo=" + sdoc + ") for subfields in " + this + " [" + getDataType() + getDataType().getClass() + "] with " + structFields.size() + " struct fields"); @@ -627,7 +627,7 @@ public class SDField extends Field implements TypedKey, ImmutableSDField { public Attribute addAttribute(Attribute attribute) { String name = attribute.getName(); - if (name == null || "".equals(name)) { + if (name == null || name.isEmpty()) { name = getName(); attribute.setName(name); } diff --git a/config-model/src/main/java/com/yahoo/schema/document/TypedKey.java b/config-model/src/main/java/com/yahoo/schema/document/TypedKey.java deleted file mode 100644 index 652d21d7f7d..00000000000 --- a/config-model/src/main/java/com/yahoo/schema/document/TypedKey.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.schema.document; - -import com.yahoo.document.DataType; - -/** - * Common interface for various typed key (or field definitions). - * Used by code which wants to use common algorithms for dealing with typed keys, like the logical mapping - * - * @author bratseth - */ -public interface TypedKey { - - String getName(); - - void setDataType(DataType type); - - DataType getDataType(); - -} diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java index 7659a1e6562..173eebe2a94 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java @@ -44,6 +44,7 @@ public class ConvertParsedFields { parsed.getGramSize().ifPresent(gramSize -> field.getMatching().setGramSize(gramSize)); parsed.getMaxLength().ifPresent(maxLength -> field.getMatching().maxLength(maxLength)); parsed.getMaxTermOccurrences().ifPresent(maxTermOccurrences -> field.getMatching().maxTermOccurrences(maxTermOccurrences)); + parsed.getMaxTokenLength().ifPresent(maxTokenLength -> field.getMatching().maxTokenLength(maxTokenLength)); parsed.getMatchAlgorithm().ifPresent (matchingAlgorithm -> field.setMatchingAlgorithm(matchingAlgorithm)); parsed.getExactTerminator().ifPresent diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java index 5ccbb7b19a4..77a10862f9c 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java @@ -39,6 +39,7 @@ public class ConvertParsedRanking { profile.inherit(name); parsed.isStrict().ifPresent(value -> profile.setStrict(value)); + parsed.isUseSignificanceModel().ifPresent(value -> profile.setUseSignificanceModel(value)); for (var constant : parsed.getConstants().values()) profile.add(constant); diff --git a/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java b/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java index 789f7023aed..063962bf0c4 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java @@ -133,9 +133,9 @@ public class IntermediateCollection { var parser = new SchemaParser(stream, deployLogger, modelProperties); try { parser.rankProfile(schema); - } catch (ParseException pe) { + } catch (ParseException | TokenMgrException e) { throw new ParseException("Failed parsing rank-profile from '" + reader.getName() + "': " + - stream.formatException(Exceptions.toMessageString(pe))); + stream.formatException(Exceptions.toMessageString(e))); } } catch (java.io.IOException ex) { throw new IllegalArgumentException("Failed reading from '" + reader.getName() + "': " + ex.getMessage()); diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedMatchSettings.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedMatchSettings.java index c7d1a215ce3..bac2c894283 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ParsedMatchSettings.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedMatchSettings.java @@ -23,6 +23,7 @@ public class ParsedMatchSettings { private Integer gramSize = null; private Integer maxLength = null; private Integer maxTermOccurrences = null; + private Integer maxTokenLength = null; Optional<MatchType> getMatchType() { return Optional.ofNullable(matchType); } Optional<Case> getMatchCase() { return Optional.ofNullable(matchCase); } @@ -31,6 +32,7 @@ public class ParsedMatchSettings { Optional<Integer> getGramSize() { return Optional.ofNullable(gramSize); } Optional<Integer> getMaxLength() { return Optional.ofNullable(maxLength); } Optional<Integer> getMaxTermOccurrences() { return Optional.ofNullable(maxTermOccurrences); } + Optional<Integer> getMaxTokenLength() { return Optional.ofNullable(maxTokenLength); } // TODO - consider allowing each set only once: void setType(MatchType value) { this.matchType = value; } @@ -40,5 +42,6 @@ public class ParsedMatchSettings { void setGramSize(int value) { this.gramSize = value; } void setMaxLength(int value) { this.maxLength = value; } void setMaxTermOccurrences(int value) { this.maxTermOccurrences = value; } + void setMaxTokenLength(int value) { this.maxTokenLength = value; } } diff --git a/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java index fbbb0c7fe83..93319e82076 100644 --- a/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java @@ -44,6 +44,7 @@ class ParsedRankProfile extends ParsedBlock { private String inheritedMatchFeatures = null; private String secondPhaseExpression = null; private Boolean strict = null; + private Boolean useSignificanceModel = null; private final List<MutateOperation> mutateOperations = new ArrayList<>(); private final List<String> inherited = new ArrayList<>(); private final Map<String, Boolean> fieldsRankFilter = new LinkedHashMap<>(); @@ -96,6 +97,8 @@ class ParsedRankProfile extends ParsedBlock { Optional<String> getSecondPhaseExpression() { return Optional.ofNullable(this.secondPhaseExpression); } Optional<Boolean> isStrict() { return Optional.ofNullable(this.strict); } + Optional<Boolean> isUseSignificanceModel() { return Optional.ofNullable(this.useSignificanceModel); } + void addSummaryFeatures(FeatureList features) { this.summaryFeatures.add(features); } void addMatchFeatures(FeatureList features) { this.matchFeatures.add(features); } void addRankFeatures(FeatureList features) { this.rankFeatures.add(features); } @@ -218,6 +221,10 @@ class ParsedRankProfile extends ParsedBlock { this.strict = strict; } + void setUseSignificanceModel(boolean useSignificanceModel) { + verifyThat(this.useSignificanceModel == null, "already has use-model"); + this.useSignificanceModel = useSignificanceModel; + } void setTermwiseLimit(double limit) { verifyThat(termwiseLimit == null, "already has termwise-limit"); this.termwiseLimit = limit; diff --git a/config-model/src/main/java/com/yahoo/schema/processing/AddDataTypeAndTransformToSummaryOfImportedFields.java b/config-model/src/main/java/com/yahoo/schema/processing/AddDataTypeAndTransformToSummaryOfImportedFields.java index 762279e3871..e66cd62caa8 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/AddDataTypeAndTransformToSummaryOfImportedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/AddDataTypeAndTransformToSummaryOfImportedFields.java @@ -32,7 +32,7 @@ public class AddDataTypeAndTransformToSummaryOfImportedFields extends Processor @Override public void process(boolean validate, boolean documentsOnly) { schema.allImportedFields() - .forEach(field -> setTransform(field)); + .forEach(this::setTransform); } private Stream<SummaryField> getSummaryFieldsForImportedField(ImmutableSDField importedField) { diff --git a/config-model/src/main/java/com/yahoo/schema/processing/AttributesImplicitWord.java b/config-model/src/main/java/com/yahoo/schema/processing/AttributesImplicitWord.java index 767593b82d0..769f0c9de92 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/AttributesImplicitWord.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/AttributesImplicitWord.java @@ -2,6 +2,7 @@ package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; +import com.yahoo.document.TensorDataType; import com.yahoo.schema.RankProfileRegistry; import com.yahoo.document.DataType; import com.yahoo.schema.Schema; @@ -45,6 +46,8 @@ public class AttributesImplicitWord extends Processor { private boolean fieldImplicitlyWordMatch(ImmutableSDField field) { // numeric types should not trigger exact-match query parsing if (field.getDataType().getPrimitiveType() instanceof NumericDataType) return false; + // Tensor type should not trigger exact-match query parsing + if (field.getDataType() instanceof TensorDataType) return false; return (! field.hasIndex() && !field.getAttributes().isEmpty() diff --git a/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java index 056c37a9830..4313ceb4be1 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.indexinglanguage.expressions.ForEachExpression; import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; +import com.yahoo.vespa.indexinglanguage.linguistics.AnnotatorConfig; import com.yahoo.vespa.model.container.search.QueryProfiles; /** @@ -75,7 +76,11 @@ public class ExactMatch extends Processor { } ScriptExpression script = field.getIndexingScript(); if (new ExpressionSearcher<>(IndexExpression.class).containedIn(script)) { - field.setIndexingScript(schema.getName(), (ScriptExpression)new MyProvider(schema).convert(field.getIndexingScript())); + var maxTokenLength = field.getMatching().maxTokenLength(); + if (maxTokenLength == null) { + maxTokenLength = AnnotatorConfig.getDefaultMaxTokenLength(); + } + field.setIndexingScript(schema.getName(), (ScriptExpression)new MyProvider(schema, maxTokenLength).convert(field.getIndexingScript())); } } @@ -85,8 +90,12 @@ public class ExactMatch extends Processor { private static class MyProvider extends TypedTransformProvider { - MyProvider(Schema schema) { + private int maxTokenLength; + + MyProvider(Schema schema, int maxTokenLength) + { super(ExactExpression.class, schema); + this.maxTokenLength = maxTokenLength; } @Override @@ -96,7 +105,7 @@ public class ExactMatch extends Processor { @Override protected Expression newTransform(DataType fieldType) { - Expression exp = new ExactExpression(); + Expression exp = new ExactExpression(maxTokenLength); if (fieldType instanceof CollectionDataType) { exp = new ForEachExpression(exp); } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/Processor.java b/config-model/src/main/java/com/yahoo/schema/processing/Processor.java index dd36bbb3b61..8b0446364ea 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/Processor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/Processor.java @@ -13,6 +13,7 @@ import com.yahoo.schema.document.RankType; import com.yahoo.schema.document.SDField; import com.yahoo.schema.document.Stemming; import com.yahoo.vespa.model.container.search.QueryProfiles; +import com.yahoo.vespa.objects.FieldBase; import java.util.Iterator; import java.util.List; @@ -110,8 +111,8 @@ public abstract class Processor { List<RankProfile.RankSetting> someRankSettings = new java.util.ArrayList<>(); for (RankProfile profile : rankProfileRegistry.rankProfilesOf(schema)) { - for (Iterator j = profile.declaredRankSettingIterator(); j.hasNext(); ) { - RankProfile.RankSetting setting = (RankProfile.RankSetting)j.next(); + for (Iterator<RankProfile.RankSetting> j = profile.declaredRankSettingIterator(); j.hasNext(); ) { + RankProfile.RankSetting setting = j.next(); if (setting.getType().equals(type)) { someRankSettings.add(setting); } @@ -128,7 +129,7 @@ public abstract class Processor { return new IllegalArgumentException(formatError(schemaName, fieldName, msg)); } - protected RuntimeException newProcessException(Schema schema, Field field, String msg) { + protected RuntimeException newProcessException(Schema schema, FieldBase field, String msg) { return newProcessException(schema.getName(), field.getName(), msg); } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java index e29f683761f..3f23cbc9b2d 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java @@ -64,12 +64,16 @@ public class TextMatch extends Processor { if (fieldMatching != null) { var maxLength = fieldMatching.maxLength(); if (maxLength != null) { - ret.setMaxTokenLength(maxLength); + ret.setMaxTokenizeLength(maxLength); } var maxTermOccurrences = fieldMatching.maxTermOccurrences(); if (maxTermOccurrences != null) { ret.setMaxTermOccurrences(maxTermOccurrences); } + var maxTokenLength = fieldMatching.maxTokenLength(); + if (maxTokenLength != null) { + ret.setMaxTokenLength(maxTokenLength); + } } return ret; } diff --git a/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java b/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java index 8ccc8870419..8be7be02135 100644 --- a/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java @@ -5,6 +5,7 @@ import com.yahoo.document.DataType; import com.yahoo.document.Field; import com.yahoo.schema.Schema; import com.yahoo.schema.document.Attribute; +import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.indexinglanguage.ValueTransformProvider; import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; import com.yahoo.vespa.indexinglanguage.expressions.Expression; @@ -29,6 +30,10 @@ public abstract class TypedTransformProvider extends ValueTransformProvider { protected final boolean requiresTransform(Expression exp) { if (exp instanceof OutputExpression) { String fieldName = ((OutputExpression)exp).getFieldName(); + if (fieldName == null) { + // Incomplete output expressions never require a transform. + return false; + } if (exp instanceof AttributeExpression) { Attribute attribute = schema.getAttribute(fieldName); if (attribute == null) @@ -42,7 +47,7 @@ public abstract class TypedTransformProvider extends ValueTransformProvider { fieldType = field.getDataType(); } else if (exp instanceof SummaryExpression) { - Field field = schema.getSummaryField(fieldName); + SummaryField field = schema.getSummaryField(fieldName); if (field == null) { // Use document field if summary field is not found var sdField = schema.getConcreteField(fieldName); diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java index 32807db8405..e48aa0eef7c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java @@ -2,19 +2,15 @@ package com.yahoo.vespa.documentmodel; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.schema.RankProfile; import com.yahoo.schema.Schema; -import com.yahoo.searchlib.rankingexpression.Reference; import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Optional; -import java.util.logging.Level; + +import static java.util.logging.Level.WARNING; /** * A document summary definition - a list of summary fields. @@ -25,7 +21,7 @@ public class DocumentSummary extends FieldView { private boolean fromDisk = false; private boolean omitSummaryFeatures = false; - private List<String> inherited = new ArrayList<>(); + private final List<String> inherited = new ArrayList<>(); private final Schema owner; @@ -115,7 +111,7 @@ public class DocumentSummary extends FieldView { /** Returns the parent of this, if any */ public List<DocumentSummary> inherited() { - return inherited.stream().map(name -> owner.getSummary(name)).toList(); + return inherited.stream().map(owner::getSummary).toList(); } @Override @@ -126,13 +122,12 @@ public class DocumentSummary extends FieldView { public void validate(DeployLogger logger) { for (var inheritedName : inherited) { var inheritedSummary = owner.getSummary(inheritedName); - if (inheritedSummary == null) { - // TODO Vespa 9: Throw IllegalArgumentException instead - logger.logApplicationPackage(Level.WARNING, - this + " inherits '" + inheritedName + "' but this" + " is not present in " + owner); - } + // TODO: Throw when no one is doing this anymore + if (inheritedName.equals("default")) + logger.logApplicationPackage(WARNING, this + " inherits '" + inheritedName + "', which makes no sense. Remove this inheritance"); + else if (inheritedSummary == null ) + throw new IllegalArgumentException(this + " inherits '" + inheritedName + "', but this is not present in " + owner); } - } } diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/FieldView.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/FieldView.java index edc25bd8246..7034365c40b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/FieldView.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/FieldView.java @@ -1,7 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.documentmodel; -import com.yahoo.document.Field; +import com.yahoo.vespa.objects.FieldBase; import java.io.Serializable; import java.util.Collection; @@ -14,7 +14,7 @@ import java.util.Map; public class FieldView implements Serializable { private final String name; - private final Map<String, Field> fields = new LinkedHashMap<>(); + private final Map<String, FieldBase> fields = new LinkedHashMap<>(); /** * Creates a view with a name @@ -24,8 +24,8 @@ public class FieldView implements Serializable { this.name = name; } public String getName() { return name; } - public Collection<Field> getFields() { return fields.values(); } - public Field get(String name) { return fields.get(name); } + public Collection<FieldBase> getFields() { return fields.values(); } + public FieldBase get(String name) { return fields.get(name); } public void remove(String name) { fields.remove(name); } /** @@ -34,7 +34,7 @@ public class FieldView implements Serializable { * @param field the field to add. * @return itself for chaining purposes. */ - public FieldView add(Field field) { + public FieldView add(FieldBase field) { if (fields.containsKey(field.getName())) { if ( ! fields.get(field.getName()).equals(field)) { throw new IllegalArgumentException( @@ -54,7 +54,7 @@ public class FieldView implements Serializable { * @return Itself for chaining. */ public FieldView add(FieldView view) { - for(Field field : view.getFields()) { + for(var field : view.getFields()) { add(field); } return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java index d50d5e36134..e529779735f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java @@ -3,10 +3,13 @@ package com.yahoo.vespa.documentmodel; import com.yahoo.document.DataType; import com.yahoo.document.Field; -import com.yahoo.schema.document.TypedKey; +import com.yahoo.vespa.objects.FieldBase; import java.io.Serializable; -import java.util.*; +import java.util.Collection; +import java.util.Iterator; +import java.util.LinkedHashSet; +import java.util.Set; import java.util.stream.Collectors; import static com.yahoo.text.Lowercase.toLowerCase; @@ -16,7 +19,7 @@ import static com.yahoo.text.Lowercase.toLowerCase; * * @author bratseth */ -public class SummaryField extends Field implements Cloneable, TypedKey { +public class SummaryField extends FieldBase implements Cloneable { /** A source (field name). */ public static class Source implements Serializable { @@ -53,6 +56,7 @@ public class SummaryField extends Field implements Cloneable, TypedKey { /** The command used per field in vsmsummary */ private VsmCommand vsmCommand = VsmCommand.NONE; + private DataType dataType; /** * The data sources for this output summary field, in prioritized order @@ -62,7 +66,7 @@ public class SummaryField extends Field implements Cloneable, TypedKey { */ private Set<Source> sources = new java.util.LinkedHashSet<>(); - private Set<String> destinations =new java.util.LinkedHashSet<>(); + private Set<String> destinations = new java.util.LinkedHashSet<>(); /** True if this field was defined implicitly */ private boolean implicit = false; @@ -84,8 +88,9 @@ public class SummaryField extends Field implements Cloneable, TypedKey { } public SummaryField(String name, DataType type, SummaryTransform transform) { - super(name, type); + super(name); this.transform=transform; + this.dataType = type; } public static SummaryField createWithUnresolvedType(String name) { @@ -106,6 +111,10 @@ public class SummaryField extends Field implements Cloneable, TypedKey { public boolean hasUnresolvedType() { return unresolvedType; } + public DataType getDataType() { + return dataType; + } + public void setTransform(SummaryTransform transform) { this.transform = transform; if (SummaryTransform.DYNAMICTEASER.equals(transform) || SummaryTransform.BOLDED.equals(transform)) { @@ -194,7 +203,7 @@ public class SummaryField extends Field implements Cloneable, TypedKey { if (merge.getTransform() != getTransform()) throw new IllegalArgumentException(merge + " conflicts with " + this + ": different transforms"); - if (!merge.getDataType().equals(getDataType())) + if (!merge.dataType.equals(dataType)) throw new IllegalArgumentException(merge + " conflicts with " + this + ": different types"); setImplicit(false); @@ -250,7 +259,7 @@ public class SummaryField extends Field implements Cloneable, TypedKey { /** Returns a string which aids locating this field in the source search definition */ public String toLocateString() { - return "summary " + getName() + " type " + toLowerCase(getDataType().getName()) + " in " + getDestinationString(); + return "summary " + getName() + " type " + toLowerCase(dataType.getName()) + " in " + getDestinationString(); } @Override @@ -290,9 +299,6 @@ public class SummaryField extends Field implements Cloneable, TypedKey { public void setResolvedDataType(DataType type) { this.dataType = type; - if (!hasForcedId()) { - this.fieldId = calculateIdV7(null); - } unresolvedType = false; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java index d8149486b32..806f14da265 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/AbstractService.java @@ -342,6 +342,7 @@ public abstract class AbstractService extends TreeConfigProducer<AnyConfigProduc return getServicePropertyString(key, null); } + @Override public String getServicePropertyString(String key, String defStr) { Object result = serviceProperties.get(key); return (result == null) ? defStr : result.toString(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/Host.java b/config-model/src/main/java/com/yahoo/vespa/model/Host.java index a8085919a98..f87f1382ffb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/Host.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/Host.java @@ -16,7 +16,7 @@ import java.util.Objects; public final class Host extends TreeConfigProducer<AnyConfigProducer> implements SentinelConfig.Producer, Comparable<Host> { // Memory needed for auxiliary processes always running on the node (config-proxy, metrics-proxy). - // Keep in sync with node-repository/ClusterModel. + // Keep in sync with node-repository/ClusterModel and startup scripts (go and bash). public static final double memoryOverheadGb = 0.7; private ConfigSentinel configSentinel = null; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java index ad728d60c5a..a513cc673dd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/LogserverContainer.java @@ -1,11 +1,9 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin; -import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.TreeConfigProducer; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.model.container.Container; /** @@ -30,9 +28,4 @@ public class LogserverContainer extends Container { return ""; } - @Override - protected String jvmOmitStackTraceInFastThrowOption(ModelContext.FeatureFlags featureFlags) { - return featureFlags.jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type.admin); - } - } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java index ca602b447fe..be38298279f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/clustercontroller/ClusterControllerContainer.java @@ -97,11 +97,6 @@ public class ClusterControllerContainer extends Container implements return ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; } - @Override - protected String jvmOmitStackTraceInFastThrowOption(ModelContext.FeatureFlags featureFlags) { - return featureFlags.jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type.admin); - } - private void configureZooKeeperServer(boolean runStandaloneZooKeeper) { if (runStandaloneZooKeeper) ContainerModelBuilder.addReconfigurableZooKeeperServerComponents(this); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java index c4702ba3543..8f262281edc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/metricsproxy/MetricsProxyContainer.java @@ -10,7 +10,6 @@ import ai.vespa.metricsproxy.rpc.RpcConnector; import ai.vespa.metricsproxy.rpc.RpcConnectorConfig; import ai.vespa.metricsproxy.service.VespaServices; import ai.vespa.metricsproxy.service.VespaServicesConfig; -import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.container.ContainerServiceType; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.provision.ApplicationId; @@ -78,11 +77,6 @@ public class MetricsProxyContainer extends Container implements } @Override - protected String jvmOmitStackTraceInFastThrowOption(ModelContext.FeatureFlags featureFlags) { - return featureFlags.jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type.admin); - } - - @Override public int getWantedPort() { return BASEPORT; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryCollector.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryCollector.java index fcd587622da..03b96b12c03 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryCollector.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryCollector.java @@ -5,6 +5,7 @@ import com.yahoo.cloud.config.OpenTelemetryConfig; import com.yahoo.config.model.ApplicationConfigProducerRoot; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.TreeConfigProducer; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Zone; import com.yahoo.vespa.model.AbstractService; import com.yahoo.vespa.model.PortAllocBridge; @@ -17,10 +18,12 @@ import java.util.Optional; public class OpenTelemetryCollector extends AbstractService implements OpenTelemetryConfig.Producer { private final Zone zone; + private final ApplicationId applicationId; public OpenTelemetryCollector(TreeConfigProducer<?> parent) { super(parent, "otelcol"); this.zone = null; + this.applicationId = null; setProp("clustertype", "admin"); setProp("clustername", "admin"); } @@ -28,6 +31,7 @@ public class OpenTelemetryCollector extends AbstractService implements OpenTelem public OpenTelemetryCollector(TreeConfigProducer<?> parent, DeployState deployState) { super(parent, "otelcol"); this.zone = deployState.zone(); + this.applicationId = deployState.getProperties().applicationId(); setProp("clustertype", "admin"); setProp("clustername", "admin"); } @@ -50,7 +54,7 @@ public class OpenTelemetryCollector extends AbstractService implements OpenTelem @Override public void getConfig(OpenTelemetryConfig.Builder builder) { - var generator = new OpenTelemetryConfigGenerator(zone); + var generator = new OpenTelemetryConfigGenerator(zone, applicationId); AnyConfigProducer pp = this; AnyConfigProducer p = pp.getParent(); while (p != null && p != pp) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java index 36eab6a04b3..3f7ca7b46a7 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGenerator.java @@ -1,16 +1,23 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.otel; +import ai.vespa.metricsproxy.metric.dimensions.PublicDimensions; import com.fasterxml.jackson.core.JsonEncoding; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; import com.yahoo.config.model.ApplicationConfigProducerRoot.StatePortInfo; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.model.Service; import java.io.ByteArrayOutputStream; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import static com.yahoo.vespa.defaults.Defaults.getDefaults; @@ -24,8 +31,12 @@ public class OpenTelemetryConfigGenerator { private final String cert_file; private final String key_file; private List<StatePortInfo> statePorts = new ArrayList<>(); + private final Zone zone; + private final ApplicationId applicationId; - OpenTelemetryConfigGenerator(Zone zone) { + OpenTelemetryConfigGenerator(Zone zone, ApplicationId applicationId) { + this.zone = zone; + this.applicationId = applicationId; boolean isCd = true; boolean isPublic = true; if (zone != null) { @@ -81,8 +92,15 @@ public class OpenTelemetryConfigGenerator { if (useTls) addTls(g); { g.writeFieldName("labels"); + var dimVals = serviceAttributes(statePort.service()); + // these will be tagged as dimension-name/label-value + // attributes on all metrics from this /state/v1 port g.writeStartObject(); - g.writeStringField("service_type", statePort.serviceType()); + for (var entry : dimVals.entrySet()) { + if (entry.getValue() != null) { + g.writeStringField(entry.getKey(), entry.getValue()); + } + } g.writeEndObject(); } g.writeEndObject(); @@ -123,6 +141,37 @@ public class OpenTelemetryConfigGenerator { } g.writeEndObject(); // file } + private void addProcessors(JsonGenerator g) throws java.io.IOException { + g.writeFieldName("processors"); + g.writeStartObject(); + addResourceProcessor(g); + g.writeEndObject(); + } + private void addResourceProcessor(JsonGenerator g) throws java.io.IOException { + g.writeFieldName("resource"); + g.writeStartObject(); + g.writeFieldName("attributes"); + g.writeStartArray(); + // common attributes for all metrics from all services; + // which application and which cloud/system/zone/environment + addAttributeInsert(g, PublicDimensions.ZONE, zoneAttr()); + addAttributeInsert(g, PublicDimensions.APPLICATION_ID, appIdAttr()); + addAttributeInsert(g, "system", systemAttr()); + addAttributeInsert(g, "tenantName", tenantAttr()); + addAttributeInsert(g, "applicationName", appNameAttr()); + addAttributeInsert(g, "instanceName", appInstanceAttr()); + addAttributeInsert(g, "cloud", cloudAttr()); + g.writeEndArray(); + g.writeEndObject(); + } + private void addAttributeInsert(JsonGenerator g, String key, String value) throws java.io.IOException { + if (value == null) return; + g.writeStartObject(); + g.writeStringField("key", key); + g.writeStringField("value", value); + g.writeStringField("action", "insert"); + g.writeEndObject(); + } private void addServiceBlock(JsonGenerator g) throws java.io.IOException { g.writeFieldName("service"); g.writeStartObject(); @@ -159,6 +208,7 @@ public class OpenTelemetryConfigGenerator { } g.writeFieldName("processors"); g.writeStartArray(); + g.writeString("resource"); g.writeEndArray(); { g.writeFieldName("exporters"); @@ -186,6 +236,7 @@ public class OpenTelemetryConfigGenerator { g.writeStartObject(); addReceivers(g); addExporters(g); + addProcessors(g); addServiceBlock(g); g.writeEndObject(); // root g.close(); @@ -203,4 +254,70 @@ public class OpenTelemetryConfigGenerator { List<String> referencedPaths() { return List.of(ca_file, cert_file, key_file); } + + private String zoneAttr() { + if (zone == null) return null; + return zone.environment().value() + "." + zone.region().value(); + } + private String appIdAttr() { + if (applicationId == null) return null; + return applicationId.toFullString(); + } + private String systemAttr() { + if (zone == null) return null; + return zone.system().value(); + } + private String tenantAttr() { + if (applicationId == null) return null; + return applicationId.tenant().value(); + } + private String appNameAttr() { + if (applicationId == null) return null; + return applicationId.application().value(); + } + private String appInstanceAttr() { + if (applicationId == null) return null; + return applicationId.instance().value(); + } + private String cloudAttr() { + if (zone == null) return null; + return zone.cloud().name().value(); + } + + private String getDeploymentCluster(ClusterSpec cluster) { + if (applicationId == null) return null; + if (zone == null) return null; + String appString = applicationId.toFullString(); + return String.join(".", appString, + zone.environment().value(), + zone.region().value(), + cluster.id().value()); + } + + private Map<String, String> serviceAttributes(Service svc) { + Map<String, String> dimvals = new LinkedHashMap<>(); + dimvals.put("instance", svc.getServiceName()); // should maybe be "local_service_name" ? + dimvals.put("instanceType", svc.getServiceType()); // maybe "local_service_type", or remove + String cName = svc.getServicePropertyString("clustername", null); + if (cName != null) { + // what about "clusterid" below, is it always the same? + dimvals.put("clustername", cName); + } + String cType = svc.getServicePropertyString("clustertype", null); + if (cType != null) { + dimvals.put("clustertype", cType); + } + var hostResource = svc.getHost(); + if (hostResource != null) { + hostResource.spec().membership().map(ClusterMembership::cluster).ifPresent(cluster -> { + dimvals.put(PublicDimensions.DEPLOYMENT_CLUSTER, getDeploymentCluster(cluster)); + // overrides value above + dimvals.put(PublicDimensions.INTERNAL_CLUSTER_TYPE, cluster.type().name()); + // alternative to above + dimvals.put(PublicDimensions.INTERNAL_CLUSTER_ID, cluster.id().value()); + cluster.group().ifPresent(group -> dimvals.put(PublicDimensions.GROUP_ID, group.toString())); + }); + } + return dimvals; + } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java index 40c9a03b126..02a6b243054 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantTensorJsonValidator.java @@ -132,7 +132,7 @@ public class ConstantTensorJsonValidator { private void consumeTopObject() throws IOException { for (var cur = parser.nextToken(); cur != JsonToken.END_OBJECT; cur = parser.nextToken()) { assertCurrentTokenIs(JsonToken.FIELD_NAME); - String fieldName = parser.getCurrentName(); + String fieldName = parser.currentName(); switch (fieldName) { case FIELD_TYPE -> consumeTypeField(); case FIELD_VALUES -> consumeValuesField(); @@ -189,7 +189,7 @@ public class ConstantTensorJsonValidator { } for (var cur = parser.nextToken(); cur != JsonToken.END_OBJECT; cur = parser.nextToken()) { assertCurrentTokenIs(JsonToken.FIELD_NAME); - validateNumeric(parser.getCurrentName(), parser.nextToken()); + validateNumeric(parser.currentName(), parser.nextToken()); } } @@ -199,7 +199,7 @@ public class ConstantTensorJsonValidator { boolean seenValue = false; for (int i = 0; i < 2; i++) { assertNextTokenIs(JsonToken.FIELD_NAME); - String fieldName = parser.getCurrentName(); + String fieldName = parser.currentName(); switch (fieldName) { case FIELD_ADDRESS -> { validateTensorAddress(new HashSet<>(tensorDimensions.keySet())); @@ -228,13 +228,13 @@ public class ConstantTensorJsonValidator { // Iterate within the address key, value pairs while ((parser.nextToken() != JsonToken.END_OBJECT)) { assertCurrentTokenIs(JsonToken.FIELD_NAME); - String dimensionName = parser.getCurrentName(); + String dimensionName = parser.currentName(); TensorType.Dimension dimension = tensorDimensions.get(dimensionName); if (dimension == null) { - throw new InvalidConstantTensorException(parser, String.format("Tensor dimension '%s' does not exist", parser.getCurrentName())); + throw new InvalidConstantTensorException(parser, String.format("Tensor dimension '%s' does not exist", dimensionName)); } if (!cellDimensions.contains(dimensionName)) { - throw new InvalidConstantTensorException(parser, String.format("Duplicate tensor dimension '%s'", parser.getCurrentName())); + throw new InvalidConstantTensorException(parser, String.format("Duplicate tensor dimension '%s'", dimensionName)); } cellDimensions.remove(dimensionName); validateLabel(dimension); @@ -300,7 +300,7 @@ public class ConstantTensorJsonValidator { } private void assertCurrentTokenIs(JsonToken wantedToken) { - assertTokenIs(parser.getCurrentToken(), wantedToken); + assertTokenIs(parser.currentToken(), wantedToken); } private void assertNextTokenIs(JsonToken wantedToken) throws IOException { @@ -316,11 +316,11 @@ public class ConstantTensorJsonValidator { static class InvalidConstantTensorException extends IllegalArgumentException { InvalidConstantTensorException(JsonParser parser, String message) { - super(message + " " + parser.getCurrentLocation().toString()); + super(message + " " + parser.currentLocation().toString()); } InvalidConstantTensorException(JsonParser parser, Exception base) { - super("Failed to parse JSON stream " + parser.getCurrentLocation().toString(), base); + super("Failed to parse JSON stream " + parser.currentLocation().toString(), base); } InvalidConstantTensorException(IOException base) { @@ -412,7 +412,7 @@ public class ConstantTensorJsonValidator { boolean seenValues = false; for (int i = 0; i < 2; i++) { assertNextTokenIs(JsonToken.FIELD_NAME); - String fieldName = parser.getCurrentName(); + String fieldName = parser.currentName(); switch (fieldName) { case FIELD_ADDRESS -> { validateTensorAddress(new HashSet<>(mappedDims)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java index 9cf5fe84c21..4900b56801c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidator.java @@ -23,21 +23,22 @@ public class JvmHeapSizeValidator implements Validator { context.model().getContainerClusters().forEach((clusterId, appCluster) -> { var mp = appCluster.getMemoryPercentage().orElse(null); if (mp == null) return; - if (mp.availableMemoryGb().isEmpty()) { + if (mp.asAbsoluteGb().isEmpty()) { context.deployState().getDeployLogger().log(Level.FINE, "Host resources unknown or percentage overridden with 'allocated-memory'"); return; } long jvmModelCost = appCluster.onnxModelCostCalculator().aggregatedModelCostInBytes(); if (jvmModelCost > 0) { - double availableMemoryGb = mp.availableMemoryGb().getAsDouble(); + double availableMemoryGb = mp.asAbsoluteGb().getAsDouble(); + int percentageOfTotal = mp.ofContainerTotal().getAsInt(); double modelCostGb = jvmModelCost / (1024D * 1024 * 1024); context.deployState().getDeployLogger().log(Level.FINE, () -> Text.format("JVM: %d%% (limit: %d%%), %.2fGB (limit: %.2fGB), ONNX: %.2fGB", - mp.percentage(), percentLimit, availableMemoryGb, gbLimit, modelCostGb)); - if (mp.percentage() < percentLimit) { + percentageOfTotal, percentLimit, availableMemoryGb, gbLimit, modelCostGb)); + if (percentageOfTotal < percentLimit) { context.illegal(Text.format("Allocated percentage of memory of JVM in cluster '%s' is too low (%d%% < %d%%). " + "Estimated cost of ONNX models is %.2fGB. Either use a node flavor with more memory or use less expensive models. " + "You may override this validation by specifying 'allocated-memory' (https://docs.vespa.ai/en/performance/container-tuning.html#jvm-heap-size).", - clusterId, mp.percentage(), percentLimit, modelCostGb)); + clusterId, percentageOfTotal, percentLimit, modelCostGb)); } if (availableMemoryGb < gbLimit) { context.illegal( diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java index 4d9386b5f19..ea579aaf5d1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/QuotaValidator.java @@ -2,12 +2,13 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.CapacityPolicies; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Exclusivity; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.QuotaExceededException; import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.Zone; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.Validation.Context; @@ -31,25 +32,35 @@ public class QuotaValidator implements Validator { @Override public void validate(Context context) { + var zone = context.deployState().zone(); + var exclusivity = new Exclusivity(zone, context.deployState().featureFlags().sharedHosts()); + var capacityPolicies = new CapacityPolicies(zone, exclusivity, context.model().applicationPackage().getApplicationId(), + context.deployState().featureFlags().adminClusterArchitecture()); var quota = context.deployState().getProperties().quota(); quota.maxClusterSize().ifPresent(maxClusterSize -> validateMaxClusterSize(maxClusterSize, context.model())); - quota.budgetAsDecimal().ifPresent(budget -> validateBudget(budget, context.model(), context.deployState().getProperties().zone())); + quota.budgetAsDecimal().ifPresent(budget -> validateBudget(budget, context, capacityPolicies)); } - private void validateBudget(BigDecimal budget, VespaModel model, Zone zone) { - var maxSpend = model.allClusters().stream() - .filter(id -> !adminClusterIds(model).contains(id)) - .map(id -> model.provisioned().all().getOrDefault(id, zeroCapacity)) - .mapToDouble(c -> c.maxResources().cost()) // TODO: This may be unspecified -> 0 - .sum(); + private void validateBudget(BigDecimal budget, Context context, + CapacityPolicies capacityPolicies) { + var zone = context.deployState().getProperties().zone(); + var application = context.model().applicationPackage().getApplicationId(); + + var maxSpend = 0.0; + for (var id : context.model().allClusters()) { + if (adminClusterIds(context.model()).contains(id)) continue; + var cluster = context.model().provisioned().clusters().get(id); + var capacity = context.model().provisioned().capacities().getOrDefault(id, zeroCapacity); + maxSpend += capacityPolicies.applyOn(capacity, cluster.isExclusive()).maxResources().cost(); + } - var actualSpend = model.allocatedHosts().getHosts().stream() + var actualSpend = context.model().allocatedHosts().getHosts().stream() .filter(hostSpec -> hostSpec.membership().get().cluster().type() != ClusterSpec.Type.admin) .mapToDouble(hostSpec -> hostSpec.advertisedResources().cost()) .sum(); if (Math.abs(actualSpend) < 0.01) { - log.warning("Deploying application " + model.applicationPackage().getApplicationId() + " with zero budget use. This is suspicious, but not blocked"); + log.warning("Deploying application " + application + " with zero budget use. This is suspicious, but not blocked"); return; } @@ -69,7 +80,7 @@ public class QuotaValidator implements Validator { /** Check that all clusters in the application do not exceed the quota max cluster size. */ private void validateMaxClusterSize(int maxClusterSize, VespaModel model) { - var invalidClusters = model.provisioned().all().entrySet().stream() + var invalidClusters = model.provisioned().capacities().entrySet().stream() .filter(entry -> entry.getValue() != null) .filter(entry -> { var cluster = entry.getValue(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index ed0804f7420..7f624032627 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -19,6 +19,7 @@ import com.yahoo.vespa.model.application.validation.change.IndexingModeChangeVal import com.yahoo.vespa.model.application.validation.change.NodeResourceChangeValidator; import com.yahoo.vespa.model.application.validation.change.RedundancyIncreaseValidator; import com.yahoo.vespa.model.application.validation.change.ResourcesReductionValidator; +import com.yahoo.vespa.model.application.validation.change.RestartOnDeployForLocalLLMValidator; import com.yahoo.vespa.model.application.validation.change.RestartOnDeployForOnnxModelChangesValidator; import com.yahoo.vespa.model.application.validation.change.StartupCommandChangeValidator; import com.yahoo.vespa.model.application.validation.change.StreamingSearchClusterChangeValidator; @@ -129,6 +130,7 @@ public class Validation { new CertificateRemovalChangeValidator().validate(execution); new RedundancyValidator().validate(execution); new RestartOnDeployForOnnxModelChangesValidator().validate(execution); + new RestartOnDeployForLocalLLMValidator().validate(execution); } public interface Context { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java index 5d7a8779005..42410dc3acf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/ResourcesReductionValidator.java @@ -60,9 +60,9 @@ public class ResourcesReductionValidator implements ChangeValidator { * This will always yield specified node resources on hosted instances and never on self-hosted instances. */ private ClusterResources clusterResources(ClusterSpec.Id id, VespaModel model) { - if ( ! model.provisioned().all().containsKey(id)) return null; + if ( ! model.provisioned().capacities().containsKey(id)) return null; - ClusterResources resources = model.provisioned().all().get(id).maxResources(); + ClusterResources resources = model.provisioned().capacities().get(id).maxResources(); if ( ! resources.nodeResources().isUnspecified()) return resources; var containerCluster = model.getContainerClusters().get(id.value()); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidator.java new file mode 100644 index 00000000000..ccfc611c3dc --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidator.java @@ -0,0 +1,55 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.Validation.ChangeContext; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; + +import java.util.HashSet; +import java.util.Set; +import java.util.logging.Logger; + +import static java.util.logging.Level.INFO; +import static java.util.stream.Collectors.toUnmodifiableSet; + +/** + * If using local LLMs, this validator will make sure that restartOnDeploy is set for + * configs for this cluster. + * + * @author lesters + */ +public class RestartOnDeployForLocalLLMValidator implements ChangeValidator { + + public static final String LOCAL_LLM_COMPONENT = ai.vespa.llm.clients.LocalLLM.class.getName(); + + private static final Logger log = Logger.getLogger(RestartOnDeployForLocalLLMValidator.class.getName()); + + @Override + public void validate(ChangeContext context) { + var previousClustersWithLocalLLM = findClustersWithLocalLLMs(context.previousModel()); + var nextClustersWithLocalLLM = findClustersWithLocalLLMs(context.model()); + + // Only restart services if we use a local LLM in both the next and previous generation + for (var clusterId : intersect(previousClustersWithLocalLLM, nextClustersWithLocalLLM)) { + String message = "Need to restart services in %s due to use of local LLM".formatted(clusterId); + context.require(new VespaRestartAction(clusterId, message)); + log.log(INFO, message); + } + } + + private Set<ClusterSpec.Id> findClustersWithLocalLLMs(VespaModel model) { + return model.getContainerClusters().values().stream() + .filter(cluster -> cluster.getAllComponents().stream() + .anyMatch(component -> component.getClassId().getName().equals(LOCAL_LLM_COMPONENT))) + .map(ApplicationContainerCluster::id) + .collect(toUnmodifiableSet()); + } + + private Set<ClusterSpec.Id> intersect(Set<ClusterSpec.Id> a, Set<ClusterSpec.Id> b) { + Set<ClusterSpec.Id> result = new HashSet<>(a); + result.retainAll(b); + return result; + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java index 008a3fc5547..e57110e44e5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidator.java @@ -115,7 +115,7 @@ public class RestartOnDeployForOnnxModelChangesValidator implements ChangeValida double memoryUsedByModels = currentModelCostInGb + nextModelCostInGb; double availableMemory = Math.max(0, totalMemory - Host.memoryOverheadGb - memoryUsedByModels); - var availableMemoryPercentage = cluster.availableMemoryPercentage(); + var availableMemoryPercentage = cluster.heapSizePercentageOfAvailable(); int memoryPercentage = (int) (availableMemory / totalMemory * availableMemoryPercentage); var prefix = "Validating Onnx models memory usage for %s".formatted(cluster); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java index d877600db13..11f4c9794aa 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java @@ -137,7 +137,7 @@ public class NodesSpecification { int defaultMinGroups = nodes.from().orElse(1) / groupSize.to().orElse(nodes.from().orElse(1)); int defaultMaxGroups = groupSize.isEmpty() ? 1 : nodes.to().orElse(1) / groupSize.from().orElse(1); - var min = new ClusterResources(nodes.from().orElse(1), groups.from().orElse(defaultMinGroups), nodeResources(nodesElement).getFirst()); + var min = new ClusterResources(nodes.from().orElse(1), groups.from().orElse(defaultMinGroups), nodeResources(nodesElement).getFirst()); var max = new ClusterResources(nodes.to().orElse(1), groups.to().orElse(defaultMaxGroups), nodeResources(nodesElement).getSecond()); return new ResourceConstraints(min, max, groupSize); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java index e3fc680ad6a..1f8d7fd7520 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainer.java @@ -82,11 +82,6 @@ public final class ApplicationContainer extends Container implements builder.myid(index()); } - @Override - protected String jvmOmitStackTraceInFastThrowOption(ModelContext.FeatureFlags featureFlags) { - return featureFlags.jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type.container); - } - @Override public Optional<String> getPreShutdownCommand() { return Optional.of(prepareStopCommand(Duration.ofMinutes(6))); } private Optional<NodeResources> realResources() { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index ed7646b3066..531dc8f0fcf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -209,22 +209,27 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat if (memoryPercentage != null) return Optional.of(JvmMemoryPercentage.of(memoryPercentage)); if (isHostedVespa()) { - int availableMemoryPercentage = availableMemoryPercentage(); - if (getContainers().isEmpty()) return Optional.of(JvmMemoryPercentage.of(availableMemoryPercentage)); // Node memory is not known - - // Node memory is known so convert available memory percentage to node memory percentage - double totalMemory = getContainers().stream().mapToDouble(c -> c.getHostResource().realResources().memoryGb()).min().orElseThrow(); - double jvmHeapDeductionGb = onnxModelCostCalculator.aggregatedModelCostInBytes() / (1024D * 1024 * 1024); - double availableMemory = Math.max(0, totalMemory - Host.memoryOverheadGb - jvmHeapDeductionGb); - int memoryPercentage = (int) (availableMemory / totalMemory * availableMemoryPercentage); - logger.log(FINE, () -> "cluster id '%s': memoryPercentage=%d, availableMemory=%f, totalMemory=%f, availableMemoryPercentage=%d, jvmHeapDeductionGb=%f" - .formatted(id(), memoryPercentage, availableMemory, totalMemory, availableMemoryPercentage, jvmHeapDeductionGb)); - return Optional.of(JvmMemoryPercentage.of(memoryPercentage, availableMemory)); + int heapSizePercentageOfAvailable = heapSizePercentageOfAvailable(); + if (getContainers().isEmpty()) return Optional.of(JvmMemoryPercentage.of(heapSizePercentageOfAvailable)); // Node memory is not known + + // Node memory is known, so compute heap size as a percentage of available memory (excluding overhead, which the startup scripts also account for) + double totalMemoryGb = getContainers().stream().mapToDouble(c -> c.getHostResource().realResources().memoryGb()).min().orElseThrow(); + double totalMemoryMinusOverhead = Math.max(0, totalMemoryGb - Host.memoryOverheadGb); + double onnxModelCostGb = onnxModelCostCalculator.aggregatedModelCostInBytes() / (1024D * 1024 * 1024); + double availableMemoryGb = Math.max(0, totalMemoryMinusOverhead - onnxModelCostGb); + int memoryPercentageOfAvailable = (int) (heapSizePercentageOfAvailable * availableMemoryGb / totalMemoryMinusOverhead); + int memoryPercentageOfTotal = (int) (heapSizePercentageOfAvailable * availableMemoryGb / totalMemoryGb); + logger.log(FINE, () -> ("cluster id '%s': memoryPercentageOfAvailable=%d, memoryPercentageOfTotal=%d, " + + "availableMemoryGb=%f, totalMemoryGb=%f, heapSizePercentageOfAvailable=%d, onnxModelCostGb=%f") + .formatted(id(), memoryPercentageOfAvailable, memoryPercentageOfTotal, + availableMemoryGb, totalMemoryGb, heapSizePercentageOfAvailable, onnxModelCostGb)); + return Optional.of(JvmMemoryPercentage.of(memoryPercentageOfAvailable, memoryPercentageOfTotal, + availableMemoryGb * heapSizePercentageOfAvailable * 1e-2)); } return Optional.empty(); } - public int availableMemoryPercentage() { + public int heapSizePercentageOfAvailable() { return getHostClusterId().isPresent() ? heapSizePercentageOfTotalAvailableMemoryWhenCombinedCluster : heapSizePercentageOfAvailableMemory; @@ -310,14 +315,14 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat public void getConfig(QrStartConfig.Builder builder) { super.getConfig(builder); var memoryPct = getMemoryPercentage().orElse(null); - int heapsize = memoryPct != null && memoryPct.availableMemoryGb().isPresent() - ? (int) (memoryPct.availableMemoryGb().getAsDouble() * 1024) : 1536; + int heapsize = memoryPct != null && memoryPct.asAbsoluteGb().isPresent() + ? (int) (memoryPct.asAbsoluteGb().getAsDouble() * 1024) : 1536; builder.jvm.verbosegc(true) .availableProcessors(0) .compressedClassSpaceSize(0) .minHeapsize(heapsize) .heapsize(heapsize); - if (memoryPct != null) builder.jvm.heapSizeAsPercentageOfPhysicalMemory(memoryPct.percentage()); + if (memoryPct != null) builder.jvm.heapSizeAsPercentageOfPhysicalMemory(memoryPct.ofContainerAvailable()); } @Override diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java index f1945ca5341..c2368fd29a2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/Container.java @@ -105,15 +105,9 @@ public abstract class Container extends AbstractService implements addBuiltinHandlers(); addChild(new SimpleComponent("com.yahoo.container.jdisc.ConfiguredApplication$ApplicationContext")); - - appendJvmOptions(jvmOmitStackTraceInFastThrowOption(deployState.featureFlags())); addEnvironmentVariable("VESPA_MALLOC_MMAP_THRESHOLD","0x1000000"); // 16M } - protected String jvmOmitStackTraceInFastThrowOption(ModelContext.FeatureFlags featureFlags) { - return featureFlags.jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type.container); - } - void setOwner(ContainerCluster<?> owner) { this.owner = owner; } /** True if this container is retired (slated for removal) */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index aeb6c030a49..00ab47e8ddb 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -73,6 +73,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.OptionalDouble; +import java.util.OptionalInt; import java.util.Set; import java.util.TreeSet; @@ -721,10 +722,10 @@ public abstract class ContainerCluster<CONTAINER extends Container> * Returns the percentage of host physical memory this application has specified for nodes in this cluster, * or empty if this is not specified by the application. */ - public record JvmMemoryPercentage(int percentage, OptionalDouble availableMemoryGb) { - static JvmMemoryPercentage of(int percentage) { return new JvmMemoryPercentage(percentage, OptionalDouble.empty()); } - static JvmMemoryPercentage of(int percentage, double availableMemoryGb) { - return new JvmMemoryPercentage(percentage, OptionalDouble.of(availableMemoryGb)); + public record JvmMemoryPercentage(int ofContainerAvailable, OptionalInt ofContainerTotal, OptionalDouble asAbsoluteGb) { // optionalInt pctOfTotal < int pctOfAvailable + static JvmMemoryPercentage of(int percentageOfAvailable) { return new JvmMemoryPercentage(percentageOfAvailable, OptionalInt.empty(), OptionalDouble.empty()); } + static JvmMemoryPercentage of(int percentageOfAvailable, int percentageOfTotal, double absoluteMemoryGb) { + return new JvmMemoryPercentage(percentageOfAvailable, OptionalInt.of(percentageOfTotal), OptionalDouble.of(absoluteMemoryGb)); } } public Optional<JvmMemoryPercentage> getMemoryPercentage() { return Optional.empty(); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/SignificanceModelRegistry.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SignificanceModelRegistry.java index c210c2621a6..693eebd75a8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/SignificanceModelRegistry.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/SignificanceModelRegistry.java @@ -33,17 +33,15 @@ public class SignificanceModelRegistry extends SimpleComponent implements Signif if (spec != null) { for (Element modelElement : XML.getChildren(spec, "model")) { - addConfig( - modelElement.getAttribute("language"), - Model.fromXml(deployState, modelElement, Set.of(SIGNIFICANCE_MODEL)).modelReference()); + addConfig(Model.fromXml(deployState, modelElement, Set.of(SIGNIFICANCE_MODEL)).modelReference()); } } } - public void addConfig(String language, ModelReference path) { + public void addConfig(ModelReference path) { configList.add( - new SignificanceModelConfig(language, path) + new SignificanceModelConfig(path) ); } @@ -53,19 +51,16 @@ public class SignificanceModelRegistry extends SimpleComponent implements Signif builder.model( configList.stream() .map(config -> new SignificanceConfig.Model.Builder() - .language(config.language) .path(config.path) ).toList() ); } - class SignificanceModelConfig { - private final String language; + static class SignificanceModelConfig { private final ModelReference path; - public SignificanceModelConfig(String language, ModelReference path) { - this.language = language; + public SignificanceModelConfig(ModelReference path) { this.path = path; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 56e2a21e38b..4983b36bee1 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -815,8 +815,9 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { !container.getHostResource().realResources().gpuResources().isZero()); onnxModel.setGpuDevice(gpuDevice, hasGpu); } - cluster.onnxModelCostCalculator().registerModel(context.getApplicationPackage().getFile(onnxModel.getFilePath()), onnxModel.onnxModelOptions()); } + for (OnnxModel onnxModel : models.asMap().values()) + cluster.onnxModelCostCalculator().registerModel(context.getApplicationPackage().getFile(onnxModel.getFilePath()), onnxModel.onnxModelOptions()); cluster.setModelEvaluation(new ContainerModelEvaluation(cluster, profiles, models)); } @@ -1037,7 +1038,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { } catch (NumberFormatException e) { throw new IllegalArgumentException("The memory percentage given for nodes in " + cluster + - " must be an integer percentage ending by the '%' sign", e); + " must be given as an integer followe by '%'", e); } } @@ -1072,9 +1073,21 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { return List.of(node); } + private static void requireFixedSizeSingularNodeIfTester(ConfigModelContext context, NodesSpecification nodes) { + if ( ! context.properties().hostedVespa() || ! context.properties().applicationId().instance().isTester()) + return; + + if ( ! nodes.maxResources().equals(nodes.minResources())) + throw new IllegalArgumentException("tester resources must be absolute, but min and max resources differ: " + nodes); + + if (nodes.maxResources().nodes() > 1) + throw new IllegalArgumentException("tester cannot run on more than 1 node, but " + nodes.maxResources().nodes() + " nodes were specified"); + } + private List<ApplicationContainer> createNodesFromNodeCount(ApplicationContainerCluster cluster, Element containerElement, Element nodesElement, ConfigModelContext context) { try { var nodesSpecification = NodesSpecification.from(new ModelElement(nodesElement), context); + requireFixedSizeSingularNodeIfTester(context, nodesSpecification); var clusterId = ClusterSpec.Id.from(cluster.name()); Map<HostResource, ClusterMembership> hosts = nodesSpecification.provision(cluster.getRoot().hostSystem(), ClusterSpec.Type.container, diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java index 15f3eece8a9..c39da176828 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DispatchTuning.java @@ -22,11 +22,6 @@ public class DispatchTuning { private final Integer maxHitsPerPartition; private final DispatchPolicy dispatchPolicy; private final Double minActiveDocsCoverage; - - public Double getTopkProbability() { - return topkProbability; - } - private final Double topkProbability; private DispatchTuning(Builder builder) { @@ -45,6 +40,8 @@ public class DispatchTuning { /** Returns the percentage of documents which must be available in a group for that group to receive queries */ public Double getMinActiveDocsCoverage() { return minActiveDocsCoverage; } + public Double getTopkProbability() { return topkProbability; } + public static class Builder { private Integer maxHitsPerPartition; @@ -71,14 +68,14 @@ public class DispatchTuning { } public static DispatchPolicy toDispatchPolicy(String policy) { - switch (policy.toLowerCase()) { - case "adaptive": case "random": return DispatchPolicy.ADAPTIVE; // TODO: Deprecate 'random' on Vespa 9 - case "round-robin": return DispatchPolicy.ROUNDROBIN; - case "latency-amortized-over-requests" : return DispatchPolicy.LATENCY_AMORTIZED_OVER_REQUESTS; - case "latency-amortized-over-time" : return DispatchPolicy.LATENCY_AMORTIZED_OVER_TIME; - case "best-of-random-2" : return DispatchPolicy.BEST_OF_RANDOM_2; - default: throw new IllegalArgumentException("Unknown dispatch policy '" + policy + "'"); - } + return switch (policy.toLowerCase()) { + case "adaptive", "random" -> DispatchPolicy.ADAPTIVE; // TODO: Deprecate 'random' on Vespa 9 + case "round-robin" -> DispatchPolicy.ROUNDROBIN; + case "latency-amortized-over-requests" -> DispatchPolicy.LATENCY_AMORTIZED_OVER_REQUESTS; + case "latency-amortized-over-time" -> DispatchPolicy.LATENCY_AMORTIZED_OVER_TIME; + case "best-of-random-2" -> DispatchPolicy.BEST_OF_RANDOM_2; + default -> throw new IllegalArgumentException("Unknown dispatch policy '" + policy + "'"); + }; } public Builder setMinActiveDocsCoverage(Double minCoverage) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java index c182f0e0507..1e35a94db59 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java @@ -35,6 +35,8 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen private final boolean hasIndexedDocumentType; private final int maxActivationInhibitedOutOfSyncGroups; private final int contentLayerMetadataFeatureLevel; + private final boolean symmetricPutAndActivateReplicaSelection; + public static class Builder extends VespaDomBuilder.DomConfigProducerBuilderBase<DistributorCluster> { ContentCluster parent; @@ -97,19 +99,22 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen var featureFlags = deployState.getProperties().featureFlags(); int maxInhibitedGroups = featureFlags.maxActivationInhibitedOutOfSyncGroups(); int contentLayerMetadataFeatureLevel = featureFlags.contentLayerMetadataFeatureLevel(); + boolean symmetricPutAndActivateReplicaSelection = featureFlags.symmetricPutAndActivateReplicaSelection(); return new DistributorCluster(parent, new BucketSplitting.Builder().build(new ModelElement(producerSpec)), gc, hasIndexedDocumentType, maxInhibitedGroups, - contentLayerMetadataFeatureLevel); + contentLayerMetadataFeatureLevel, + symmetricPutAndActivateReplicaSelection); } } private DistributorCluster(ContentCluster parent, BucketSplitting bucketSplitting, GcOptions gc, boolean hasIndexedDocumentType, int maxActivationInhibitedOutOfSyncGroups, - int contentLayerMetadataFeatureLevel) + int contentLayerMetadataFeatureLevel, + boolean symmetricPutAndActivateReplicaSelection) { super(parent, "distributor"); this.parent = parent; @@ -118,6 +123,7 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen this.hasIndexedDocumentType = hasIndexedDocumentType; this.maxActivationInhibitedOutOfSyncGroups = maxActivationInhibitedOutOfSyncGroups; this.contentLayerMetadataFeatureLevel = contentLayerMetadataFeatureLevel; + this.symmetricPutAndActivateReplicaSelection = symmetricPutAndActivateReplicaSelection; } @Override @@ -132,6 +138,7 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen if (contentLayerMetadataFeatureLevel > 0) { builder.enable_operation_cancellation(true); } + builder.symmetric_put_and_activate_replica_selection(symmetricPutAndActivateReplicaSelection); bucketSplitting.getConfig(builder); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index bac86e37e8f..d37e5d5382a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -46,6 +46,7 @@ import com.yahoo.vespa.model.content.IndexedHierarchicDistributionValidator; import com.yahoo.vespa.model.content.Redundancy; import com.yahoo.vespa.model.content.ReservedDocumentTypeNameValidator; import com.yahoo.vespa.model.content.StorageGroup; +import com.yahoo.vespa.model.content.StorageNode; import com.yahoo.vespa.model.content.engines.PersistenceEngine; import com.yahoo.vespa.model.content.engines.ProtonEngine; import com.yahoo.vespa.model.content.storagecluster.StorageCluster; @@ -137,6 +138,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem c.rootGroup = new StorageGroup.Builder(contentElement, context).buildRootGroup(deployState, c, c.search.isStreaming()); c.clusterControllerConfig = createClusterControllerConfig(contentElement, deployState, c, resourceLimits); validateThatGroupSiblingsAreUnique(c.clusterId, c.rootGroup); + warnIfDistributionKeyRangeIsSuboptimal(c.clusterId, c.rootGroup, deployState); c.search.handleRedundancy(c.redundancy); setupSearchCluster(c.search, contentElement, deployState.getDeployLogger()); @@ -247,7 +249,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem for (ContainerModel containerModel : containers) { Optional<String> hostClusterId = containerModel.getCluster().getHostClusterId(); if (hostClusterId.isPresent() && hostClusterId.get().equals(clusterId) && containerModel.getCluster().getMemoryPercentage().isPresent()) { - return containerModel.getCluster().getMemoryPercentage().get().percentage() * 0.01; + return containerModel.getCluster().getMemoryPercentage().get().ofContainerAvailable() * 0.01; } } return 0.0; @@ -275,6 +277,38 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem } } + private static class HighestDistributionKeyAggregator { + public int nodeCount = 0; + public int highestNodeDistributionKey = 0; + + void aggregateNodeStats(StorageGroup group) { + for (StorageNode n : group.getNodes()) { + nodeCount++; + highestNodeDistributionKey = Math.max(highestNodeDistributionKey, n.getDistributionKey()); + } + for (StorageGroup g : group.getSubgroups()) { + aggregateNodeStats(g); + } + } + } + + private void warnIfDistributionKeyRangeIsSuboptimal(String clusterId, StorageGroup rootGroup, DeployState deployState) { + if (rootGroup == null) { + return; // Unit testing case + } + var aggr = new HighestDistributionKeyAggregator(); + aggr.aggregateNodeStats(rootGroup); + int warnThreshold = 100; // ... Not scientifically chosen + if ((aggr.highestNodeDistributionKey - aggr.nodeCount) >= warnThreshold) { + deployState.getDeployLogger().logApplicationPackage(WARNING, + ("Content cluster '%s' has %d node(s), but the highest distribution key is %d. " + + "Having much higher distribution keys than the number of nodes is not recommended, " + + "as it may negatively affect performance. " + + "See https://docs.vespa.ai/en/reference/services-content.html#node") + .formatted(clusterId, aggr.nodeCount, aggr.highestNodeDistributionKey)); + } + } + private void addClusterControllers(ConfigModelContext context, ModelElement contentElement, ContentCluster contentCluster, diff --git a/config-model/src/main/javacc/SchemaParser.jj b/config-model/src/main/javacc/SchemaParser.jj index 255cc3cde70..1365c133932 100644 --- a/config-model/src/main/javacc/SchemaParser.jj +++ b/config-model/src/main/javacc/SchemaParser.jj @@ -183,11 +183,14 @@ TOKEN : | < GRAM_SIZE: "gram-size" > | < MAX_LENGTH: "max-length" > | < MAX_OCCURRENCES: "max-occurrences" > +| < MAX_TOKEN_LENGTH: "max-token-length" > | < PREFIX: "prefix" > | < SUBSTRING: "substring" > | < SUFFIX: "suffix" > | < CONSTANT: "constant"> | < ONNX_MODEL: "onnx-model"> +| < SIGNIFICANCE: "significance"> +| < USE_MODEL: "use-model"> | < INTRAOP_THREADS: "intraop-threads"> | < INTEROP_THREADS: "interop-threads"> | < GPU_DEVICE: "gpu-device"> @@ -1366,7 +1369,8 @@ void matchType(ParsedMatchSettings matchInfo) : { } */ void matchItem(ParsedMatchSettings matchInfo) : { } { - ( matchType(matchInfo) | exactTerminator(matchInfo) | gramSize(matchInfo) | matchSize(matchInfo) | maxTermOccurrences(matchInfo)) + ( matchType(matchInfo) | exactTerminator(matchInfo) | gramSize(matchInfo) | matchSize(matchInfo) | + maxTermOccurrences(matchInfo) | maxTokenLength(matchInfo) ) } void exactTerminator(ParsedMatchSettings matchInfo) : @@ -1411,6 +1415,16 @@ void maxTermOccurrences(ParsedMatchSettings matchInfo) : } } +void maxTokenLength(ParsedMatchSettings matchInfo) : +{ + int maxTokenLength; +} +{ + <MAX_TOKEN_LENGTH> <COLON> maxTokenLength = integer() { + matchInfo.setMaxTokenLength(maxTokenLength); + } +} + /** * Consumes a rank statement of a field element. * @@ -1761,7 +1775,8 @@ void rankProfileItem(ParsedSchema schema, ParsedRankProfile profile) : { } | matchFeatures(profile) | summaryFeatures(profile) | onnxModelInProfile(profile) - | strict(profile) ) + | strict(profile) + | significance(profile)) } /** @@ -2115,6 +2130,22 @@ void strict(ParsedRankProfile profile) : ) } +void significance(ParsedRankProfile profile) : +{} +{ + <SIGNIFICANCE> lbrace() (significanceItem(profile) (<NL>)*)* <RBRACE> + {} +} + +void significanceItem(ParsedRankProfile profile) : +{} +{ + <USE_MODEL> <COLON> ( + ( <TRUE> { profile.setUseSignificanceModel(true); } ) | + ( <FALSE> { profile.setUseSignificanceModel(false); } ) + ) +} + /** * Consumes a match-features block of a rank profile. * @@ -2710,6 +2741,7 @@ String identifierWithDash() : | <TARGET_HITS_MAX_ADJUSTMENT_FACTOR> | <TERMWISE_LIMIT> | <UPPER_BOUND> + | <USE_MODEL> ) { return token.image; } } @@ -2812,6 +2844,7 @@ String identifier() : { } | <STEMMING> | <STRENGTH> | <STRICT> + | <SIGNIFICANCE> | <STRING> | <STRUCT> | <SUBSTRING> diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 08092f10020..c79a7b38d09 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -138,7 +138,7 @@ Threadpool = element threadpool { } Significance = element significance { - element model { attribute language { xsd:string } & ModelReference }* + element model { ModelReference }* } Clients = element clients { diff --git a/config-model/src/main/resources/schema/deployment.rnc b/config-model/src/main/resources/schema/deployment.rnc index 3491d868f20..f79fc614a53 100644 --- a/config-model/src/main/resources/schema/deployment.rnc +++ b/config-model/src/main/resources/schema/deployment.rnc @@ -2,6 +2,8 @@ # RELAX NG Compact Syntax # Vespa Deployment file +include "common.rnc" + start = element deployment { attribute version { "1.0" } & attribute major-version { text }? & @@ -100,7 +102,7 @@ Test = element test { attribute tester-flavor { xsd:string }? & attribute cloud-account { xsd:string }? & attribute empty-host-ttl { xsd:string }? & - text + Tester? } Staging = element staging { @@ -108,7 +110,7 @@ Staging = element staging { attribute tester-flavor { xsd:string }? & attribute cloud-account { xsd:string }? & attribute empty-host-ttl { xsd:string }? & - text + Tester? } Dev = element dev { @@ -129,7 +131,8 @@ Prod = element prod { Region* & Delay* & ProdTest* & - ParallelSteps* + ParallelSteps* & + Tester? } ProdTest = element test { @@ -197,3 +200,7 @@ MemberRegion = element region { attribute fraction { xsd:double }? & text } + +Tester = element tester { + Nodes? +}
\ No newline at end of file diff --git a/config-model/src/test/cfg/application/ml_serving/services.xml b/config-model/src/test/cfg/application/ml_serving/services.xml index 3a5a4438c78..b1271b1297f 100644 --- a/config-model/src/test/cfg/application/ml_serving/services.xml +++ b/config-model/src/test/cfg/application/ml_serving/services.xml @@ -3,7 +3,13 @@ <services version="1.0"> <container version="1.0"> - <model-evaluation/> + <model-evaluation> + <onnx> + <models> + <model name="sqrt" /> <!-- list one of the models --> + </models> + </onnx> + </model-evaluation> <nodes> <node hostalias="node1" /> </nodes> diff --git a/config-model/src/test/cfg/significance/services.xml b/config-model/src/test/cfg/significance/services.xml index 6991f5498fb..ffdb73bfc2e 100644 --- a/config-model/src/test/cfg/significance/services.xml +++ b/config-model/src/test/cfg/significance/services.xml @@ -8,9 +8,9 @@ <container version="1.0"> <search> <significance> - <model language="en" model-id="idf-wiki-english" path="models/idf-english-wiki.json.zst"/> - <model language="no" path="models/idf-norwegian-wiki.json.zst" /> - <model language="ru" url="https://some/uri/blob.json" /> + <model model-id="idf-wiki-english" path="models/idf-english-wiki.json.zst"/> + <model path="models/idf-norwegian-wiki.json.zst" /> + <model url="https://some/uri/blob.json" /> </significance> </search> </container> diff --git a/config-model/src/test/derived/advanced/ilscripts.cfg b/config-model/src/test/derived/advanced/ilscripts.cfg index 51a49502b64..d633cd97f0c 100644 --- a/config-model/src/test/derived/advanced/ilscripts.cfg +++ b/config-model/src/test/derived/advanced/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "advanced" ilscript[].docfield[] "debug_src" diff --git a/config-model/src/test/derived/annotationsimplicitstruct/ilscripts.cfg b/config-model/src/test/derived/annotationsimplicitstruct/ilscripts.cfg index 767c3af3c19..53dc789fbb7 100644 --- a/config-model/src/test/derived/annotationsimplicitstruct/ilscripts.cfg +++ b/config-model/src/test/derived/annotationsimplicitstruct/ilscripts.cfg @@ -1,3 +1,4 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "annotationsimplicitstruct" diff --git a/config-model/src/test/derived/annotationsinheritance/ilscripts.cfg b/config-model/src/test/derived/annotationsinheritance/ilscripts.cfg index d8e6c882b80..b0a69c5408a 100644 --- a/config-model/src/test/derived/annotationsinheritance/ilscripts.cfg +++ b/config-model/src/test/derived/annotationsinheritance/ilscripts.cfg @@ -1,3 +1,4 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "annotationsinheritance" diff --git a/config-model/src/test/derived/annotationsinheritance2/ilscripts.cfg b/config-model/src/test/derived/annotationsinheritance2/ilscripts.cfg index ae4ea621583..5ec1f839429 100644 --- a/config-model/src/test/derived/annotationsinheritance2/ilscripts.cfg +++ b/config-model/src/test/derived/annotationsinheritance2/ilscripts.cfg @@ -1,3 +1,4 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "annotationsinheritance2" diff --git a/config-model/src/test/derived/annotationsreference/ilscripts.cfg b/config-model/src/test/derived/annotationsreference/ilscripts.cfg index 812f5e44545..eaa20043be8 100644 --- a/config-model/src/test/derived/annotationsreference/ilscripts.cfg +++ b/config-model/src/test/derived/annotationsreference/ilscripts.cfg @@ -1,3 +1,4 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "annotationsreference" diff --git a/config-model/src/test/derived/annotationssimple/ilscripts.cfg b/config-model/src/test/derived/annotationssimple/ilscripts.cfg index 9d0962df5be..af179221eb4 100644 --- a/config-model/src/test/derived/annotationssimple/ilscripts.cfg +++ b/config-model/src/test/derived/annotationssimple/ilscripts.cfg @@ -1,3 +1,4 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "annotationssimple" diff --git a/config-model/src/test/derived/arrays/ilscripts.cfg b/config-model/src/test/derived/arrays/ilscripts.cfg index 98cff642d9e..3f2dae48552 100644 --- a/config-model/src/test/derived/arrays/ilscripts.cfg +++ b/config-model/src/test/derived/arrays/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "arrays" ilscript[].docfield[] "tags" diff --git a/config-model/src/test/derived/attributeprefetch/ilscripts.cfg b/config-model/src/test/derived/attributeprefetch/ilscripts.cfg index dec054b33f0..5a3784f7cb9 100644 --- a/config-model/src/test/derived/attributeprefetch/ilscripts.cfg +++ b/config-model/src/test/derived/attributeprefetch/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "prefetch" ilscript[].docfield[] "singlebyte" diff --git a/config-model/src/test/derived/attributes/ilscripts.cfg b/config-model/src/test/derived/attributes/ilscripts.cfg index 6d3ef2799d9..58279759e5f 100644 --- a/config-model/src/test/derived/attributes/ilscripts.cfg +++ b/config-model/src/test/derived/attributes/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "attributes" ilscript[].docfield[] "a1" diff --git a/config-model/src/test/derived/attributes/index-info.cfg b/config-model/src/test/derived/attributes/index-info.cfg index 1d4e8f485b3..245cff48d15 100644 --- a/config-model/src/test/derived/attributes/index-info.cfg +++ b/config-model/src/test/derived/attributes/index-info.cfg @@ -175,8 +175,6 @@ indexinfo[].command[].indexname "a13" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "a13" indexinfo[].command[].command "type tensor(x{})" -indexinfo[].command[].indexname "a13" -indexinfo[].command[].command "word" indexinfo[].command[].indexname "a7_arr" indexinfo[].command[].command "lowercase" indexinfo[].command[].indexname "a7_arr" diff --git a/config-model/src/test/derived/bolding_dynamic_summary/ilscripts.cfg b/config-model/src/test/derived/bolding_dynamic_summary/ilscripts.cfg index c20c321ebcf..0b925da4778 100644 --- a/config-model/src/test/derived/bolding_dynamic_summary/ilscripts.cfg +++ b/config-model/src/test/derived/bolding_dynamic_summary/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "test" ilscript[].docfield[] "str_1" diff --git a/config-model/src/test/derived/complex/ilscripts.cfg b/config-model/src/test/derived/complex/ilscripts.cfg index 4405d2fda40..7d025e15703 100644 --- a/config-model/src/test/derived/complex/ilscripts.cfg +++ b/config-model/src/test/derived/complex/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "complex" ilscript[].docfield[] "title" diff --git a/config-model/src/test/derived/emptydefault/ilscripts.cfg b/config-model/src/test/derived/emptydefault/ilscripts.cfg index e4242153bce..bbb8e5c556c 100644 --- a/config-model/src/test/derived/emptydefault/ilscripts.cfg +++ b/config-model/src/test/derived/emptydefault/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "emptydefault" ilscript[].docfield[] "one" diff --git a/config-model/src/test/derived/exactmatch/ilscripts.cfg b/config-model/src/test/derived/exactmatch/ilscripts.cfg index 21dfbd1371b..1d1bd6d5e8a 100644 --- a/config-model/src/test/derived/exactmatch/ilscripts.cfg +++ b/config-model/src/test/derived/exactmatch/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "exactmatch" ilscript[].docfield[] "tag" diff --git a/config-model/src/test/derived/hnsw_index/ilscripts.cfg b/config-model/src/test/derived/hnsw_index/ilscripts.cfg index e48f116f468..c811b93c3df 100644 --- a/config-model/src/test/derived/hnsw_index/ilscripts.cfg +++ b/config-model/src/test/derived/hnsw_index/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "test" ilscript[].docfield[] "t1" diff --git a/config-model/src/test/derived/id/ilscripts.cfg b/config-model/src/test/derived/id/ilscripts.cfg index d3ab29f6cd8..121e305059e 100644 --- a/config-model/src/test/derived/id/ilscripts.cfg +++ b/config-model/src/test/derived/id/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "id" ilscript[].docfield[] "uri" diff --git a/config-model/src/test/derived/imported_position_field_summary/schema-info.cfg b/config-model/src/test/derived/imported_position_field_summary/schema-info.cfg index f820ad9720b..5a474f62e07 100644 --- a/config-model/src/test/derived/imported_position_field_summary/schema-info.cfg +++ b/config-model/src/test/derived/imported_position_field_summary/schema-info.cfg @@ -53,6 +53,8 @@ schema[].summaryclass[].fields[].dynamic false schema[].rankprofile[].name "default" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "unranked" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false diff --git a/config-model/src/test/derived/indexswitches/ilscripts.cfg b/config-model/src/test/derived/indexswitches/ilscripts.cfg index 472c1f95cb0..454f675c0a2 100644 --- a/config-model/src/test/derived/indexswitches/ilscripts.cfg +++ b/config-model/src/test/derived/indexswitches/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "indexswitches" ilscript[].docfield[] "title" diff --git a/config-model/src/test/derived/inheritance/ilscripts.cfg b/config-model/src/test/derived/inheritance/ilscripts.cfg index d4c804773f0..c966f32a502 100644 --- a/config-model/src/test/derived/inheritance/ilscripts.cfg +++ b/config-model/src/test/derived/inheritance/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "child" ilscript[].docfield[] "onlygrandparent" diff --git a/config-model/src/test/derived/language/ilscripts.cfg b/config-model/src/test/derived/language/ilscripts.cfg index 1860f180839..d0abc08f1e0 100644 --- a/config-model/src/test/derived/language/ilscripts.cfg +++ b/config-model/src/test/derived/language/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "language" ilscript[].docfield[] "language" diff --git a/config-model/src/test/derived/lowercase/ilscripts.cfg b/config-model/src/test/derived/lowercase/ilscripts.cfg index 8ba4bfa3349..49515e50df4 100644 --- a/config-model/src/test/derived/lowercase/ilscripts.cfg +++ b/config-model/src/test/derived/lowercase/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "lowercase" ilscript[].docfield[] "single_field_source" diff --git a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg index 0cdf921de25..4a6de4154f8 100644 --- a/config-model/src/test/derived/multiplesummaries/ilscripts.cfg +++ b/config-model/src/test/derived/multiplesummaries/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "multiplesummaries" ilscript[].docfield[] "a" diff --git a/config-model/src/test/derived/music/ilscripts.cfg b/config-model/src/test/derived/music/ilscripts.cfg index f90cdb15baa..f79e8824b69 100644 --- a/config-model/src/test/derived/music/ilscripts.cfg +++ b/config-model/src/test/derived/music/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "music" ilscript[].docfield[] "bgndata" diff --git a/config-model/src/test/derived/nearestneighbor_streaming/vsmfields.cfg b/config-model/src/test/derived/nearestneighbor_streaming/vsmfields.cfg index ab9a96f819b..ec06d01f05a 100644 --- a/config-model/src/test/derived/nearestneighbor_streaming/vsmfields.cfg +++ b/config-model/src/test/derived/nearestneighbor_streaming/vsmfields.cfg @@ -3,25 +3,25 @@ searchall 1 fieldspec[].name "vec_a" fieldspec[].searchmethod NEAREST_NEIGHBOR fieldspec[].arg1 "EUCLIDEAN" -fieldspec[].normalize LOWERCASE +fieldspec[].normalize LOWERCASE_AND_FOLD fieldspec[].maxlength 1048576 fieldspec[].fieldtype ATTRIBUTE fieldspec[].name "vec_b" fieldspec[].searchmethod NEAREST_NEIGHBOR fieldspec[].arg1 "ANGULAR" -fieldspec[].normalize LOWERCASE +fieldspec[].normalize LOWERCASE_AND_FOLD fieldspec[].maxlength 1048576 fieldspec[].fieldtype ATTRIBUTE fieldspec[].name "vec_c" fieldspec[].searchmethod NEAREST_NEIGHBOR fieldspec[].arg1 "INNERPRODUCT" -fieldspec[].normalize LOWERCASE +fieldspec[].normalize LOWERCASE_AND_FOLD fieldspec[].maxlength 1048576 fieldspec[].fieldtype ATTRIBUTE fieldspec[].name "vec_d" fieldspec[].searchmethod NONE fieldspec[].arg1 "" -fieldspec[].normalize LOWERCASE +fieldspec[].normalize LOWERCASE_AND_FOLD fieldspec[].maxlength 1048576 fieldspec[].fieldtype ATTRIBUTE documenttype[].name "test" diff --git a/config-model/src/test/derived/neuralnet_noqueryprofile/schema-info.cfg b/config-model/src/test/derived/neuralnet_noqueryprofile/schema-info.cfg index 728856abbf2..8f59c21e97f 100644 --- a/config-model/src/test/derived/neuralnet_noqueryprofile/schema-info.cfg +++ b/config-model/src/test/derived/neuralnet_noqueryprofile/schema-info.cfg @@ -156,6 +156,7 @@ schema[].summaryclass[].fields[].dynamic false schema[].rankprofile[].name "default" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].input[].name "query(W_0)" schema[].rankprofile[].input[].type "tensor(hidden[9],x[9])" schema[].rankprofile[].input[].name "query(b_0)" @@ -173,9 +174,11 @@ schema[].rankprofile[].input[].type "tensor()" schema[].rankprofile[].name "unranked" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "defaultRankProfile" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].input[].name "query(W_0)" schema[].rankprofile[].input[].type "tensor(hidden[9],x[9])" schema[].rankprofile[].input[].name "query(b_0)" @@ -193,6 +196,7 @@ schema[].rankprofile[].input[].type "tensor()" schema[].rankprofile[].name "neuralNetworkProfile" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].input[].name "query(W_0)" schema[].rankprofile[].input[].type "tensor(hidden[9],x[9])" schema[].rankprofile[].input[].name "query(b_0)" diff --git a/config-model/src/test/derived/newrank/ilscripts.cfg b/config-model/src/test/derived/newrank/ilscripts.cfg index b02e09a0496..487d2fca902 100644 --- a/config-model/src/test/derived/newrank/ilscripts.cfg +++ b/config-model/src/test/derived/newrank/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "newrank" ilscript[].docfield[] "bgndata" diff --git a/config-model/src/test/derived/orderilscripts/ilscripts.cfg b/config-model/src/test/derived/orderilscripts/ilscripts.cfg index 0ed1589af0a..4918e23efc6 100644 --- a/config-model/src/test/derived/orderilscripts/ilscripts.cfg +++ b/config-model/src/test/derived/orderilscripts/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "orderilscripts" ilscript[].docfield[] "foo" diff --git a/config-model/src/test/derived/position_array/ilscripts.cfg b/config-model/src/test/derived/position_array/ilscripts.cfg index ecafbc4a025..3f7611b25d8 100644 --- a/config-model/src/test/derived/position_array/ilscripts.cfg +++ b/config-model/src/test/derived/position_array/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "position_array" ilscript[].docfield[] "pos" diff --git a/config-model/src/test/derived/position_attribute/ilscripts.cfg b/config-model/src/test/derived/position_attribute/ilscripts.cfg index d2fc8503ce5..fbd1a293418 100644 --- a/config-model/src/test/derived/position_attribute/ilscripts.cfg +++ b/config-model/src/test/derived/position_attribute/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "position_attribute" ilscript[].docfield[] "pos" diff --git a/config-model/src/test/derived/position_extra/ilscripts.cfg b/config-model/src/test/derived/position_extra/ilscripts.cfg index a86dcec92ec..4645798723c 100644 --- a/config-model/src/test/derived/position_extra/ilscripts.cfg +++ b/config-model/src/test/derived/position_extra/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "position_extra" ilscript[].docfield[] "pos_str" diff --git a/config-model/src/test/derived/prefixexactattribute/ilscripts.cfg b/config-model/src/test/derived/prefixexactattribute/ilscripts.cfg index 40c7843a0a4..2d1904cf9d8 100644 --- a/config-model/src/test/derived/prefixexactattribute/ilscripts.cfg +++ b/config-model/src/test/derived/prefixexactattribute/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "prefixexactattribute" ilscript[].docfield[] "indexfield0" diff --git a/config-model/src/test/derived/rankingexpression/schema-info.cfg b/config-model/src/test/derived/rankingexpression/schema-info.cfg index 5bf01f10ede..f78eb7de310 100644 --- a/config-model/src/test/derived/rankingexpression/schema-info.cfg +++ b/config-model/src/test/derived/rankingexpression/schema-info.cfg @@ -148,96 +148,125 @@ schema[].summaryclass[].fields[].dynamic false schema[].rankprofile[].name "default" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures true +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "unranked" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "static" schema[].rankprofile[].hasSummaryFeatures true schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "overflow" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "duplicates" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "whitespace1" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "whitespace2" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros" schema[].rankprofile[].hasSummaryFeatures true schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros2" schema[].rankprofile[].hasSummaryFeatures true schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros3" schema[].rankprofile[].hasSummaryFeatures true schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros3-inherited" schema[].rankprofile[].hasSummaryFeatures true schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros-inherited" schema[].rankprofile[].hasSummaryFeatures true schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros-inherited2" schema[].rankprofile[].hasSummaryFeatures true schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros-inherited3" schema[].rankprofile[].hasSummaryFeatures true schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros-refering-macros" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros-refering-macros-inherited" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros-refering-macros-inherited2" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "macros-refering-macros-inherited-two-levels" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "withmf" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "withboolean" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "withglobalphase" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "layered" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].input[].name "query(v)" schema[].rankprofile[].input[].type "tensor(v[3])" schema[].rankprofile[].name "withtfl" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].input[].name "query(v)" schema[].rankprofile[].input[].type "tensor(v[3])" schema[].rankprofile[].name "withtfl2" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].input[].name "query(v)" schema[].rankprofile[].input[].type "tensor(v[3])" schema[].rankprofile[].name "withnorm" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "withfusion" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "function-with-arg-as-summary-feature" schema[].rankprofile[].hasSummaryFeatures true schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "function-with-arg-in-global-phase" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "withstringcompare" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].input[].name "query(myquerystring)" schema[].rankprofile[].input[].type "string" schema[].rankprofile[].input[].name "query(mybadlong)" diff --git a/config-model/src/test/derived/rankprofilemodularity/schema-info.cfg b/config-model/src/test/derived/rankprofilemodularity/schema-info.cfg index 377c10d3293..68892737e63 100644 --- a/config-model/src/test/derived/rankprofilemodularity/schema-info.cfg +++ b/config-model/src/test/derived/rankprofilemodularity/schema-info.cfg @@ -18,24 +18,32 @@ schema[].summaryclass[].fields[].dynamic false schema[].rankprofile[].name "default" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "unranked" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "in_schema0" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "in_schema1" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "in_schema2" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "in_schema3" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "outside_schema1" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "outside_schema2" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false diff --git a/config-model/src/test/derived/rankprofilemodularity2/invalid_comment.profile b/config-model/src/test/derived/rankprofilemodularity2/invalid_comment.profile new file mode 100644 index 00000000000..40e77c8c6be --- /dev/null +++ b/config-model/src/test/derived/rankprofilemodularity2/invalid_comment.profile @@ -0,0 +1,8 @@ +rank-profile outside_schema1 { + + // Comment with wrong comment character + function foo() { + expression: now + } + +} diff --git a/config-model/src/test/derived/ranktypes/ilscripts.cfg b/config-model/src/test/derived/ranktypes/ilscripts.cfg index adcd2f70c70..22526d1aa23 100644 --- a/config-model/src/test/derived/ranktypes/ilscripts.cfg +++ b/config-model/src/test/derived/ranktypes/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "ranktypes" ilscript[].docfield[] "title" diff --git a/config-model/src/test/derived/schemainheritance/ilscripts.cfg b/config-model/src/test/derived/schemainheritance/ilscripts.cfg index f7324920fe7..b1ba947f1dc 100644 --- a/config-model/src/test/derived/schemainheritance/ilscripts.cfg +++ b/config-model/src/test/derived/schemainheritance/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "child" ilscript[].docfield[] "pf1" diff --git a/config-model/src/test/derived/schemainheritance/schema-info.cfg b/config-model/src/test/derived/schemainheritance/schema-info.cfg index 9fe71780c7a..466e66ad0bb 100644 --- a/config-model/src/test/derived/schemainheritance/schema-info.cfg +++ b/config-model/src/test/derived/schemainheritance/schema-info.cfg @@ -116,12 +116,16 @@ schema[].summaryclass[].fields[].dynamic false schema[].rankprofile[].name "default" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "unranked" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "child_profile" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false schema[].rankprofile[].name "parent_profile" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false +schema[].rankprofile[].significance.useModel false diff --git a/config-model/src/test/derived/structanyorder/ilscripts.cfg b/config-model/src/test/derived/structanyorder/ilscripts.cfg index c07f04b3021..a806bc1b712 100644 --- a/config-model/src/test/derived/structanyorder/ilscripts.cfg +++ b/config-model/src/test/derived/structanyorder/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "annotationsimplicitstruct" ilscript[].docfield[] "structfield" diff --git a/config-model/src/test/derived/tensor/index-info.cfg b/config-model/src/test/derived/tensor/index-info.cfg index c9ce2433e17..2402f074837 100644 --- a/config-model/src/test/derived/tensor/index-info.cfg +++ b/config-model/src/test/derived/tensor/index-info.cfg @@ -9,26 +9,18 @@ indexinfo[].command[].indexname "f2" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "f2" indexinfo[].command[].command "type tensor<float>(x[2],y[1])" -indexinfo[].command[].indexname "f2" -indexinfo[].command[].command "word" indexinfo[].command[].indexname "f3" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "f3" indexinfo[].command[].command "type tensor(x{})" -indexinfo[].command[].indexname "f3" -indexinfo[].command[].command "word" indexinfo[].command[].indexname "f4" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "f4" indexinfo[].command[].command "type tensor(x[10],y[10])" -indexinfo[].command[].indexname "f4" -indexinfo[].command[].command "word" indexinfo[].command[].indexname "f5" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "f5" indexinfo[].command[].command "type tensor<float>(x[10])" -indexinfo[].command[].indexname "f5" -indexinfo[].command[].command "word" indexinfo[].command[].indexname "f6" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "f6" @@ -39,5 +31,3 @@ indexinfo[].command[].indexname "f7" indexinfo[].command[].command "attribute" indexinfo[].command[].indexname "f7" indexinfo[].command[].command "type tensor<int8>(p{},x[5])" -indexinfo[].command[].indexname "f7" -indexinfo[].command[].command "word" diff --git a/config-model/src/test/derived/tokenization/ilscripts.cfg b/config-model/src/test/derived/tokenization/ilscripts.cfg index c08b6a54c83..cad8ec81879 100644 --- a/config-model/src/test/derived/tokenization/ilscripts.cfg +++ b/config-model/src/test/derived/tokenization/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "tokenization" ilscript[].docfield[] "text" diff --git a/config-model/src/test/derived/types/ilscripts.cfg b/config-model/src/test/derived/types/ilscripts.cfg index 17bed90deb4..73befb221ce 100644 --- a/config-model/src/test/derived/types/ilscripts.cfg +++ b/config-model/src/test/derived/types/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "types" ilscript[].docfield[] "abyte" diff --git a/config-model/src/test/derived/uri_array/ilscripts.cfg b/config-model/src/test/derived/uri_array/ilscripts.cfg index 3dd97e5c11f..0dc87b513ce 100644 --- a/config-model/src/test/derived/uri_array/ilscripts.cfg +++ b/config-model/src/test/derived/uri_array/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "uri_array" ilscript[].docfield[] "my_uri" diff --git a/config-model/src/test/derived/uri_wset/ilscripts.cfg b/config-model/src/test/derived/uri_wset/ilscripts.cfg index 48e07ef9959..cc45ee5ad8f 100644 --- a/config-model/src/test/derived/uri_wset/ilscripts.cfg +++ b/config-model/src/test/derived/uri_wset/ilscripts.cfg @@ -1,4 +1,5 @@ maxtermoccurrences 10000 +maxtokenlength 1000 fieldmatchmaxlength 1000000 ilscript[].doctype "uri_wset" ilscript[].docfield[] "my_uri" diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 5ead9812b56..2846925b9c3 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -151,7 +151,7 @@ public class ModelProvisioningTest { assertEquals("-Xlog:gc", mydisc2.getContainers().get(1).getJvmOptions()); assertEquals("lib/blablamalloc.so", mydisc2.getContainers().get(0).getPreLoad()); assertEquals("lib/blablamalloc.so", mydisc2.getContainers().get(1).getPreLoad()); - assertEquals(45, mydisc2.getMemoryPercentage().get().percentage()); + assertEquals(45, mydisc2.getMemoryPercentage().get().ofContainerAvailable()); assertEquals(Optional.of("-XX:+UseParNewGC"), mydisc2.getJvmGCOptions()); QrStartConfig.Builder qrStartBuilder = new QrStartConfig.Builder(); mydisc2.getConfig(qrStartBuilder); @@ -234,8 +234,8 @@ public class ModelProvisioningTest { assertEquals(2, model.getContentClusters().get("content1").getRootGroup().getNodes().size(), "Nodes in content1"); assertEquals(1, model.getContainerClusters().get("container1").getContainers().size(), "Nodes in container1"); assertEquals(2, model.getContentClusters().get("content").getRootGroup().getNodes().size(), "Nodes in cluster without ID"); - assertEquals(65, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size for container1"); - assertEquals(84, physicalMemoryPercentage(model.getContainerClusters().get("container2")), "Heap size for container2"); + assertEquals(85, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size for container1"); + assertEquals(85, physicalMemoryPercentage(model.getContainerClusters().get("container2")), "Heap size for container2"); assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Type.content, model); assertProvisioned(1, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model); assertProvisioned(2, ClusterSpec.Id.from("content"), ClusterSpec.Type.content, model); @@ -287,8 +287,8 @@ public class ModelProvisioningTest { VespaModel model = tester.createModel(xmlWithNodes, true, deployStateWithClusterEndpoints("container1").deployLogger(logger)); assertEquals(2, model.getContentClusters().get("content1").getRootGroup().getNodes().size(), "Nodes in content1"); assertEquals(2, model.getContainerClusters().get("container1").getContainers().size(), "Nodes in container1"); - assertEquals(18, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size is lowered with combined clusters"); - assertEquals(2025077080L, protonMemorySize(model.getContentClusters().get("content1")), "Memory for proton is lowered to account for the jvm heap"); + assertEquals(24, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size is lowered with combined clusters"); + assertEquals(1876900708, protonMemorySize(model.getContentClusters().get("content1")), "Memory for proton is lowered to account for the jvm heap"); assertProvisioned(0, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model); assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Id.from("container1"), ClusterSpec.Type.combined, model); var msgs = logger.msgs().stream().filter(m -> m.level().equals(Level.WARNING)).toList(); @@ -356,7 +356,7 @@ public class ModelProvisioningTest { VespaModel model = tester.createModel(xmlWithNodes, true, deployStateWithClusterEndpoints("container1")); assertEquals(2, model.getContentClusters().get("content1").getRootGroup().getNodes().size(), "Nodes in content1"); assertEquals(2, model.getContainerClusters().get("container1").getContainers().size(), "Nodes in container1"); - assertEquals(65, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size is normal"); + assertEquals(85, physicalMemoryPercentage(model.getContainerClusters().get("container1")), "Heap size is normal"); assertEquals((long) ((3 - memoryOverheadGb) * (Math.pow(1024, 3))), protonMemorySize(model.getContentClusters().get("content1")), "Memory for proton is normal"); } diff --git a/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java index 8ffbab84fd7..539ec91ee5f 100644 --- a/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java @@ -6,16 +6,13 @@ import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.model.test.utils.DeployLoggerStub; import com.yahoo.vespa.objects.FieldBase; import com.yahoo.yolean.Exceptions; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import static com.yahoo.config.model.test.TestUtil.joinLines; import java.util.Collection; import java.util.List; -import java.util.Optional; import java.util.logging.Level; -import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.*; @@ -233,16 +230,12 @@ public class SummaryTestCase { " document-summary test_summary inherits nonesuch {" + " }" + "}"); - DeployLoggerStub logger = new DeployLoggerStub(); - ApplicationBuilder.createFromStrings(logger, schema); - assertEquals("document-summary 'test_summary' inherits 'nonesuch' but this is not present in schema 'test'", - logger.entries.get(0).message); - // fail("Expected failure"); + ApplicationBuilder.createFromString(schema); + fail("Expected failure"); } catch (IllegalArgumentException e) { - fail(); - // assertEquals("document-summary 'test_summary' inherits nonesuch but this is not present in schema 'test'", - // e.getMessage()); + assertEquals("document-summary 'test_summary' inherits 'nonesuch', but this is not present in schema 'test'", + e.getMessage()); } } diff --git a/config-model/src/test/java/com/yahoo/schema/parser/IntermediateCollectionTestCase.java b/config-model/src/test/java/com/yahoo/schema/parser/IntermediateCollectionTestCase.java index 8d2c98439ca..af6fb82ee7b 100644 --- a/config-model/src/test/java/com/yahoo/schema/parser/IntermediateCollectionTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/parser/IntermediateCollectionTestCase.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.schema.parser; +import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.io.reader.NamedReader; import static com.yahoo.config.model.test.TestUtil.joinLines; @@ -232,5 +233,16 @@ public class IntermediateCollectionTestCase { assertTrue(ex.getMessage().startsWith("Inheritance/reference cycle for documents: ")); } + @Test + void can_detect_errors_in_rank_profile_outside_schema() { + var collection = new IntermediateCollection(); + collection.addSchemaFromFile("src/test/derived/rankprofilemodularity/test.sd"); + var exception = assertThrows(ParseException.class, () -> { + collection.addRankProfileFile("test", "src/test/derived/rankprofilemodularity2/invalid_comment.profile"); + }); + var message = exception.getMessage(); + assertTrue(message.contains("Failed parsing rank-profile from 'src/test/derived/rankprofilemodularity2/invalid_comment.profile'")); + assertTrue(message.contains("Lexical error at line 3, column 6"), message); + } } diff --git a/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java b/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java index 5a2dc218da7..4186e352388 100644 --- a/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java @@ -121,6 +121,39 @@ public class SchemaParserTestCase { } @Test + void significance_can_be_parsed() throws Exception { + String input = """ + schema foo { + rank-profile significance-ranking-0 inherits default { + significance { + use-model: true + } + } + rank-profile significance-ranking-1 { + significance { + use-model: false + } + } + } + """; + + ParsedSchema schema = parseString(input); + assertEquals("foo", schema.name()); + var rplist = schema.getRankProfiles(); + assertEquals(2, rplist.size()); + + var rp0 = rplist.get(0); + assertEquals("significance-ranking-0", rp0.name()); + assertTrue(rp0.isUseSignificanceModel().isPresent()); + assertTrue(rp0.isUseSignificanceModel().get()); + + var rp1 = rplist.get(1); + assertEquals("significance-ranking-1", rp1.name()); + assertTrue(rp1.isUseSignificanceModel().isPresent()); + assertFalse(rp1.isUseSignificanceModel().get()); + } + + @Test void maxOccurrencesCanBeParsed() throws Exception { String input = joinLines ("schema foo {", @@ -137,6 +170,23 @@ public class SchemaParserTestCase { assertEquals(11, field.matchSettings().getMaxTermOccurrences().get()); } + @Test + void maxTokenLengthCanBeParsed() throws Exception { + String input = joinLines + ("schema foo {", + " document foo {", + " field bar type string {", + " indexing: summary | index", + " match { max-token-length: 11 }", + " }", + " }", + "}"); + ParsedSchema schema = parseString(input); + var field = schema.getDocument().getFields().get(0); + assertEquals("bar", field.name()); + assertEquals(11, field.matchSettings().getMaxTokenLength().get()); + } + void checkFileParses(String fileName) throws Exception { var schema = parseFile(fileName); assertNotNull(schema); diff --git a/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java index de99d46b9ca..355a810f5ff 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java @@ -10,6 +10,7 @@ import com.yahoo.schema.Schema; import com.yahoo.schema.ApplicationBuilder; import com.yahoo.schema.AbstractSchemaTestCase; import com.yahoo.schema.document.BooleanIndexDefinition; +import com.yahoo.schema.document.MatchType; import com.yahoo.schema.document.SDDocumentType; import com.yahoo.schema.document.SDField; import com.yahoo.vespa.documentmodel.SummaryField; @@ -155,6 +156,24 @@ public class IndexingScriptRewriterTestCase extends AbstractSchemaTestCase { field); } + @Test + void requireThatMaxTokenLengthIsPropagated() { + var field = new SDField("test", DataType.STRING); + field.getMatching().maxTokenLength(10); + field.parseIndexingScript("test", "{ summary | index }"); + assertIndexingScript("{ input test | tokenize normalize stem:\"BEST\" max-token-length:10 | summary test | index test; }", + field); + } + + @Test + void requireThatMaxTokenLengthIsPropagatedForWordMatch() { + var field = new SDField("test", DataType.STRING); + field.getMatching().maxTokenLength(10).setType(MatchType.WORD); + field.parseIndexingScript("test", "{ summary | index }"); + assertIndexingScript("{ input test | exact max-token-length:10 | summary test | index test; }", + field); + } + private static void assertIndexingScript(String expectedScript, SDField unprocessedField) { assertEquals(expectedScript, processField(unprocessedField).toString()); diff --git a/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java index 9eca2106c5e..5ea097f13fb 100644 --- a/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java @@ -9,6 +9,7 @@ import org.junit.jupiter.api.Test; import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; public class SummaryConsistencyTestCase { @@ -42,4 +43,24 @@ public class SummaryConsistencyTestCase { Schema schema = ApplicationBuilder.createFromString(sd).getSchema(); assertEquals(SummaryTransform.ATTRIBUTECOMBINER, schema.getSummaryField("elem_array_unfiltered").getTransform()); } + + @Test + void testDocumentSummaryWithInheritanceOfNonExistingSummary() { + String schemaString = """ + schema foo { + document foo { + field foo type string { + indexing: summary + } + } + document-summary foo_summary inherits non-existent { + summary foo { + source: foo + } + } + } + """; + assertThrows(IllegalArgumentException.class, () -> ApplicationBuilder.createFromString(schemaString).getSchema()); + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java index 4df9f261dfe..7aa6eb76995 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ClusterInfoTest.java @@ -258,7 +258,7 @@ public class ClusterInfoTest { .provisioned(provisioner.provisioned()) .build(); new VespaModel(new NullConfigModelRegistry(), deployState); - return deployState.provisioned().all(); + return deployState.provisioned().capacities(); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGeneratorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGeneratorTest.java index 7c4968aac84..c24fcb27dc9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGeneratorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/admin/otel/OpenTelemetryConfigGeneratorTest.java @@ -2,8 +2,24 @@ package com.yahoo.vespa.model.admin.otel; import com.yahoo.config.model.ApplicationConfigProducerRoot.StatePortInfo; +import com.yahoo.config.model.producer.TreeConfigProducer; +import com.yahoo.config.model.test.MockRoot; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterMembership; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.HostSpec; +import com.yahoo.config.provision.NodeResources; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.Zone; +import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.Host; +import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.AbstractService; +import com.yahoo.vespa.model.PortAllocBridge; import org.junit.jupiter.api.Test; import java.util.List; +import java.util.Optional; import static org.junit.jupiter.api.Assertions.*; /** @@ -13,10 +29,45 @@ public class OpenTelemetryConfigGeneratorTest { @Test void testBuildsYaml() { - var generator = new OpenTelemetryConfigGenerator(null); - generator.addStatePorts(List.of(new StatePortInfo("localhost", 19098, "config-sentinel", "sentinel"))); + var mockZone = new Zone(SystemName.PublicCd, Environment.prod, RegionName.from("mock")); + var app = ApplicationId.from("mytenant", "myapp", "myinstance"); + var generator = new OpenTelemetryConfigGenerator(mockZone, app); + var root = new MockRoot(); + + var mockHost = new Host(root, "localhost2.local"); + var mockVersion = new com.yahoo.component.Version(8); + var mockCluster = ClusterMembership.from("container/feeding/2/3", mockVersion, Optional.empty()); + var noResource = NodeResources.unspecified(); + var mockHostSpec = new HostSpec("localhost1.local", + noResource, noResource, noResource, + mockCluster, + Optional.empty(), Optional.empty(), Optional.empty()); + var mockHostResource = new HostResource(mockHost, mockHostSpec); + var mockSvc1 = new MockService(root, "sentinel"); + mockSvc1.setHostResource(mockHostResource); + var mockPort1 = new StatePortInfo("localhost", 19098, mockSvc1); + + var mockSvc2 = new MockService(root, "searchnode"); + mockSvc2.setProp("clustername", "mycluster"); + mockSvc2.setProp("clustertype", "mockup"); + var mockPort2 = new StatePortInfo("other.host.local", 19102, mockSvc2); + + generator.addStatePorts(List.of(mockPort1, mockPort2)); String yaml = generator.generate(); + // System.err.println(">>>\n" + yaml + "\n<<<"); assertTrue(yaml.contains("sentinel")); } + static class MockService extends AbstractService { + private final String name; + public MockService(TreeConfigProducer<?> parent, String name) { + super(parent, name); + this.name = name; + } + public String getServiceName() { return name; } + public String getServiceType() { return "dummy"; } + @Override public int getPortCount() { return 0; } + @Override public void allocatePorts(int start, PortAllocBridge from) { } + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java index ae1db366c9f..2e51a425f6d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ComplexFieldsValidatorTestCase.java @@ -143,6 +143,32 @@ public class ComplexFieldsValidatorTestCase { } @Test + void logs_warning_when_complex_fields_have_struct_fields_with_index_and_exact_match() throws IOException, SAXException { + var logger = new MyLogger(); + createModelAndValidate(joinLines( + "schema test {", + " document test {", + " field nesteds type array<nested> {", + " struct-field foo {", + " indexing: attribute | index", + " match {", + " exact", + " exact-terminator: '@@'", + " }", + " }", + " }", + " struct nested {", + " field foo type string {}", + " }", + " }", + "}"), logger); + assertTrue(logger.message.toString().contains("For cluster 'mycluster', schema 'test': " + + "The following complex fields have struct fields with 'indexing: index' which is " + + "not supported and has no effect: nesteds (nesteds.foo). " + + "Remove setting or change to 'indexing: attribute' if needed for matching.")); + } + + @Test void validation_passes_when_only_supported_struct_field_attributes_are_used() throws IOException, SAXException { createModelAndValidate(joinLines("search test {", " document test {", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java index 45125c8eb68..340968f89d1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/JvmHeapSizeValidatorTest.java @@ -45,10 +45,10 @@ class JvmHeapSizeValidatorTest { @Test void fails_on_too_low_heap_size() throws IOException, SAXException { - var deployState = createDeployState(2.2, 1024L * 1024 * 1024); + var deployState = createDeployState(2.3, 1024L * 1024 * 1024); var model = new VespaModel(new NullConfigModelRegistry(), deployState); ValidationTester.expect(new JvmHeapSizeValidator(), model, deployState, - "Allocated memory to JVM in cluster 'container' is too low (0.50GB < 0.60GB). Estimated cost of ONNX models is 1.00GB."); + "Allocated memory to JVM in cluster 'container' is too low (0.51GB < 0.60GB). Estimated cost of ONNX models is 1.00GB."); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java index 89f81dfdaef..590433757c3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/QuotaValidatorTest.java @@ -54,6 +54,12 @@ public class QuotaValidatorTest { } @Test + void test_deploy_within_quota_budget_because_in_dev() { + var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota).setZone(devZone)); + tester.deploy(null, getServices(10), Environment.dev, null, CONTAINER_CLUSTER); + } + + @Test void test_deploy_above_quota_budget_in_publiccd() { var tester = new ValidationTester(13, false, new TestProperties().setHostedVespa(true).setQuota(quota.withBudget(BigDecimal.ONE)).setZone(publicCdZone)); try { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidatorTest.java new file mode 100644 index 00000000000..13e91f60712 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForLocalLLMValidatorTest.java @@ -0,0 +1,79 @@ +// Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation.change; + +import com.yahoo.config.model.api.ConfigChangeAction; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.ValidationTester; +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author lesters + */ +public class RestartOnDeployForLocalLLMValidatorTest { + + private static final String LOCAL_LLM_COMPONENT = RestartOnDeployForLocalLLMValidator.LOCAL_LLM_COMPONENT; + + @Test + void validate_no_restart_on_deploy() { + VespaModel current = createModel(); + VespaModel next = createModel(withComponent(LOCAL_LLM_COMPONENT)); + List<ConfigChangeAction> result = validateModel(current, next); + assertEquals(0, result.size()); + } + + @Test + void validate_restart_on_deploy() { + VespaModel current = createModel(withComponent(LOCAL_LLM_COMPONENT)); + VespaModel next = createModel(withComponent(LOCAL_LLM_COMPONENT)); + List<ConfigChangeAction> result = validateModel(current, next); + assertEquals(1, result.size()); + assertTrue(result.get(0).validationId().isEmpty()); + assertEquals("Need to restart services in cluster 'cluster1' due to use of local LLM", result.get(0).getMessage()); + } + + private static List<ConfigChangeAction> validateModel(VespaModel current, VespaModel next) { + return ValidationTester.validateChanges(new RestartOnDeployForLocalLLMValidator(), + next, + deployStateBuilder().previousModel(current).build()); + } + + private static VespaModel createModel(String component) { + var xml = """ + <services version='1.0'> + <container id='cluster1' version='1.0'> + <http> + <server id='server1' port='8080'/> + </http> + %s + </container> + </services> + """.formatted(component); + DeployState.Builder builder = deployStateBuilder(); + return new VespaModelCreatorWithMockPkg(null, xml).create(builder); + } + + private static VespaModel createModel() { + return createModel(""); + } + + private static String withComponent(String componentClass) { + return "<component id='llm' class='%s' />".formatted(componentClass); + } + + private static DeployState.Builder deployStateBuilder() { + return new DeployState.Builder().properties(new TestProperties()); + } + + private static void assertStartsWith(String expected, List<ConfigChangeAction> result) { + assertTrue(result.get(0).getMessage().startsWith(expected)); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java index 4c0786ea879..484a938476d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/RestartOnDeployForOnnxModelChangesValidatorTest.java @@ -33,7 +33,7 @@ public class RestartOnDeployForOnnxModelChangesValidatorTest { // Must be so large that changing model set or options requires restart (due to using more memory than available), // but not so large that deployment will not work at all with one model - private static final long defaultCost = 723456789; + private static final long defaultCost = 635241309; private static final long defaultHash = 0; @@ -47,8 +47,8 @@ public class RestartOnDeployForOnnxModelChangesValidatorTest { @Test void validate_changed_estimated_cost() { - VespaModel current = createModel(onnxModelCost(70000000, defaultHash)); - VespaModel next = createModel(onnxModelCost(723456789, defaultHash)); + VespaModel current = createModel(onnxModelCost(defaultCost, defaultHash)); + VespaModel next = createModel(onnxModelCost(19 * defaultCost / 20, defaultHash)); List<ConfigChangeAction> result = validateModel(current, next); assertEquals(1, result.size()); assertTrue(result.get(0).validationId().isEmpty()); @@ -58,8 +58,8 @@ public class RestartOnDeployForOnnxModelChangesValidatorTest { @Test void validate_changed_estimated_cost_non_hosted() { boolean hosted = false; - VespaModel current = createModel(onnxModelCost(70000000, defaultHash), hosted); - VespaModel next = createModel(onnxModelCost(723456789, defaultHash), hosted); + VespaModel current = createModel(onnxModelCost(defaultCost, defaultHash), hosted); + VespaModel next = createModel(onnxModelCost(19 * defaultCost / 20, defaultHash), hosted); List<ConfigChangeAction> result = validateModel(current, next, hosted); assertEquals(0, result.size()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java index 8778f0c26c0..0677cabafb0 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/ContentClusterFixture.java @@ -29,11 +29,18 @@ public abstract class ContentClusterFixture { nextCluster = createCluster(nextSd); } + protected ContentClusterFixture(ContentCluster currentCluster, ContentCluster nextCluster) { + this.currentCluster = currentCluster; + this.nextCluster = nextCluster; + } + public ContentClusterFixture(String entireSd) throws Exception { - currentCluster = new ContentClusterBuilder().build( - ContentClusterUtils.createMockRoot(List.of(entireSd))); - nextCluster = new ContentClusterBuilder().build( - ContentClusterUtils.createMockRoot(List.of(entireSd))); + currentCluster = createClusterFromEntireSd(entireSd); + nextCluster = createClusterFromEntireSd(entireSd); + } + + protected static ContentCluster createClusterFromEntireSd(String sdContent) throws Exception { + return new ContentClusterBuilder().build(ContentClusterUtils.createMockRoot(List.of(sdContent))); } private static ContentCluster createCluster(String sdContent) throws Exception { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java index cd54a20523f..247f01068fa 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidatorTest.java @@ -39,6 +39,21 @@ public class IndexingScriptChangeValidatorTest { } } + private static class ComplexFixture extends ContentClusterFixture { + IndexingScriptChangeValidator validator; + public ComplexFixture(String currentSd, String nextSd) throws Exception { + super(createClusterFromEntireSd(currentSd), createClusterFromEntireSd(nextSd)); + validator = new IndexingScriptChangeValidator(ClusterSpec.Id.from("test"), + currentDb().getDerivedConfiguration().getSchema(), + nextDb().getDerivedConfiguration().getSchema()); + } + + @Override + public List<VespaConfigChangeAction> validate() { + return validator.validate(); + } + } + private static class ScriptFixture { private final ScriptExpression currentScript; @@ -56,6 +71,9 @@ public class IndexingScriptChangeValidatorTest { private static final String FIELD = "field f1 type string"; private static final String FIELD_F2 = "field f2 type string"; + private static final String TENSOR_FIELD_F1 = "field f1 type tensor(x[2])"; + private static final String TENSOR_FIELD_F2 = "field f2 type tensor(x[2])"; + private static final String TENSOR_FIELD_F3 = "field f3 type tensor(x[2])"; private static VespaConfigChangeAction expectedReindexingAction(String changedMsg, String fromScript, String toScript) { return expectedReindexingAction("f1", changedMsg, fromScript, toScript); @@ -115,6 +133,28 @@ public class IndexingScriptChangeValidatorTest { } @Test + void requireThatAddingIndexAspectForExtraTensorFieldWithChangedInputRequireReindexing() throws Exception { + new ComplexFixture(joinLines("schema test {", + " document test {", + " " + TENSOR_FIELD_F1 + " { }", + " " + TENSOR_FIELD_F2 + " { }", + " }", + " " + TENSOR_FIELD_F3 + " { indexing: input f1 | attribute }", + "}"), + joinLines("schema test {", + " document test {", + " " + TENSOR_FIELD_F1 + " { }", + " " + TENSOR_FIELD_F2 + " { }", + " }", + " " + TENSOR_FIELD_F3 + " { indexing: input f2 | index | attribute }", + "}")). + assertValidation(List.of(expectedReindexingAction("f3", "add index aspect", + "{ input f1 | attribute f3; }", + "{ input f2 | index f3 | attribute f3; }"))); + } + + + @Test void requireThatSettingDynamicSummaryIsOk() throws Exception { new Fixture(FIELD + " { indexing: summary }", FIELD + " { indexing: summary \n summary: dynamic }"). diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java index 2ea4249883d..9cf34b55ebf 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java @@ -221,24 +221,6 @@ public class ContainerClusterTest { } @Test - void requireThatJvmOmitStackTraceInFastThrowOptionWorks() { - // Empty option if option not set in property - MockRoot root = createRoot(new DeployState.Builder().build()); - ApplicationContainerCluster cluster = newClusterWithSearch(root); - addContainer(root, cluster, "c1", "host-c1"); - ApplicationContainer container = cluster.getContainers().get(0); - assertEquals("", container.getJvmOptions()); - - String jvmOption = "-XX:-foo"; - DeployState deployState = new DeployState.Builder().properties(new TestProperties().setJvmOmitStackTraceInFastThrowOption(jvmOption)).build(); - root = createRoot(deployState); - cluster = newClusterWithSearch(root); - addContainer(root, cluster, "c1", "host-c1"); - container = cluster.getContainers().get(0); - assertEquals(jvmOption, container.getJvmOptions()); - } - - @Test void requireThatWeCanHandleNullJvmOptions() { MockRoot root = createRoot(false); ApplicationContainerCluster cluster = newClusterWithSearch(root); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java index ea43f5c8124..b782366655f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/SchemaChainsTest.java @@ -142,7 +142,7 @@ public class SchemaChainsTest extends SchemaChainsTestBase { assertTrue(chain.phases().isEmpty()); assertEquals(1, chain.inherits().size()); assertEquals("native", chain.inherits(0)); - assertEquals(10, chain.components().size()); + assertEquals(11, chain.components().size()); assertEquals("com.yahoo.prelude.querytransform.PhrasingSearcher@vespa", chain.components(0)); assertEquals("com.yahoo.prelude.searcher.FieldCollapsingSearcher@vespa", chain.components(1)); assertEquals("com.yahoo.search.yql.MinimalQueryInserter@vespa", chain.components(2)); @@ -153,13 +153,14 @@ public class SchemaChainsTest extends SchemaChainsTestBase { assertEquals("com.yahoo.prelude.semantics.SemanticSearcher@vespa", chain.components(7)); assertEquals("com.yahoo.search.grouping.GroupingQueryParser@vespa", chain.components(8)); assertEquals("com.yahoo.search.querytransform.WeakAndReplacementSearcher@vespa", chain.components(9)); + assertEquals("com.yahoo.search.searchers.OpportunisticWeakAndSearcher@vespa", chain.components(10)); assertTrue(chain.excludes().isEmpty()); assertEquals(ChainsConfig.Chains.Type.SEARCH, chain.type()); } @Test public void require_all_default_chains_are_correct() { - assertEquals(63, chainsConfig.components().size()); + assertEquals(64, chainsConfig.components().size()); assertEquals(10, chainsConfig.chains().size()); validateVespaPhasesChain(findChain("vespaPhases")); validateNativeChain(findChain("native")); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index e704da08d18..60ea37cad3f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -17,6 +17,7 @@ import com.yahoo.config.model.provision.InMemoryProvisioner; import com.yahoo.config.model.provision.SingleNodeProvisioner; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.model.test.MockRoot; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; @@ -49,6 +50,7 @@ import com.yahoo.vespa.model.container.component.Handler; import com.yahoo.vespa.model.content.utils.ContentClusterUtils; import com.yahoo.vespa.model.test.VespaModelTester; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; +import com.yahoo.yolean.Exceptions; import org.junit.jupiter.api.Test; import org.w3c.dom.Element; import org.xml.sax.SAXException; @@ -598,7 +600,7 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { .setCloudAccount(cloudAccount)) .build()); assertEquals(2, model.hostSystem().getHosts().size()); - assertEquals(List.of(cloudAccount), model.provisioned().all().values() + assertEquals(List.of(cloudAccount), model.provisioned().capacities().values() .stream() .map(capacity -> capacity.cloudAccount().get()) .toList()); @@ -741,6 +743,29 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { } @Test + void testerContainer() { + createModelWithTesterNodes("<nodes count='1' docker-image='foo/bar/baz'><resources vcpu='0.1' memory='1Gb' disk='1Gb'/></nodes>"); + + assertEquals("In container cluster 'default': tester cannot run on more than 1 node, but 2 nodes were specified", + Exceptions.toMessageString(assertThrows(IllegalArgumentException.class, + () -> createModelWithTesterNodes("<nodes count='2'/>")))); + + assertEquals("In container cluster 'default': tester resources must be absolute, but min and max resources differ: specification of dedicated " + + "min 1 nodes with [vcpu: 0.0, memory: 1.0 Gb, disk: 0.0 Gb, bandwidth: 0.3 Gbps, architecture: any] " + + "max 1 nodes with [vcpu: 0.0, memory: 2.0 Gb, disk: 0.0 Gb, bandwidth: 0.3 Gbps, architecture: any]", + Exceptions.toMessageString(assertThrows(IllegalArgumentException.class, + () -> createModelWithTesterNodes("<nodes><resources memory='[1Gb, 2Gb]'/></nodes>")))); + } + + void createModelWithTesterNodes(String testerNodesXml) { + String containerXml = "<container id='default' version='1.0'>%s</container>".formatted(testerNodesXml); + VespaModelTester tester = new VespaModelTester(); + tester.setApplicationId("t", "a", "i-t"); + tester.addHosts(3); + tester.createModel(containerXml, true); + } + + @Test void cluster_with_zookeeper() { Function<Integer, String> servicesXml = (nodeCount) -> "<container version='1.0' id='default'>" + "<nodes count='" + nodeCount + "'/>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 786caa4b317..5dca5a9eaa1 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.content; +import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.ModelContext; @@ -48,6 +49,8 @@ import java.util.Map; import java.util.Optional; import java.util.OptionalInt; import java.util.Set; +import java.util.function.Consumer; +import java.util.logging.Level; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -941,8 +944,19 @@ public class ContentClusterTest extends ContentBaseTest { } private static ContentCluster createOneNodeCluster(String clusterXml, TestProperties props, Optional<Flavor> flavor) throws Exception { + return createOneNodeCluster(clusterXml, props, flavor, new StringBuffer()); + } + + private static ContentCluster createOneNodeCluster(String clusterXml, TestProperties props, + Optional<Flavor> flavor, StringBuffer deployWarningsBuffer) throws Exception { + DeployLogger logger = (level, message) -> { + if (level == Level.WARNING) { // only care about warnings + deployWarningsBuffer.append("%s\n".formatted(message)); + } + }; DeployState.Builder deployStateBuilder = new DeployState.Builder() - .properties(props); + .properties(props) + .deployLogger(logger); MockRoot root = flavor.isPresent() ? ContentClusterUtils.createMockRoot(new SingleNodeProvisioner(flavor.get()), List.of(), deployStateBuilder) : @@ -1471,13 +1485,18 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(expectedGroupsAllowedDown, config.max_number_of_groups_allowed_to_be_down()); } - private boolean resolveDistributorOperationCancellationConfig(Integer featureLevel) throws Exception { + private StorDistributormanagerConfig resolveDistributorConfig(Consumer<TestProperties> propertyMutator) throws Exception { var properties = new TestProperties(); - if (featureLevel != null) { - properties.setContentLayerMetadataFeatureLevel(featureLevel); - } - var cfg = resolveStorDistributormanagerConfig(properties); - return cfg.enable_operation_cancellation(); + propertyMutator.accept(properties); + return resolveStorDistributormanagerConfig(properties); + } + + private boolean resolveDistributorOperationCancellationConfig(Integer featureLevel) throws Exception { + return resolveDistributorConfig((props) -> { + if (featureLevel != null) { + props.setContentLayerMetadataFeatureLevel(featureLevel); + } + }).enable_operation_cancellation(); } @Test @@ -1488,6 +1507,21 @@ public class ContentClusterTest extends ContentBaseTest { assertTrue(resolveDistributorOperationCancellationConfig(2)); } + private boolean resolveDistributorSymmetricReplicaSelectionConfig(Boolean flagValue) throws Exception { + return resolveDistributorConfig((props) -> { + if (flagValue != null) { + props.setSymmetricPutAndActivateReplicaSelection(flagValue); + } + }).symmetric_put_and_activate_replica_selection(); + } + + @Test + void distributor_symmetric_replica_selection_config_controlled_by_properties() throws Exception { + assertFalse(resolveDistributorSymmetricReplicaSelectionConfig(null)); // defaults to false + assertFalse(resolveDistributorSymmetricReplicaSelectionConfig(false)); + assertTrue(resolveDistributorSymmetricReplicaSelectionConfig(true)); + } + @Test void node_distribution_key_outside_legal_range_is_disallowed() { // Only [0, UINT16_MAX - 1] is a valid range. UINT16_MAX is a special content layer-internal @@ -1506,6 +1540,42 @@ public class ContentClusterTest extends ContentBaseTest { } } + private String createClusterAndGetDeploymentWarnings(String xml) { + var warningBuf = new StringBuffer(); + try { + createOneNodeCluster(xml, new TestProperties(), Optional.empty(), warningBuf); + } catch (Exception e) { + throw new RuntimeException(e); + } + return warningBuf.toString(); + }; + + private static String clusterXmlWithNodeDistributionKey(int key) { + return "<content version=\"1.0\" id=\"mockcluster\">" + + " <redundancy>1</redundancy>" + + " <documents/>" + + " <group>" + + " <node distribution-key=\"%d\" hostalias=\"mockhost\"/>".formatted(key) + + " </group>" + + "</content>"; + } + + @Test + void distribution_key_much_higher_than_node_count_logs_deployment_warning() { + // "much higher" is somewhat arbitrary, but needs to be kept in track with threshold in `ContentCluster`. + String warnings = createClusterAndGetDeploymentWarnings(clusterXmlWithNodeDistributionKey(101)); + assertEquals(warnings, "Content cluster 'mockcluster' has 1 node(s), but the highest distribution " + + "key is 101. Having much higher distribution keys than the number of nodes " + + "is not recommended, as it may negatively affect performance. " + + "See https://docs.vespa.ai/en/reference/services-content.html#node\n"); + } + + @Test + void distribution_key_not_much_higher_than_node_count_does_not_log_deployment_warning() { + String warnings = createClusterAndGetDeploymentWarnings(clusterXmlWithNodeDistributionKey(100)); + assertEquals(warnings, ""); + } + private String servicesWithGroups(int groupCount, double minGroupUpRatio) { String services = String.format("<?xml version='1.0' encoding='UTF-8' ?>" + "<services version='1.0'>" + diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java index 4fd61f59ed7..6114cc7d8bf 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java @@ -8,9 +8,14 @@ import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter; import ai.vespa.rankingexpression.importer.vespa.VespaImporter; import ai.vespa.rankingexpression.importer.xgboost.XGBoostImporter; import com.yahoo.config.FileReference; +import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.model.ApplicationPackageTester; import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ContainerEndpoint; +import com.yahoo.config.model.api.OnnxModelCost; +import com.yahoo.config.model.api.OnnxModelCost.Calculator; +import com.yahoo.config.model.api.OnnxModelCost.ModelInfo; +import com.yahoo.config.model.api.OnnxModelOptions; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.io.IOUtils; @@ -22,7 +27,10 @@ import org.xml.sax.SAXException; import java.io.IOException; import java.io.UncheckedIOException; +import java.net.URI; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; @@ -47,6 +55,7 @@ public class ImportedModelTester { private final String modelName; private final Path applicationDir; private final DeployState deployState; + public final Calculator calculator = new MockCalculator(); public ImportedModelTester(String modelName, Path applicationDir) { this(modelName, applicationDir, new DeployState.Builder()); @@ -58,6 +67,7 @@ public class ImportedModelTester { deployState = deployStateBuilder.applicationPackage(ApplicationPackageTester.create(applicationDir.toString()).app()) .endpoints(Set.of(new ContainerEndpoint("container", ApplicationClusterEndpoint.Scope.zone, List.of("default.example.com")))) .modelImporters(importers) + .onnxModelCost((pkg, app, cluster) -> calculator) .build(); } @@ -98,4 +108,19 @@ public class ImportedModelTester { } } + public static class MockCalculator implements OnnxModelCost.Calculator { + private final Map<String, ModelInfo> models = new HashMap<>(); + @Override public long aggregatedModelCostInBytes() { return models.size(); } + @Override public void registerModel(ApplicationFile path, OnnxModelOptions onnxModelOptions) { + models.put(path.toString(), new ModelInfo(path.toString(), 1, 1, onnxModelOptions)); + } + @Override public void registerModel(URI uri, OnnxModelOptions onnxModelOptions) { + models.put(uri.toString(), new ModelInfo(uri.toString(), 1, 1, onnxModelOptions)); + } + @Override public Map<String, ModelInfo> models() { return Map.copyOf(models); } + @Override public void setRestartOnDeploy() { } + @Override public boolean restartOnDeploy() { return false; } + @Override public void store() { } + } + } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java index cc33c8561fc..bced4c546c6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ModelEvaluationTest.java @@ -55,6 +55,7 @@ public class ModelEvaluationTest { RankProfilesConfig config = new RankProfilesConfig(b); assertEquals(0, config.rankprofile().size()); + assertEquals(0, tester.calculator.aggregatedModelCostInBytes()); } finally { IOUtils.recursiveDeleteDir(appDir.append(ApplicationPackage.MODELS_GENERATED_DIR).toFile()); @@ -69,6 +70,7 @@ public class ModelEvaluationTest { try { ImportedModelTester tester = new ImportedModelTester("ml_serving", appDir); assertHasMlModels(tester.createVespaModel(), appDir); + assertEquals(3, tester.calculator.aggregatedModelCostInBytes()); // At this point the expression is stored - copy application to another location which do not have a models dir storedAppDir.toFile().mkdirs(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/significance/test/SignificanceModelTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/significance/test/SignificanceModelTestCase.java index 00e95a34287..26e8c67a226 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/significance/test/SignificanceModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/significance/test/SignificanceModelTestCase.java @@ -37,9 +37,6 @@ public class SignificanceModelTestCase { ApplicationContainerCluster containerCluster = vespaModel.getContainerClusters().get("container"); var significanceConfig = assertSignificancePresent(containerCluster); assertEquals(3, significanceConfig.model().size()); - assertEquals("en", significanceConfig.model().get(0).language()); - assertEquals("no", significanceConfig.model().get(1).language()); - assertEquals("ru", significanceConfig.model().get(2).language()); assertEquals("models/idf-norwegian-wiki.json.zst", modelReference(significanceConfig.model().get(1), "path").path().orElseThrow().value()); assertEquals("https://some/uri/blob.json", modelReference(significanceConfig.model().get(2), "path").url().orElseThrow().value()); diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index 7333ef5a87b..a413ec7753b 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -168,7 +168,7 @@ </threadpool> <significance> - <model language="en" model-id="idf-wiki-simple-english" path="models/idf-simple-english-wiki.json.zst" /> + <model model-id="idf-wiki-simple-english" path="models/idf-simple-english-wiki.json.zst" /> </significance> </search> |